stroblme
34243e6902
save_artifact/load_artifact outside a node use the worker's own store code in $FLUKSIO_ARTIFACT_DIR or <tmp>/fluksio-artifacts, so a node can be called by hand. --follow prints status changes and each node's resource wait; the placer's waiting snapshot now carries the run. fluksio cancel <run>...
101 lines
3.5 KiB
Python
101 lines
3.5 KiB
Python
"""The Fluksio engine, and the decorators that declare flows in your own code.
|
|
|
|
Two audiences, one import name. In a research repository, ``from fluksio
|
|
import Port, node, Flow`` is the authoring API: it says which of your existing
|
|
functions are nodes and which nodes make up a flow, and ``fluksio sync``
|
|
uploads what it finds. Those names come from :mod:`fluksio.sdk`, which imports
|
|
nothing but the standard library. ``FLOWS`` and ``build`` come with them, so a
|
|
test can check the declarations without an engine.
|
|
|
|
Inside a node, ``import fluksio`` is not this package at all: the worker
|
|
installs a reporter of its own under that name before any node code runs, so
|
|
what a node gets is :mod:`fluksio_worker.worker_main`'s ``emit`` and
|
|
``save_artifact`` — and inert copies of the decorators, since a module the
|
|
node imports may well declare them at its top. Everywhere else the functions
|
|
below stand in, so a node called directly — in a test, in a notebook — behaves
|
|
like the real one: ``emit`` does nothing and the artifact helpers use a local
|
|
directory.
|
|
|
|
Nothing is imported from the rest of the package here. Every ``from fluksio.x
|
|
import y`` in the engine passes through this module, and an import cycle or a
|
|
second of start-up cost would both begin here.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
from importlib.metadata import PackageNotFoundError
|
|
from importlib.metadata import version as _version
|
|
from typing import Any
|
|
|
|
from fluksio.sdk import FLOWS, Flow, Port, build, node, use
|
|
|
|
try:
|
|
__version__ = _version("fluksio")
|
|
except PackageNotFoundError: # pragma: no cover - a checkout that was never installed
|
|
__version__ = "0.0.0+unknown"
|
|
|
|
__all__ = [
|
|
"FLOWS",
|
|
"Flow",
|
|
"Port",
|
|
"build",
|
|
"emit",
|
|
"load_artifact",
|
|
"node",
|
|
"save_artifact",
|
|
"use",
|
|
]
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def emit(**ports: Any) -> None:
|
|
"""Publish values on a node's declared output ports, mid-run.
|
|
|
|
Outside a node there is nowhere to publish to, and this does nothing on
|
|
purpose: a node function called directly — in a test, in a notebook, under
|
|
a debugger — is ordinary Python, and having it die on a progress report
|
|
would defeat the point of the code staying yours.
|
|
"""
|
|
logger.debug("fluksio.emit(%s) outside a node", ", ".join(ports))
|
|
|
|
|
|
def _local_store() -> Any:
|
|
"""The worker's own artifact code, pointed at a directory of this machine.
|
|
|
|
``$FLUKSIO_ARTIFACT_DIR`` when it is set, which is how a worker finds the
|
|
engine's store, and a directory in the temporary location otherwise. The
|
|
references that come out are the ones a node gets inside the engine, so
|
|
what a node returns can be handed to the next one by hand.
|
|
"""
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
from fluksio_worker.worker_main import ARTIFACT_DIR_ENV, _Reporter
|
|
|
|
os.environ.setdefault(
|
|
ARTIFACT_DIR_ENV, str(Path(tempfile.gettempdir()) / "fluksio-artifacts")
|
|
)
|
|
return _Reporter("fluksio")
|
|
|
|
|
|
def save_artifact(
|
|
source: Any,
|
|
name: str = "",
|
|
media_type: str = "application/octet-stream",
|
|
volatile: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Put bytes or a file in the artifact store and return a reference to them.
|
|
|
|
Outside a node the store is a local directory, see :func:`_local_store`.
|
|
"""
|
|
reference: dict[str, Any] = _local_store().save_artifact(
|
|
source, name, media_type, volatile
|
|
)
|
|
return reference
|
|
|
|
|
|
def load_artifact(ref: dict[str, Any]) -> str:
|
|
"""Fetch what an artifact reference points at, and return a path to it."""
|
|
return str(_local_store().load_artifact(ref))
|