"""The Fluksio engine, and the decorators that declare flows in your own code. Two audiences, one import name. In a research repository, ``from fluksio import Port, node, Flow`` is the authoring API: it says which of your existing functions are nodes and which nodes make up a flow, and ``fluksio sync`` uploads what it finds. Those names come from :mod:`fluksio.sdk`, which imports nothing but the standard library. ``FLOWS`` and ``build`` come with them, so a test can check the declarations without an engine. Inside a node, ``import fluksio`` is not this package at all: the worker installs a reporter of its own under that name before any node code runs, so what a node gets is :mod:`fluksio_worker.worker_main`'s ``emit`` and ``save_artifact`` — and inert copies of the decorators, since a module the node imports may well declare them at its top. Everywhere else the functions below stand in, so a node called directly — in a test, in a notebook — behaves like the real one: ``emit`` does nothing and the artifact helpers use a local directory. Nothing is imported from the rest of the package here. Every ``from fluksio.x import y`` in the engine passes through this module, and an import cycle or a second of start-up cost would both begin here. """ import logging import os from importlib.metadata import PackageNotFoundError from importlib.metadata import version as _version from typing import Any from fluksio.sdk import FLOWS, Flow, Port, build, node, use try: __version__ = _version("fluksio") except PackageNotFoundError: # pragma: no cover - a checkout that was never installed __version__ = "0.0.0+unknown" __all__ = [ "FLOWS", "Flow", "Port", "build", "emit", "load_artifact", "node", "save_artifact", "use", ] logger = logging.getLogger(__name__) def emit(**ports: Any) -> None: """Publish values on a node's declared output ports, mid-run. Outside a node there is nowhere to publish to, and this does nothing on purpose: a node function called directly — in a test, in a notebook, under a debugger — is ordinary Python, and having it die on a progress report would defeat the point of the code staying yours. """ logger.debug("fluksio.emit(%s) outside a node", ", ".join(ports)) def _local_store() -> Any: """The worker's own artifact code, pointed at a directory of this machine. ``$FLUKSIO_ARTIFACT_DIR`` when it is set, which is how a worker finds the engine's store, and a directory in the temporary location otherwise. The references that come out are the ones a node gets inside the engine, so what a node returns can be handed to the next one by hand. """ import tempfile from pathlib import Path from fluksio_worker.worker_main import ARTIFACT_DIR_ENV, _Reporter os.environ.setdefault( ARTIFACT_DIR_ENV, str(Path(tempfile.gettempdir()) / "fluksio-artifacts") ) return _Reporter("fluksio") def save_artifact( source: Any, name: str = "", media_type: str = "application/octet-stream", volatile: bool = False, ) -> dict[str, Any]: """Put bytes or a file in the artifact store and return a reference to them. Outside a node the store is a local directory, see :func:`_local_store`. """ reference: dict[str, Any] = _local_store().save_artifact( source, name, media_type, volatile ) return reference def load_artifact(ref: dict[str, Any]) -> str: """Fetch what an artifact reference points at, and return a path to it.""" return str(_local_store().load_artifact(ref))