SDK: artifacts work outside a node, run --follow prints status and queue waits, fluksio cancel

save_artifact/load_artifact outside a node use the worker's own store code in
$FLUKSIO_ARTIFACT_DIR or <tmp>/fluksio-artifacts, so a node can be called by
hand. --follow prints status changes and each node's resource wait; the
placer's waiting snapshot now carries the run. fluksio cancel <run>...
This commit is contained in:
2026-10-28 08:07:35 +01:00
parent 4c2df681d3
commit 34243e6902
14 changed files with 274 additions and 27 deletions
+33 -11
View File
@@ -11,9 +11,10 @@ Inside a node, ``import fluksio`` is not this package at all: the worker
installs a reporter of its own under that name before any node code runs, so
what a node gets is :mod:`fluksio_worker.worker_main`'s ``emit`` and
``save_artifact`` — and inert copies of the decorators, since a module the
node imports may well declare them at its top. The stubs below stand in the
same place everywhere else, and say so rather than failing as a missing
attribute.
node imports may well declare them at its top. Everywhere else the functions
below stand in, so a node called directly — in a test, in a notebook — behaves
like the real one: ``emit`` does nothing and the artifact helpers use a local
directory.
Nothing is imported from the rest of the package here. Every ``from fluksio.x
import y`` in the engine passes through this module, and an import cycle or a
@@ -21,6 +22,7 @@ second of start-up cost would both begin here.
"""
import logging
import os
from importlib.metadata import PackageNotFoundError
from importlib.metadata import version as _version
from typing import Any
@@ -46,11 +48,6 @@ __all__ = [
logger = logging.getLogger(__name__)
_OUTSIDE = (
"fluksio.{name}() only works inside a node: the worker running it installs "
"the real one. There is nothing to {verb} out here."
)
def emit(**ports: Any) -> None:
"""Publish values on a node's declared output ports, mid-run.
@@ -63,16 +60,41 @@ def emit(**ports: Any) -> None:
logger.debug("fluksio.emit(%s) outside a node", ", ".join(ports))
def _local_store() -> Any:
"""The worker's own artifact code, pointed at a directory of this machine.
``$FLUKSIO_ARTIFACT_DIR`` when it is set, which is how a worker finds the
engine's store, and a directory in the temporary location otherwise. The
references that come out are the ones a node gets inside the engine, so
what a node returns can be handed to the next one by hand.
"""
import tempfile
from pathlib import Path
from fluksio_worker.worker_main import ARTIFACT_DIR_ENV, _Reporter
os.environ.setdefault(
ARTIFACT_DIR_ENV, str(Path(tempfile.gettempdir()) / "fluksio-artifacts")
)
return _Reporter("fluksio")
def save_artifact(
source: Any,
name: str = "",
media_type: str = "application/octet-stream",
volatile: bool = False,
) -> dict[str, Any]:
"""Put bytes in the artifact store and return a reference to them."""
raise RuntimeError(_OUTSIDE.format(name="save_artifact", verb="save to"))
"""Put bytes or a file in the artifact store and return a reference to them.
Outside a node the store is a local directory, see :func:`_local_store`.
"""
reference: dict[str, Any] = _local_store().save_artifact(
source, name, media_type, volatile
)
return reference
def load_artifact(ref: dict[str, Any]) -> str:
"""Fetch what an artifact reference points at, and return a path to it."""
raise RuntimeError(_OUTSIDE.format(name="load_artifact", verb="load from"))
return str(_local_store().load_artifact(ref))