SDK: sync and run import the declaring module in a child process
The command line no longer imports a research module, so import-time GPU work ends with a short-lived child instead of holding a CUDA context for the length of a followed run. fluksio.sdk.declare starts the child, which writes each flow's document and shims as JSON; Declared stands in for Flow in sync(). Errors keep their type and message, a child's traceback reaches the user. Claude-Session: https://claude.ai/code/session_01SSi5AvZ7pFmySK8zy3NPMh
This commit is contained in:
@@ -14,6 +14,7 @@ have to resolve inside a worker, where ``fluksio`` is the reporter module.
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import copy
|
||||
import inspect
|
||||
import json
|
||||
import re
|
||||
@@ -27,6 +28,7 @@ from fluksio.sdk.imports import digest_of, reached
|
||||
|
||||
__all__ = [
|
||||
"FLOWS",
|
||||
"Declared",
|
||||
"Flow",
|
||||
"Port",
|
||||
"SyncError",
|
||||
@@ -750,18 +752,27 @@ class Flow:
|
||||
],
|
||||
"outputs": self.outputs,
|
||||
}
|
||||
if origin is not None:
|
||||
declared_in = _relative(self.declared_in, str(origin.get("repo") or ""))
|
||||
doc["origin"] = origin | {
|
||||
"declared_in": declared_in,
|
||||
"module": self.module if declared_in else "",
|
||||
}
|
||||
return doc
|
||||
return _stamped(doc, origin, self.declared_in, self.module)
|
||||
|
||||
def shims(self) -> dict[str, str]:
|
||||
"""One generated node body per node, keyed by node id."""
|
||||
return {spec.id: _shim(spec) for spec in self.nodes}
|
||||
|
||||
def declaration(self) -> dict[str, Any]:
|
||||
"""Everything `sync` takes of this flow, as JSON a process can hand on.
|
||||
|
||||
The document is worked out here, where the code was imported, and the
|
||||
origin is stamped onto it later by :class:`Declared`: only the process
|
||||
that runs `sync` knows which repository it is stamping with.
|
||||
"""
|
||||
return {
|
||||
"name": self.name,
|
||||
"document": self.document(),
|
||||
"shims": self.shims(),
|
||||
"declared_in": self.declared_in,
|
||||
"module": self.module,
|
||||
}
|
||||
|
||||
def submit(
|
||||
self, *, seed: int | None = None, client: Any = None, **params: Any
|
||||
) -> Any:
|
||||
@@ -783,6 +794,47 @@ class Flow:
|
||||
return f"Flow({self.name!r}, nodes={[n.id for n in self.nodes]})"
|
||||
|
||||
|
||||
def _stamped(
|
||||
doc: dict[str, Any], origin: dict[str, Any] | None, declared_in: str, module: str
|
||||
) -> dict[str, Any]:
|
||||
"""Where the flow came from, added to its document."""
|
||||
if origin is not None:
|
||||
inside = _relative(declared_in, str(origin.get("repo") or ""))
|
||||
doc["origin"] = origin | {
|
||||
"declared_in": inside,
|
||||
"module": module if inside else "",
|
||||
}
|
||||
return doc
|
||||
|
||||
|
||||
class Declared:
|
||||
"""A flow another process imported: what `sync` needs, without the code.
|
||||
|
||||
Stands where a :class:`Flow` does for :func:`fluksio.sdk.client.sync`, so
|
||||
`fluksio sync` can read declarations in a child process and never import
|
||||
the module itself. Built from :meth:`Flow.declaration`.
|
||||
"""
|
||||
|
||||
def __init__(self, data: dict[str, Any]) -> None:
|
||||
self.name: str = data["name"]
|
||||
self._data = data
|
||||
|
||||
def document(self, origin: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
return _stamped(
|
||||
copy.deepcopy(self._data["document"]),
|
||||
origin,
|
||||
self._data["declared_in"],
|
||||
self._data["module"],
|
||||
)
|
||||
|
||||
def shims(self) -> dict[str, str]:
|
||||
shims: dict[str, str] = dict(self._data["shims"])
|
||||
return shims
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Declared({self.name!r})"
|
||||
|
||||
|
||||
def build(target: Flow, origin: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""The flow document for a flow — checked, without a version."""
|
||||
return target.document(origin)
|
||||
|
||||
Reference in New Issue
Block a user