Add a Python SDK: flows declared in your own repository
A data scientist keeps their code where it is and decorates it: `@node` declares a function's ports beside the function, `Flow(name, nodes=[...])` says which of them make a flow, and `use(fn, wire=..., **settings)` rebinds one for a single flow. `fluksio sync` uploads the document plus a generated import shim per node, so the store still holds a complete, runnable, git-versioned definition while the code it imports stays theirs. `fluksio login|run|runs` and `flow.submit().wait()` are the client half, over the run endpoints that already existed. Runs record the user repository's commit beside the store's, so "what code produced this number" is answerable on the side that now holds the code. - `fluksio/sdk/`: ports, decorators, the flow builder and its checks, the shim generator, an HTTP client and sync. Standard library only at import, so `from fluksio import node` in a training script pulls in no engine. - `FlowDef.origin` marks a flow code-defined; `Run.origin_commit` carries the repository's commit; `POST /modules/refresh` retires the workers without an install, which every sync calls — a worker holds the imported package in memory, so an edit to it is invisible until the process goes. - The canvas shows a generated body read-only and names the repository to edit instead; a body edited there stops the next sync rather than being discarded. - The worker's reporter carries inert `Port`, `node`, `use` and `Flow`, since the shim imports a module whose first line declares them. - `examples/myresearch` is the worked example, `make sync-example` uploads it. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012ue1tkFWB1bcGy3aWhCKpU
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,657 @@
|
||||
"""Declare flows and nodes in the repository you already have.
|
||||
|
||||
A research repository keeps its functions where they are; the decorators here
|
||||
say which of them are nodes and which nodes make up a flow. ``fluksio sync``
|
||||
uploads the flow document and, per node, a generated import shim — so the flow
|
||||
store still holds a complete, runnable, git-versioned definition, and the code
|
||||
it imports stays yours.
|
||||
|
||||
Nothing here imports the engine. A ``from fluksio import node`` in a training
|
||||
script must not drag FastAPI and SQLAlchemy in behind it, and the same names
|
||||
have to resolve inside a worker, where ``fluksio`` is the reporter module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import re
|
||||
import warnings
|
||||
from collections.abc import Callable, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any, TypeVar
|
||||
|
||||
__all__ = [
|
||||
"FLOWS",
|
||||
"Flow",
|
||||
"Port",
|
||||
"SyncError",
|
||||
"build",
|
||||
"node",
|
||||
"shims",
|
||||
"use",
|
||||
]
|
||||
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
||||
#: Mirrors :class:`fluksio.flow.messages.DType`. Mirrored rather than imported:
|
||||
#: importing it would pull the engine into a research process.
|
||||
DTYPES = frozenset(
|
||||
{"float", "int", "str", "bool", "json", "series", "record", "list", "artifact"}
|
||||
)
|
||||
#: What a list may hold, mirroring ``messages._ITEM_TYPES``.
|
||||
ITEM_TYPES = frozenset({"float", "int", "str", "bool", "json", "record"})
|
||||
#: Settings the engine reads itself, mirroring ``nodes.base.RESERVED_SETTINGS``.
|
||||
RESERVED_SETTINGS = frozenset({"synchronous"})
|
||||
#: Mirrors ``fluksio.flow.schemas.NAME_PATTERN``.
|
||||
NAME_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$")
|
||||
#: First line of every generated node body. Its absence is how ``sync`` knows a
|
||||
#: node's source was edited on the canvas.
|
||||
MARKER = "# generated by fluksio sync"
|
||||
|
||||
|
||||
class SyncError(Exception):
|
||||
"""A declaration the engine would refuse, caught where it was written."""
|
||||
|
||||
|
||||
#: Every flow declared in the modules imported so far. ``fluksio sync`` reads it.
|
||||
FLOWS: dict[str, Flow] = {}
|
||||
|
||||
|
||||
def _name(value: str, what: str) -> str:
|
||||
if not NAME_PATTERN.match(value):
|
||||
raise SyncError(
|
||||
f"{what} '{value}': use lowercase letters, digits and underscores, "
|
||||
"starting with a letter"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
class Port:
|
||||
"""One port of a node, and the message it binds to.
|
||||
|
||||
A thin mirror of :class:`fluksio.flow.messages.MessageSpec`. ``name`` is
|
||||
the wiring — two nodes are connected because one provides the name another
|
||||
requires — and ``port`` is what the function calls its parameter, which
|
||||
defaults to the name.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
dtype: str | None = None,
|
||||
*,
|
||||
port: str = "",
|
||||
stream: bool = False,
|
||||
initial: Any = None,
|
||||
item: str | None = None,
|
||||
interval: float = 0.0,
|
||||
trigger: bool = True,
|
||||
) -> None:
|
||||
#: Whether the caller said the type or left it to be resolved from
|
||||
#: whoever provides the message. A message has one type, declared where
|
||||
#: it is produced; requiring it does not mean declaring it again.
|
||||
self.explicit = dtype is not None
|
||||
dtype = dtype or "float"
|
||||
if dtype not in DTYPES:
|
||||
raise SyncError(f"port '{name}': unknown type '{dtype}'")
|
||||
if item is not None and item not in ITEM_TYPES:
|
||||
raise SyncError(f"port '{name}': a list cannot hold '{item}' items")
|
||||
if interval < 0:
|
||||
raise SyncError(f"port '{name}': interval cannot be negative")
|
||||
self.name = name
|
||||
self.dtype = dtype
|
||||
self.port = port or name.rsplit(".", 1)[-1]
|
||||
self.stream = stream
|
||||
self.initial = initial
|
||||
self.item = item
|
||||
self.interval = interval
|
||||
self.trigger = trigger
|
||||
if not self.port.isidentifier():
|
||||
raise SyncError(
|
||||
f"port '{name}': '{self.port}' is not a Python identifier, so no "
|
||||
"parameter can carry it — pass port='...'"
|
||||
)
|
||||
|
||||
def spec(self) -> dict[str, Any]:
|
||||
"""The ``MessageSpec`` this port serialises to."""
|
||||
out: dict[str, Any] = {
|
||||
"name": self.name,
|
||||
"port": self.port,
|
||||
"dtype": self.dtype,
|
||||
"interval": self.interval,
|
||||
"trigger": self.trigger,
|
||||
"stream": self.stream,
|
||||
}
|
||||
if self.item is not None:
|
||||
out["item"] = self.item
|
||||
return out
|
||||
|
||||
def replace(self, name: str = "") -> Port:
|
||||
"""A copy of this port, optionally bound to a different message."""
|
||||
copy = Port(
|
||||
name or self.name,
|
||||
self.dtype,
|
||||
port=self.port,
|
||||
stream=self.stream,
|
||||
initial=self.initial,
|
||||
item=self.item,
|
||||
interval=self.interval,
|
||||
trigger=self.trigger,
|
||||
)
|
||||
copy.explicit = self.explicit
|
||||
return copy
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Port({self.name!r}, {self.dtype!r})"
|
||||
|
||||
|
||||
def _ports(value: Sequence[str | Port] | str | Port) -> tuple[list[Port], bool]:
|
||||
"""Normalise a ports declaration; the flag says it was a single port.
|
||||
|
||||
A single ``Port`` rather than a list of one is the opt-in to a bare return:
|
||||
the shim wraps the value in the dict the engine expects. It has to be
|
||||
explicit, because an artifact reference is itself a dict and a dict whose
|
||||
keys match no port is dropped in silence.
|
||||
"""
|
||||
if isinstance(value, (str, Port)):
|
||||
return [value if isinstance(value, Port) else Port(value)], True
|
||||
return [item if isinstance(item, Port) else Port(item) for item in value], False
|
||||
|
||||
|
||||
class NodeSpec:
|
||||
"""What ``@node`` recorded about a function."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fn: Callable[..., Any],
|
||||
*,
|
||||
id: str,
|
||||
requires: list[Port],
|
||||
provides: list[Port],
|
||||
single: bool,
|
||||
settings: dict[str, Any],
|
||||
title: str,
|
||||
timeout: float | None,
|
||||
device: str | None,
|
||||
device_policy: str,
|
||||
) -> None:
|
||||
self.fn = fn
|
||||
self.id = id
|
||||
self.requires = requires
|
||||
self.provides = provides
|
||||
self.single = single
|
||||
self.settings = settings
|
||||
self.title = title
|
||||
self.timeout = timeout
|
||||
self.device = device
|
||||
self.device_policy = device_policy
|
||||
|
||||
def rebind(
|
||||
self, *, id: str = "", wire: dict[str, str] | None = None, **settings: Any
|
||||
) -> NodeSpec:
|
||||
"""A copy of this node wired and configured for one flow."""
|
||||
wire = wire or {}
|
||||
known = {port.port for port in self.requires} | {p.port for p in self.provides}
|
||||
unknown = sorted(set(wire) - known)
|
||||
if unknown:
|
||||
raise SyncError(
|
||||
f"node '{self.id}': wire={{'{unknown[0]}': ...}} names no port of it "
|
||||
f"(it has {', '.join(sorted(known)) or 'none'})"
|
||||
)
|
||||
|
||||
# Copied rather than shared: the same decorated function appears in
|
||||
# several flows, and each resolves its own port types.
|
||||
def rewire(ports: list[Port]) -> list[Port]:
|
||||
return [port.replace(wire.get(port.port, "")) for port in ports]
|
||||
|
||||
return NodeSpec(
|
||||
self.fn,
|
||||
id=_name(id, "node id") if id else self.id,
|
||||
requires=rewire(self.requires),
|
||||
provides=rewire(self.provides),
|
||||
single=self.single,
|
||||
settings={**self.settings, **settings},
|
||||
title=self.title,
|
||||
timeout=self.timeout,
|
||||
device=self.device,
|
||||
device_policy=self.device_policy,
|
||||
)
|
||||
|
||||
|
||||
class NodeUse:
|
||||
"""One use of a decorated function in one flow — see :func:`use`."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fn: Callable[..., Any],
|
||||
*,
|
||||
id: str = "",
|
||||
wire: dict[str, str] | None = None,
|
||||
settings: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
self.fn = fn
|
||||
self.id = id
|
||||
self.wire = wire
|
||||
self.settings = settings or {}
|
||||
|
||||
|
||||
def node(
|
||||
*,
|
||||
requires: Sequence[str | Port] = (),
|
||||
provides: Sequence[str | Port] | str | Port = (),
|
||||
id: str = "",
|
||||
settings: dict[str, Any] | None = None,
|
||||
title: str = "",
|
||||
timeout: float | None = None,
|
||||
device: str | None = None,
|
||||
device_policy: str = "require",
|
||||
) -> Callable[[F], F]:
|
||||
"""Mark a function as a node, declaring its ports.
|
||||
|
||||
The function is returned untouched: it is still callable, testable and
|
||||
importable as what it was. A bare string in ``requires`` or ``provides`` is
|
||||
shorthand for a ``float`` port of that name.
|
||||
|
||||
Parameters with a default that are not ports become the node's settings, so
|
||||
the canvas can tune them without editing code. ``device`` picks the worker
|
||||
the code runs on — ``"gpu"``, say — and ``device_policy="prefer"`` runs it
|
||||
locally when no such worker is attached rather than waiting for one.
|
||||
"""
|
||||
if device_policy not in ("require", "prefer"):
|
||||
raise SyncError("device_policy is 'require' or 'prefer'")
|
||||
|
||||
def decorate(fn: F) -> F:
|
||||
required, _ = _ports(requires)
|
||||
provided, single = _ports(provides)
|
||||
spec = NodeSpec(
|
||||
fn,
|
||||
id=_name(id or fn.__name__, "node id"),
|
||||
requires=required,
|
||||
provides=provided,
|
||||
single=single,
|
||||
settings=dict(settings or {}),
|
||||
title=title,
|
||||
timeout=timeout,
|
||||
device=device,
|
||||
device_policy=device_policy,
|
||||
)
|
||||
_check_signature(spec)
|
||||
fn.__fluksio__ = spec # type: ignore[attr-defined]
|
||||
return fn
|
||||
|
||||
return decorate
|
||||
|
||||
|
||||
def use(
|
||||
fn: Callable[..., Any],
|
||||
*,
|
||||
id: str = "",
|
||||
wire: dict[str, str] | None = None,
|
||||
**settings: Any,
|
||||
) -> NodeUse:
|
||||
"""The same node, wired or configured differently in this flow.
|
||||
|
||||
``wire={"dataset": "augmented"}`` binds the ``dataset`` port to another
|
||||
message; keyword arguments override settings. Give ``id`` when one function
|
||||
appears twice in one flow.
|
||||
"""
|
||||
return NodeUse(fn, id=id, wire=wire, settings=settings)
|
||||
|
||||
|
||||
def _spec_of(entry: Callable[..., Any] | NodeUse) -> NodeSpec:
|
||||
fn = entry.fn if isinstance(entry, NodeUse) else entry
|
||||
spec = getattr(fn, "__fluksio__", None)
|
||||
if not isinstance(spec, NodeSpec):
|
||||
raise SyncError(
|
||||
f"{getattr(fn, '__name__', fn)!r} is not a node — decorate it with @node()"
|
||||
)
|
||||
if isinstance(entry, NodeUse):
|
||||
return spec.rebind(id=entry.id, wire=entry.wire, **entry.settings)
|
||||
return spec.rebind()
|
||||
|
||||
|
||||
def _check_signature(spec: NodeSpec) -> None:
|
||||
"""Whether the declaration matches the function it is on.
|
||||
|
||||
Raised where the decorator is rather than where the flow is: this is a
|
||||
disagreement between a function and its own ports, and the engine would
|
||||
otherwise only find it when the node is first called.
|
||||
"""
|
||||
fn, where = spec.fn, f"node '{spec.id}'"
|
||||
if inspect.iscoroutinefunction(fn):
|
||||
raise SyncError(f"{where}: async functions cannot be nodes")
|
||||
if fn.__module__ == "__main__":
|
||||
raise SyncError(
|
||||
f"{where}: {fn.__name__}() is defined in a script run directly, so the "
|
||||
"generated node could not import it — put it in an importable module"
|
||||
)
|
||||
generator = inspect.isgeneratorfunction(fn)
|
||||
if generator and spec.single:
|
||||
raise SyncError(
|
||||
f"{where}: {fn.__name__}() yields, so it says which port each value is "
|
||||
f"on — pass provides=[Port(...)] rather than a single port"
|
||||
)
|
||||
|
||||
signature = inspect.signature(fn)
|
||||
parameters = signature.parameters
|
||||
var_keyword = any(
|
||||
p.kind is inspect.Parameter.VAR_KEYWORD for p in parameters.values()
|
||||
)
|
||||
ports = {port.port for port in spec.requires}
|
||||
|
||||
for port in ports:
|
||||
if port not in parameters and not var_keyword:
|
||||
raise SyncError(
|
||||
f"{where}: port '{port}' has no parameter to arrive in — "
|
||||
f"{fn.__name__}({', '.join(parameters)}) takes no such argument"
|
||||
)
|
||||
for name, parameter in parameters.items():
|
||||
if parameter.kind is inspect.Parameter.VAR_POSITIONAL:
|
||||
raise SyncError(f"{where}: *{name} cannot be filled from ports")
|
||||
if parameter.kind is inspect.Parameter.VAR_KEYWORD or name in ports:
|
||||
continue
|
||||
if parameter.default is inspect.Parameter.empty:
|
||||
raise SyncError(
|
||||
f"{where}: parameter '{name}' is neither a port nor a setting — "
|
||||
f"add Port('{name}', ...) to requires=, or give it a default"
|
||||
)
|
||||
|
||||
|
||||
def _check(spec: NodeSpec, mode: str) -> dict[str, Any]:
|
||||
"""The node as stored, with the settings and wiring of one flow.
|
||||
|
||||
Mirrors ``FlowController._build_node`` and ``flow.runs.batch_issues``. Kept
|
||||
apart from the signature check because ``use(fn, epochs=3)`` adds settings
|
||||
per flow, so this is the first point at which they are all known.
|
||||
"""
|
||||
fn, where = spec.fn, f"node '{spec.id}'"
|
||||
parameters = inspect.signature(fn).parameters
|
||||
var_keyword = any(
|
||||
p.kind is inspect.Parameter.VAR_KEYWORD for p in parameters.values()
|
||||
)
|
||||
ports = {port.port for port in spec.requires}
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
for name, parameter in parameters.items():
|
||||
if name in ports or parameter.default is inspect.Parameter.empty:
|
||||
continue
|
||||
if name in RESERVED_SETTINGS:
|
||||
continue
|
||||
if _serialisable(parameter.default):
|
||||
params[name] = parameter.default
|
||||
for name, value in spec.settings.items():
|
||||
if name in ports:
|
||||
raise SyncError(
|
||||
f"{where}: '{name}' is both a port and a setting — a node reads its "
|
||||
"settings beside its ports, so the names have to differ"
|
||||
)
|
||||
if name in RESERVED_SETTINGS:
|
||||
raise SyncError(f"{where}: '{name}' is a setting the engine reads itself")
|
||||
if name not in parameters and not var_keyword:
|
||||
raise SyncError(
|
||||
f"{where}: setting '{name}' is not a parameter of {fn.__name__}()"
|
||||
)
|
||||
if not _serialisable(value):
|
||||
raise SyncError(
|
||||
f"{where}: setting '{name}' is not JSON — settings are stored with "
|
||||
"the flow, so they have to survive a round trip"
|
||||
)
|
||||
params[name] = value
|
||||
|
||||
if mode == "batch":
|
||||
for port in spec.requires + spec.provides:
|
||||
if port.interval > 0 and not port.stream:
|
||||
raise SyncError(
|
||||
f"{where}: port '{port.port}' has interval={port.interval} but a "
|
||||
"batch run delivers once — drop it, or declare stream=True"
|
||||
)
|
||||
|
||||
return {
|
||||
"id": spec.id,
|
||||
"type": "python",
|
||||
"title": spec.title,
|
||||
"params": params,
|
||||
"requires": [port.spec() for port in spec.requires],
|
||||
"provides": [port.spec() for port in spec.provides],
|
||||
"timeout": spec.timeout,
|
||||
"device": spec.device,
|
||||
"device_policy": spec.device_policy,
|
||||
}
|
||||
|
||||
|
||||
def _serialisable(value: Any) -> bool:
|
||||
try:
|
||||
json.dumps(value)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class Flow:
|
||||
"""A flow: which nodes are in it, what it takes and what it reports."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
*,
|
||||
nodes: Sequence[Callable[..., Any] | NodeUse],
|
||||
mode: str = "batch",
|
||||
inputs: Sequence[Port] = (),
|
||||
outputs: Sequence[str] = (),
|
||||
title: str = "",
|
||||
) -> None:
|
||||
"""Declare a flow over decorated functions, and register it for sync.
|
||||
|
||||
``nodes`` is the membership: any decorated function, from anywhere in
|
||||
the repository. Wiring is not membership — nodes connect because one
|
||||
provides a message another requires, never because one imported the
|
||||
other. Wrap an entry in :func:`use` to rebind or reconfigure it here
|
||||
without touching the other flows using it.
|
||||
"""
|
||||
if mode not in ("batch", "live"):
|
||||
raise SyncError(f"flow '{name}': mode is 'batch' or 'live'")
|
||||
self.name = _name(name, "flow name")
|
||||
self.title = title
|
||||
self.mode = mode
|
||||
self.inputs = list(inputs)
|
||||
self.outputs = list(outputs)
|
||||
self.nodes = [_spec_of(entry) for entry in nodes]
|
||||
self._resolve_types()
|
||||
self._validate()
|
||||
self._nodes = [_check(spec, mode) for spec in self.nodes]
|
||||
self._register()
|
||||
|
||||
def _register(self) -> None:
|
||||
"""Put this flow where `fluksio sync` will find it."""
|
||||
existing = FLOWS.get(self.name)
|
||||
if existing is not None and existing is not self:
|
||||
warnings.warn(
|
||||
f"two flows are called '{self.name}'; the later one wins, and only "
|
||||
"it will be synced",
|
||||
stacklevel=3,
|
||||
)
|
||||
FLOWS[self.name] = self
|
||||
|
||||
def _resolve_types(self) -> None:
|
||||
"""Give every required port the type of the message it reads.
|
||||
|
||||
A message has one type, declared once by whoever produces it. Requiring
|
||||
it is not a second declaration, so `requires=["dataset"]` takes the
|
||||
artifact type from the node providing `dataset` rather than defaulting
|
||||
to a float the engine would then refuse at run time. Spelling the type
|
||||
out on both sides is still allowed — and a disagreement is an error,
|
||||
because one of the two is wrong about what it is handling.
|
||||
"""
|
||||
declared: dict[str, Port] = {}
|
||||
for port in self.inputs:
|
||||
declared.setdefault(port.name, port)
|
||||
for spec in self.nodes:
|
||||
for port in spec.provides:
|
||||
declared.setdefault(port.name, port)
|
||||
for spec in self.nodes:
|
||||
for port in spec.requires:
|
||||
source = declared.get(port.name)
|
||||
if source is None:
|
||||
continue
|
||||
if not port.explicit:
|
||||
port.dtype = source.dtype
|
||||
port.item = source.item
|
||||
elif port.dtype != source.dtype:
|
||||
raise SyncError(
|
||||
f"flow '{self.name}': node '{spec.id}' reads '{port.name}' as "
|
||||
f"{port.dtype}, but it is provided as {source.dtype}"
|
||||
)
|
||||
|
||||
def _validate(self) -> None:
|
||||
seen: set[str] = set()
|
||||
for spec in self.nodes:
|
||||
if spec.id in seen:
|
||||
raise SyncError(
|
||||
f"flow '{self.name}': two nodes are called '{spec.id}' — "
|
||||
"pass use(fn, id='...') to tell them apart"
|
||||
)
|
||||
seen.add(spec.id)
|
||||
|
||||
produced: dict[str, list[str]] = {}
|
||||
for spec in self.nodes:
|
||||
for port in spec.provides:
|
||||
produced.setdefault(port.name, []).append(spec.id)
|
||||
for name, writers in produced.items():
|
||||
if len(writers) > 1:
|
||||
warnings.warn(
|
||||
f"flow '{self.name}': {', '.join(writers)} all publish "
|
||||
f"'{name}', so a reader gets whichever ran last",
|
||||
stacklevel=4,
|
||||
)
|
||||
|
||||
available = set(produced) | {port.name for port in self.inputs}
|
||||
for spec in self.nodes:
|
||||
for port in spec.requires:
|
||||
if "." in port.name or port.name in available:
|
||||
continue
|
||||
raise SyncError(
|
||||
f"flow '{self.name}': node '{spec.id}' requires '{port.name}', "
|
||||
"which no node in it provides and no input declares"
|
||||
)
|
||||
for output in self.outputs:
|
||||
if output not in available:
|
||||
raise SyncError(
|
||||
f"flow '{self.name}': output '{output}' is not produced by any "
|
||||
"of its nodes"
|
||||
)
|
||||
|
||||
def document(self, origin: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""The flow document to store, without a version — sync sets that."""
|
||||
doc: dict[str, Any] = {
|
||||
"name": self.name,
|
||||
"title": self.title,
|
||||
"mode": self.mode,
|
||||
"nodes": self._nodes,
|
||||
"inputs": [
|
||||
{"spec": port.spec(), "initial": port.initial} for port in self.inputs
|
||||
],
|
||||
"outputs": self.outputs,
|
||||
}
|
||||
if origin is not None:
|
||||
doc["origin"] = origin
|
||||
return doc
|
||||
|
||||
def shims(self) -> dict[str, str]:
|
||||
"""One generated node body per node, keyed by node id."""
|
||||
return {spec.id: _shim(spec) for spec in self.nodes}
|
||||
|
||||
def submit(
|
||||
self, *, seed: int | None = None, client: Any = None, **params: Any
|
||||
) -> Any:
|
||||
"""Start a run of this flow and hand back a handle to it."""
|
||||
from fluksio.sdk.client import Client
|
||||
|
||||
return (client or Client()).submit(self.name, params, seed=seed)
|
||||
|
||||
def runs(self, limit: int = 20, client: Any = None) -> list[dict[str, Any]]:
|
||||
"""This flow's runs, newest first."""
|
||||
from fluksio.sdk.client import Client
|
||||
|
||||
rows: list[dict[str, Any]] = (client or Client()).runs(
|
||||
flow=self.name, limit=limit
|
||||
)
|
||||
return rows
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Flow({self.name!r}, nodes={[n.id for n in self.nodes]})"
|
||||
|
||||
|
||||
def build(target: Flow, origin: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""The flow document for a flow — checked, without a version."""
|
||||
return target.document(origin)
|
||||
|
||||
|
||||
def shims(target: Flow) -> dict[str, str]:
|
||||
"""The generated node bodies for a flow."""
|
||||
return target.shims()
|
||||
|
||||
|
||||
def import_root(fn: Callable[..., Any]) -> str:
|
||||
"""The directory that has to be on the path for ``fn`` to be importable.
|
||||
|
||||
Its module's dotted name says how deep in a package it sits, so counting
|
||||
the segments back up from the file lands on the directory the import
|
||||
resolves against — which is not always the repository root, and is what a
|
||||
generated shim actually needs.
|
||||
"""
|
||||
source = inspect.getsourcefile(fn)
|
||||
if not source:
|
||||
return ""
|
||||
path = Path(source).resolve()
|
||||
for _ in fn.__module__.split("."):
|
||||
path = path.parent
|
||||
return str(path)
|
||||
|
||||
|
||||
def _shim(spec: NodeSpec) -> str:
|
||||
"""The node body the store keeps: an import of the real function.
|
||||
|
||||
The store therefore still holds a complete, runnable definition — the body
|
||||
simply happens to be generated, which is why it says so and says where the
|
||||
real thing is.
|
||||
"""
|
||||
fn = spec.fn
|
||||
where = inspect.getsourcefile(fn) or fn.__module__
|
||||
repo = import_root(fn)
|
||||
ports = [port.port for port in spec.requires]
|
||||
signature = ", ".join(ports + ["**settings"])
|
||||
arguments = ", ".join([f"{port}={port}" for port in ports] + ["**settings"])
|
||||
call = f"{fn.__name__}({arguments})"
|
||||
|
||||
lines = [
|
||||
f"{MARKER} from {where} — edit that file instead",
|
||||
"import sys",
|
||||
"",
|
||||
]
|
||||
if repo:
|
||||
# ponytail: the repository is on the path because the shim puts it
|
||||
# there. An editable install (`-e /repo` in the Modules manifest, or a
|
||||
# VCS requirement on a remote worker) makes these three lines a no-op.
|
||||
lines += [
|
||||
f"_REPO = {repo!r}",
|
||||
"if _REPO not in sys.path:",
|
||||
" sys.path.insert(0, _REPO)",
|
||||
"",
|
||||
]
|
||||
lines += [
|
||||
f"from {fn.__module__} import {fn.__name__}",
|
||||
"",
|
||||
"",
|
||||
f"def process({signature}):",
|
||||
]
|
||||
if inspect.isgeneratorfunction(fn):
|
||||
# `return (yield from ...)`, not a bare `yield from`: the delegating
|
||||
# form is what carries the generator's own return value out, and that
|
||||
# return value is everything the node produces at the end.
|
||||
lines.append(f" return (yield from {call})")
|
||||
elif spec.single:
|
||||
lines.append(f" return {{{spec.provides[0].port!r}: {call}}}")
|
||||
else:
|
||||
lines.append(f" return {call}")
|
||||
return "\n".join(lines) + "\n"
|
||||
@@ -0,0 +1,306 @@
|
||||
"""The `fluksio sync`, `run`, `runs` and `login` commands.
|
||||
|
||||
Kept beside the SDK rather than in `fluksio.cli`: these are the client half of
|
||||
the tool, and none of them needs the engine to be importable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import getpass
|
||||
import importlib
|
||||
import json
|
||||
import pkgutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fluksio.sdk import FLOWS, Flow, SyncError
|
||||
from fluksio.sdk.client import (
|
||||
ApiError,
|
||||
Client,
|
||||
config_path,
|
||||
login,
|
||||
origin_of,
|
||||
repo_root,
|
||||
sync,
|
||||
)
|
||||
|
||||
__all__ = ["add_parsers", "discover"]
|
||||
|
||||
|
||||
def _say(message: str = "") -> None:
|
||||
print(message)
|
||||
|
||||
|
||||
def _fail(message: str) -> int:
|
||||
print(f"fluksio: {message}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Discovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _package_of(directory: Path) -> tuple[str, str]:
|
||||
"""The path root and dotted name of a package directory."""
|
||||
parts = [directory.name]
|
||||
parent = directory.parent
|
||||
while (parent / "__init__.py").exists():
|
||||
parts.append(parent.name)
|
||||
parent = parent.parent
|
||||
return str(parent), ".".join(reversed(parts))
|
||||
|
||||
|
||||
def _module_of(path: Path) -> tuple[str, str]:
|
||||
"""The path root and dotted name of a module file."""
|
||||
parts = [path.stem]
|
||||
directory = path.parent
|
||||
while (directory / "__init__.py").exists():
|
||||
parts.append(directory.name)
|
||||
directory = directory.parent
|
||||
return str(directory), ".".join(reversed(parts))
|
||||
|
||||
|
||||
def _import(root: str, dotted: str) -> None:
|
||||
if root not in sys.path:
|
||||
sys.path.insert(0, root)
|
||||
importlib.import_module(dotted)
|
||||
|
||||
|
||||
def discover(targets: list[str]) -> list[Flow]:
|
||||
"""Import what was named and hand back the flows it declared.
|
||||
|
||||
Imported by dotted name with its root on the path, never from a file
|
||||
location: the generated node bodies import the same way, and a module
|
||||
loaded under a different name would generate an import that does not
|
||||
resolve.
|
||||
"""
|
||||
for target in targets:
|
||||
path = Path(target)
|
||||
if not path.exists():
|
||||
_import(str(Path.cwd()), target)
|
||||
continue
|
||||
path = path.resolve()
|
||||
if path.is_file():
|
||||
_import(*_module_of(path))
|
||||
continue
|
||||
if (path / "__init__.py").exists():
|
||||
root, dotted = _package_of(path)
|
||||
_import(root, dotted)
|
||||
package = sys.modules[dotted]
|
||||
for info in pkgutil.walk_packages(package.__path__, f"{dotted}."):
|
||||
importlib.import_module(info.name)
|
||||
continue
|
||||
for module in sorted(path.glob("*.py")):
|
||||
_import(*_module_of(module))
|
||||
return list(FLOWS.values())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Commands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def cmd_login(args: argparse.Namespace) -> int:
|
||||
email = args.email or input("Email: ")
|
||||
password = args.password or getpass.getpass("Password: ")
|
||||
try:
|
||||
login(args.url, email, password)
|
||||
except ApiError as exc:
|
||||
return _fail(f"could not log in: {exc.detail}")
|
||||
_say(f"Logged in to {args.url}; the token is in {config_path()}.")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_sync(args: argparse.Namespace) -> int:
|
||||
targets = args.targets or ["."]
|
||||
try:
|
||||
flows = discover(targets)
|
||||
except (ImportError, SyncError) as exc:
|
||||
return _fail(str(exc))
|
||||
if not flows:
|
||||
return _fail(
|
||||
f"no flows declared in {', '.join(targets)} — a flow is a `flow(...)` "
|
||||
"call at module level"
|
||||
)
|
||||
|
||||
repo = repo_root(targets[0])
|
||||
origin = origin_of(repo)
|
||||
if origin["dirty"]:
|
||||
_say(f"warning: {repo} has uncommitted changes, so the stamp says -dirty")
|
||||
if not origin["commit"]:
|
||||
_say(f"warning: {repo} is not a git repository, so runs cannot name a commit")
|
||||
|
||||
if args.dry_run:
|
||||
for target in flows:
|
||||
_say(f"=== flow {target.name}")
|
||||
_say(json.dumps(target.document(origin), indent=2))
|
||||
for node_id, code in target.shims().items():
|
||||
_say(f"=== {target.name}.{node_id}")
|
||||
_say(code)
|
||||
return 0
|
||||
|
||||
try:
|
||||
client = Client(url=args.url, token=args.token)
|
||||
reports = sync(
|
||||
flows,
|
||||
client,
|
||||
origin=origin,
|
||||
publish=not args.no_publish,
|
||||
force=args.force,
|
||||
)
|
||||
except (SyncError, ApiError) as exc:
|
||||
return _fail(str(exc))
|
||||
|
||||
for report in reports:
|
||||
if report.unchanged:
|
||||
_say(f" {report.flow}: unchanged")
|
||||
continue
|
||||
what = "created" if report.created else "updated"
|
||||
detail = ", ".join(report.changed)
|
||||
state = "published" if report.published else "draft"
|
||||
_say(f" {report.flow}: {what} ({detail}) — {state}")
|
||||
stamp = origin["commit"][:7] + ("-dirty" if origin["dirty"] else "")
|
||||
_say(f"Stamped with {stamp or 'no commit'} from {repo}.")
|
||||
return 0
|
||||
|
||||
|
||||
def _coerce(value: str, dtype: str) -> Any:
|
||||
if dtype == "int":
|
||||
return int(value)
|
||||
if dtype == "float":
|
||||
return float(value)
|
||||
if dtype == "bool":
|
||||
return value.lower() in ("true", "1", "yes", "on")
|
||||
if dtype == "str":
|
||||
return value
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
|
||||
"""Turn `--lr 0.05` into a typed parameter, using the flow's own inputs."""
|
||||
types = {
|
||||
str(entry["spec"]["name"]): str(entry["spec"].get("dtype", "float"))
|
||||
for entry in definition.get("inputs") or []
|
||||
}
|
||||
params: dict[str, Any] = {}
|
||||
pending: str | None = None
|
||||
for token in rest:
|
||||
if token.startswith("--"):
|
||||
if pending is not None:
|
||||
# A flag with no value is a flag: `--resume` means true.
|
||||
params[pending] = True
|
||||
name, sep, value = token[2:].partition("=")
|
||||
# Only the name is spelled with dashes; a value may hold one, and
|
||||
# `--lr=1e-4` is the case that says so.
|
||||
pending = name.replace("-", "_")
|
||||
if sep:
|
||||
params[pending] = _coerce(value, types.get(pending, "json"))
|
||||
pending = None
|
||||
continue
|
||||
if pending is None:
|
||||
raise SyncError(f"unexpected argument '{token}'")
|
||||
params[pending] = _coerce(token, types.get(pending, "json"))
|
||||
pending = None
|
||||
if pending is not None:
|
||||
params[pending] = True
|
||||
unknown = sorted(set(params) - set(types))
|
||||
if unknown:
|
||||
raise SyncError(
|
||||
f"'{unknown[0]}' is not an input of this flow (it takes "
|
||||
f"{', '.join(sorted(types)) or 'none'})"
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
|
||||
try:
|
||||
client = Client(url=args.url, token=args.token)
|
||||
stored = client.get_flow(args.flow)
|
||||
if stored is None:
|
||||
return _fail(f"no flow '{args.flow}' on that engine")
|
||||
params = _params(stored.get("definition") or {}, rest)
|
||||
handle = client.submit(args.flow, params, seed=args.seed)
|
||||
except (SyncError, ApiError) as exc:
|
||||
return _fail(str(exc))
|
||||
_say(f"{handle.id} queued {json.dumps(params)}")
|
||||
if not args.wait:
|
||||
return 0
|
||||
handle.wait(timeout=args.timeout)
|
||||
_say(f"{handle.id} {handle.status} {json.dumps(handle.result)}")
|
||||
return 0 if handle.status == "ok" else 1
|
||||
|
||||
|
||||
def cmd_runs(args: argparse.Namespace) -> int:
|
||||
try:
|
||||
rows = Client(url=args.url, token=args.token).runs(
|
||||
flow=args.flow, limit=args.limit
|
||||
)
|
||||
except (SyncError, ApiError) as exc:
|
||||
return _fail(str(exc))
|
||||
for row in rows:
|
||||
commit = (row.get("origin_commit") or "")[:7]
|
||||
_say(
|
||||
f"{row['id']} {row['status']:<9} {row['flow']:<16} "
|
||||
f"{row['duration_ms'] / 1000:7.1f}s {commit:<8} {json.dumps(row['params'])}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wiring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def add_parsers(subparsers: Any) -> None:
|
||||
"""Register the client commands on `fluksio`'s parser."""
|
||||
|
||||
def with_engine(sub: argparse.ArgumentParser) -> None:
|
||||
sub.add_argument(
|
||||
"--url", default="", help="the engine (default: the last login)"
|
||||
)
|
||||
sub.add_argument("--token", default="", help="override the stored token")
|
||||
|
||||
parser = subparsers.add_parser("login", help="store a token for an engine")
|
||||
parser.add_argument("--url", default="http://localhost:8000")
|
||||
parser.add_argument("--email", default="")
|
||||
parser.add_argument("--password", default="")
|
||||
parser.set_defaults(func=cmd_login)
|
||||
|
||||
parser = subparsers.add_parser(
|
||||
"sync", help="upload the flows declared in your own code"
|
||||
)
|
||||
parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help="modules, packages or directories to import (default: .)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run", action="store_true", help="print what would be uploaded"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-publish", action="store_true", help="leave the changes as a draft"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force", action="store_true", help="overwrite work done on the canvas"
|
||||
)
|
||||
with_engine(parser)
|
||||
parser.set_defaults(func=cmd_sync)
|
||||
|
||||
parser = subparsers.add_parser(
|
||||
"run", help="start a run, passing the flow's inputs as --name value"
|
||||
)
|
||||
parser.add_argument("flow")
|
||||
parser.add_argument("--seed", type=int, default=None)
|
||||
parser.add_argument("--wait", action="store_true", help="block until it finishes")
|
||||
parser.add_argument("--timeout", type=float, default=0.0)
|
||||
with_engine(parser)
|
||||
parser.set_defaults(func=cmd_run)
|
||||
|
||||
parser = subparsers.add_parser("runs", help="the runs an engine has recorded")
|
||||
parser.add_argument("--flow", default="")
|
||||
parser.add_argument("--limit", type=int, default=20)
|
||||
with_engine(parser)
|
||||
parser.set_defaults(func=cmd_runs)
|
||||
@@ -0,0 +1,411 @@
|
||||
"""Talking to an engine: uploading declared flows, and running them.
|
||||
|
||||
Two halves of one job. :func:`sync` puts what the decorators declared into the
|
||||
flow store; :class:`Client` submits runs and reads back what they produced, so
|
||||
an experiment is started and inspected from the same script that defines it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fluksio.sdk import MARKER, Flow, SyncError
|
||||
|
||||
__all__ = ["Client", "RunHandle", "SyncReport", "config_path", "login", "sync"]
|
||||
|
||||
API = "/api/v1"
|
||||
#: A run is over when it reaches one of these.
|
||||
DONE = frozenset({"ok", "error", "cancelled", "abandoned"})
|
||||
|
||||
|
||||
def config_path() -> Path:
|
||||
"""Where ``fluksio login`` leaves the engine it talked to."""
|
||||
base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config")
|
||||
return Path(base) / "fluksio" / "client.json"
|
||||
|
||||
|
||||
def _stored() -> dict[str, str]:
|
||||
path = config_path()
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(path.read_text())
|
||||
except ValueError:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
"""The engine refused, with what it said."""
|
||||
|
||||
def __init__(self, status: int, detail: Any) -> None:
|
||||
super().__init__(f"{status}: {detail}")
|
||||
self.status = status
|
||||
self.detail = detail
|
||||
|
||||
|
||||
class Conflict(ApiError):
|
||||
"""Someone else saved in between — the 409 the optimistic lock answers."""
|
||||
|
||||
def __init__(self, detail: Any) -> None:
|
||||
super().__init__(409, detail)
|
||||
self.current_version = 0
|
||||
if isinstance(detail, dict):
|
||||
self.current_version = int(detail.get("current_version") or 0)
|
||||
|
||||
|
||||
class Client:
|
||||
"""An authenticated engine, addressed over its HTTP API."""
|
||||
|
||||
def __init__(
|
||||
self, url: str = "", token: str = "", http: Any = None, timeout: float = 30.0
|
||||
) -> None:
|
||||
stored = _stored()
|
||||
self.url = url or os.environ.get("FLUKSIO_URL") or stored.get("url") or ""
|
||||
self.token = (
|
||||
token or os.environ.get("FLUKSIO_TOKEN") or stored.get("token") or ""
|
||||
)
|
||||
if http is None:
|
||||
if not self.url:
|
||||
raise SyncError(
|
||||
"No engine to talk to. Run `fluksio login --url http://…`, or "
|
||||
"set FLUKSIO_URL and FLUKSIO_TOKEN."
|
||||
)
|
||||
import httpx
|
||||
|
||||
http = httpx.Client(base_url=self.url, timeout=timeout)
|
||||
self.http = http
|
||||
if self.token:
|
||||
self.http.headers["Authorization"] = f"Bearer {self.token}"
|
||||
|
||||
# -- plumbing ----------------------------------------------------------
|
||||
|
||||
def _call(self, method: str, path: str, **kwargs: Any) -> Any:
|
||||
response = self.http.request(method, f"{API}{path}", **kwargs)
|
||||
if response.status_code == 409:
|
||||
raise Conflict(_detail(response))
|
||||
if response.status_code >= 400:
|
||||
raise ApiError(response.status_code, _detail(response))
|
||||
if response.status_code == 204 or not response.content:
|
||||
return None
|
||||
return response.json()
|
||||
|
||||
# -- flows -------------------------------------------------------------
|
||||
|
||||
def get_flow(self, name: str) -> dict[str, Any] | None:
|
||||
"""The stored flow, draft included, or ``None`` if there is none."""
|
||||
try:
|
||||
result: dict[str, Any] = self._call("GET", f"/flows/{name}")
|
||||
except ApiError as exc:
|
||||
if exc.status == 404:
|
||||
return None
|
||||
raise
|
||||
return result
|
||||
|
||||
def put_flow(self, document: dict[str, Any]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = self._call(
|
||||
"PUT", f"/flows/{document['name']}", json=document
|
||||
)
|
||||
return result
|
||||
|
||||
def get_source(self, flow: str, node: str) -> str:
|
||||
result = self._call("GET", f"/flows/{flow}/nodes/{node}/source")
|
||||
return str(result.get("code", ""))
|
||||
|
||||
def put_source(self, flow: str, node: str, code: str) -> dict[str, Any]:
|
||||
result: dict[str, Any] = self._call(
|
||||
"PUT", f"/flows/{flow}/nodes/{node}/source", json={"code": code}
|
||||
)
|
||||
return result
|
||||
|
||||
def publish(self, name: str, version: int) -> dict[str, Any]:
|
||||
result: dict[str, Any] = self._call(
|
||||
"POST", f"/flows/{name}/publish", json={"version": version}
|
||||
)
|
||||
return result
|
||||
|
||||
def refresh_modules(self) -> None:
|
||||
"""Retire the engine's workers, so the next run imports the code as it is."""
|
||||
self._call("POST", "/modules/refresh")
|
||||
|
||||
# -- runs --------------------------------------------------------------
|
||||
|
||||
def submit(
|
||||
self, flow: str, params: dict[str, Any] | None = None, seed: int | None = None
|
||||
) -> RunHandle:
|
||||
row = self._call(
|
||||
"POST", f"/runs/flows/{flow}", json={"params": params or {}, "seed": seed}
|
||||
)
|
||||
return RunHandle(self, row["id"], row)
|
||||
|
||||
def run(self, run_id: str) -> dict[str, Any]:
|
||||
result: dict[str, Any] = self._call("GET", f"/runs/{run_id}")
|
||||
return result
|
||||
|
||||
def runs(self, flow: str = "", limit: int = 20, **filters: Any) -> Any:
|
||||
query = {"limit": limit, **filters}
|
||||
if flow:
|
||||
query["flow"] = flow
|
||||
return self._call("GET", "/runs", params=query)
|
||||
|
||||
def metrics(self, run_id: str, name: str = "", stride: int = 1) -> Any:
|
||||
query: dict[str, Any] = {"stride": stride}
|
||||
if name:
|
||||
query["name"] = name
|
||||
return self._call("GET", f"/runs/{run_id}/metrics", params=query)
|
||||
|
||||
def compare(self, ids: Iterable[str], metric: str) -> Any:
|
||||
return self._call(
|
||||
"GET",
|
||||
"/runs/series/compare",
|
||||
params={"ids": ",".join(ids), "metric": metric},
|
||||
)
|
||||
|
||||
def cancel(self, run_id: str) -> Any:
|
||||
return self._call("POST", f"/runs/{run_id}/cancel")
|
||||
|
||||
def download(self, digest: str) -> bytes:
|
||||
response = self.http.request("GET", f"{API}/artifacts/{digest}")
|
||||
if response.status_code >= 400:
|
||||
raise ApiError(response.status_code, _detail(response))
|
||||
return bytes(response.content)
|
||||
|
||||
|
||||
def _detail(response: Any) -> Any:
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
return response.text
|
||||
return body.get("detail", body) if isinstance(body, dict) else body
|
||||
|
||||
|
||||
class RunHandle:
|
||||
"""One run, and the answers it accumulates."""
|
||||
|
||||
def __init__(self, client: Client, run_id: str, row: dict[str, Any]) -> None:
|
||||
self.client = client
|
||||
self.id = run_id
|
||||
self._row = row
|
||||
|
||||
def refresh(self) -> RunHandle:
|
||||
self._row = self.client.run(self.id)
|
||||
return self
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return str(self._row.get("status", ""))
|
||||
|
||||
@property
|
||||
def done(self) -> bool:
|
||||
return self.status in DONE
|
||||
|
||||
@property
|
||||
def result(self) -> dict[str, Any]:
|
||||
"""What the flow's outputs held when it finished."""
|
||||
result = self._row.get("result")
|
||||
return result if isinstance(result, dict) else {}
|
||||
|
||||
@property
|
||||
def artifacts(self) -> list[dict[str, Any]]:
|
||||
rows = self._row.get("artifacts")
|
||||
return rows if isinstance(rows, list) else []
|
||||
|
||||
def wait(self, timeout: float = 0.0, poll: float = 1.0) -> RunHandle:
|
||||
"""Block until the run is over, or ``timeout`` seconds have passed."""
|
||||
deadline = time.monotonic() + timeout if timeout else 0.0
|
||||
while True:
|
||||
self.refresh()
|
||||
if self.done:
|
||||
return self
|
||||
if deadline and time.monotonic() > deadline:
|
||||
raise TimeoutError(f"run {self.id} is still {self.status}")
|
||||
time.sleep(poll)
|
||||
|
||||
def metrics(self, name: str = "", stride: int = 1) -> list[dict[str, Any]]:
|
||||
"""A streamed port's whole series — a run's metrics are its outputs."""
|
||||
points: list[dict[str, Any]] = self.client.metrics(self.id, name, stride)
|
||||
return points
|
||||
|
||||
def download(self, name: str) -> bytes:
|
||||
"""The bytes of an artifact this run produced."""
|
||||
for row in self.artifacts:
|
||||
if row.get("name") == name:
|
||||
return self.client.download(str(row["digest"]))
|
||||
have = ", ".join(str(row.get("name")) for row in self.artifacts) or "none"
|
||||
raise KeyError(f"run {self.id} has no artifact '{name}' (it has {have})")
|
||||
|
||||
def __getitem__(self, key: str) -> Any:
|
||||
return self._row[key]
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"RunHandle({self.id!r}, status={self.status!r})"
|
||||
|
||||
|
||||
def login(url: str, email: str, password: str, timeout: float = 30.0) -> str:
|
||||
"""Exchange credentials for a token and remember the engine."""
|
||||
import httpx
|
||||
|
||||
response = httpx.post(
|
||||
f"{url.rstrip('/')}{API}/login/access-token",
|
||||
data={"username": email, "password": password},
|
||||
timeout=timeout,
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
raise ApiError(response.status_code, _detail(response))
|
||||
token = str(response.json()["access_token"])
|
||||
path = config_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"url": url.rstrip("/"), "token": token}, indent=2))
|
||||
path.chmod(0o600)
|
||||
return token
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# sync
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SyncReport:
|
||||
"""What one flow's sync did, for the CLI to print."""
|
||||
|
||||
def __init__(self, flow: str) -> None:
|
||||
self.flow = flow
|
||||
self.created = False
|
||||
self.changed: list[str] = []
|
||||
self.published = False
|
||||
|
||||
@property
|
||||
def unchanged(self) -> bool:
|
||||
return not self.created and not self.changed
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"SyncReport({self.flow!r}, changed={self.changed})"
|
||||
|
||||
|
||||
def origin_of(repo: str) -> dict[str, Any]:
|
||||
"""Where this flow came from, so a run can name the code that produced it."""
|
||||
commit, dirty = "", False
|
||||
if repo:
|
||||
head = _git(repo, "rev-parse", "HEAD")
|
||||
if head is not None:
|
||||
commit = head
|
||||
dirty = bool(_git(repo, "status", "--porcelain"))
|
||||
return {"kind": "python", "repo": repo, "commit": commit, "dirty": dirty}
|
||||
|
||||
|
||||
def _git(repo: str, *args: str) -> str | None:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "-C", repo, *args], capture_output=True, text=True, timeout=10
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
return result.stdout.strip() if result.returncode == 0 else None
|
||||
|
||||
|
||||
def repo_root(start: str | Path) -> str:
|
||||
"""The git repository a path sits in, or its directory if there is none."""
|
||||
directory = Path(start).resolve()
|
||||
if directory.is_file():
|
||||
directory = directory.parent
|
||||
found = _git(str(directory), "rev-parse", "--show-toplevel")
|
||||
return found or str(directory)
|
||||
|
||||
|
||||
def sync(
|
||||
flows: Iterable[Flow],
|
||||
client: Client,
|
||||
*,
|
||||
repo: str = "",
|
||||
origin: dict[str, Any] | None = None,
|
||||
publish: bool = True,
|
||||
force: bool = False,
|
||||
) -> list[SyncReport]:
|
||||
"""Put declared flows into the store, refusing to overwrite canvas work.
|
||||
|
||||
The order matters: the document first, so the nodes it names exist, then
|
||||
each node's generated body, then one publish. Both writes no-op on
|
||||
identical content, so a second sync with nothing changed commits nothing.
|
||||
"""
|
||||
stamp = origin if origin is not None else origin_of(repo)
|
||||
reports = []
|
||||
for target in flows:
|
||||
reports.append(_sync_one(target, client, stamp, publish=publish, force=force))
|
||||
# Always, even when nothing here changed. A worker holds the imported
|
||||
# package in `sys.modules` for as long as it lives, so an edit to the
|
||||
# caller's own code is invisible until the process is retired — and that
|
||||
# edit is invisible to this function too, since it changes no shim and no
|
||||
# document. Refusing to refresh on a "no-op" sync would skip exactly the
|
||||
# case the refresh exists for.
|
||||
client.refresh_modules()
|
||||
return reports
|
||||
|
||||
|
||||
def _sync_one(
|
||||
target: Flow,
|
||||
client: Client,
|
||||
origin: dict[str, Any],
|
||||
*,
|
||||
publish: bool,
|
||||
force: bool,
|
||||
) -> SyncReport:
|
||||
report = SyncReport(target.name)
|
||||
stored = client.get_flow(target.name)
|
||||
version = 1
|
||||
if stored is None:
|
||||
report.created = True
|
||||
else:
|
||||
definition = stored.get("definition") or {}
|
||||
version = int(definition.get("version") or 1)
|
||||
if not force:
|
||||
_refuse_on_drift(client, target, definition)
|
||||
|
||||
saved = client.put_flow(target.document(origin) | {"version": version})
|
||||
stored_version = int((saved.get("definition") or {}).get("version") or version)
|
||||
if report.created or stored_version != version:
|
||||
# The store bumps the version only when the content actually changed,
|
||||
# so this is its own no-op detection rather than a second guess at it.
|
||||
report.changed.append("flow")
|
||||
version = stored_version
|
||||
|
||||
for node_id, code in target.shims().items():
|
||||
if not report.created and client.get_source(target.name, node_id) == code:
|
||||
continue
|
||||
client.put_source(target.name, node_id, code)
|
||||
report.changed.append(node_id)
|
||||
|
||||
if publish and (client.get_flow(target.name) or {}).get("has_draft"):
|
||||
client.publish(target.name, version)
|
||||
report.published = True
|
||||
return report
|
||||
|
||||
|
||||
def _refuse_on_drift(client: Client, target: Flow, definition: dict[str, Any]) -> None:
|
||||
"""Stop before overwriting work that was done somewhere else.
|
||||
|
||||
Two ways a stored flow is not ours to replace: it was drawn on the canvas
|
||||
and has no origin at all, or one of its node bodies no longer carries the
|
||||
line saying it was generated — which means somebody edited the code there.
|
||||
"""
|
||||
if not definition.get("origin"):
|
||||
raise SyncError(
|
||||
f"flow '{target.name}' was not created by sync, so replacing it would "
|
||||
"discard whoever drew it. Rename yours, or pass --force."
|
||||
)
|
||||
for stored_node in definition.get("nodes") or []:
|
||||
node_id = str(stored_node.get("id"))
|
||||
try:
|
||||
code = client.get_source(target.name, node_id)
|
||||
except ApiError:
|
||||
continue
|
||||
if code and not code.startswith(MARKER):
|
||||
raise SyncError(
|
||||
f"node '{target.name}.{node_id}' was edited on the canvas, and "
|
||||
"syncing would throw that edit away. Copy it out, or pass --force."
|
||||
)
|
||||
Reference in New Issue
Block a user