Clients take a flow's id or key: one resolver in the SDK, id first, explicit id:/key:, a warning for reads and a refusal for changes when a name means two flows, deleted flows found by their last key; sync creates by POST, retries a create race through the drift checks and reports the id; MCP gets create_flow and resolves every flow argument with the caller's token; displays show flow_name; an engine from before ids is named as such

This commit is contained in:
2026-11-02 19:39:14 +01:00
parent fd637e0f1e
commit 53916d869b
20 changed files with 816 additions and 147 deletions
+109 -39
View File
@@ -18,7 +18,9 @@ from pathlib import Path
from typing import Any
from urllib.parse import quote
from fluksio import __version__
from fluksio.sdk import MARKER, Declared, Flow, SyncError
from fluksio.sdk.resolve import Resolved, resolve
__all__ = [
"Client",
@@ -62,6 +64,15 @@ WAIT_TOLERANCE = 5
#: Runs per `POST /runs/import`, which is what the engine caps one request at.
IMPORT_PAGE = 200
#: What to do about an engine from before flow ids. It lists flows without an
#: id and has no route to create one by, so this client cannot tell which flow
#: a name means, nor make one.
OLD_ENGINE = (
"this engine is older than flow ids, and this client "
f"({__version__}) needs them. Upgrade the engine first: "
"`pip install -U fluksio` where it runs"
)
#: What a project-local instance is called, beside `.venv` and `.git`.
DATA_DIR_NAME = ".fluksio"
@@ -251,13 +262,49 @@ class Client:
"""The listing of the flow holding the key *key*, or ``None``."""
result = self._call("GET", "/flows/", params={"name": key})
found = list(result.get("data") or [])
if found and "id" not in found[0]:
# It ignored the filter too, so this is every flow, not the one.
raise SyncError(OLD_ENGINE)
return found[0] if found else None
def create_flow(self, document: dict[str, Any]) -> dict[str, Any]:
"""A new flow, under the id the engine issues for it."""
result: dict[str, Any] = self._call("POST", "/flows/", json=document)
try:
result: dict[str, Any] = self._call("POST", "/flows/", json=document)
except ApiError as exc:
if exc.status in (404, 405):
raise SyncError(OLD_ENGINE) from exc
raise
return result
def resolve_flow(
self, flow: str, *, mutation: bool = False, history: bool = False
) -> Resolved:
"""The flow *flow* names, by its id or its key (`fluksio.sdk.resolve`).
``history`` counts deleted flows too, under the key their runs last
had, for a question about the past. A name two flows answer to is
refused when ``mutation`` says the call changes something, and comes
back with a warning otherwise, for the caller to show.
"""
rows = self.flows()
if any("id" not in row for row in rows):
raise SyncError(OLD_ENGINE)
flows = [(str(row["id"]), str(row["name"])) for row in rows]
if history:
flows += [
(str(row["flow"]), str(row.get("flow_name") or row["flow"]))
for row in self._call("GET", "/runs/overview")
]
return resolve(flow, flows, mutation=mutation)
def _flow_id(self, flow: str, **how: bool) -> str:
"""The id behind *flow*, warning when it could have meant another."""
found = self.resolve_flow(flow, **how)
if found.warning:
warnings.warn(found.warning, stacklevel=3)
return found.id
def put_flow(self, document: dict[str, Any]) -> dict[str, Any]:
"""Save over the flow *document* names by its id, or by its key for
one whose id is its key."""
@@ -338,14 +385,15 @@ class Client:
) -> list[dict[str, Any]]:
"""What went wrong, or who changed what. Newest first.
Engine-wide, and narrowed by flow, by run or by time. ``run`` takes a
run id and answers what the engine recorded during it. For the failure
that ended a run, :attr:`RunHandle.failures` is still the shorter road:
it reads the run's own node rows, which carry the traceback too.
Engine-wide, and narrowed by flow (its id or key), by run or by time.
``run`` takes a run id and answers what the engine recorded during it.
For the failure that ended a run, :attr:`RunHandle.failures` is still
the shorter road: it reads the run's own node rows, which carry the
traceback too.
"""
query: dict[str, Any] = {"kind": kind, "limit": limit}
if flow:
query["flow"] = flow
query["flow"] = self._flow_id(flow, history=True)
if run:
query["run"] = run
for name, value in (("since", since), ("until", until)):
@@ -366,8 +414,9 @@ class Client:
no_cache: bool = False,
cause: str = "sdk",
) -> RunHandle:
"""Queue a run. ``cause`` is what the history records it as coming
from — a script is the default, `fluksio run` says so itself.
"""Queue a run of the flow *flow*, its id or key. ``cause`` is what
the history records it as coming from — a script is the default,
`fluksio run` says so itself.
Safe to retry: the key is minted here, once per call, so an attempt
that timed out on the way back is answered with the run it made rather
@@ -375,7 +424,7 @@ class Client:
"""
row = self._call(
"POST",
f"/runs/flows/{seg(flow)}",
f"/runs/flows/{seg(self._flow_id(flow, mutation=True))}",
idempotent=True,
json={
"params": params or {},
@@ -401,7 +450,7 @@ class Client:
keyed = [{**entry, "idempotency_key": uuid.uuid4().hex} for entry in entries]
rows = self._call(
"POST",
f"/runs/flows/{seg(flow)}/sweep",
f"/runs/flows/{seg(self._flow_id(flow, mutation=True))}/sweep",
idempotent=True,
json={"runs": keyed, "no_cache": no_cache},
)
@@ -410,10 +459,17 @@ class Client:
def import_runs(self, entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Record runs executed somewhere else, a page at a time.
Each entry is the shape `POST /runs/import` takes. Safe to send again:
one whose `external_id` is already on the engine answers
`created: False` and adds nothing.
Each entry is the shape `POST /runs/import` takes, but its `flow` may
be the key as well as the id. Safe to send again: one whose
`external_id` is already on the engine answers `created: False` and
adds nothing.
"""
named = {str(entry["flow"]) for entry in entries if entry.get("flow")}
ids = {flow: self._flow_id(flow, mutation=True) for flow in named}
entries = [
{**entry, "flow": ids[str(entry["flow"])]} if entry.get("flow") else entry
for entry in entries
]
answers: list[dict[str, Any]] = []
for start in range(0, len(entries), IMPORT_PAGE):
answers.extend(
@@ -431,9 +487,10 @@ class Client:
return result
def runs(self, flow: str = "", limit: int = 20, **filters: Any) -> Any:
"""Runs, newest first, of every flow or of *flow*, its id or key."""
query = {"limit": limit, **filters}
if flow:
query["flow"] = flow
query["flow"] = self._flow_id(flow, history=True)
return self._call("GET", "/runs", params=query)
def metrics(self, run_id: str, name: str = "", stride: int = 1) -> Any:
@@ -456,7 +513,7 @@ class Client:
if named:
query["ids"] = ",".join(named)
if flow:
query["flow"] = flow
query["flow"] = self._flow_id(flow, history=True)
return self._call("GET", "/runs/metrics/names", params=query)
def compare(self, ids: Iterable[str], metric: str, x: str = "") -> Any:
@@ -482,7 +539,8 @@ class Client:
"""Runs' series as long rows: run, name, step, ts, value.
``pd.DataFrame(client.export_metrics(flow="train"))`` is every loss
curve of that flow with the run id on each row. ``name`` keeps the
curve of that flow with the run id on each row. ``flow`` is its id or
key. ``name`` keeps the
metrics it lists, ``stride`` thins each curve, and the selection is
narrowed the way :meth:`runs` is — ``status=``, ``group=``,
``since=``, ``until=``.
@@ -530,7 +588,7 @@ class Client:
and that is a warning, since the rows alone look complete.
"""
if flow:
query["flow"] = flow
query["flow"] = self._flow_id(flow, history=True)
named = ",".join(ids)
if named:
query["ids"] = named
@@ -728,6 +786,9 @@ class SyncReport:
def __init__(self, flow: str) -> None:
self.flow = flow
#: The id this engine holds the flow under. Another engine's differs:
#: the key is what instances share.
self.id = ""
self.created = False
self.changed: list[str] = []
self.published = False
@@ -835,30 +896,41 @@ def _sync_one(
force: bool,
) -> SyncReport:
report = SyncReport(target.name)
# The declared key is what an instance shares; the id is this engine's,
# and every request after the lookup names the flow by it.
found = client.find_flow(target.name)
flow_id = str(found["id"]) if found else ""
stored = client.get_flow(flow_id) if flow_id else None
document = target.document(origin)
# The declared key is what instances share; the id is this engine's, and
# every request after the lookup names the flow by it. Looked up a second
# time only when another sync created the flow in between, and then its
# flow is checked as any stored one is before it is written over.
raced = False
version = 1
# What the drift check already read, so the write loop below does not ask
# the engine for every node's body a second time.
known: dict[str, str] = {}
if stored is None:
while True:
found = client.find_flow(target.name)
flow_id = str(found["id"]) if found else ""
stored = client.get_flow(flow_id) if flow_id else None
if stored is not None:
definition = stored.get("definition") or {}
version = int(definition.get("version") or 1)
if not force:
known = _refuse_on_drift(client, target, flow_id, definition)
saved = client.put_flow(document | {"id": flow_id, "version": version})
break
try:
saved = client.create_flow(document)
except Conflict:
# The key was free a moment ago. Anything else the engine refuses
# a create for is refused again, and that one is raised.
if raced:
raise
raced = True
continue
report.created = True
else:
definition = stored.get("definition") or {}
version = int(definition.get("version") or 1)
if not force:
known = _refuse_on_drift(client, target, definition)
document = target.document(origin)
flow_id = str((saved.get("definition") or {}).get("id") or "")
break
report.id = flow_id
moved = _moved_code(stored, document)
if stored is None:
saved = client.create_flow(document)
flow_id = str((saved.get("definition") or {}).get("id") or target.name)
else:
saved = client.put_flow(document | {"id": flow_id, "version": version})
# An engine older than the per-node digest parses the document without
# those fields and stores it without them, silently: no error, no 404, and
# a cache still keyed on the whole repository.
@@ -930,7 +1002,7 @@ def _moved_code(stored: dict[str, Any] | None, document: dict[str, Any]) -> list
def _refuse_on_drift(
client: Client, target: Flow | Declared, definition: dict[str, Any]
client: Client, target: Flow | Declared, flow_id: str, definition: dict[str, Any]
) -> dict[str, str]:
"""Stop before overwriting work that was done somewhere else.
@@ -956,9 +1028,7 @@ def _refuse_on_drift(
for stored_node in definition.get("nodes") or []:
node_id = str(stored_node.get("id"))
try:
entry = client.get_source_entry(
str(definition.get("id") or target.name), node_id
)
entry = client.get_source_entry(flow_id, node_id)
except ApiError:
continue
if entry.get("missing"):