Clients take a flow's id or key: one resolver in the SDK, id first, explicit id:/key:, a warning for reads and a refusal for changes when a name means two flows, deleted flows found by their last key; sync creates by POST, retries a create race through the drift checks and reports the id; MCP gets create_flow and resolves every flow argument with the caller's token; displays show flow_name; an engine from before ids is named as such

This commit is contained in:
2026-11-02 19:39:14 +01:00
parent fd637e0f1e
commit 53916d869b
20 changed files with 816 additions and 147 deletions
+11 -6
View File
@@ -776,18 +776,23 @@ class Flow:
def submit(
self, *, seed: int | None = None, client: Any = None, **params: Any
) -> Any:
"""Start a run of this flow and hand back a handle to it."""
"""Start a run of this flow and hand back a handle to it.
By its key, which is all a declaration knows, and as a key: an id
that happens to be spelled the same is another flow's.
"""
from fluksio.sdk.client import Client
return (client or Client()).submit(self.name, params, seed=seed)
return (client or Client()).submit(f"key:{self.name}", params, seed=seed)
def runs(self, limit: int = 20, client: Any = None) -> list[dict[str, Any]]:
"""This flow's runs, newest first."""
"""This flow's runs, newest first: the flow holding its key now, not
one deleted before it that held the key too."""
from fluksio.sdk.client import Client
rows: list[dict[str, Any]] = (client or Client()).runs(
flow=self.name, limit=limit
)
engine = client or Client()
flow_id = engine.resolve_flow(f"key:{self.name}").id
rows: list[dict[str, Any]] = engine.runs(flow=f"id:{flow_id}", limit=limit)
return rows
def __repr__(self) -> str:
+47 -17
View File
@@ -43,6 +43,7 @@ from fluksio.sdk.client import (
sync,
)
from fluksio.sdk.declare import ImportFailed, declare, declare_module
from fluksio.sdk.resolve import Unresolved
__all__ = ["add_parsers", "discover"]
@@ -72,6 +73,21 @@ def _fail(message: str) -> int:
NO_ENGINE = "Is one running? `fluksio serve` starts one."
@contextmanager
def _plain_warnings() -> Iterator[None]:
"""What the SDK warns about, one line each on stderr.
A terminal wants the sentence, not where in the library it was raised.
"""
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
try:
yield
finally:
for warning in caught:
print(f"fluksio: warning: {warning.message}", file=sys.stderr)
def _unreachable(exc: Exception, note: str = NO_ENGINE) -> int:
"""The engine did not answer. Say so as a sentence, not a traceback."""
return _fail(
@@ -387,15 +403,19 @@ def cmd_sync(args: argparse.Namespace) -> int:
return _unreachable(exc, "Nothing was published; sync again when it is back.")
for report in reports:
# The id too when it is not the key, since that is what a URL holds.
name = (
report.flow if report.id == report.flow else f"{report.flow} ({report.id})"
)
if report.unchanged:
_say(f" {report.flow}: unchanged")
_say(f" {name}: unchanged")
continue
what = "created" if report.created else "updated"
detail = ", ".join(report.changed)
# Nothing to publish is not the same as left as a draft, and saying
# the second when it was the first reads as work not finished.
state = "published" if report.published else "draft" if report.drafted else ""
_say(f" {report.flow}: {what} ({detail})" + (f" — {state}" if state else ""))
_say(f" {name}: {what} ({detail})" + (f" — {state}" if state else ""))
if any(report.forgot_code for report in reports):
engine = _engine_version(client)
_say(
@@ -533,7 +553,13 @@ def _declaration(client: Client, flow: str) -> tuple[str, str, Path] | None:
"""
if not flow:
return None
stored = client.get_flow(flow) or {}
try:
# As the run that follows will: a name two flows answer to is refused
# there, and not guessed at here.
flow_id = client.resolve_flow(flow, mutation=True).id
except Unresolved:
return None
stored = client.get_flow(flow_id) or {}
origin = (stored.get("definition") or {}).get("origin") or {}
declared_in = str(origin.get("declared_in") or "")
module = str(origin.get("module") or "")
@@ -716,7 +742,9 @@ def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
with _client_for(args) as client:
if not args.no_sync:
_sync_first(client, args.sync, args.flow)
stored = client.get_flow(args.flow)
# After the sync, which may be what creates it.
flow_id = client.resolve_flow(args.flow, mutation=True).id
stored = client.get_flow(flow_id)
if stored is None:
return _fail(f"no flow '{args.flow}' on that engine")
definition = stored.get("definition") or {}
@@ -732,7 +760,7 @@ def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
):
params = _ask_params(definition)
handle = client.submit(
args.flow,
f"id:{flow_id}",
params,
seed=args.seed,
no_cache=args.no_cache,
@@ -938,7 +966,7 @@ def _status_screen(client: Client) -> Any:
state, "red" if state == "error" else "yellow"
),
),
Text(str(row.get("flow", "")), style="dim"),
Text(str(row.get("flow_name") or row.get("flow", "")), style="dim"),
Text(_dur(row.get("duration_ms")), style="dim"),
Text(
_ago(row.get("finished_at") or row.get("created_at")), style="dim"
@@ -952,7 +980,9 @@ def _status_screen(client: Client) -> Any:
broken.add_column("")
for event in failures:
where = " ".join(
str(event.get(key, "")) for key in ("flow", "node") if event.get(key)
str(event.get(key, ""))
for key in ("flow_name", "node")
if event.get(key)
)
broken.add_row(
Text(f" {where}", style="red"),
@@ -1076,7 +1106,8 @@ def _stamp(row: dict[str, Any]) -> str:
def cmd_runs(args: argparse.Namespace) -> int:
try:
with _client_for(args, retries=0) as client:
rows = client.runs(flow=args.flow, limit=args.limit)
with _plain_warnings():
rows = client.runs(flow=args.flow, limit=args.limit)
declared = _declared(client, {str(row["flow"]) for row in rows})
except (SyncError, ApiError) as exc:
return _fail(str(exc))
@@ -1100,8 +1131,9 @@ def cmd_runs(args: argparse.Namespace) -> int:
params = json.dumps(shown)
if len(params) > room:
params = params[: room - 3] + "..."
flow = row.get("flow_name") or row["flow"]
_say(
f"{row['id']} {_status(row['status'], 9)} {row['flow']:<16} "
f"{row['id']} {_status(row['status'], 9)} {flow:<16} "
f"{_dur(row['duration_ms']):>8} {_ago(row.get('created_at')):>9} "
f"{_stamp(row):<22} {params}"
)
@@ -1269,11 +1301,12 @@ def cmd_sweep(args: argparse.Namespace) -> int:
with _client_for(args) as client:
if not args.no_sync:
_sync_first(client, args.sync, args.flow)
stored = client.get_flow(args.flow)
flow_id = client.resolve_flow(args.flow, mutation=True).id
stored = client.get_flow(flow_id)
if stored is None:
return _fail(f"no flow '{args.flow}' on that engine")
entries = _grid(stored.get("definition") or {}, args.param, args.seed)
handles = client.sweep(args.flow, entries, no_cache=args.no_cache)
handles = client.sweep(f"id:{flow_id}", entries, no_cache=args.no_cache)
for handle, entry in zip(handles, entries, strict=True):
_say(f"{handle.id} queued {json.dumps(entry['params'])}")
if not wait:
@@ -1401,13 +1434,10 @@ def _export(
try:
with _client_for(args, retries=0) as client:
try:
if getattr(args, "list_names", False):
return _list_names(client, args)
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
with _plain_warnings():
if getattr(args, "list_names", False):
return _list_names(client, args)
rows = fetch(client)
for warning in caught:
print(f"fluksio: warning: {warning.message}", file=sys.stderr)
except ApiError as exc:
if exc.status != 404:
raise
+109 -39
View File
@@ -18,7 +18,9 @@ from pathlib import Path
from typing import Any
from urllib.parse import quote
from fluksio import __version__
from fluksio.sdk import MARKER, Declared, Flow, SyncError
from fluksio.sdk.resolve import Resolved, resolve
__all__ = [
"Client",
@@ -62,6 +64,15 @@ WAIT_TOLERANCE = 5
#: Runs per `POST /runs/import`, which is what the engine caps one request at.
IMPORT_PAGE = 200
#: What to do about an engine from before flow ids. It lists flows without an
#: id and has no route to create one by, so this client cannot tell which flow
#: a name means, nor make one.
OLD_ENGINE = (
"this engine is older than flow ids, and this client "
f"({__version__}) needs them. Upgrade the engine first: "
"`pip install -U fluksio` where it runs"
)
#: What a project-local instance is called, beside `.venv` and `.git`.
DATA_DIR_NAME = ".fluksio"
@@ -251,13 +262,49 @@ class Client:
"""The listing of the flow holding the key *key*, or ``None``."""
result = self._call("GET", "/flows/", params={"name": key})
found = list(result.get("data") or [])
if found and "id" not in found[0]:
# It ignored the filter too, so this is every flow, not the one.
raise SyncError(OLD_ENGINE)
return found[0] if found else None
def create_flow(self, document: dict[str, Any]) -> dict[str, Any]:
"""A new flow, under the id the engine issues for it."""
result: dict[str, Any] = self._call("POST", "/flows/", json=document)
try:
result: dict[str, Any] = self._call("POST", "/flows/", json=document)
except ApiError as exc:
if exc.status in (404, 405):
raise SyncError(OLD_ENGINE) from exc
raise
return result
def resolve_flow(
self, flow: str, *, mutation: bool = False, history: bool = False
) -> Resolved:
"""The flow *flow* names, by its id or its key (`fluksio.sdk.resolve`).
``history`` counts deleted flows too, under the key their runs last
had, for a question about the past. A name two flows answer to is
refused when ``mutation`` says the call changes something, and comes
back with a warning otherwise, for the caller to show.
"""
rows = self.flows()
if any("id" not in row for row in rows):
raise SyncError(OLD_ENGINE)
flows = [(str(row["id"]), str(row["name"])) for row in rows]
if history:
flows += [
(str(row["flow"]), str(row.get("flow_name") or row["flow"]))
for row in self._call("GET", "/runs/overview")
]
return resolve(flow, flows, mutation=mutation)
def _flow_id(self, flow: str, **how: bool) -> str:
"""The id behind *flow*, warning when it could have meant another."""
found = self.resolve_flow(flow, **how)
if found.warning:
warnings.warn(found.warning, stacklevel=3)
return found.id
def put_flow(self, document: dict[str, Any]) -> dict[str, Any]:
"""Save over the flow *document* names by its id, or by its key for
one whose id is its key."""
@@ -338,14 +385,15 @@ class Client:
) -> list[dict[str, Any]]:
"""What went wrong, or who changed what. Newest first.
Engine-wide, and narrowed by flow, by run or by time. ``run`` takes a
run id and answers what the engine recorded during it. For the failure
that ended a run, :attr:`RunHandle.failures` is still the shorter road:
it reads the run's own node rows, which carry the traceback too.
Engine-wide, and narrowed by flow (its id or key), by run or by time.
``run`` takes a run id and answers what the engine recorded during it.
For the failure that ended a run, :attr:`RunHandle.failures` is still
the shorter road: it reads the run's own node rows, which carry the
traceback too.
"""
query: dict[str, Any] = {"kind": kind, "limit": limit}
if flow:
query["flow"] = flow
query["flow"] = self._flow_id(flow, history=True)
if run:
query["run"] = run
for name, value in (("since", since), ("until", until)):
@@ -366,8 +414,9 @@ class Client:
no_cache: bool = False,
cause: str = "sdk",
) -> RunHandle:
"""Queue a run. ``cause`` is what the history records it as coming
from — a script is the default, `fluksio run` says so itself.
"""Queue a run of the flow *flow*, its id or key. ``cause`` is what
the history records it as coming from — a script is the default,
`fluksio run` says so itself.
Safe to retry: the key is minted here, once per call, so an attempt
that timed out on the way back is answered with the run it made rather
@@ -375,7 +424,7 @@ class Client:
"""
row = self._call(
"POST",
f"/runs/flows/{seg(flow)}",
f"/runs/flows/{seg(self._flow_id(flow, mutation=True))}",
idempotent=True,
json={
"params": params or {},
@@ -401,7 +450,7 @@ class Client:
keyed = [{**entry, "idempotency_key": uuid.uuid4().hex} for entry in entries]
rows = self._call(
"POST",
f"/runs/flows/{seg(flow)}/sweep",
f"/runs/flows/{seg(self._flow_id(flow, mutation=True))}/sweep",
idempotent=True,
json={"runs": keyed, "no_cache": no_cache},
)
@@ -410,10 +459,17 @@ class Client:
def import_runs(self, entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Record runs executed somewhere else, a page at a time.
Each entry is the shape `POST /runs/import` takes. Safe to send again:
one whose `external_id` is already on the engine answers
`created: False` and adds nothing.
Each entry is the shape `POST /runs/import` takes, but its `flow` may
be the key as well as the id. Safe to send again: one whose
`external_id` is already on the engine answers `created: False` and
adds nothing.
"""
named = {str(entry["flow"]) for entry in entries if entry.get("flow")}
ids = {flow: self._flow_id(flow, mutation=True) for flow in named}
entries = [
{**entry, "flow": ids[str(entry["flow"])]} if entry.get("flow") else entry
for entry in entries
]
answers: list[dict[str, Any]] = []
for start in range(0, len(entries), IMPORT_PAGE):
answers.extend(
@@ -431,9 +487,10 @@ class Client:
return result
def runs(self, flow: str = "", limit: int = 20, **filters: Any) -> Any:
"""Runs, newest first, of every flow or of *flow*, its id or key."""
query = {"limit": limit, **filters}
if flow:
query["flow"] = flow
query["flow"] = self._flow_id(flow, history=True)
return self._call("GET", "/runs", params=query)
def metrics(self, run_id: str, name: str = "", stride: int = 1) -> Any:
@@ -456,7 +513,7 @@ class Client:
if named:
query["ids"] = ",".join(named)
if flow:
query["flow"] = flow
query["flow"] = self._flow_id(flow, history=True)
return self._call("GET", "/runs/metrics/names", params=query)
def compare(self, ids: Iterable[str], metric: str, x: str = "") -> Any:
@@ -482,7 +539,8 @@ class Client:
"""Runs' series as long rows: run, name, step, ts, value.
``pd.DataFrame(client.export_metrics(flow="train"))`` is every loss
curve of that flow with the run id on each row. ``name`` keeps the
curve of that flow with the run id on each row. ``flow`` is its id or
key. ``name`` keeps the
metrics it lists, ``stride`` thins each curve, and the selection is
narrowed the way :meth:`runs` is — ``status=``, ``group=``,
``since=``, ``until=``.
@@ -530,7 +588,7 @@ class Client:
and that is a warning, since the rows alone look complete.
"""
if flow:
query["flow"] = flow
query["flow"] = self._flow_id(flow, history=True)
named = ",".join(ids)
if named:
query["ids"] = named
@@ -728,6 +786,9 @@ class SyncReport:
def __init__(self, flow: str) -> None:
self.flow = flow
#: The id this engine holds the flow under. Another engine's differs:
#: the key is what instances share.
self.id = ""
self.created = False
self.changed: list[str] = []
self.published = False
@@ -835,30 +896,41 @@ def _sync_one(
force: bool,
) -> SyncReport:
report = SyncReport(target.name)
# The declared key is what an instance shares; the id is this engine's,
# and every request after the lookup names the flow by it.
found = client.find_flow(target.name)
flow_id = str(found["id"]) if found else ""
stored = client.get_flow(flow_id) if flow_id else None
document = target.document(origin)
# The declared key is what instances share; the id is this engine's, and
# every request after the lookup names the flow by it. Looked up a second
# time only when another sync created the flow in between, and then its
# flow is checked as any stored one is before it is written over.
raced = False
version = 1
# What the drift check already read, so the write loop below does not ask
# the engine for every node's body a second time.
known: dict[str, str] = {}
if stored is None:
while True:
found = client.find_flow(target.name)
flow_id = str(found["id"]) if found else ""
stored = client.get_flow(flow_id) if flow_id else None
if stored is not None:
definition = stored.get("definition") or {}
version = int(definition.get("version") or 1)
if not force:
known = _refuse_on_drift(client, target, flow_id, definition)
saved = client.put_flow(document | {"id": flow_id, "version": version})
break
try:
saved = client.create_flow(document)
except Conflict:
# The key was free a moment ago. Anything else the engine refuses
# a create for is refused again, and that one is raised.
if raced:
raise
raced = True
continue
report.created = True
else:
definition = stored.get("definition") or {}
version = int(definition.get("version") or 1)
if not force:
known = _refuse_on_drift(client, target, definition)
document = target.document(origin)
flow_id = str((saved.get("definition") or {}).get("id") or "")
break
report.id = flow_id
moved = _moved_code(stored, document)
if stored is None:
saved = client.create_flow(document)
flow_id = str((saved.get("definition") or {}).get("id") or target.name)
else:
saved = client.put_flow(document | {"id": flow_id, "version": version})
# An engine older than the per-node digest parses the document without
# those fields and stores it without them, silently: no error, no 404, and
# a cache still keyed on the whole repository.
@@ -930,7 +1002,7 @@ def _moved_code(stored: dict[str, Any] | None, document: dict[str, Any]) -> list
def _refuse_on_drift(
client: Client, target: Flow | Declared, definition: dict[str, Any]
client: Client, target: Flow | Declared, flow_id: str, definition: dict[str, Any]
) -> dict[str, str]:
"""Stop before overwriting work that was done somewhere else.
@@ -956,9 +1028,7 @@ def _refuse_on_drift(
for stored_node in definition.get("nodes") or []:
node_id = str(stored_node.get("id"))
try:
entry = client.get_source_entry(
str(definition.get("id") or target.name), node_id
)
entry = client.get_source_entry(flow_id, node_id)
except ApiError:
continue
if entry.get("missing"):
+87
View File
@@ -0,0 +1,87 @@
"""Which flow an argument names: its id or its key, id first.
The API takes ids and only ids. People type keys, and scripts written before
ids existed pass keys, so the CLI, the SDK and the MCP tools answer the same
question here, with the same words.
Pure: the caller fetches what it may see with its own credentials and hands
it over as ``(id, key)`` pairs. The SDK carries no engine code, so this lives
with it, and the server's MCP tools import it from here.
"""
from __future__ import annotations
import difflib
from collections.abc import Iterable
from dataclasses import dataclass
from fluksio.sdk import SyncError
__all__ = ["Resolved", "Unresolved", "resolve"]
#: The explicit forms. A colon is legal in neither an id nor a key, so they
#: cannot clash with a real value.
ID = "id:"
KEY = "key:"
class Unresolved(SyncError):
"""Nothing, or more than one thing, answers to the argument."""
@dataclass(frozen=True)
class Resolved:
id: str
#: Empty for an explicit id nobody lists any more.
key: str
#: Set when the argument was one flow's id and another's key, and the
#: id was taken. Read-only callers show it; a change never gets this far.
warning: str = ""
def resolve(
value: str, flows: Iterable[tuple[str, str]], *, mutation: bool = False
) -> Resolved:
"""The flow *value* names among *flows*.
``id:<x>`` is taken as said, listed or not: history outlives its flow,
and the engine answers for an id nobody holds. ``key:<x>`` is the flow
holding that key. A plain string is an id if one has it, else a key.
A plain string that is one flow's id and another's key picks the id with
a warning, or is refused when *mutation* says the call changes something.
A key several flows answer to, which history can hold, is always refused.
"""
keys: dict[str, str] = {}
for flow_id, key in flows:
keys.setdefault(flow_id, key)
if value.startswith(ID):
named = value[len(ID) :]
return Resolved(named, keys.get(named, ""))
explicit = value.startswith(KEY)
key = value[len(KEY) :] if explicit else value
holders = [flow_id for flow_id, held in keys.items() if held == key]
if not explicit and value in keys:
others = [flow_id for flow_id in holders if flow_id != value]
if not others:
return Resolved(value, keys[value])
both = (
f"'{value}' is the id of '{keys[value]}' and the key of "
f"'{', '.join(others)}'."
)
if mutation:
raise Unresolved(f"{both} Pass id:{value} or key:{value}.")
return Resolved(
value, keys[value], f"{both} Using the id. Pass key:{value} for the other."
)
if len(holders) == 1:
return Resolved(holders[0], key)
if holders:
raise Unresolved(
f"Several flows have had the key '{key}': {', '.join(holders)}. "
"Pass id:<one of them>."
)
close = difflib.get_close_matches(key, sorted(set(keys.values())), n=3)
hint = f" Closest keys: {', '.join(close)}." if close else ""
what = "key" if explicit else "id or key"
raise Unresolved(f"No flow with the {what} '{key}'.{hint}")