Clients take a flow's id or key: one resolver in the SDK, id first, explicit id:/key:, a warning for reads and a refusal for changes when a name means two flows, deleted flows found by their last key; sync creates by POST, retries a create race through the drift checks and reports the id; MCP gets create_flow and resolves every flow argument with the caller's token; displays show flow_name; an engine from before ids is named as such
This commit is contained in:
+109
-39
@@ -18,7 +18,9 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from fluksio import __version__
|
||||
from fluksio.sdk import MARKER, Declared, Flow, SyncError
|
||||
from fluksio.sdk.resolve import Resolved, resolve
|
||||
|
||||
__all__ = [
|
||||
"Client",
|
||||
@@ -62,6 +64,15 @@ WAIT_TOLERANCE = 5
|
||||
#: Runs per `POST /runs/import`, which is what the engine caps one request at.
|
||||
IMPORT_PAGE = 200
|
||||
|
||||
#: What to do about an engine from before flow ids. It lists flows without an
|
||||
#: id and has no route to create one by, so this client cannot tell which flow
|
||||
#: a name means, nor make one.
|
||||
OLD_ENGINE = (
|
||||
"this engine is older than flow ids, and this client "
|
||||
f"({__version__}) needs them. Upgrade the engine first: "
|
||||
"`pip install -U fluksio` where it runs"
|
||||
)
|
||||
|
||||
|
||||
#: What a project-local instance is called, beside `.venv` and `.git`.
|
||||
DATA_DIR_NAME = ".fluksio"
|
||||
@@ -251,13 +262,49 @@ class Client:
|
||||
"""The listing of the flow holding the key *key*, or ``None``."""
|
||||
result = self._call("GET", "/flows/", params={"name": key})
|
||||
found = list(result.get("data") or [])
|
||||
if found and "id" not in found[0]:
|
||||
# It ignored the filter too, so this is every flow, not the one.
|
||||
raise SyncError(OLD_ENGINE)
|
||||
return found[0] if found else None
|
||||
|
||||
def create_flow(self, document: dict[str, Any]) -> dict[str, Any]:
|
||||
"""A new flow, under the id the engine issues for it."""
|
||||
result: dict[str, Any] = self._call("POST", "/flows/", json=document)
|
||||
try:
|
||||
result: dict[str, Any] = self._call("POST", "/flows/", json=document)
|
||||
except ApiError as exc:
|
||||
if exc.status in (404, 405):
|
||||
raise SyncError(OLD_ENGINE) from exc
|
||||
raise
|
||||
return result
|
||||
|
||||
def resolve_flow(
|
||||
self, flow: str, *, mutation: bool = False, history: bool = False
|
||||
) -> Resolved:
|
||||
"""The flow *flow* names, by its id or its key (`fluksio.sdk.resolve`).
|
||||
|
||||
``history`` counts deleted flows too, under the key their runs last
|
||||
had, for a question about the past. A name two flows answer to is
|
||||
refused when ``mutation`` says the call changes something, and comes
|
||||
back with a warning otherwise, for the caller to show.
|
||||
"""
|
||||
rows = self.flows()
|
||||
if any("id" not in row for row in rows):
|
||||
raise SyncError(OLD_ENGINE)
|
||||
flows = [(str(row["id"]), str(row["name"])) for row in rows]
|
||||
if history:
|
||||
flows += [
|
||||
(str(row["flow"]), str(row.get("flow_name") or row["flow"]))
|
||||
for row in self._call("GET", "/runs/overview")
|
||||
]
|
||||
return resolve(flow, flows, mutation=mutation)
|
||||
|
||||
def _flow_id(self, flow: str, **how: bool) -> str:
|
||||
"""The id behind *flow*, warning when it could have meant another."""
|
||||
found = self.resolve_flow(flow, **how)
|
||||
if found.warning:
|
||||
warnings.warn(found.warning, stacklevel=3)
|
||||
return found.id
|
||||
|
||||
def put_flow(self, document: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Save over the flow *document* names by its id, or by its key for
|
||||
one whose id is its key."""
|
||||
@@ -338,14 +385,15 @@ class Client:
|
||||
) -> list[dict[str, Any]]:
|
||||
"""What went wrong, or who changed what. Newest first.
|
||||
|
||||
Engine-wide, and narrowed by flow, by run or by time. ``run`` takes a
|
||||
run id and answers what the engine recorded during it. For the failure
|
||||
that ended a run, :attr:`RunHandle.failures` is still the shorter road:
|
||||
it reads the run's own node rows, which carry the traceback too.
|
||||
Engine-wide, and narrowed by flow (its id or key), by run or by time.
|
||||
``run`` takes a run id and answers what the engine recorded during it.
|
||||
For the failure that ended a run, :attr:`RunHandle.failures` is still
|
||||
the shorter road: it reads the run's own node rows, which carry the
|
||||
traceback too.
|
||||
"""
|
||||
query: dict[str, Any] = {"kind": kind, "limit": limit}
|
||||
if flow:
|
||||
query["flow"] = flow
|
||||
query["flow"] = self._flow_id(flow, history=True)
|
||||
if run:
|
||||
query["run"] = run
|
||||
for name, value in (("since", since), ("until", until)):
|
||||
@@ -366,8 +414,9 @@ class Client:
|
||||
no_cache: bool = False,
|
||||
cause: str = "sdk",
|
||||
) -> RunHandle:
|
||||
"""Queue a run. ``cause`` is what the history records it as coming
|
||||
from — a script is the default, `fluksio run` says so itself.
|
||||
"""Queue a run of the flow *flow*, its id or key. ``cause`` is what
|
||||
the history records it as coming from — a script is the default,
|
||||
`fluksio run` says so itself.
|
||||
|
||||
Safe to retry: the key is minted here, once per call, so an attempt
|
||||
that timed out on the way back is answered with the run it made rather
|
||||
@@ -375,7 +424,7 @@ class Client:
|
||||
"""
|
||||
row = self._call(
|
||||
"POST",
|
||||
f"/runs/flows/{seg(flow)}",
|
||||
f"/runs/flows/{seg(self._flow_id(flow, mutation=True))}",
|
||||
idempotent=True,
|
||||
json={
|
||||
"params": params or {},
|
||||
@@ -401,7 +450,7 @@ class Client:
|
||||
keyed = [{**entry, "idempotency_key": uuid.uuid4().hex} for entry in entries]
|
||||
rows = self._call(
|
||||
"POST",
|
||||
f"/runs/flows/{seg(flow)}/sweep",
|
||||
f"/runs/flows/{seg(self._flow_id(flow, mutation=True))}/sweep",
|
||||
idempotent=True,
|
||||
json={"runs": keyed, "no_cache": no_cache},
|
||||
)
|
||||
@@ -410,10 +459,17 @@ class Client:
|
||||
def import_runs(self, entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Record runs executed somewhere else, a page at a time.
|
||||
|
||||
Each entry is the shape `POST /runs/import` takes. Safe to send again:
|
||||
one whose `external_id` is already on the engine answers
|
||||
`created: False` and adds nothing.
|
||||
Each entry is the shape `POST /runs/import` takes, but its `flow` may
|
||||
be the key as well as the id. Safe to send again: one whose
|
||||
`external_id` is already on the engine answers `created: False` and
|
||||
adds nothing.
|
||||
"""
|
||||
named = {str(entry["flow"]) for entry in entries if entry.get("flow")}
|
||||
ids = {flow: self._flow_id(flow, mutation=True) for flow in named}
|
||||
entries = [
|
||||
{**entry, "flow": ids[str(entry["flow"])]} if entry.get("flow") else entry
|
||||
for entry in entries
|
||||
]
|
||||
answers: list[dict[str, Any]] = []
|
||||
for start in range(0, len(entries), IMPORT_PAGE):
|
||||
answers.extend(
|
||||
@@ -431,9 +487,10 @@ class Client:
|
||||
return result
|
||||
|
||||
def runs(self, flow: str = "", limit: int = 20, **filters: Any) -> Any:
|
||||
"""Runs, newest first, of every flow or of *flow*, its id or key."""
|
||||
query = {"limit": limit, **filters}
|
||||
if flow:
|
||||
query["flow"] = flow
|
||||
query["flow"] = self._flow_id(flow, history=True)
|
||||
return self._call("GET", "/runs", params=query)
|
||||
|
||||
def metrics(self, run_id: str, name: str = "", stride: int = 1) -> Any:
|
||||
@@ -456,7 +513,7 @@ class Client:
|
||||
if named:
|
||||
query["ids"] = ",".join(named)
|
||||
if flow:
|
||||
query["flow"] = flow
|
||||
query["flow"] = self._flow_id(flow, history=True)
|
||||
return self._call("GET", "/runs/metrics/names", params=query)
|
||||
|
||||
def compare(self, ids: Iterable[str], metric: str, x: str = "") -> Any:
|
||||
@@ -482,7 +539,8 @@ class Client:
|
||||
"""Runs' series as long rows: run, name, step, ts, value.
|
||||
|
||||
``pd.DataFrame(client.export_metrics(flow="train"))`` is every loss
|
||||
curve of that flow with the run id on each row. ``name`` keeps the
|
||||
curve of that flow with the run id on each row. ``flow`` is its id or
|
||||
key. ``name`` keeps the
|
||||
metrics it lists, ``stride`` thins each curve, and the selection is
|
||||
narrowed the way :meth:`runs` is — ``status=``, ``group=``,
|
||||
``since=``, ``until=``.
|
||||
@@ -530,7 +588,7 @@ class Client:
|
||||
and that is a warning, since the rows alone look complete.
|
||||
"""
|
||||
if flow:
|
||||
query["flow"] = flow
|
||||
query["flow"] = self._flow_id(flow, history=True)
|
||||
named = ",".join(ids)
|
||||
if named:
|
||||
query["ids"] = named
|
||||
@@ -728,6 +786,9 @@ class SyncReport:
|
||||
|
||||
def __init__(self, flow: str) -> None:
|
||||
self.flow = flow
|
||||
#: The id this engine holds the flow under. Another engine's differs:
|
||||
#: the key is what instances share.
|
||||
self.id = ""
|
||||
self.created = False
|
||||
self.changed: list[str] = []
|
||||
self.published = False
|
||||
@@ -835,30 +896,41 @@ def _sync_one(
|
||||
force: bool,
|
||||
) -> SyncReport:
|
||||
report = SyncReport(target.name)
|
||||
# The declared key is what an instance shares; the id is this engine's,
|
||||
# and every request after the lookup names the flow by it.
|
||||
found = client.find_flow(target.name)
|
||||
flow_id = str(found["id"]) if found else ""
|
||||
stored = client.get_flow(flow_id) if flow_id else None
|
||||
document = target.document(origin)
|
||||
# The declared key is what instances share; the id is this engine's, and
|
||||
# every request after the lookup names the flow by it. Looked up a second
|
||||
# time only when another sync created the flow in between, and then its
|
||||
# flow is checked as any stored one is before it is written over.
|
||||
raced = False
|
||||
version = 1
|
||||
# What the drift check already read, so the write loop below does not ask
|
||||
# the engine for every node's body a second time.
|
||||
known: dict[str, str] = {}
|
||||
if stored is None:
|
||||
while True:
|
||||
found = client.find_flow(target.name)
|
||||
flow_id = str(found["id"]) if found else ""
|
||||
stored = client.get_flow(flow_id) if flow_id else None
|
||||
if stored is not None:
|
||||
definition = stored.get("definition") or {}
|
||||
version = int(definition.get("version") or 1)
|
||||
if not force:
|
||||
known = _refuse_on_drift(client, target, flow_id, definition)
|
||||
saved = client.put_flow(document | {"id": flow_id, "version": version})
|
||||
break
|
||||
try:
|
||||
saved = client.create_flow(document)
|
||||
except Conflict:
|
||||
# The key was free a moment ago. Anything else the engine refuses
|
||||
# a create for is refused again, and that one is raised.
|
||||
if raced:
|
||||
raise
|
||||
raced = True
|
||||
continue
|
||||
report.created = True
|
||||
else:
|
||||
definition = stored.get("definition") or {}
|
||||
version = int(definition.get("version") or 1)
|
||||
if not force:
|
||||
known = _refuse_on_drift(client, target, definition)
|
||||
|
||||
document = target.document(origin)
|
||||
flow_id = str((saved.get("definition") or {}).get("id") or "")
|
||||
break
|
||||
report.id = flow_id
|
||||
moved = _moved_code(stored, document)
|
||||
if stored is None:
|
||||
saved = client.create_flow(document)
|
||||
flow_id = str((saved.get("definition") or {}).get("id") or target.name)
|
||||
else:
|
||||
saved = client.put_flow(document | {"id": flow_id, "version": version})
|
||||
# An engine older than the per-node digest parses the document without
|
||||
# those fields and stores it without them, silently: no error, no 404, and
|
||||
# a cache still keyed on the whole repository.
|
||||
@@ -930,7 +1002,7 @@ def _moved_code(stored: dict[str, Any] | None, document: dict[str, Any]) -> list
|
||||
|
||||
|
||||
def _refuse_on_drift(
|
||||
client: Client, target: Flow | Declared, definition: dict[str, Any]
|
||||
client: Client, target: Flow | Declared, flow_id: str, definition: dict[str, Any]
|
||||
) -> dict[str, str]:
|
||||
"""Stop before overwriting work that was done somewhere else.
|
||||
|
||||
@@ -956,9 +1028,7 @@ def _refuse_on_drift(
|
||||
for stored_node in definition.get("nodes") or []:
|
||||
node_id = str(stored_node.get("id"))
|
||||
try:
|
||||
entry = client.get_source_entry(
|
||||
str(definition.get("id") or target.name), node_id
|
||||
)
|
||||
entry = client.get_source_entry(flow_id, node_id)
|
||||
except ApiError:
|
||||
continue
|
||||
if entry.get("missing"):
|
||||
|
||||
Reference in New Issue
Block a user