A batch run skips the nodes waiting on a failed one and ends error; a run whose nodes cannot load fails with the load error before anything executes; cancelling a run stops its pipeline without announcing a pause of the live flow; batch_issues refuses a delay node's interval; the client accepts an engine URL with or without /api/v1; RedisState's compare-and-swap records the names it writes

This commit is contained in:
2026-11-04 21:38:21 +01:00
parent eb17528ce6
commit d028bf326a
11 changed files with 221 additions and 32 deletions
+23 -1
View File
@@ -148,6 +148,17 @@ class RebuildBusy(RuntimeError):
""" """
class NodesNotLoaded(RuntimeError):
"""A run's flow has nodes that could not be built, so none of it runs."""
def __init__(self, errors: dict[str, str]) -> None:
#: The load error of each node, by its qualified id.
self.errors = errors
node, error = next(iter(errors.items()))
more = f" and {len(errors) - 1} more" if len(errors) > 1 else ""
super().__init__(f"Node '{node}'{more} could not be loaded: {error}")
class NodeStatus(str, Enum): class NodeStatus(str, Enum):
ACTIVE = "active" ACTIVE = "active"
ERROR = "error" ERROR = "error"
@@ -2010,10 +2021,21 @@ class FlowController:
and webhooks would only be a second copy of what the live pipeline and webhooks would only be a second copy of what the live pipeline
already holds. The state is the run's, which is what keeps two runs of already holds. The state is the run's, which is what keeps two runs of
one flow from overwriting each other's messages. one flow from overwriting each other's messages.
:raises NodesNotLoaded: When a node could not be built. The live engine
leaves such a node out, but a run without it would finish as if
the flow had done its work.
""" """
nodes, _loaded, initial_values, _inputs = self._build_flows( nodes, loaded, initial_values, _inputs = self._build_flows(
[(flow, draft)], run=run [(flow, draft)], run=run
) )
errors = {
entry.id: entry.error or "unknown error"
for entry in loaded.values()
if entry.status is NodeStatus.ERROR
}
if errors:
raise NodesNotLoaded(errors)
pipeline = Pipeline( pipeline = Pipeline(
nodes=nodes, nodes=nodes,
state=state, state=state,
+18 -3
View File
@@ -147,6 +147,8 @@ class NodeOutcome(BaseModel):
artifacts: dict[str, dict[str, Any]] = {} artifacts: dict[str, dict[str, Any]] = {}
#: Restored from an earlier run rather than executed. #: Restored from an earlier run rather than executed.
cached: bool = False cached: bool = False
#: Not executed, because a node it waits on failed or was skipped itself.
skipped: bool = False
#: Which run the restored values came from — and, because a restored row #: Which run the restored values came from — and, because a restored row
#: holds no series of its own, which run this node's metrics live in. #: holds no series of its own, which run this node's metrics live in.
cached_from: str = "" cached_from: str = ""
@@ -1071,7 +1073,8 @@ class Pipeline:
overrides: dict[str, Any] | None = None, overrides: dict[str, Any] | None = None,
inputs: dict[str, Any] | None = None, inputs: dict[str, Any] | None = None,
) -> dict[str, Any] | None: ) -> dict[str, Any] | None:
"""Run one node and record its outputs. Never raises. """Run one node and record its outputs. Never raises: None means it
failed, and a node that published nothing answers ``{}``.
``inputs`` is what the readiness check already read — the same keys, a ``inputs`` is what the readiness check already read — the same keys, a
moment earlier. Reading them again here was the second of two MGETs moment earlier. Reading them again here was the second of two MGETs
@@ -1106,7 +1109,7 @@ class Pipeline:
if key: if key:
hit, restored = self._from_cache(node, key, state, entry_id) hit, restored = self._from_cache(node, key, state, entry_id)
if hit: if hit:
return restored return restored or {}
with logs.capture(collected): with logs.capture(collected):
result = node.execute(inputs) result = node.execute(inputs)
@@ -1169,7 +1172,7 @@ class Pipeline:
output_values=result, output_values=result,
) )
) )
return result return result or {}
except Exception as exc: except Exception as exc:
# One failing node must not take the rest of the graph down. # One failing node must not take the rest of the graph down.
error = self.publish_error(node, exc, collected, entry_id) error = self.publish_error(node, exc, collected, entry_id)
@@ -1431,6 +1434,11 @@ class Pipeline:
ready_nodes = deque(n for n, count in in_degree.items() if count == 0) ready_nodes = deque(n for n, count in in_degree.items() if count == 0)
blocked: set[Node] = set() blocked: set[Node] = set()
# A batch run starts from an empty state, so a node whose producer
# failed would be called without that input. It is skipped instead,
# and so is everything waiting on it. A live wave still has the last
# values to run on.
failed: set[Node] | None = set() if self.run_id else None
gate_revision = self._gate_revision gate_revision = self._gate_revision
node_futures: dict[Future[dict[str, Any] | None], Node] = {} node_futures: dict[Future[dict[str, Any] | None], Node] = {}
finished: SimpleQueue[Future[dict[str, Any] | None]] = SimpleQueue() finished: SimpleQueue[Future[dict[str, Any] | None]] = SimpleQueue()
@@ -1469,6 +1477,11 @@ class Pipeline:
if n.passive: if n.passive:
complete(n) complete(n)
continue continue
if failed is not None and not failed.isdisjoint(deps[n]):
failed.add(n)
self._observe(NodeOutcome(node=n.id, ok=False, skipped=True))
complete(n)
continue
if fresh is not None and untouched(n): if fresh is not None and untouched(n):
complete(n) complete(n)
continue continue
@@ -1497,6 +1510,8 @@ class Pipeline:
future = finished.get() future = finished.get()
n = node_futures.pop(future) n = node_futures.pop(future)
result = future.result() result = future.result()
if result is None and failed is not None:
failed.add(n)
if result is not None and fresh is not None: if result is not None and fresh is not None:
fresh.update(result) fresh.update(result)
# Silent, failed and throttled producers still release their # Silent, failed and throttled producers still release their
+46 -22
View File
@@ -51,7 +51,12 @@ from sqlmodel import Session, col, select
from fluksio.core.db import engine as db_engine from fluksio.core.db import engine as db_engine
from fluksio.flow import tombstones from fluksio.flow import tombstones
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext from fluksio.flow.controller import (
NODE_TYPES,
FlowController,
NodesNotLoaded,
RunContext,
)
from fluksio.flow.messages import qualify, requalify from fluksio.flow.messages import qualify, requalify
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
from fluksio.flow.queue import WorkItem, WorkQueue from fluksio.flow.queue import WorkItem, WorkQueue
@@ -137,8 +142,9 @@ def batch_issues(
history keeps every value, and the interval only thins what is published. history keeps every value, and the interval only thins what is published.
Anywhere else it is a message quietly going missing. Anywhere else it is a message quietly going missing.
A delay node is fine; without a queue to defer into it simply sleeps, A delay node's ``interval`` is the same limit, put on every output it
which in a run is what was asked for. has. Its ``delay`` is fine: without a queue to defer into it simply
sleeps, which in a run is what was asked for.
The other is a node with no body. The store answers a new node's template The other is a node with no body. The store answers a new node's template
when nothing was ever written for one, so such a node runs — and returns when nothing was ever written for one, so such a node runs — and returns
@@ -148,8 +154,11 @@ def batch_issues(
""" """
issues: list[str] = [] issues: list[str] = []
for node in flow.nodes: for node in flow.nodes:
for spec in list(node.requires) + list(node.provides): held = float(node.params.get("interval") or 0) if node.type == "delay" else 0
if spec.interval > 0 and not spec.stream: limits = [(spec, spec.interval) for spec in node.requires]
limits += [(spec, max(spec.interval, held)) for spec in node.provides]
for spec, interval in limits:
if interval > 0 and not spec.stream:
issues.append( issues.append(
f"Node '{node.id}' rate-limits '{spec.port or spec.name}'. " f"Node '{node.id}' rate-limits '{spec.port or spec.name}'. "
"A run has no timer to release what that holds back, so " "A run has no timer to release what that holds back, so "
@@ -1236,8 +1245,10 @@ class RunService:
return True return True
# The gate first, so nothing new is submitted while the running nodes # The gate first, so nothing new is submitted while the running nodes
# are being killed; a gated node is never handed to the executor, so # are being killed; a gated node is never handed to the executor, so
# the graph drains instead of going further. # the graph drains instead of going further. Stopped rather than
pipeline.pause(self._key_of(run_id)) # paused: a pause announces itself on the engine's bus, under the key
# the live flow shares, so every open editor showed that one paused.
pipeline.set_disabled({self._key_of(run_id)})
workers = self.controller.workers workers = self.controller.workers
if workers is not None: if workers is not None:
# Keyed by run, so a sweep cancelling one config leaves the others # Keyed by run, so a sweep cancelling one config leaves the others
@@ -1502,7 +1513,7 @@ class RunService:
def observe(outcome: NodeOutcome) -> None: def observe(outcome: NodeOutcome) -> None:
nonlocal errors nonlocal errors
if not outcome.ok: if not outcome.ok and not outcome.skipped:
errors += 1 errors += 1
self._record_node(run, key, outcome) self._record_node(run, key, outcome)
@@ -1537,19 +1548,26 @@ class RunService:
digests = node_digests(flow) digests = node_digests(flow)
run.code_digest = self._restamp(run, run_digest(flow)) run.code_digest = self._restamp(run, run_digest(flow))
state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}") state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}")
pipeline = self.controller.build_run_pipeline( try:
flow, pipeline = self.controller.build_run_pipeline(
state=state, flow,
draft=run.draft, state=state,
observer=observe, draft=run.draft,
emission_observer=sink.handle, observer=observe,
run=RunContext( emission_observer=sink.handle,
run_id=run_id, run=RunContext(
code_digest=run.code_digest, run_id=run_id,
node_digests=digests, code_digest=run.code_digest,
), node_digests=digests,
run_cache=None if run.no_cache else self._cache, ),
) run_cache=None if run.no_cache else self._cache,
)
except NodesNotLoaded as exc:
# Nothing of it runs; the finally below records it.
for node, error in exc.errors.items():
observe(NodeOutcome(node=node, ok=False, error=error))
status, reason = "error", str(exc)[:1024]
return
with self._lock: with self._lock:
if run_id in self._cancelled: if run_id in self._cancelled:
# Cancelled while it was being built, so nothing of it # Cancelled while it was being built, so nothing of it
@@ -1627,10 +1645,16 @@ class RunService:
run_id = run.id run_id = run.id
node = requalify(outcome.node, key, run.flow_id)[:255] node = requalify(outcome.node, key, run.flow_id)[:255]
outputs = _cacheable(outcome) outputs = _cacheable(outcome)
if outcome.cached:
status = "cached"
elif outcome.ok:
status = "ok"
else:
status = "skipped" if outcome.skipped else "error"
row = RunNode( row = RunNode(
run_id=run_id, run_id=run_id,
node=node, node=node,
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"), status=status,
started_at=datetime.now(UTC), started_at=datetime.now(UTC),
duration_ms=outcome.duration_ms, duration_ms=outcome.duration_ms,
error=outcome.error[:ERROR_CAP], error=outcome.error[:ERROR_CAP],
+1
View File
@@ -855,6 +855,7 @@ class RedisState(StateBackend):
pipe.setex(self._key(key), self._ttl, data) pipe.setex(self._key(key), self._ttl, data)
else: else:
pipe.set(self._key(key), data) pipe.set(self._key(key), data)
self._note(pipe, updates)
pipe.execute() pipe.execute()
return True return True
+10 -2
View File
@@ -170,6 +170,11 @@ class Conflict(ApiError):
self.current_version = int(detail.get("current_version") or 0) self.current_version = int(detail.get("current_version") or 0)
def engine_url(url: str) -> str:
"""An engine's address, whether it was given with the API's path or not."""
return url.rstrip("/").removesuffix(API).rstrip("/")
class Client: class Client:
"""An authenticated engine, addressed over its HTTP API.""" """An authenticated engine, addressed over its HTTP API."""
@@ -182,7 +187,9 @@ class Client:
retries: int = RETRIES, retries: int = RETRIES,
) -> None: ) -> None:
stored = _stored() stored = _stored()
self.url = url or os.environ.get("FLUKSIO_URL") or stored.get("url") or "" self.url = engine_url(
url or os.environ.get("FLUKSIO_URL") or stored.get("url") or ""
)
self.token = ( self.token = (
token or os.environ.get("FLUKSIO_TOKEN") or stored.get("token") or "" token or os.environ.get("FLUKSIO_TOKEN") or stored.get("token") or ""
) )
@@ -766,8 +773,9 @@ def login(
"""Exchange credentials for a token and remember the engine.""" """Exchange credentials for a token and remember the engine."""
import httpx import httpx
url = engine_url(url)
response = httpx.post( response = httpx.post(
f"{url.rstrip('/')}{API}/login/access-token", f"{url}{API}/login/access-token",
data={"username": email, "password": password}, data={"username": email, "password": password},
timeout=timeout, timeout=timeout,
) )
+85 -1
View File
@@ -19,7 +19,7 @@ from fluksio.core.db import engine as db_engine
from fluksio.flow.artifacts import ArtifactStore from fluksio.flow.artifacts import ArtifactStore
from fluksio.flow.controller import FlowController from fluksio.flow.controller import FlowController
from fluksio.flow.messages import DType, MessageSpec from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, NodeOutcome from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, NodeOutcome, Pipeline
from fluksio.flow.runs import ( from fluksio.flow.runs import (
LEASE_STALE_S, LEASE_STALE_S,
OUTPUT_CAP, OUTPUT_CAP,
@@ -729,6 +729,57 @@ def test_a_draft_run_executes_the_draft(tmp_path):
session.commit() session.commit()
def test_a_failed_stage_skips_what_waits_on_it_and_fails_the_run(tmp_path):
"""Stages after a failure were called without their input and raised
TypeError, and a run whose only node could not load finished `ok`."""
model, score = (MessageSpec(name=n, dtype=DType.FLOAT) for n in ("model", "score"))
staged = FlowDef(
name="staged",
mode="batch",
nodes=[
NodeDef(id="fit", provides=[model]),
NodeDef(id="readout", requires=[model], provides=[score]),
NodeDef(id="evaluate", requires=[score]),
],
)
broken = FlowDef(name="broken", mode="batch", nodes=[NodeDef(id="fit")])
store = FlowStore(tmp_path / "flows")
store.write_flow(staged)
store.write_flow(broken)
store.write_node_source("staged", "fit", "def process():\n raise ValueError\n")
store.write_node_source(
"staged", "readout", "def process(model):\n return {'score': model}\n"
)
store.write_node_source("staged", "evaluate", "def process(score):\n pass\n")
store.write_node_source("broken", "fit", "def process(:\n")
service = RunService(controller=FlowController(store), queue=_Collect())
made = [service.submit(name, {}).id for name in ("staged", "broken")]
try:
for run_id in made:
service._drive(run_id)
with Session(db_engine) as session:
ran, failed = (session.get(Run, run_id) for run_id in made)
assert ran.status == failed.status == "error"
assert failed.status_reason.startswith("Node 'broken.fit' could not")
rows = session.exec(select(RunNode).where(col(RunNode.run_id).in_(made)))
assert {row.node: row.status for row in rows} == {
"staged.fit": "error",
"staged.readout": "skipped",
"staged.evaluate": "skipped",
"broken.fit": "error",
}
finally:
with Session(db_engine) as session:
for node in session.exec(
select(RunNode).where(col(RunNode.run_id).in_(made))
).all():
session.delete(node)
for run in session.exec(select(Run).where(col(Run.id).in_(made))).all():
session.delete(run)
session.commit()
def test_a_retry_is_a_new_run_that_names_the_one_it_repeats(): def test_a_retry_is_a_new_run_that_names_the_one_it_repeats():
"""The way back from a run an engine restart interrupted. """The way back from a run an engine restart interrupted.
@@ -905,6 +956,39 @@ def test_a_run_waiting_for_a_worker_keeps_its_lease_and_can_be_cancelled(
session.commit() session.commit()
class _Bus:
def __init__(self):
self.events = []
def publish(self, event):
self.events.append(event)
def test_cancelling_a_run_does_not_pause_the_live_flow_it_shares_a_key_with():
"""A run's pipeline publishes on the engine's bus under the flow's key, so
its pause marked the live flow paused in every open editor."""
class _Idle(_OneFlow):
workers = None
remote = None
service = RunService(
controller=_Idle(FlowDef(name="study", mode="batch")), queue=_Collect()
)
run = service.submit("study", {})
bus = _Bus()
pipeline = Pipeline(events=bus)
service._active[run.id] = pipeline
try:
assert service.cancel(run.id)
assert pipeline.is_disabled("study")
assert not [e for e in bus.events if e["type"] == "flow_paused"]
finally:
with Session(db_engine) as session:
session.delete(session.get(Run, run.id))
session.commit()
def test_a_key_nobody_used_submits_normally( def test_a_key_nobody_used_submits_normally(
client, superuser_token_headers, monkeypatch client, superuser_token_headers, monkeypatch
): ):
+6
View File
@@ -32,3 +32,9 @@ def test_a_ttl_expires_counters_and_the_names_set(state):
keys = state.keys() keys = state.keys()
assert "__names__" in keys and "__version__:heat.c" in keys assert "__names__" in keys and "__version__:heat.c" in keys
assert all(state._client.ttl(state._key(k)) > 0 for k in keys) assert all(state._client.ttl(state._key(k)) > 0 for k in keys)
def test_a_compare_and_swap_names_what_it_wrote(state):
state.set("heat.a", 1)
assert state.compare_and_swap_multi({"heat.a": 1}, {"heat.b": 2})
assert "heat.b" in state.message_names()
+10
View File
@@ -346,6 +346,16 @@ def test_a_rate_limited_port_cannot_be_run_as_a_batch():
assert not batch_issues(double_flow()) assert not batch_issues(double_flow())
def test_a_delay_nodes_interval_cannot_be_run_as_a_batch():
flow = double_flow()
flow.nodes[0].type = "delay"
flow.nodes[0].params = {"interval": 5}
# The node puts it on every output, where it holds values back the same way.
assert batch_issues(flow)
flow.nodes[0].params = {"delay": 5}
assert not batch_issues(flow)
def test_a_node_with_no_stored_code_cannot_be_run(): def test_a_node_with_no_stored_code_cannot_be_run():
"""The store answers a template for one, and a template publishes nothing.""" """The store answers a template for one, and a template publishes nothing."""
+13
View File
@@ -33,6 +33,19 @@ def a_client(handler, monkeypatch, **kwargs):
return Client(url="http://engine", token="t", http=http, **kwargs) return Client(url="http://engine", token="t", http=http, **kwargs)
@pytest.mark.parametrize(
"url",
[
"http://e:8000",
"http://e:8000/",
"http://e:8000/api/v1",
"http://e:8000/api/v1/",
],
)
def test_an_engine_url_may_carry_the_api_path(url):
assert Client(url=url, token="t").url == "http://e:8000"
def test_idempotent_get_is_tried_again(monkeypatch): def test_idempotent_get_is_tried_again(monkeypatch):
calls = [] calls = []
+2 -1
View File
@@ -151,7 +151,8 @@ Client commands use the nearest `.fluksio/client.json`, or the global
configuration when there is no project instance. Legacy client configuration configuration when there is no project instance. Legacy client configuration
is a fallback. is a fallback.
`FLUKSIO_URL` and `FLUKSIO_TOKEN` override saved values; command options `FLUKSIO_URL` and `FLUKSIO_TOKEN` override saved values; command options
`--url` and `--token` override both. `--url` and `--token` override both. An engine URL works with or without
the `/api/v1` path.
A client needs an engine at least as new as itself. An engine from before A client needs an engine at least as new as itself. An engine from before
flow ids cannot create flows the way this client does, and the client says flow ids cannot create flows the way this client does, and the client says
+7 -2
View File
@@ -18,9 +18,14 @@ A batch flow does not need activation and does not open subscriptions,
schedules or webhooks. A run executes each node once after its dependencies schedules or webhooks. A run executes each node once after its dependencies
finish. Missing parameters use their declared initial values. finish. Missing parameters use their declared initial values.
When a node fails, the nodes that wait on it are recorded as `skipped` and
the run ends `error`. A node that cannot be loaded fails the run before
anything executes, with the node's load error as the reason.
Non-streaming ports cannot use `interval` in a batch flow: there is no timer Non-streaming ports cannot use `interval` in a batch flow: there is no timer
to deliver a delayed value. Streaming ports can use it to reduce display to deliver a delayed value. The same goes for a delay node's `interval`
updates while keeping every metric point. setting. Streaming ports can use it to reduce display updates while keeping
every metric point.
## Submitting ## Submitting