A batch run skips the nodes waiting on a failed one and ends error; a run whose nodes cannot load fails with the load error before anything executes; cancelling a run stops its pipeline without announcing a pause of the live flow; batch_issues refuses a delay node's interval; the client accepts an engine URL with or without /api/v1; RedisState's compare-and-swap records the names it writes

This commit is contained in:
2026-11-04 21:38:21 +01:00
parent eb17528ce6
commit d028bf326a
11 changed files with 221 additions and 32 deletions
+23 -1
View File
@@ -148,6 +148,17 @@ class RebuildBusy(RuntimeError):
"""
class NodesNotLoaded(RuntimeError):
"""A run's flow has nodes that could not be built, so none of it runs."""
def __init__(self, errors: dict[str, str]) -> None:
#: The load error of each node, by its qualified id.
self.errors = errors
node, error = next(iter(errors.items()))
more = f" and {len(errors) - 1} more" if len(errors) > 1 else ""
super().__init__(f"Node '{node}'{more} could not be loaded: {error}")
class NodeStatus(str, Enum):
ACTIVE = "active"
ERROR = "error"
@@ -2010,10 +2021,21 @@ class FlowController:
and webhooks would only be a second copy of what the live pipeline
already holds. The state is the run's, which is what keeps two runs of
one flow from overwriting each other's messages.
:raises NodesNotLoaded: When a node could not be built. The live engine
leaves such a node out, but a run without it would finish as if
the flow had done its work.
"""
nodes, _loaded, initial_values, _inputs = self._build_flows(
nodes, loaded, initial_values, _inputs = self._build_flows(
[(flow, draft)], run=run
)
errors = {
entry.id: entry.error or "unknown error"
for entry in loaded.values()
if entry.status is NodeStatus.ERROR
}
if errors:
raise NodesNotLoaded(errors)
pipeline = Pipeline(
nodes=nodes,
state=state,
+18 -3
View File
@@ -147,6 +147,8 @@ class NodeOutcome(BaseModel):
artifacts: dict[str, dict[str, Any]] = {}
#: Restored from an earlier run rather than executed.
cached: bool = False
#: Not executed, because a node it waits on failed or was skipped itself.
skipped: bool = False
#: Which run the restored values came from — and, because a restored row
#: holds no series of its own, which run this node's metrics live in.
cached_from: str = ""
@@ -1071,7 +1073,8 @@ class Pipeline:
overrides: dict[str, Any] | None = None,
inputs: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Run one node and record its outputs. Never raises.
"""Run one node and record its outputs. Never raises: None means it
failed, and a node that published nothing answers ``{}``.
``inputs`` is what the readiness check already read — the same keys, a
moment earlier. Reading them again here was the second of two MGETs
@@ -1106,7 +1109,7 @@ class Pipeline:
if key:
hit, restored = self._from_cache(node, key, state, entry_id)
if hit:
return restored
return restored or {}
with logs.capture(collected):
result = node.execute(inputs)
@@ -1169,7 +1172,7 @@ class Pipeline:
output_values=result,
)
)
return result
return result or {}
except Exception as exc:
# One failing node must not take the rest of the graph down.
error = self.publish_error(node, exc, collected, entry_id)
@@ -1431,6 +1434,11 @@ class Pipeline:
ready_nodes = deque(n for n, count in in_degree.items() if count == 0)
blocked: set[Node] = set()
# A batch run starts from an empty state, so a node whose producer
# failed would be called without that input. It is skipped instead,
# and so is everything waiting on it. A live wave still has the last
# values to run on.
failed: set[Node] | None = set() if self.run_id else None
gate_revision = self._gate_revision
node_futures: dict[Future[dict[str, Any] | None], Node] = {}
finished: SimpleQueue[Future[dict[str, Any] | None]] = SimpleQueue()
@@ -1469,6 +1477,11 @@ class Pipeline:
if n.passive:
complete(n)
continue
if failed is not None and not failed.isdisjoint(deps[n]):
failed.add(n)
self._observe(NodeOutcome(node=n.id, ok=False, skipped=True))
complete(n)
continue
if fresh is not None and untouched(n):
complete(n)
continue
@@ -1497,6 +1510,8 @@ class Pipeline:
future = finished.get()
n = node_futures.pop(future)
result = future.result()
if result is None and failed is not None:
failed.add(n)
if result is not None and fresh is not None:
fresh.update(result)
# Silent, failed and throttled producers still release their
+46 -22
View File
@@ -51,7 +51,12 @@ from sqlmodel import Session, col, select
from fluksio.core.db import engine as db_engine
from fluksio.flow import tombstones
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
from fluksio.flow.controller import (
NODE_TYPES,
FlowController,
NodesNotLoaded,
RunContext,
)
from fluksio.flow.messages import qualify, requalify
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
from fluksio.flow.queue import WorkItem, WorkQueue
@@ -137,8 +142,9 @@ def batch_issues(
history keeps every value, and the interval only thins what is published.
Anywhere else it is a message quietly going missing.
A delay node is fine; without a queue to defer into it simply sleeps,
which in a run is what was asked for.
A delay node's ``interval`` is the same limit, put on every output it
has. Its ``delay`` is fine: without a queue to defer into it simply
sleeps, which in a run is what was asked for.
The other is a node with no body. The store answers a new node's template
when nothing was ever written for one, so such a node runs — and returns
@@ -148,8 +154,11 @@ def batch_issues(
"""
issues: list[str] = []
for node in flow.nodes:
for spec in list(node.requires) + list(node.provides):
if spec.interval > 0 and not spec.stream:
held = float(node.params.get("interval") or 0) if node.type == "delay" else 0
limits = [(spec, spec.interval) for spec in node.requires]
limits += [(spec, max(spec.interval, held)) for spec in node.provides]
for spec, interval in limits:
if interval > 0 and not spec.stream:
issues.append(
f"Node '{node.id}' rate-limits '{spec.port or spec.name}'. "
"A run has no timer to release what that holds back, so "
@@ -1236,8 +1245,10 @@ class RunService:
return True
# The gate first, so nothing new is submitted while the running nodes
# are being killed; a gated node is never handed to the executor, so
# the graph drains instead of going further.
pipeline.pause(self._key_of(run_id))
# the graph drains instead of going further. Stopped rather than
# paused: a pause announces itself on the engine's bus, under the key
# the live flow shares, so every open editor showed that one paused.
pipeline.set_disabled({self._key_of(run_id)})
workers = self.controller.workers
if workers is not None:
# Keyed by run, so a sweep cancelling one config leaves the others
@@ -1502,7 +1513,7 @@ class RunService:
def observe(outcome: NodeOutcome) -> None:
nonlocal errors
if not outcome.ok:
if not outcome.ok and not outcome.skipped:
errors += 1
self._record_node(run, key, outcome)
@@ -1537,19 +1548,26 @@ class RunService:
digests = node_digests(flow)
run.code_digest = self._restamp(run, run_digest(flow))
state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}")
pipeline = self.controller.build_run_pipeline(
flow,
state=state,
draft=run.draft,
observer=observe,
emission_observer=sink.handle,
run=RunContext(
run_id=run_id,
code_digest=run.code_digest,
node_digests=digests,
),
run_cache=None if run.no_cache else self._cache,
)
try:
pipeline = self.controller.build_run_pipeline(
flow,
state=state,
draft=run.draft,
observer=observe,
emission_observer=sink.handle,
run=RunContext(
run_id=run_id,
code_digest=run.code_digest,
node_digests=digests,
),
run_cache=None if run.no_cache else self._cache,
)
except NodesNotLoaded as exc:
# Nothing of it runs; the finally below records it.
for node, error in exc.errors.items():
observe(NodeOutcome(node=node, ok=False, error=error))
status, reason = "error", str(exc)[:1024]
return
with self._lock:
if run_id in self._cancelled:
# Cancelled while it was being built, so nothing of it
@@ -1627,10 +1645,16 @@ class RunService:
run_id = run.id
node = requalify(outcome.node, key, run.flow_id)[:255]
outputs = _cacheable(outcome)
if outcome.cached:
status = "cached"
elif outcome.ok:
status = "ok"
else:
status = "skipped" if outcome.skipped else "error"
row = RunNode(
run_id=run_id,
node=node,
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
status=status,
started_at=datetime.now(UTC),
duration_ms=outcome.duration_ms,
error=outcome.error[:ERROR_CAP],
+1
View File
@@ -855,6 +855,7 @@ class RedisState(StateBackend):
pipe.setex(self._key(key), self._ttl, data)
else:
pipe.set(self._key(key), data)
self._note(pipe, updates)
pipe.execute()
return True