A batch run skips the nodes waiting on a failed one and ends error; a run whose nodes cannot load fails with the load error before anything executes; cancelling a run stops its pipeline without announcing a pause of the live flow; batch_issues refuses a delay node's interval; the client accepts an engine URL with or without /api/v1; RedisState's compare-and-swap records the names it writes

This commit is contained in:
2026-11-04 21:38:21 +01:00
parent eb17528ce6
commit d028bf326a
11 changed files with 221 additions and 32 deletions
+46 -22
View File
@@ -51,7 +51,12 @@ from sqlmodel import Session, col, select
from fluksio.core.db import engine as db_engine
from fluksio.flow import tombstones
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
from fluksio.flow.controller import (
NODE_TYPES,
FlowController,
NodesNotLoaded,
RunContext,
)
from fluksio.flow.messages import qualify, requalify
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
from fluksio.flow.queue import WorkItem, WorkQueue
@@ -137,8 +142,9 @@ def batch_issues(
history keeps every value, and the interval only thins what is published.
Anywhere else it is a message quietly going missing.
A delay node is fine; without a queue to defer into it simply sleeps,
which in a run is what was asked for.
A delay node's ``interval`` is the same limit, put on every output it
has. Its ``delay`` is fine: without a queue to defer into it simply
sleeps, which in a run is what was asked for.
The other is a node with no body. The store answers a new node's template
when nothing was ever written for one, so such a node runs — and returns
@@ -148,8 +154,11 @@ def batch_issues(
"""
issues: list[str] = []
for node in flow.nodes:
for spec in list(node.requires) + list(node.provides):
if spec.interval > 0 and not spec.stream:
held = float(node.params.get("interval") or 0) if node.type == "delay" else 0
limits = [(spec, spec.interval) for spec in node.requires]
limits += [(spec, max(spec.interval, held)) for spec in node.provides]
for spec, interval in limits:
if interval > 0 and not spec.stream:
issues.append(
f"Node '{node.id}' rate-limits '{spec.port or spec.name}'. "
"A run has no timer to release what that holds back, so "
@@ -1236,8 +1245,10 @@ class RunService:
return True
# The gate first, so nothing new is submitted while the running nodes
# are being killed; a gated node is never handed to the executor, so
# the graph drains instead of going further.
pipeline.pause(self._key_of(run_id))
# the graph drains instead of going further. Stopped rather than
# paused: a pause announces itself on the engine's bus, under the key
# the live flow shares, so every open editor showed that one paused.
pipeline.set_disabled({self._key_of(run_id)})
workers = self.controller.workers
if workers is not None:
# Keyed by run, so a sweep cancelling one config leaves the others
@@ -1502,7 +1513,7 @@ class RunService:
def observe(outcome: NodeOutcome) -> None:
nonlocal errors
if not outcome.ok:
if not outcome.ok and not outcome.skipped:
errors += 1
self._record_node(run, key, outcome)
@@ -1537,19 +1548,26 @@ class RunService:
digests = node_digests(flow)
run.code_digest = self._restamp(run, run_digest(flow))
state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}")
pipeline = self.controller.build_run_pipeline(
flow,
state=state,
draft=run.draft,
observer=observe,
emission_observer=sink.handle,
run=RunContext(
run_id=run_id,
code_digest=run.code_digest,
node_digests=digests,
),
run_cache=None if run.no_cache else self._cache,
)
try:
pipeline = self.controller.build_run_pipeline(
flow,
state=state,
draft=run.draft,
observer=observe,
emission_observer=sink.handle,
run=RunContext(
run_id=run_id,
code_digest=run.code_digest,
node_digests=digests,
),
run_cache=None if run.no_cache else self._cache,
)
except NodesNotLoaded as exc:
# Nothing of it runs; the finally below records it.
for node, error in exc.errors.items():
observe(NodeOutcome(node=node, ok=False, error=error))
status, reason = "error", str(exc)[:1024]
return
with self._lock:
if run_id in self._cancelled:
# Cancelled while it was being built, so nothing of it
@@ -1627,10 +1645,16 @@ class RunService:
run_id = run.id
node = requalify(outcome.node, key, run.flow_id)[:255]
outputs = _cacheable(outcome)
if outcome.cached:
status = "cached"
elif outcome.ok:
status = "ok"
else:
status = "skipped" if outcome.skipped else "error"
row = RunNode(
run_id=run_id,
node=node,
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
status=status,
started_at=datetime.now(UTC),
duration_ms=outcome.duration_ms,
error=outcome.error[:ERROR_CAP],