A batch run skips the nodes waiting on a failed one and ends error; a run whose nodes cannot load fails with the load error before anything executes; cancelling a run stops its pipeline without announcing a pause of the live flow; batch_issues refuses a delay node's interval; the client accepts an engine URL with or without /api/v1; RedisState's compare-and-swap records the names it writes
This commit is contained in:
@@ -148,6 +148,17 @@ class RebuildBusy(RuntimeError):
|
||||
"""
|
||||
|
||||
|
||||
class NodesNotLoaded(RuntimeError):
|
||||
"""A run's flow has nodes that could not be built, so none of it runs."""
|
||||
|
||||
def __init__(self, errors: dict[str, str]) -> None:
|
||||
#: The load error of each node, by its qualified id.
|
||||
self.errors = errors
|
||||
node, error = next(iter(errors.items()))
|
||||
more = f" and {len(errors) - 1} more" if len(errors) > 1 else ""
|
||||
super().__init__(f"Node '{node}'{more} could not be loaded: {error}")
|
||||
|
||||
|
||||
class NodeStatus(str, Enum):
|
||||
ACTIVE = "active"
|
||||
ERROR = "error"
|
||||
@@ -2010,10 +2021,21 @@ class FlowController:
|
||||
and webhooks would only be a second copy of what the live pipeline
|
||||
already holds. The state is the run's, which is what keeps two runs of
|
||||
one flow from overwriting each other's messages.
|
||||
|
||||
:raises NodesNotLoaded: When a node could not be built. The live engine
|
||||
leaves such a node out, but a run without it would finish as if
|
||||
the flow had done its work.
|
||||
"""
|
||||
nodes, _loaded, initial_values, _inputs = self._build_flows(
|
||||
nodes, loaded, initial_values, _inputs = self._build_flows(
|
||||
[(flow, draft)], run=run
|
||||
)
|
||||
errors = {
|
||||
entry.id: entry.error or "unknown error"
|
||||
for entry in loaded.values()
|
||||
if entry.status is NodeStatus.ERROR
|
||||
}
|
||||
if errors:
|
||||
raise NodesNotLoaded(errors)
|
||||
pipeline = Pipeline(
|
||||
nodes=nodes,
|
||||
state=state,
|
||||
|
||||
@@ -147,6 +147,8 @@ class NodeOutcome(BaseModel):
|
||||
artifacts: dict[str, dict[str, Any]] = {}
|
||||
#: Restored from an earlier run rather than executed.
|
||||
cached: bool = False
|
||||
#: Not executed, because a node it waits on failed or was skipped itself.
|
||||
skipped: bool = False
|
||||
#: Which run the restored values came from — and, because a restored row
|
||||
#: holds no series of its own, which run this node's metrics live in.
|
||||
cached_from: str = ""
|
||||
@@ -1071,7 +1073,8 @@ class Pipeline:
|
||||
overrides: dict[str, Any] | None = None,
|
||||
inputs: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Run one node and record its outputs. Never raises.
|
||||
"""Run one node and record its outputs. Never raises: None means it
|
||||
failed, and a node that published nothing answers ``{}``.
|
||||
|
||||
``inputs`` is what the readiness check already read — the same keys, a
|
||||
moment earlier. Reading them again here was the second of two MGETs
|
||||
@@ -1106,7 +1109,7 @@ class Pipeline:
|
||||
if key:
|
||||
hit, restored = self._from_cache(node, key, state, entry_id)
|
||||
if hit:
|
||||
return restored
|
||||
return restored or {}
|
||||
|
||||
with logs.capture(collected):
|
||||
result = node.execute(inputs)
|
||||
@@ -1169,7 +1172,7 @@ class Pipeline:
|
||||
output_values=result,
|
||||
)
|
||||
)
|
||||
return result
|
||||
return result or {}
|
||||
except Exception as exc:
|
||||
# One failing node must not take the rest of the graph down.
|
||||
error = self.publish_error(node, exc, collected, entry_id)
|
||||
@@ -1431,6 +1434,11 @@ class Pipeline:
|
||||
|
||||
ready_nodes = deque(n for n, count in in_degree.items() if count == 0)
|
||||
blocked: set[Node] = set()
|
||||
# A batch run starts from an empty state, so a node whose producer
|
||||
# failed would be called without that input. It is skipped instead,
|
||||
# and so is everything waiting on it. A live wave still has the last
|
||||
# values to run on.
|
||||
failed: set[Node] | None = set() if self.run_id else None
|
||||
gate_revision = self._gate_revision
|
||||
node_futures: dict[Future[dict[str, Any] | None], Node] = {}
|
||||
finished: SimpleQueue[Future[dict[str, Any] | None]] = SimpleQueue()
|
||||
@@ -1469,6 +1477,11 @@ class Pipeline:
|
||||
if n.passive:
|
||||
complete(n)
|
||||
continue
|
||||
if failed is not None and not failed.isdisjoint(deps[n]):
|
||||
failed.add(n)
|
||||
self._observe(NodeOutcome(node=n.id, ok=False, skipped=True))
|
||||
complete(n)
|
||||
continue
|
||||
if fresh is not None and untouched(n):
|
||||
complete(n)
|
||||
continue
|
||||
@@ -1497,6 +1510,8 @@ class Pipeline:
|
||||
future = finished.get()
|
||||
n = node_futures.pop(future)
|
||||
result = future.result()
|
||||
if result is None and failed is not None:
|
||||
failed.add(n)
|
||||
if result is not None and fresh is not None:
|
||||
fresh.update(result)
|
||||
# Silent, failed and throttled producers still release their
|
||||
|
||||
@@ -51,7 +51,12 @@ from sqlmodel import Session, col, select
|
||||
from fluksio.core.db import engine as db_engine
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
|
||||
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
|
||||
from fluksio.flow.controller import (
|
||||
NODE_TYPES,
|
||||
FlowController,
|
||||
NodesNotLoaded,
|
||||
RunContext,
|
||||
)
|
||||
from fluksio.flow.messages import qualify, requalify
|
||||
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
|
||||
from fluksio.flow.queue import WorkItem, WorkQueue
|
||||
@@ -137,8 +142,9 @@ def batch_issues(
|
||||
history keeps every value, and the interval only thins what is published.
|
||||
Anywhere else it is a message quietly going missing.
|
||||
|
||||
A delay node is fine; without a queue to defer into it simply sleeps,
|
||||
which in a run is what was asked for.
|
||||
A delay node's ``interval`` is the same limit, put on every output it
|
||||
has. Its ``delay`` is fine: without a queue to defer into it simply
|
||||
sleeps, which in a run is what was asked for.
|
||||
|
||||
The other is a node with no body. The store answers a new node's template
|
||||
when nothing was ever written for one, so such a node runs — and returns
|
||||
@@ -148,8 +154,11 @@ def batch_issues(
|
||||
"""
|
||||
issues: list[str] = []
|
||||
for node in flow.nodes:
|
||||
for spec in list(node.requires) + list(node.provides):
|
||||
if spec.interval > 0 and not spec.stream:
|
||||
held = float(node.params.get("interval") or 0) if node.type == "delay" else 0
|
||||
limits = [(spec, spec.interval) for spec in node.requires]
|
||||
limits += [(spec, max(spec.interval, held)) for spec in node.provides]
|
||||
for spec, interval in limits:
|
||||
if interval > 0 and not spec.stream:
|
||||
issues.append(
|
||||
f"Node '{node.id}' rate-limits '{spec.port or spec.name}'. "
|
||||
"A run has no timer to release what that holds back, so "
|
||||
@@ -1236,8 +1245,10 @@ class RunService:
|
||||
return True
|
||||
# The gate first, so nothing new is submitted while the running nodes
|
||||
# are being killed; a gated node is never handed to the executor, so
|
||||
# the graph drains instead of going further.
|
||||
pipeline.pause(self._key_of(run_id))
|
||||
# the graph drains instead of going further. Stopped rather than
|
||||
# paused: a pause announces itself on the engine's bus, under the key
|
||||
# the live flow shares, so every open editor showed that one paused.
|
||||
pipeline.set_disabled({self._key_of(run_id)})
|
||||
workers = self.controller.workers
|
||||
if workers is not None:
|
||||
# Keyed by run, so a sweep cancelling one config leaves the others
|
||||
@@ -1502,7 +1513,7 @@ class RunService:
|
||||
|
||||
def observe(outcome: NodeOutcome) -> None:
|
||||
nonlocal errors
|
||||
if not outcome.ok:
|
||||
if not outcome.ok and not outcome.skipped:
|
||||
errors += 1
|
||||
self._record_node(run, key, outcome)
|
||||
|
||||
@@ -1537,19 +1548,26 @@ class RunService:
|
||||
digests = node_digests(flow)
|
||||
run.code_digest = self._restamp(run, run_digest(flow))
|
||||
state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}")
|
||||
pipeline = self.controller.build_run_pipeline(
|
||||
flow,
|
||||
state=state,
|
||||
draft=run.draft,
|
||||
observer=observe,
|
||||
emission_observer=sink.handle,
|
||||
run=RunContext(
|
||||
run_id=run_id,
|
||||
code_digest=run.code_digest,
|
||||
node_digests=digests,
|
||||
),
|
||||
run_cache=None if run.no_cache else self._cache,
|
||||
)
|
||||
try:
|
||||
pipeline = self.controller.build_run_pipeline(
|
||||
flow,
|
||||
state=state,
|
||||
draft=run.draft,
|
||||
observer=observe,
|
||||
emission_observer=sink.handle,
|
||||
run=RunContext(
|
||||
run_id=run_id,
|
||||
code_digest=run.code_digest,
|
||||
node_digests=digests,
|
||||
),
|
||||
run_cache=None if run.no_cache else self._cache,
|
||||
)
|
||||
except NodesNotLoaded as exc:
|
||||
# Nothing of it runs; the finally below records it.
|
||||
for node, error in exc.errors.items():
|
||||
observe(NodeOutcome(node=node, ok=False, error=error))
|
||||
status, reason = "error", str(exc)[:1024]
|
||||
return
|
||||
with self._lock:
|
||||
if run_id in self._cancelled:
|
||||
# Cancelled while it was being built, so nothing of it
|
||||
@@ -1627,10 +1645,16 @@ class RunService:
|
||||
run_id = run.id
|
||||
node = requalify(outcome.node, key, run.flow_id)[:255]
|
||||
outputs = _cacheable(outcome)
|
||||
if outcome.cached:
|
||||
status = "cached"
|
||||
elif outcome.ok:
|
||||
status = "ok"
|
||||
else:
|
||||
status = "skipped" if outcome.skipped else "error"
|
||||
row = RunNode(
|
||||
run_id=run_id,
|
||||
node=node,
|
||||
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
|
||||
status=status,
|
||||
started_at=datetime.now(UTC),
|
||||
duration_ms=outcome.duration_ms,
|
||||
error=outcome.error[:ERROR_CAP],
|
||||
|
||||
@@ -855,6 +855,7 @@ class RedisState(StateBackend):
|
||||
pipe.setex(self._key(key), self._ttl, data)
|
||||
else:
|
||||
pipe.set(self._key(key), data)
|
||||
self._note(pipe, updates)
|
||||
|
||||
pipe.execute()
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user