A batch run skips the nodes waiting on a failed one and ends error; a run whose nodes cannot load fails with the load error before anything executes; cancelling a run stops its pipeline without announcing a pause of the live flow; batch_issues refuses a delay node's interval; the client accepts an engine URL with or without /api/v1; RedisState's compare-and-swap records the names it writes
This commit is contained in:
@@ -51,7 +51,12 @@ from sqlmodel import Session, col, select
|
||||
from fluksio.core.db import engine as db_engine
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
|
||||
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
|
||||
from fluksio.flow.controller import (
|
||||
NODE_TYPES,
|
||||
FlowController,
|
||||
NodesNotLoaded,
|
||||
RunContext,
|
||||
)
|
||||
from fluksio.flow.messages import qualify, requalify
|
||||
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
|
||||
from fluksio.flow.queue import WorkItem, WorkQueue
|
||||
@@ -137,8 +142,9 @@ def batch_issues(
|
||||
history keeps every value, and the interval only thins what is published.
|
||||
Anywhere else it is a message quietly going missing.
|
||||
|
||||
A delay node is fine; without a queue to defer into it simply sleeps,
|
||||
which in a run is what was asked for.
|
||||
A delay node's ``interval`` is the same limit, put on every output it
|
||||
has. Its ``delay`` is fine: without a queue to defer into it simply
|
||||
sleeps, which in a run is what was asked for.
|
||||
|
||||
The other is a node with no body. The store answers a new node's template
|
||||
when nothing was ever written for one, so such a node runs — and returns
|
||||
@@ -148,8 +154,11 @@ def batch_issues(
|
||||
"""
|
||||
issues: list[str] = []
|
||||
for node in flow.nodes:
|
||||
for spec in list(node.requires) + list(node.provides):
|
||||
if spec.interval > 0 and not spec.stream:
|
||||
held = float(node.params.get("interval") or 0) if node.type == "delay" else 0
|
||||
limits = [(spec, spec.interval) for spec in node.requires]
|
||||
limits += [(spec, max(spec.interval, held)) for spec in node.provides]
|
||||
for spec, interval in limits:
|
||||
if interval > 0 and not spec.stream:
|
||||
issues.append(
|
||||
f"Node '{node.id}' rate-limits '{spec.port or spec.name}'. "
|
||||
"A run has no timer to release what that holds back, so "
|
||||
@@ -1236,8 +1245,10 @@ class RunService:
|
||||
return True
|
||||
# The gate first, so nothing new is submitted while the running nodes
|
||||
# are being killed; a gated node is never handed to the executor, so
|
||||
# the graph drains instead of going further.
|
||||
pipeline.pause(self._key_of(run_id))
|
||||
# the graph drains instead of going further. Stopped rather than
|
||||
# paused: a pause announces itself on the engine's bus, under the key
|
||||
# the live flow shares, so every open editor showed that one paused.
|
||||
pipeline.set_disabled({self._key_of(run_id)})
|
||||
workers = self.controller.workers
|
||||
if workers is not None:
|
||||
# Keyed by run, so a sweep cancelling one config leaves the others
|
||||
@@ -1502,7 +1513,7 @@ class RunService:
|
||||
|
||||
def observe(outcome: NodeOutcome) -> None:
|
||||
nonlocal errors
|
||||
if not outcome.ok:
|
||||
if not outcome.ok and not outcome.skipped:
|
||||
errors += 1
|
||||
self._record_node(run, key, outcome)
|
||||
|
||||
@@ -1537,19 +1548,26 @@ class RunService:
|
||||
digests = node_digests(flow)
|
||||
run.code_digest = self._restamp(run, run_digest(flow))
|
||||
state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}")
|
||||
pipeline = self.controller.build_run_pipeline(
|
||||
flow,
|
||||
state=state,
|
||||
draft=run.draft,
|
||||
observer=observe,
|
||||
emission_observer=sink.handle,
|
||||
run=RunContext(
|
||||
run_id=run_id,
|
||||
code_digest=run.code_digest,
|
||||
node_digests=digests,
|
||||
),
|
||||
run_cache=None if run.no_cache else self._cache,
|
||||
)
|
||||
try:
|
||||
pipeline = self.controller.build_run_pipeline(
|
||||
flow,
|
||||
state=state,
|
||||
draft=run.draft,
|
||||
observer=observe,
|
||||
emission_observer=sink.handle,
|
||||
run=RunContext(
|
||||
run_id=run_id,
|
||||
code_digest=run.code_digest,
|
||||
node_digests=digests,
|
||||
),
|
||||
run_cache=None if run.no_cache else self._cache,
|
||||
)
|
||||
except NodesNotLoaded as exc:
|
||||
# Nothing of it runs; the finally below records it.
|
||||
for node, error in exc.errors.items():
|
||||
observe(NodeOutcome(node=node, ok=False, error=error))
|
||||
status, reason = "error", str(exc)[:1024]
|
||||
return
|
||||
with self._lock:
|
||||
if run_id in self._cancelled:
|
||||
# Cancelled while it was being built, so nothing of it
|
||||
@@ -1627,10 +1645,16 @@ class RunService:
|
||||
run_id = run.id
|
||||
node = requalify(outcome.node, key, run.flow_id)[:255]
|
||||
outputs = _cacheable(outcome)
|
||||
if outcome.cached:
|
||||
status = "cached"
|
||||
elif outcome.ok:
|
||||
status = "ok"
|
||||
else:
|
||||
status = "skipped" if outcome.skipped else "error"
|
||||
row = RunNode(
|
||||
run_id=run_id,
|
||||
node=node,
|
||||
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
|
||||
status=status,
|
||||
started_at=datetime.now(UTC),
|
||||
duration_ms=outcome.duration_ms,
|
||||
error=outcome.error[:ERROR_CAP],
|
||||
|
||||
Reference in New Issue
Block a user