Merge remote-tracking branch 'origin/main' into HEAD
Docs / docs (push) Successful in 36s
Playwright Tests / test-playwright (1, 2) (push) Successful in 4m11s
Playwright Tests / test-playwright (2, 2) (push) Successful in 2m34s
pre-commit / pre-commit (push) Successful in 3m35s
Test Backend / test-backend (3.12) (push) Failing after 6m25s
Test Backend / test-backend (3.14) (push) Failing after 6m2s
Compose Smoke Test / test-compose (push) Successful in 38s
Playwright Tests / merge-reports (push) Successful in 1m52s

This commit is contained in:
2026-11-10 18:32:06 +01:00
29 changed files with 796 additions and 132 deletions
+30 -3
View File
@@ -148,6 +148,17 @@ class RebuildBusy(RuntimeError):
"""
class NodesNotLoaded(RuntimeError):
"""A run's flow has nodes that could not be built, so none of it runs."""
def __init__(self, errors: dict[str, str]) -> None:
#: The load error of each node, by its qualified id.
self.errors = errors
node, error = next(iter(errors.items()))
more = f" and {len(errors) - 1} more" if len(errors) > 1 else ""
super().__init__(f"Node '{node}'{more} could not be loaded: {error}")
class NodeStatus(str, Enum):
ACTIVE = "active"
ERROR = "error"
@@ -730,8 +741,13 @@ class FlowController:
try:
# Work already claimed belongs to the pipeline it was claimed
# against; let it finish there before swapping the graph out.
if self.execution is not None:
await run_in_threadpool(self.execution.pause_intake)
# Work that does not finish in time keeps the old graph, which
# nothing has touched yet.
if self.execution is not None and not await run_in_threadpool(
self.execution.pause_intake
):
self.execution.resume_intake()
raise RebuildBusy("Claimed work is still running on the pipeline")
await self._teardown()
# A fresh supervisor per build, so a flow quarantined by the last
# one gets another chance once its author has changed something.
@@ -2010,10 +2026,21 @@ class FlowController:
and webhooks would only be a second copy of what the live pipeline
already holds. The state is the run's, which is what keeps two runs of
one flow from overwriting each other's messages.
:raises NodesNotLoaded: When a node could not be built. The live engine
leaves such a node out, but a run without it would finish as if
the flow had done its work.
"""
nodes, _loaded, initial_values, _inputs = self._build_flows(
nodes, loaded, initial_values, _inputs = self._build_flows(
[(flow, draft)], run=run
)
errors = {
entry.id: entry.error or "unknown error"
for entry in loaded.values()
if entry.status is NodeStatus.ERROR
}
if errors:
raise NodesNotLoaded(errors)
pipeline = Pipeline(
nodes=nodes,
state=state,
+37 -4
View File
@@ -64,6 +64,10 @@ BAR_ROWS = 8
#: client (``ui/core/counter.ts``, ``MAX_COUNTERS``).
COUNTERS = 3
#: How many windows one querying chart offers, stacked down the tile's edge.
#: Mirrored in the client (``ChartWidget.tsx``, ``MAX_RANGES``).
CHART_RANGES = 8
#: Resolved out here on purpose: the store has a ``list`` method, which
#: shadows the builtin for any annotation written inside the class.
Bindings = list[dict[str, Any]]
@@ -259,10 +263,15 @@ class WidgetDef(BaseModel):
A chart comes in two kinds. The default reads what the engine kept for a
message. One with ``source: "query"`` asks instead, and its config is
``{source, request, request_dtype: "record", message, dtype: "series",
refresh_s, range_s}``: it publishes ``{range_s, interval_s}`` to
refresh_s, range_s, ranges}``: it publishes ``{range_s, interval_s}`` to
``request`` exactly as a slider publishes a value, and draws the ``series``
a flow answers with on ``message``.
``ranges`` lists the windows its rail offers, up to ``CHART_RANGES`` of
``{label, range_s, interval_s}``, and ``range_s`` names the one it opens on.
Without them it offers 1h, 6h, 24h and 7d. ``range_s: 0`` asks for the
whole history, and the flow answering decides how far back that reaches.
A colour widget picks what it publishes with ``format``, because fixtures
differ and a change node per tile is not the answer:
@@ -448,6 +457,28 @@ class WidgetDef(BaseModel):
raise ValueError(
f"a querying chart's {key} must be '{want}', not '{bound}'"
)
ranges = self.config.get("ranges") or []
if not isinstance(ranges, list) or len(ranges) > CHART_RANGES:
raise ValueError(
f"a querying chart offers a list of at most {CHART_RANGES} ranges"
)
for entry in ranges:
window = entry if isinstance(entry, dict) else {}
label = window.get("label")
span, step = window.get("range_s"), window.get("interval_s")
# `type() is int`, because a bool is an int to isinstance.
if not (
isinstance(label, str)
and label.strip()
and type(span) is int
and span >= 0
and type(step) is int
and step > 0
):
raise ValueError(
"a querying chart's range needs a label, a whole range_s "
f"of 0 or more and a whole interval_s above 0, not {entry!r}"
)
return self
if self.type == "bar" and len(self.bar_rows) > BAR_ROWS:
@@ -961,21 +992,23 @@ class DashboardStore:
self._dump(shown.model_copy(update={"version": top + 1}))
)
draft.unlink()
self.flows._commit(f"Discard draft of dashboard '{name}'")
return self.read(name)
shown = self.read(name)
self.flows._commit(f"Discard draft of dashboard {named(shown.name, name)}")
return shown
def delete(self, name: str) -> None:
if not self.exists(name):
raise DashboardNotFound(name)
path = self._file(name)
with self.flows._write_lock:
key = next((k for k, i in self.key_index().items() if i == name), name)
path.unlink(missing_ok=True)
self._draft_file(name).unlink(missing_ok=True)
try:
path.parent.rmdir()
except OSError:
pass
self.flows._commit(f"Delete dashboard '{name}'")
self.flows._commit(f"Delete dashboard {named(key, name)}")
def duplicate(self, name: str) -> DashboardDef:
"""Copy the working copy into a new dashboard, as a draft only.
+18 -5
View File
@@ -255,14 +255,26 @@ class ExecutionService:
with self._inflight_lock:
self._inflight_lock.notify_all()
def pause_intake(self) -> None:
def pause_intake(self) -> bool:
"""Stop claiming, and wait for what is already claimed to finish.
Called around a rebuild: items claimed against the old pipeline should
finish there rather than half-run against the new one.
finish there rather than half-run against the new one. That includes
a claim still under way: the consumer holds its lock from the claim to
the dispatch, so once that lock is free here the claim is counted, and
every later pass finds the intake closed.
False when that took longer than `DRAIN_TIMEOUT_S`. The intake stays
closed either way; the caller decides whether to go on.
"""
self._intake.clear()
# So a consumer waiting for capacity sees the intake closed.
self._capacity_changed()
deadline = time.monotonic() + DRAIN_TIMEOUT_S
if not self._consumer_lock.acquire(timeout=DRAIN_TIMEOUT_S):
logger.warning("Rebuild did not wait out a claim")
return False
self._consumer_lock.release()
with self._inflight_lock:
while self._running:
remaining = deadline - time.monotonic()
@@ -270,8 +282,9 @@ class ExecutionService:
logger.warning(
"Rebuild did not wait out %d cascades", self._running.total()
)
return
return False
self._inflight_lock.wait(remaining)
return True
def resume_intake(self) -> None:
self._intake.set()
@@ -284,8 +297,8 @@ class ExecutionService:
"""Stop claiming, promoting, touching and reclaiming, once every
claimed cascade has finished. For a rename, which rewrites the queue.
Unlike `pause_intake` this also waits out a claim already blocking
when intake clears, and a timer pass. It waits without a limit and
Unlike `pause_intake` this also waits out a timer pass, and keeps the
consumer lock until `release`. It waits without a limit and
tells ``report`` the nodes whose cascades are still running each time
they change. Setting ``abort`` gives up: claiming resumes and it
returns False. True means held until `release`.
+41 -3
View File
@@ -148,6 +148,8 @@ class NodeOutcome(BaseModel):
artifacts: dict[str, dict[str, Any]] = {}
#: Restored from an earlier run rather than executed.
cached: bool = False
#: Not executed, because a node it waits on failed or was skipped itself.
skipped: bool = False
#: Which run the restored values came from — and, because a restored row
#: holds no series of its own, which run this node's metrics live in.
cached_from: str = ""
@@ -1073,7 +1075,8 @@ class Pipeline:
inputs: dict[str, Any] | None = None,
arrived: list[str] | None = None,
) -> dict[str, Any] | None:
"""Run one node and record its outputs. Never raises.
"""Run one node and record its outputs. Never raises: None means it
failed, and a node that published nothing answers ``{}``.
``inputs`` is what the readiness check already read — the same keys, a
moment earlier. Reading them again here was the second of two MGETs
@@ -1108,7 +1111,7 @@ class Pipeline:
if key:
hit, restored = self._from_cache(node, key, state, entry_id)
if hit:
return restored
return restored or {}
token = ARRIVED.set(arrived)
try:
@@ -1175,7 +1178,7 @@ class Pipeline:
output_values=result,
)
)
return result
return result or {}
except Exception as exc:
# One failing node must not take the rest of the graph down.
error = self.publish_error(node, exc, collected, entry_id)
@@ -1437,6 +1440,11 @@ class Pipeline:
ready_nodes = deque(n for n, count in in_degree.items() if count == 0)
blocked: set[Node] = set()
# A batch run starts from an empty state, so a node whose producer
# failed would be called without that input. It is skipped instead,
# and so is everything waiting on it. A live wave still has the last
# values to run on.
failed: set[Node] | None = set() if self.run_id else None
gate_revision = self._gate_revision
node_futures: dict[Future[dict[str, Any] | None], Node] = {}
finished: SimpleQueue[Future[dict[str, Any] | None]] = SimpleQueue()
@@ -1475,10 +1483,26 @@ class Pipeline:
if n.passive:
complete(n)
continue
if failed is not None and not failed.isdisjoint(deps[n]):
failed.add(n)
self._observe(NodeOutcome(node=n.id, ok=False, skipped=True))
complete(n)
continue
if fresh is not None and untouched(n):
complete(n)
continue
ready_inputs: dict[str, Any] | None = None
if failed is not None:
# A live wave's readiness rule: a node runs once every
# triggering input has a value. A run has nothing to wait
# for, so the node fails instead.
ready_inputs = state.get_present(list(n.requires))
missing = self._no_value(n, ready_inputs)
if missing:
failed.add(n)
self._observe(NodeOutcome(node=n.id, ok=False, error=missing))
complete(n)
continue
if check_ready:
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
if not ready:
@@ -1518,6 +1542,8 @@ class Pipeline:
future = finished.get()
n = node_futures.pop(future)
result = future.result()
if result is None and failed is not None:
failed.add(n)
if result is not None and fresh is not None:
fresh.update(result)
# Silent, failed and throttled producers still release their
@@ -1536,6 +1562,18 @@ class Pipeline:
return state
def _no_value(self, node: Node, values: dict[str, Any]) -> str:
"""The first triggering input with nothing in ``values``, as an error,
or "" when there is none."""
for name, spec in node.requires.items():
if spec.trigger and name not in values:
producers = ", ".join(
f"'{p.local_id}'" for p in self.produces.get(name, ())
)
where = f" from {producers}" if producers else ""
return f"No value on input '{spec.port or name}'{where}"
return ""
def run(
self, inputs: dict[str, Any] | None = None, nodes: set[Node] | None = None
) -> StateBackend:
+46 -22
View File
@@ -51,7 +51,12 @@ from sqlmodel import Session, col, select
from fluksio.core.db import engine as db_engine
from fluksio.flow import tombstones
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
from fluksio.flow.controller import (
NODE_TYPES,
FlowController,
NodesNotLoaded,
RunContext,
)
from fluksio.flow.messages import qualify, requalify
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
from fluksio.flow.queue import WorkItem, WorkQueue
@@ -137,8 +142,9 @@ def batch_issues(
history keeps every value, and the interval only thins what is published.
Anywhere else it is a message quietly going missing.
A delay node is fine; without a queue to defer into it simply sleeps,
which in a run is what was asked for.
A delay node's ``interval`` is the same limit, put on every output it
has. Its ``delay`` is fine: without a queue to defer into it simply
sleeps, which in a run is what was asked for.
The other is a node with no body. The store answers a new node's template
when nothing was ever written for one, so such a node runs — and returns
@@ -148,8 +154,11 @@ def batch_issues(
"""
issues: list[str] = []
for node in flow.nodes:
for spec in list(node.requires) + list(node.provides):
if spec.interval > 0 and not spec.stream:
held = float(node.params.get("interval") or 0) if node.type == "delay" else 0
limits = [(spec, spec.interval) for spec in node.requires]
limits += [(spec, max(spec.interval, held)) for spec in node.provides]
for spec, interval in limits:
if interval > 0 and not spec.stream:
issues.append(
f"Node '{node.id}' rate-limits '{spec.port or spec.name}'. "
"A run has no timer to release what that holds back, so "
@@ -1236,8 +1245,10 @@ class RunService:
return True
# The gate first, so nothing new is submitted while the running nodes
# are being killed; a gated node is never handed to the executor, so
# the graph drains instead of going further.
pipeline.pause(self._key_of(run_id))
# the graph drains instead of going further. Stopped rather than
# paused: a pause announces itself on the engine's bus, under the key
# the live flow shares, so every open editor showed that one paused.
pipeline.set_disabled({self._key_of(run_id)})
workers = self.controller.workers
if workers is not None:
# Keyed by run, so a sweep cancelling one config leaves the others
@@ -1502,7 +1513,7 @@ class RunService:
def observe(outcome: NodeOutcome) -> None:
nonlocal errors
if not outcome.ok:
if not outcome.ok and not outcome.skipped:
errors += 1
self._record_node(run, key, outcome)
@@ -1537,19 +1548,26 @@ class RunService:
digests = node_digests(flow)
run.code_digest = self._restamp(run, run_digest(flow))
state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}")
pipeline = self.controller.build_run_pipeline(
flow,
state=state,
draft=run.draft,
observer=observe,
emission_observer=sink.handle,
run=RunContext(
run_id=run_id,
code_digest=run.code_digest,
node_digests=digests,
),
run_cache=None if run.no_cache else self._cache,
)
try:
pipeline = self.controller.build_run_pipeline(
flow,
state=state,
draft=run.draft,
observer=observe,
emission_observer=sink.handle,
run=RunContext(
run_id=run_id,
code_digest=run.code_digest,
node_digests=digests,
),
run_cache=None if run.no_cache else self._cache,
)
except NodesNotLoaded as exc:
# Nothing of it runs; the finally below records it.
for node, error in exc.errors.items():
observe(NodeOutcome(node=node, ok=False, error=error))
status, reason = "error", str(exc)[:1024]
return
with self._lock:
if run_id in self._cancelled:
# Cancelled while it was being built, so nothing of it
@@ -1627,10 +1645,16 @@ class RunService:
run_id = run.id
node = requalify(outcome.node, key, run.flow_id)[:255]
outputs = _cacheable(outcome)
if outcome.cached:
status = "cached"
elif outcome.ok:
status = "ok"
else:
status = "skipped" if outcome.skipped else "error"
row = RunNode(
run_id=run_id,
node=node,
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
status=status,
started_at=datetime.now(UTC),
duration_ms=outcome.duration_ms,
error=outcome.error[:ERROR_CAP],
+37 -30
View File
@@ -639,6 +639,14 @@ class RedisState(StateBackend):
names = [k for k in keys if not k.startswith("__")]
if names:
pipe.sadd(self._names_key(), *names)
if self._ttl:
pipe.expire(self._names_key(), self._ttl)
def _incr(self, pipe: Any, key: str) -> None:
"""INCR a counter in the caller's pipeline, expiring with the values."""
pipe.incr(self._key(key))
if self._ttl:
pipe.expire(self._key(key), self._ttl)
def set(self, key: str, value: Any) -> None:
data = self._serialize(value)
@@ -747,7 +755,9 @@ class RedisState(StateBackend):
def increment(self, key: str) -> int:
"""Atomically increment a counter using Redis INCR."""
return cast(int, self._client.incr(self._key(key)))
pipe = self._client.pipeline()
self._incr(pipe, key)
return cast(int, pipe.execute()[0])
def increment_multi(self, keys: list[str]) -> None:
"""Bump every counter in one round trip rather than one INCR each."""
@@ -755,7 +765,7 @@ class RedisState(StateBackend):
return
pipe = self._client.pipeline()
for key in keys:
pipe.incr(self._key(key))
self._incr(pipe, key)
pipe.execute()
def get_multi(self, keys: list[str]) -> dict[str, Any]:
@@ -824,34 +834,31 @@ class RedisState(StateBackend):
watch_keys = [self._key(k) for k in expected.keys()]
try:
# Start watching the keys for changes
self._client.watch(*watch_keys)
# WATCH has to go through the pipeline: it pins one connection, and
# the client-level call is a no-op in redis-py 7. Leaving the block
# resets the pipeline, which unwatches.
with self._client.pipeline() as pipe:
try:
pipe.watch(*watch_keys) # type: ignore[no-untyped-call]
# A watching pipeline runs commands immediately.
current = cast(list[bytes | None], pipe.mget(watch_keys))
for raw, exp_value in zip(current, expected.values(), strict=True):
if self._deserialize(raw) != exp_value:
return False
# Check current values
current = self.get_multi(list(expected.keys()))
for key, exp_value in expected.items():
if current.get(key) != exp_value:
self._client.unwatch()
return False
# All values match - execute the transaction
pipe = self._client.pipeline(True) # True = use MULTI/EXEC
for key, value in updates.items():
data = self._serialize(value)
if self._ttl:
pipe.setex(self._key(key), self._ttl, data)
else:
pipe.set(self._key(key), data)
pipe.execute()
return True
except redis.WatchError:
# Another client modified one of the watched keys
return False
pipe.multi()
for key, value in updates.items():
data = self._serialize(value)
if self._ttl:
pipe.setex(self._key(key), self._ttl, data)
else:
pipe.set(self._key(key), data)
self._note(pipe, updates)
pipe.execute()
return True
except redis.WatchError:
# Another client modified one of the watched keys
return False
def close(self) -> None:
"""Give the connection pool back."""
@@ -916,7 +923,7 @@ class RedisState(StateBackend):
if self._ttl:
pipe.expire(history_key, self._ttl)
for key in counters:
pipe.incr(self._key(key))
self._incr(pipe, key)
self._note(pipe, values)
pipe.execute()
+9 -5
View File
@@ -202,12 +202,14 @@ def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str:
class FlowNotFound(KeyError):
def __init__(self, name: str) -> None:
def __init__(self, name: str, published: bool = False) -> None:
super().__init__(name)
self.name = name
self.published = published
def __str__(self) -> str:
return f"No flow named '{self.name}'"
what = "published flow" if self.published else "flow"
return f"No {what} with the id '{self.name}'"
class FlowExists(ValueError):
@@ -720,7 +722,7 @@ class FlowStore:
return self._load(path)
path = self._flow_file(name)
if not path.exists():
raise FlowNotFound(name)
raise FlowNotFound(name, published=not draft)
return self._load(path)
def key_index(self) -> dict[str, str]:
@@ -782,7 +784,9 @@ class FlowStore:
and returned as a copy so a caller sorting it cannot disturb the next.
"""
with self._read_all_lock:
if self._read_all_at != self.revision:
# Taken before scanning, so a commit landing meanwhile scans again.
at = self.revision
if self._read_all_at != at:
flows = []
for path in sorted(self.root.glob("*/flow.json")):
name = path.parent.name
@@ -791,7 +795,7 @@ class FlowStore:
except Exception:
logger.exception("Skipping unreadable flow '%s'", name)
self._read_all = flows
self._read_all_at = self.revision
self._read_all_at = at
return list(self._read_all)
def write_flow(self, flow: FlowDef) -> bool:
+3 -1
View File
@@ -428,8 +428,10 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
app.state.device_store = devices
controller.devices = devices
# What the sockets want and the flows read, mapped onto the devices'
# on-demand ports.
# on-demand ports. It is process-wide, so it lets go of this
# controller when the engine stops.
demand.bind(controller)
started.append(lambda: demand.bind(None))
# Charts need a deeper series than the default; tell the engine
# before it starts recording.
controller.set_history_limits(dashboards.history_requirements())
+5 -1
View File
@@ -877,12 +877,16 @@ def _shim(spec: NodeSpec) -> str:
The import is aliased because ``process``'s parameters are the node's port
names, and a port may be named after the very function it feeds — which
would otherwise shadow the import inside the body.
Only the triggering ports are named. The engine leaves out an input that
has no value yet, which only a port that does not trigger can lack, so
those travel in ``**settings`` and the function's own default applies.
"""
_refuse_main(spec)
fn = spec.fn
where = inspect.getsourcefile(fn) or fn.__module__
repo = import_root(fn)
ports = [port.port for port in spec.requires]
ports = [port.port for port in spec.requires if port.trigger]
signature = ", ".join(ports + ["**settings"])
arguments = ", ".join([f"{port}={port}" for port in ports] + ["**settings"])
call = f"_impl({arguments})"
+10 -2
View File
@@ -170,6 +170,11 @@ class Conflict(ApiError):
self.current_version = int(detail.get("current_version") or 0)
def engine_url(url: str) -> str:
"""An engine's address, whether it was given with the API's path or not."""
return url.rstrip("/").removesuffix(API).rstrip("/")
class Client:
"""An authenticated engine, addressed over its HTTP API."""
@@ -182,7 +187,9 @@ class Client:
retries: int = RETRIES,
) -> None:
stored = _stored()
self.url = url or os.environ.get("FLUKSIO_URL") or stored.get("url") or ""
self.url = engine_url(
url or os.environ.get("FLUKSIO_URL") or stored.get("url") or ""
)
self.token = (
token or os.environ.get("FLUKSIO_TOKEN") or stored.get("token") or ""
)
@@ -766,8 +773,9 @@ def login(
"""Exchange credentials for a token and remember the engine."""
import httpx
url = engine_url(url)
response = httpx.post(
f"{url.rstrip('/')}{API}/login/access-token",
f"{url}{API}/login/access-token",
data={"username": email, "password": password},
timeout=timeout,
)