Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Give the controller a per-flow rebuild, not yet called by anything

Browse Source 
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01StpRc2C6au1WJ1EUU7fsfu
This commit is contained in:
stroblme
2026-08-23 17:51:18 +02:00
co-authored by Claude Opus 5
parent 9cf4c4e714
commit a4a9f2adff
2 changed files with 301 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/fluksio/flow/controller.py
+200 -19
View File
@@ -85,6 +85,13 @@ NODE_STOP_TIMEOUT = 5.0
# that never ends.
REBUILD_WAIT = 15.0
# How long a caller waits for a *flow* rebuild that is already running. A
# per-flow rebuild reconnects one flow's nodes rather than the installation's,
# so this is a queueing budget — several of them back to back, which is what
# seeding does — not the room a single one needs. Fifteen seconds was sized for
# the whole-pipeline rebuild and would let a wedge sit unreported.
FLOW_REBUILD_WAIT = 5.0
class RebuildBusy(RuntimeError):
"""A rebuild could not start because the one before it has not finished.
@@ -343,6 +350,9 @@ class FlowController:
self.disabled: set[str] = set()
#: Flows that only run when a run asks them to.
self.batch: set[str] = set()
#: Each flow's declared inputs, kept so rebuilding one flow can
#: validate the whole graph without re-reading every flow off the disk.
self._flow_inputs: dict[str, dict[str, bool]] = {}
self.supervisor = Supervisor(events)
self.history_limits: dict[str, int] = {}
self._lock = asyncio.Lock()
@@ -420,22 +430,27 @@ class FlowController:
await run_in_threadpool(self.store.write_enabled, flow, enabled)
await self.reload()
async def reload(self, wait: float | None = REBUILD_WAIT) -> None:
"""Rebuild the whole pipeline from what is currently stored.
async def _acquire(self, wait: float | None) -> None:
"""Take the rebuild lock, or say the engine is busy.
Only one rebuild runs at a time. A caller waits *wait* seconds for the
one in front of it and then gives up with ``RebuildBusy`` — hanging on
a rebuild that is stuck is worse than saying so. ``None`` waits.
Only one rebuild runs at a time whatever its scope. A caller waits
*wait* seconds for the one in front of it and then gives up with
``RebuildBusy`` — hanging on a rebuild that is stuck is worse than
saying so. ``None`` waits.
"""
if wait is None:
await self._lock.acquire()
else:
try:
await asyncio.wait_for(self._lock.acquire(), wait)
except asyncio.TimeoutError:
raise RebuildBusy(
f"A pipeline rebuild is still running after {wait:.0f}s"
) from None
return
try:
await asyncio.wait_for(self._lock.acquire(), wait)
except asyncio.TimeoutError:
raise RebuildBusy(
f"A pipeline rebuild is still running after {wait:.0f}s"
) from None
async def reload(self, wait: float | None = REBUILD_WAIT) -> None:
"""Rebuild the whole pipeline from what is currently stored."""
await self._acquire(wait)
try:
# Work already claimed belongs to the pipeline it was claimed
# against; let it finish there before swapping the graph out.
@@ -463,6 +478,9 @@ class FlowController:
nodes, loaded, initial_values, flow_inputs = await run_in_threadpool(
self._build_flows, [(flow, False) for flow in published]
)
self._flow_inputs = {
flow.name: _declared_inputs(flow)[0] for flow in published
}
# A rebuild is a fresh set of nodes, but not a fresh history: every
# publish rebuilds every flow, so dropping the failures here would
@@ -510,6 +528,159 @@ class FlowController:
}
)
async def reload_flow(
self, name: str, wait: float | None = FLOW_REBUILD_WAIT
) -> None:
"""Rebuild one flow, leaving every other flow's nodes connected.
A flow is not a subgraph — its nodes can read and write messages
another flow owns — but the wiring is derived from message names, so
swapping one flow's nodes into the graph and deriving the edges again
is enough. What that saves is the reconnecting: the cost of a rebuild
on a populated installation is every node opening its socket again,
and only one flow's have changed.
A flow the store no longer has is taken out instead of replaced.
The failure shape is the whole-pipeline one narrowed to a flow, and
deliberately no better: a build that raises changes nothing, and
anything failing after the teardown leaves that flow's nodes stopped
in the graph — which is exactly what a failed ``reload`` leaves behind
for all of them.
"""
pipeline = self.pipeline
if pipeline is None:
# Nothing built yet, so there is nothing to splice one flow into.
await self.reload(wait=wait)
return
await self._acquire(wait)
try:
# Built before anything is stopped, so the window in which the
# flow is not running is its own teardown and nothing more. Off
# the loop for the same reason the full build is: compiling a
# python node waits for a worker slot.
built = await run_in_threadpool(self._build_one, name)
await self._teardown(name)
if built is None:
pipeline.remove_flow(name)
self.loaded = {
node_id: entry
for node_id, entry in self.loaded.items()
if entry.flow != name
}
# Or the flow would go on declaring inputs nothing provides
# any more, and the consumers it left behind would look fine.
self._flow_inputs.pop(name, None)
self.disabled = self.disabled - {name}
self.batch = self.batch - {name}
else:
flow, nodes, loaded, initial_values, enabled = built
pipeline.replace_flow(name, nodes, initial_values)
# A rebuild is a fresh set of nodes, but not a fresh history:
# a failure nobody has dismissed is the operator's to keep.
for node_id, entry in loaded.items():
previous = self.loaded.get(node_id)
if previous is not None and previous.last_error:
entry.last_error = previous.last_error
entry.last_error_ts = previous.last_error_ts
# Rebound rather than mutated: request threads and the failure
# watcher read these without holding anything.
self.loaded = {
**{
node_id: entry
for node_id, entry in self.loaded.items()
if entry.flow != name
},
**loaded,
}
self._flow_inputs[name] = _declared_inputs(flow)[0]
self.disabled = (
self.disabled - {name} if enabled else self.disabled | {name}
)
self.batch = (
self.batch | {name} if flow.mode == "batch" else self.batch - {name}
)
pipeline.set_disabled(self.disabled)
# Over the whole graph, because it has to be: taking a producer out
# of one flow is what leaves another flow's input unconnected.
self.issues = _collect_issues(
self.loaded, pipeline, self._all_flow_inputs()
)
await self._activate(name)
# This flow's pause is cleared by the replace, so no resume will
# ever come for what it parked. Release it here or it is lost.
self._release_parked(name)
finally:
self._lock.release()
self._publish(
{
"type": "pipeline_rebuilt",
"issues": [issue.model_dump() for issue in self.issues],
"nodes": [status.model_dump() for status in self.node_statuses()],
# Every flow's, not this one's: the canvas replaces its paused
# set from this, so a narrowed list would clear the markers of
# flows the rebuild never touched.
"paused": self.paused_flows(),
}
)
async def reload_failed_flows(self) -> list[str]:
"""Rebuild only the flows holding a node that would not load.
What installing a module wants: a node red because its import was
missing is built again against the packages just installed, and every
other flow keeps the connections it already has.
"""
broken = sorted(
{
entry.flow
for entry in self.loaded.values()
if entry.status is NodeStatus.ERROR
}
)
for flow in broken:
await self.reload_flow(flow)
return broken
async def reload_lib_users(self, ref: str) -> list[str]:
"""Rebuild the flows running a shared node whose source has changed."""
flows = sorted({usage.split(".", 1)[0] for usage in self.store.usages(ref)})
for flow in flows:
await self.reload_flow(flow)
return flows
def _build_one(
self, name: str
) -> tuple[FlowDef, list[Node], dict[str, LoadedNode], dict[str, Any], bool] | None:
"""Read one published flow and build its nodes. None when it is gone.
Blocking: this is the part of a rebuild that talks to the worker pool.
"""
try:
flow = self.store.read_flow(name)
except FlowNotFound:
return None
nodes, loaded, initial_values, _ = self._build_flows([(flow, False)])
return flow, nodes, loaded, initial_values, self.store.read_enabled(name)
def _all_flow_inputs(self) -> dict[str, bool]:
"""Every flow's declared inputs, merged the way one build would see them.
In name order, which is the order ``read_all`` returns flows in, so a
name two flows both declare resolves to the same one either way.
"""
merged: dict[str, bool] = {}
for name in sorted(self._flow_inputs):
merged.update(self._flow_inputs[name])
return merged
async def _teardown(self, flow: str | None = None) -> None:
"""Stop everything the previous pipeline started, or one flow's share."""
for entry in self.loaded.values():
@@ -583,13 +754,9 @@ class FlowController:
loaded[entry.id] = entry
if entry.node is not None:
nodes.append(entry.node)
for flow_input in flow.inputs:
name = qualify(flow.name, flow_input.spec.name)
if not name:
continue
flow_inputs[name] = flow_input.initial is not None
if flow_input.initial is not None:
initial_values[name] = flow_input.initial
declared, initial = _declared_inputs(flow)
flow_inputs.update(declared)
initial_values.update(initial)
return nodes, loaded, initial_values, flow_inputs
@@ -1225,6 +1392,20 @@ def with_settings(
return call
def _declared_inputs(flow: FlowDef) -> tuple[dict[str, bool], dict[str, Any]]:
"""A flow's declared inputs, and the ones that start with a value."""
declared: dict[str, bool] = {}
initial: dict[str, Any] = {}
for flow_input in flow.inputs:
name = qualify(flow.name, flow_input.spec.name)
if not name:
continue
declared[name] = flow_input.initial is not None
if flow_input.initial is not None:
initial[name] = flow_input.initial
return declared, initial
def _collect_issues(
loaded: dict[str, LoadedNode],
pipeline: Pipeline,