Flows and dashboards are created by POST under an issued or superuser-pinned id; PUT only updates; deleted flows leave a tombstone with their key; the controller stores by id and runs by key, hooks live under /hooks/<flow id>; a key with state or queued work behind it is refused until cleared

This commit is contained in:
2026-11-02 17:41:23 +01:00
parent fae85f5ecb
commit 08a6376836
63 changed files with 2832 additions and 936 deletions
+92 -40
View File
@@ -72,6 +72,7 @@ from fluksio.flow.pipeline import (
ValueSource,
)
from fluksio.flow.placement import Placer
from fluksio.flow.queue import WorkItem, owned_by
from fluksio.flow.remote import RemoteWorkerHub
from fluksio.flow.resources import (
ResourceAccountant,
@@ -160,7 +161,10 @@ class LoadedNode:
"""
id: str
#: The flow's key, which ``id`` is qualified with.
flow: str
#: The flow's id, which its store paths and API routes use.
flow_id: str = ""
status: NodeStatus = NodeStatus.ACTIVE
node: Node | None = None
error: str | None = None
@@ -556,6 +560,9 @@ class FlowController:
self.disabled: set[str] = set()
#: Flows that only run when a run asks them to.
self.batch: set[str] = set()
#: The key each built flow runs under, by its id. What a rebuild of a
#: flow that is gone takes out, since its document can no longer say.
self._flow_keys: dict[str, str] = {}
#: Each flow's declared inputs, kept so rebuilding one flow can
#: validate the whole graph without re-reading every flow off the disk.
self._flow_inputs: dict[str, dict[str, bool]] = {}
@@ -642,15 +649,16 @@ class FlowController:
entry.last_error_ts = None
entry.failures_in_a_row = 0
async def set_enabled(self, flow: str, enabled: bool) -> None:
async def set_enabled(self, flow_id: str, flow: str, enabled: bool) -> None:
"""Stop or start one flow, without building anything.
Its nodes are built whether or not it runs — being stopped means
having no subscriptions, schedules or webhooks, not being absent — so
all this has to do is the lifecycle call and the gate that goes with
it. Nothing is compiled and no other flow is touched.
it. Nothing is compiled and no other flow is touched. *flow_id* is
where that is stored, *flow* the key the gate goes by.
"""
await run_in_threadpool(self.store.write_enabled, flow, enabled)
await run_in_threadpool(self.store.write_enabled, flow_id, enabled)
if self.pipeline is None:
# Nothing built yet, so there are no nodes to start or stop.
await self.reload()
@@ -726,10 +734,9 @@ class FlowController:
self.supervisor = Supervisor(self.events)
published = self.store.read_all()
self._flow_keys = {flow.id: flow.name for flow in published}
self.disabled = {
flow.name
for flow in published
if not self.store.read_enabled(flow.name)
flow.name for flow in published if not self.store.read_enabled(flow.id)
}
# A batch flow is built — the canvas draws it and validation covers
# it — but never activated: it runs when a run asks it to, and a
@@ -795,7 +802,7 @@ class FlowController:
self._rewired()
async def reload_flow(
self, name: str, wait: float | None = FLOW_REBUILD_WAIT
self, flow_id: str, wait: float | None = FLOW_REBUILD_WAIT
) -> None:
"""Rebuild one flow, leaving every other flow's nodes connected.
@@ -806,7 +813,8 @@ class FlowController:
on a populated instance is every node opening its socket again,
and only one flow's have changed.
A flow the store no longer has is taken out instead of replaced.
A flow the store no longer has is taken out instead of replaced. The
flow is named by its id; everything about running it goes by its key.
The failure shape is the whole-pipeline one narrowed to a flow, and
deliberately no better: a build that raises changes nothing, and
@@ -826,11 +834,14 @@ class FlowController:
# flow is not running is its own teardown and nothing more. Off
# the loop for the same reason the full build is: compiling a
# python node waits for a worker slot.
built = await run_in_threadpool(self._build_one, name)
built = await run_in_threadpool(self._build_one, flow_id)
# A flow never built has nothing to take out, so no key either.
name = built[0].name if built else self._flow_keys.pop(flow_id, "")
await self._teardown(name)
if built is None:
pipeline.remove_flow(name)
if name:
pipeline.remove_flow(name)
self.loaded = {
node_id: entry
for node_id, entry in self.loaded.items()
@@ -843,6 +854,7 @@ class FlowController:
self.batch = self.batch - {name}
else:
flow, nodes, loaded, initial_values, enabled = built
self._flow_keys[flow_id] = name
pipeline.replace_flow(name, nodes, initial_values)
# A rebuild is a fresh set of nodes, but not a fresh history:
@@ -953,7 +965,7 @@ class FlowController:
"""
broken = sorted(
{
entry.flow
entry.flow_id
for entry in self.loaded.values()
if entry.status is NodeStatus.ERROR
}
@@ -963,7 +975,10 @@ class FlowController:
return broken
async def reload_lib_users(self, ref: str) -> list[str]:
"""Rebuild the flows running a shared node whose source has changed."""
"""Rebuild the flows running a shared node whose source has changed.
Returns their ids, which is what `usages` qualifies with.
"""
usages = self.store.usages(ref, draft=False)
flows = sorted({usage.split(".", 1)[0] for usage in usages})
for flow in flows:
@@ -971,18 +986,18 @@ class FlowController:
return flows
def _build_one(
self, name: str
self, flow_id: str
) -> tuple[FlowDef, list[Node], dict[str, LoadedNode], dict[str, Any], bool] | None:
"""Read one published flow and build its nodes. None when it is gone.
Blocking: this is the part of a rebuild that talks to the worker pool.
"""
try:
flow = self.store.read_flow(name)
flow = self.store.read_flow(flow_id)
except FlowNotFound:
return None
nodes, loaded, initial_values, _ = self._build_flows([(flow, False)])
return flow, nodes, loaded, initial_values, self.store.read_enabled(name)
return flow, nodes, loaded, initial_values, self.store.read_enabled(flow_id)
def _all_flow_inputs(self) -> dict[str, bool]:
"""Every flow's declared inputs, merged the way one build would see them.
@@ -1145,7 +1160,7 @@ class FlowController:
for flow, draft in flows:
for node_def in flow.nodes:
entry = self._build_node(flow.name, node_def, draft=draft, run=run)
entry = self._build_node(flow, node_def, draft=draft, run=run)
loaded[entry.id] = entry
if entry.node is not None:
nodes.append(entry.node)
@@ -1157,13 +1172,16 @@ class FlowController:
def _build_node(
self,
flow: str,
flow_def: FlowDef,
node_def: NodeDef,
draft: bool = False,
run: RunContext | None = None,
) -> LoadedNode:
"""One node of *flow_def*: its code read by the flow's id, and run,
named and wired by its key."""
flow, flow_id = flow_def.name, flow_def.id
node_id = f"{flow}.{node_def.id}"
entry = LoadedNode(id=node_id, flow=flow)
entry = LoadedNode(id=node_id, flow=flow, flow_id=flow_id)
try:
node_type = NODE_TYPES.get(node_def.type)
if node_type is None:
@@ -1192,19 +1210,21 @@ class FlowController:
code = self.store.read_lib_source(node_def.source_ref)
else:
owner, local = flow, node_def.id
code = self.store.read_node_source(flow, node_def.id, draft=draft)
code = self.store.read_node_source(
flow_id, node_def.id, draft=draft
)
# The store answers the new-node template when nothing was
# written, which is what an editor should open with and not
# something to run. A draft legitimately has none yet.
entry.missing_source = not draft and not self.store.has_node_source(
flow, node_def.id
flow_id, node_def.id
)
# Only a body somebody wrote: the template a new node opens
# with takes nothing, and a node declared before it is written
# is reported as missing its source rather than as wrong.
written = bool(node_def.source_ref) or self.store.has_node_source(
flow, node_def.id, draft=draft
flow_id, node_def.id, draft=draft
)
if written:
gap = _signature_gap(
@@ -1376,8 +1396,10 @@ class FlowController:
node._on_health = self._health_changed
if _is_hook(node):
# Hooks live under their flow, away from the JSON API.
node.url = f"{HOOK_PREFIX}/{flow}/{node.url.lstrip('/')}"
# Hooks live under their flow, away from the JSON API: under
# its id, so a URL handed out keeps working when the key
# changes.
node.url = f"{HOOK_PREFIX}/{flow_id}/{node.url.lstrip('/')}"
entry.node = node
except (SecretNotFound, LibNotFound) as exc:
@@ -1493,7 +1515,7 @@ class FlowController:
issue for issue in self.issues if not issue.flow or issue.flow == flow
] + self._health_issues(flow)
def preview(self, name: str) -> Preview:
def preview(self, flow_id: str) -> Preview:
"""Build a flow's unpublished draft without deploying it.
The draft stands in for its published self among the other flows, so
@@ -1501,13 +1523,15 @@ class FlowController:
graph is thrown away afterwards and seeded into a throwaway state, so
nothing here touches what is running.
"""
flows: list[tuple[FlowDef, bool]] = [
(flow, False) for flow in self.store.read_all() if flow.name != name
]
try:
flows.append((self.store.read_flow(name, draft=True), True))
draft = self.store.read_flow(flow_id, draft=True)
except FlowNotFound:
return Preview(nodes=[], issues=[])
name = draft.name
flows: list[tuple[FlowDef, bool]] = [
(flow, False) for flow in self.store.read_all() if flow.id != flow_id
]
flows.append((draft, True))
nodes, loaded, initial_values, flow_inputs = self._build_flows(flows)
pipeline = Pipeline(
@@ -1856,21 +1880,35 @@ class FlowController:
return self.execution.stats() if self.execution is not None else {}
def forget_flow(self, flow: str) -> None:
"""Drop what a deleted flow left behind. Blocking."""
"""Drop what a deleted flow left behind, by its key. Blocking.
After its nodes are torn down, or they put some of it straight back.
"""
if self.execution is not None:
self.execution.queue.clear_flow(flow)
prefix = f"{flow}."
with self.state.lock():
stale = [
key
for key in self.state.keys()
# Both the messages themselves and the engine's own bookkeeping
# about them, which is keyed by message name too.
if key.startswith(prefix) or f":{prefix}" in key
]
for key in stale:
for key in [k for k in self.state.keys() if _in_namespace(k, flow)]:
self.state.delete(key)
def key_residue(self, flow: str) -> list[str]:
"""What still lives under the flow key *flow*. Blocking.
State and live queue work in its namespace: left by a flow that held
the key before, or put there by anything still writing to it. A new
flow taking the key would inherit it, so creating one is refused while
this is not empty, and so is renaming a flow onto it. Work that was
acknowledged, dead-lettered or marked done is history and not counted.
Names what it found, state keys first.
"""
found = [key for key in self.state.keys() if _in_namespace(key, flow)]
if self.execution is not None:
found += [
f"queued work of '{item.node}'"
for item in self.execution.queue.live_items()
if _touches(item, flow)
]
return found
def run_flow(self, flow: str, inputs: dict[str, Any] | None = None) -> None:
"""Run every node of one flow. Blocking — call from a worker thread."""
if self.pipeline is None:
@@ -1912,7 +1950,7 @@ class FlowController:
pipeline.history_limits = self.history_limits
return pipeline
def run_preview(self, flow_name: str, inputs: dict[str, Any] | None = None) -> None:
def run_preview(self, flow_id: str, inputs: dict[str, Any] | None = None) -> None:
"""Run a flow's unpublished draft once, as the editor shows it.
The nodes are built and run but never activated, so a draft never
@@ -1920,7 +1958,7 @@ class FlowController:
that. State and events are the live ones, so values land on the canvas
exactly as they do for a published run.
"""
flow = self.store.read_flow(flow_name, draft=True)
flow = self.store.read_flow(flow_id, draft=True)
nodes, _, initial_values, _ = self._build_flows([(flow, True)])
pipeline = Pipeline(
nodes=nodes,
@@ -1960,6 +1998,20 @@ class FlowController:
self.events.publish(event)
def _in_namespace(state_key: str, flow: str) -> bool:
"""Whether a state key is about the flow key *flow*: one of its messages,
or the engine's own bookkeeping on one of its messages or nodes, which is
keyed by those names too."""
prefix = f"{flow}."
return state_key.startswith(prefix) or f":{prefix}" in state_key
def _touches(item: WorkItem, flow: str) -> bool:
"""Whether a queued item names anything of the flow key *flow*: as its
flow, its node or one of the messages it carries."""
return owned_by(item, flow) or any(flow_of(name) == flow for name in item.outputs)
def _bound(specs: list[MessageSpec]) -> list[MessageSpec]:
"""Ports without a message name are not wired into the graph."""
return [spec for spec in specs if spec.name]
+102 -33
View File
@@ -28,11 +28,15 @@ from fluksio.flow.messages import DType, qualify
from fluksio.flow.schemas import FlowDef, _validate_id, _validate_name
from fluksio.flow.store import (
FlowStore,
IdTaken,
StaleVersion,
_updated_at,
_version,
checked,
claim,
copy_name,
named,
new_id,
read_keys,
)
@@ -619,6 +623,27 @@ class DashboardDef(BaseModel):
return shapes
class DashboardCreate(BaseModel):
"""A new dashboard: one empty grid under a key, filled in by saves after."""
#: Empty for an id the server issues, or one a superuser pins, as for a
#: flow (`FlowCreate`). Dashboards keep no history, so it only has to be free.
id: str = ""
name: str
#: Empty reads the key as words.
title: str = ""
@field_validator("id")
@classmethod
def _check_id(cls, value: str) -> str:
return _validate_id(value)
@field_validator("name")
@classmethod
def _check_name(cls, value: str) -> str:
return _validate_name(value)
class DashboardSummary(BaseModel):
"""A dashboard in a list, without its contents."""
@@ -766,14 +791,17 @@ class DashboardStore:
self._keys = (at, keys)
return keys
def _claim(self, defn: DashboardDef) -> DashboardDef:
def _claim(self, defn: DashboardDef, create: bool = False) -> DashboardDef:
"""*defn* with its id, once it is certain to be a save the store allows.
Under the flow store's write lock; see `claim`. An id left out is the
key, which is how every dashboard is created for now.
Under the flow store's write lock; see `claim`. A save is to a
dashboard that exists, named by its id or, from before ids, its key.
Only *create* may make a new one.
"""
dash_id = checked(defn.id or defn.name)
current = self.read(dash_id, draft=True) if self.exists(dash_id) else None
if current is None and not create:
raise DashboardNotFound(dash_id)
claim(
dash_id,
defn.name,
@@ -807,21 +835,50 @@ class DashboardStore:
"""Publish a dashboard directly, skipping the draft.
The API never does: it creates a draft and promotes it. This is for a
caller that already has the finished document — a test, or a seed.
caller that already has the finished document, a test or a seed, and
creates one too, under the id it carries or else its key.
"""
with self._lock, self.flows._write_lock:
defn = self._claim(defn)
defn = self._claim(defn, create=True)
path = self._file(defn.id)
current = 0
if path.exists():
current = DashboardDef.model_validate_json(path.read_text()).version
current = _version(path)
if base_version is not None and base_version != current:
raise StaleVersion(defn.name, current)
saved = defn.model_copy(update={"version": current + 1, "has_draft": False})
# Above the draft too, which an editor may be holding.
top = max(current, _version(self._draft_file(defn.id)))
saved = defn.model_copy(update={"version": top + 1, "has_draft": False})
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(self._dump(saved))
self.flows._commit(f"Save dashboard '{defn.name}'")
self.flows._commit(f"Save dashboard {named(defn.name, defn.id)}")
return saved
def create(
self, defn: DashboardDef, pin: str = "", published: bool = False
) -> DashboardDef:
"""A new dashboard, only a draft unless *published* says otherwise.
Its id is issued here unless *pin* names one, a superuser's to ask for
as for a flow (`FlowStore.create`). Dashboards have no history to
adopt, so a pin only has to be free.
"""
with self._lock, self.flows._write_lock:
keys = self.key_index()
if pin:
dash_id = checked(pin)
if (self.root / dash_id).exists():
raise IdTaken(dash_id, "dashboard")
else:
dash_id = new_id("d", lambda c: (self.root / c).exists() or c in keys)
defn = self._claim(defn.model_copy(update={"id": dash_id}), create=True)
if keys.get(dash_id, dash_id) != dash_id:
raise IdTaken(dash_id, "dashboard")
saved = defn.model_copy(update={"version": 1, "has_draft": not published})
path = self._file(dash_id) if published else self._draft_file(dash_id)
path.parent.mkdir(parents=True)
path.write_text(self._dump(saved))
self.flows._commit(f"Create dashboard {named(defn.name, dash_id)}")
return saved
def write_draft(
@@ -829,23 +886,19 @@ class DashboardStore:
) -> DashboardDef:
"""Save unpublished changes, refusing to overwrite someone else's.
``base_version`` is the version the editor last saw — of the working
copy, which is the draft once there is one, and 0 for a dashboard that
does not exist yet: creating one is its first draft.
``base_version`` is the version the editor last saw, of the working
copy, which is the draft once there is one. The dashboard has to
exist: creating one is `create`.
"""
with self._lock, self.flows._write_lock:
defn = self._claim(defn)
current = 0
if self.exists(defn.id):
current = self.read(defn.id, draft=True).version
current = self.read(defn.id, draft=True).version
if base_version is not None and base_version != current:
raise StaleVersion(defn.name, current)
saved = defn.model_copy(update={"version": current + 1, "has_draft": True})
path = self._draft_file(defn.id)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(self._dump(saved))
self.flows._commit(f"Update draft of dashboard '{defn.name}'")
self._draft_file(defn.id).write_text(self._dump(saved))
self.flows._commit(f"Update draft of dashboard {named(defn.name, defn.id)}")
return saved
def publish(self, name: str, base_version: int | None = None) -> DashboardDef:
@@ -859,14 +912,24 @@ class DashboardStore:
if draft.exists():
self._file(name).write_text(self._dump(current))
draft.unlink()
self.flows._commit(f"Publish dashboard '{name}'")
self.flows._commit(f"Publish dashboard {named(current.name, name)}")
return current.model_copy(update={"has_draft": False})
def discard_draft(self, name: str) -> DashboardDef:
"""Throw the unpublished changes away and go back to what is shown."""
"""Throw the unpublished changes away and go back to what is shown.
What is shown comes back a version above the draft, as a flow's does
(`FlowStore.discard_draft`).
"""
with self._lock, self.flows._write_lock:
draft = self._draft_file(name)
if draft.exists():
shown = self.read(name)
top = _version(draft)
if shown.version <= top:
self._file(name).write_text(
self._dump(shown.model_copy(update={"version": top + 1}))
)
draft.unlink()
self.flows._commit(f"Discard draft of dashboard '{name}'")
return self.read(name)
@@ -887,27 +950,32 @@ class DashboardStore:
def duplicate(self, name: str) -> DashboardDef:
"""Copy the working copy into a new dashboard, as a draft only.
Named ``{name}_copy``, then ``_copy_2`` and on. Unpublished like a new
dashboard, so no panel shows it until someone publishes it.
Under an id of its own and the key ``{name}_copy``, then ``_copy_2``
and on. Unpublished like a new dashboard, so no panel shows it until
someone publishes it.
"""
with self._lock, self.flows._write_lock:
source = self.read(name, draft=True)
# The copy's id is its key, so it has to be free as both.
keys = self.key_index()
new_name = copy_name(source.name, lambda n: self.exists(n) or n in keys)
new_name = copy_name(source.name, lambda n: n in keys)
copy_id = new_id("d", lambda c: (self.root / c).exists() or c in keys)
copy = source.model_copy(
update={
"id": new_name,
"id": copy_id,
"name": new_name,
"title": f"{source.title} (copy)" if source.title else "",
"version": 1,
"has_draft": True,
}
)
path = self._draft_file(new_name)
path.parent.mkdir(parents=True, exist_ok=True)
copy = self._claim(copy, create=True)
path = self._draft_file(copy_id)
path.parent.mkdir(parents=True)
path.write_text(self._dump(copy))
self.flows._commit(f"Duplicate dashboard '{name}' as '{new_name}'")
self.flows._commit(
f"Duplicate dashboard {named(source.name, name)} as "
f"{named(new_name, copy_id)}"
)
return copy
def bindings_for(self, flow: str) -> Bindings:
@@ -984,11 +1052,11 @@ class DashboardStore:
return limits
def default_dashboard(name: str) -> DashboardDef:
"""A new dashboard: one page, one section, nothing in it yet."""
def default_dashboard(name: str, title: str = "") -> DashboardDef:
"""A new dashboard: one empty grid, titled after its key unless told."""
return DashboardDef(
name=name,
title=name.replace("_", " ").capitalize(),
title=title or name.replace("_", " ").capitalize(),
)
@@ -1094,6 +1162,7 @@ __all__ = [
"INPUT_WIDGETS",
"SETTING_DTYPES",
"WIDGET_DTYPES",
"DashboardCreate",
"DashboardDef",
"DashboardExists",
"DashboardNotFound",
+1 -1
View File
@@ -445,7 +445,7 @@ class DeviceStore:
if self._wiring_at != revision:
found: dict[str, list[Binding]] = {}
for flow in self.flows.read_all():
if not self.flows.read_enabled(flow.name):
if not self.flows.read_enabled(flow.id):
continue
for binding in bindings_of(flow):
found.setdefault(binding.device, []).append(binding)
+4
View File
@@ -24,6 +24,7 @@ from sqlmodel import Session, col
from fluksio.core.config import settings
from fluksio.core.db import engine
from fluksio.flow import tombstones
from fluksio.flow.events import EventBus
from fluksio.models import EngineEvent, FlowRun, MetricBucket
@@ -456,6 +457,9 @@ class MetricsCollector:
session.execute(delete(MetricBucket).where(col(MetricBucket.bucket) < cutoff))
session.execute(delete(EngineEvent).where(col(EngineEvent.ts) < cutoff))
session.execute(delete(FlowRun).where(col(FlowRun.started_at) < cutoff))
# After the rows above: a deleted flow's name is kept as long as any
# of its history is.
tombstones.prune(session)
# A cascade still open long after it started did not finish; saying so
# is more honest than leaving it running forever. A batch run is not
# swept: it is allowed to take hours, and `Run` has its own lease sweep
+2 -2
View File
@@ -161,7 +161,7 @@ class HttpNode(Node):
url: str = Field(
description=(
"A path such as /tick, served as a webhook at /hooks/<flow>/<path>, "
"A path such as /tick, served as a webhook at /hooks/<flow id>/<path>, "
"or the full URL a sender requests."
)
)
@@ -197,7 +197,7 @@ class HttpNode(Node):
default="",
description=(
"Shared secret for a webhook, appended to its URL: "
"/hooks/<flow>/<url>/<secret>. Empty leaves the webhook open "
"/hooks/<flow id>/<url>/<secret>. Empty leaves the webhook open "
"to anyone."
),
json_schema_extra={"x-secret": True},
+3 -3
View File
@@ -31,7 +31,7 @@ class WebSocketNode(Node):
The URL picks the role: ``ws://`` or ``wss://`` dials out and holds the
connection, a path such as ``/ticker`` is served at
``/hooks/<flow>/<path>[/<secret>]`` while the flow runs. The ports pick the
``/hooks/<flow id>/<path>[/<secret>]`` while the flow runs. The ports pick the
direction, as on the mqtt node: outputs only make a **listener**, inputs a
**sender**.
@@ -79,7 +79,7 @@ class WebSocketNode(Node):
url: str = Field(
description=(
"ws:// or wss:// to connect to a server, or a path such as "
"/ticker to serve one at /hooks/<flow>/<path>."
"/ticker to serve one at /hooks/<flow id>/<path>."
)
)
headers: dict[str, str] = Field(
@@ -100,7 +100,7 @@ class WebSocketNode(Node):
default="",
description=(
"Shared secret for a served socket, appended to its URL: "
"/hooks/<flow>/<url>/<secret>. Empty leaves it open to anyone."
"/hooks/<flow id>/<url>/<secret>. Empty leaves it open to anyone."
),
json_schema_extra={"x-secret": True},
)
+103 -1
View File
@@ -151,6 +151,17 @@ class WorkItem:
)
def owned_by(item: WorkItem, flow: str) -> bool:
"""Whether *item* is work of the flow with key *flow*, not one reaching it.
A run item names its run's flow by identity rather than by key, and a
run is never removed with a flow's queued work, so it is nobody's here.
"""
return item.kind != "run" and (
item.flow == flow or item.node.split(".", 1)[0] == flow
)
class WorkQueue(ABC):
"""What the execution service pulls from."""
@@ -236,7 +247,21 @@ class WorkQueue(ABC):
@abstractmethod
def clear_flow(self, flow: str) -> None:
"""Forget anything held for a flow that no longer exists."""
"""Drop the work of a flow that no longer exists (`owned_by`).
What it parked, what waits on a timer, and what is queued or claimed
and not yet acknowledged, so a new flow under the same key never runs
it. A cascade already running finishes; its acknowledgement then has
nothing left to remove.
"""
@abstractmethod
def live_items(self) -> list[WorkItem]:
"""Every item still to run: queued, claimed, delayed or parked.
Not what was acknowledged, dead-lettered or marked done, which is
history. Reads the whole queue, so it is for rare questions.
"""
@abstractmethod
def stats(self) -> dict[str, Any]:
@@ -388,6 +413,23 @@ class MemoryWorkQueue(WorkQueue):
def clear_flow(self, flow: str) -> None:
with self._lock:
self._parked.pop(flow, None)
for lane in (self._items, self._due):
kept = [item for item in lane if not owned_by(item, flow)]
lane.clear()
lane.extend(kept)
self._delayed = [e for e in self._delayed if not owned_by(e[2], flow)]
heapq.heapify(self._delayed)
def live_items(self) -> list[WorkItem]:
"""What is waiting. A claimed item is gone from here once it is handed
out, so unlike Redis this cannot say what is running."""
with self._lock:
return [
*self._items,
*self._due,
*(entry[2] for entry in self._delayed),
*(item for held in self._parked.values() for item in held),
]
def backlog(self) -> int:
with self._lock:
@@ -683,8 +725,68 @@ class RedisWorkQueue(WorkQueue):
pipe = self._redis.pipeline()
pipe.delete(self._parked_key(flow))
pipe.srem(self._parked_flows_key, flow)
# Timers before the streams: one promoted in between is in the due
# stream by the time that is read.
for raw in cast(list[str], self._redis.zrange(self._delayed_key, 0, -1)):
if owned_by(WorkItem.from_fields(orjson.loads(raw), ""), flow):
pipe.zrem(self._delayed_key, raw)
for stream in (self._stream, self._due_stream):
ids = [
entry_id
for entry_id, fields in self._live_entries(stream)
if owned_by(WorkItem.from_fields(fields, entry_id), flow)
]
if ids:
pipe.xack(stream, GROUP, *ids)
pipe.xdel(stream, *ids)
pipe.execute()
def _live_entries(self, stream: str) -> list[tuple[str, dict[str, str]]]:
"""The entries of *stream* still to run: claimed and unacknowledged,
then never handed out. Acknowledged ones stay in a stream until it is
trimmed, and are history."""
group = self._group_info(stream)
if not group:
return []
last = str(group.get("last-delivered-id") or "0-0")
live: list[tuple[str, dict[str, str]]] = []
pending = int(group.get("pending") or 0)
if pending:
rows = cast(
list[dict[str, Any]],
self._redis.xpending_range(
stream, GROUP, min="-", max="+", count=pending
),
)
ids = {row["message_id"] for row in rows}
if ids:
# In id order, so the first is the oldest still held.
entries = self._redis.xrange(
stream, min=rows[0]["message_id"], max=last
)
live += [(i, f) for i, f in cast(Any, entries) if i in ids]
live += cast(Any, self._redis.xrange(stream, min=f"({last}", max="+"))
return live
def live_items(self) -> list[WorkItem]:
items = [
WorkItem.from_fields(fields, prefix + entry_id)
for stream, prefix in ((self._due_stream, DUE_PREFIX), (self._stream, ""))
for entry_id, fields in self._live_entries(stream)
]
items += [
WorkItem.from_fields(orjson.loads(raw), "")
for raw in cast(list[str], self._redis.zrange(self._delayed_key, 0, -1))
]
for flow in cast(set[str], self._redis.smembers(self._parked_flows_key)):
items += [
WorkItem.from_fields(orjson.loads(raw), "")
for raw in cast(
list[str], self._redis.lrange(self._parked_key(flow), 0, -1)
)
]
return items
def _group_info(self, stream: str = "") -> dict[str, Any]:
"""This consumer group's row of ``XINFO GROUPS``, or an empty one."""
try:
+1 -1
View File
@@ -155,7 +155,7 @@ def batch_issues(
"the value would be dropped. Remove the interval, or mark "
"the port as streaming if it is a curve being thinned out."
)
if store is not None and _has_no_body(store, flow.name, node, draft):
if store is not None and _has_no_body(store, flow.id, node, draft):
issues.append(
f"Node '{node.id}' has no stored code. It would run as an "
"empty node and publish nothing, so the run is refused."
+30
View File
@@ -352,6 +352,36 @@ class FlowDef(BaseModel):
return _validate_name(value)
class FlowCreate(BaseModel):
"""A new flow as a client hands it over: a document but for what the
server owns. The fields mean what they do on `FlowDef`, but ``id``."""
#: Empty for an id the server issues. Anything else pins it, which only a
#: superuser may: an id that is free, or whose history this flow resumes.
id: str = ""
name: str
title: str = ""
nodes: list[NodeDef] = Field(default_factory=list)
inputs: list[FlowInput] = Field(default_factory=list)
mode: Literal["live", "batch"] = "live"
outputs: list[str] = Field(default_factory=list)
origin: FlowOrigin | None = None
@field_validator("id")
@classmethod
def _check_id(cls, value: str) -> str:
return _validate_id(value)
@field_validator("name")
@classmethod
def _check_name(cls, value: str) -> str:
return _validate_name(value)
def document(self) -> FlowDef:
"""The flow to store, before the store gives it an id."""
return FlowDef.model_validate(self.model_dump(exclude={"id"}))
class NodeSource(BaseModel):
"""The Python source of a node."""
+208 -68
View File
@@ -14,8 +14,9 @@ unpublished — which is what every flow written before this existed looks like.
A flow is two names. Its id is which flow it is: the directory, the argument
every method here takes, and never changed once given. Its key, ``name``, is
the namespace of its messages and unique among flows. A flow from before ids
existed has its key as its id (`fluksio.flow.identity`), and so does every new
one for now. Published and draft documents always share both.
existed has its key as its id (`fluksio.flow.identity`). A new one is given an
opaque id by `create`, or the one a superuser pinned. Published and draft
documents always share both.
"""
from __future__ import annotations
@@ -31,10 +32,13 @@ import threading
from collections.abc import Callable, Iterable
from datetime import UTC, datetime
from pathlib import Path
from typing import Literal
from typing import TYPE_CHECKING, Literal
from fluksio.flow.schemas import FlowDef, _validate_name
if TYPE_CHECKING:
from fluksio.flow.tombstones import Tombstones
logger = logging.getLogger(__name__)
#: Where the sources of nodes shared between flows live.
@@ -97,6 +101,37 @@ class IdentityChange(ValueError):
"""
class RecordedKey(IdentityChange):
"""A pinned id restored under another key than its history is filed under.
The history names the old key's messages, so taking a new key with it is
a rename, not a create.
"""
def __init__(self, obj_id: str, key: str) -> None:
super().__init__(obj_id)
self.id = obj_id
self.key = key
def __str__(self) -> str:
return (
f"'{self.id}' has history filed under the key '{self.key}', so it "
"can only be restored with that key"
)
class IdTaken(ValueError):
"""A pinned id that is in use: an object's, a directory's or another key."""
def __init__(self, obj_id: str, kind: str = "flow") -> None:
super().__init__(obj_id)
self.id = obj_id
self.kind = kind
def __str__(self) -> str:
return f"The id '{self.id}' is already taken by a {self.kind}"
def claim(
obj_id: str,
key: str,
@@ -146,6 +181,11 @@ def read_keys(directories: Iterable[Path], document: str) -> dict[str, str]:
return keys
def named(key: str, obj_id: str) -> str:
"""How a commit message names an object: by key, and by id when it differs."""
return f"'{key}'" if key == obj_id else f"'{key}' ({obj_id})"
def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str:
"""A fresh id for a flow (``f``) or a dashboard (``d``), e.g. ``f_k3q7m2xa``.
@@ -222,6 +262,14 @@ def _updated_at(paths: Iterable[Path]) -> datetime | None:
return datetime.fromtimestamp(max(stamps), UTC) if stamps else None
def _version(path: Path) -> int:
"""The version a stored document carries, 0 for none or an unreadable one."""
try:
return int(json.loads(path.read_text()).get("version", 1))
except (OSError, ValueError, TypeError, AttributeError):
return 0
def _same_content(left: FlowDef, right: FlowDef) -> bool:
"""Equal but for the version counter, which the server owns."""
return left.model_copy(update={"version": 0}) == right.model_copy(
@@ -252,8 +300,11 @@ class FlowStore:
#: a store reached 1898 commits, 5863 loose objects and no pack at all.
_GC_EVERY = 500
def __init__(self, root: Path) -> None:
def __init__(self, root: Path, tombstones: Tombstones | None = None) -> None:
self.root = root
#: Where a deleted flow's key is kept, and what history an id has.
#: None leaves both out: a store without a database, as in tests.
self.tombstones = tombstones
self.root.mkdir(parents=True, exist_ok=True)
# Draft writes are check-and-set, so two clients saving at once must not
# interleave between reading the current version and writing the next.
@@ -679,14 +730,16 @@ class FlowStore:
self._keys = (at, keys)
return keys
def _claim(self, flow: FlowDef) -> FlowDef:
def _claim(self, flow: FlowDef, create: bool = False) -> FlowDef:
"""*flow* with its id, once it is certain to be a save the store allows.
Under `_write_lock`. An id left out is the key, which is how every
flow is created for now.
Under `_write_lock`. A save is to a flow that exists, named by its id
or, from before ids, by its key. Only *create* may make a new one.
"""
flow_id = checked(flow.id or flow.name)
current = self.read_flow(flow_id, draft=True) if self.exists(flow_id) else None
if current is None and not create:
raise FlowNotFound(flow_id)
claim(
flow_id,
flow.name,
@@ -696,6 +749,22 @@ class FlowStore:
)
return flow.model_copy(update={"id": flow_id})
def _issue(self, keys: dict[str, str]) -> str:
"""A fresh flow id. Under `_write_lock`, with *keys* the key index.
Free as a directory, a key and in history: an id a deleted flow's runs
are still filed under would hand them to the new one.
"""
graves = self.tombstones
return new_id(
"f",
lambda c: (
self._flow_dir(c).exists()
or c in keys
or (graves is not None and graves.recorded(c) is not None)
),
)
def read_all(self) -> list[FlowDef]:
"""Every published flow — what the engine runs.
@@ -718,9 +787,14 @@ class FlowStore:
return list(self._read_all)
def write_flow(self, flow: FlowDef) -> bool:
"""Publish a flow directly. Returns False when nothing actually changed."""
"""Publish a flow directly. Returns False when nothing actually changed.
Creates one too, under the id it carries or else its key: what a test
or a seed holding a finished document wants. The API creates only
through `create`.
"""
with self._write_lock:
flow = self._claim(flow)
flow = self._claim(flow, create=True)
path = self._flow_file(flow.id)
content = flow.model_dump_json(indent=2) + "\n"
if path.exists() and path.read_text() == content:
@@ -728,122 +802,188 @@ class FlowStore:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content)
self._commit(f"Update flow '{flow.name}'")
self._commit(f"Update flow {named(flow.name, flow.id)}")
return True
def create(self, flow: FlowDef, pin: str = "") -> FlowDef:
"""A new flow, only a draft until someone publishes it.
Its id is issued here unless *pin* names one, which is a superuser's
to ask for: the Demo, a test, a restore. A pinned id whose history
outlived its flow adopts that history, so it has to take the key the
history is filed under, and its tombstone goes once the document is on
disk.
"""
with self._write_lock:
keys = self.key_index()
if pin:
flow_id = checked(pin)
# The directory rather than `exists`: anything left in it would
# become the new flow's.
if self._flow_dir(flow_id).exists():
raise IdTaken(flow_id)
recorded = (
self.tombstones.recorded(flow_id) if self.tombstones else None
)
if recorded is not None and recorded != flow.name:
raise RecordedKey(flow_id, recorded)
else:
flow_id = self._issue(keys)
flow = self._claim(flow.model_copy(update={"id": flow_id}), create=True)
# Another flow's key as this one's id would make naming either
# ambiguous. Checked after the key, which is the likelier mistake.
if keys.get(flow_id, flow_id) != flow_id:
raise IdTaken(flow_id)
stored = flow.model_copy(update={"version": 1})
path = self._draft_file(flow_id)
path.parent.mkdir(parents=True)
path.write_text(stored.model_dump_json(indent=2) + "\n")
self._commit(f"Create flow {named(flow.name, flow_id)}")
if pin and self.tombstones is not None:
self.tombstones.lift(flow_id)
return stored
def write_draft(self, flow: FlowDef, base_version: int) -> FlowDef:
"""Save unpublished changes, refusing to overwrite someone else's.
``base_version`` is the version the editor last saw. It has to match
what is stored, otherwise another client saved in between and this
write would silently discard their work.
write would silently discard their work. The flow has to exist:
creating one is `create`.
"""
with self._write_lock:
flow = self._claim(flow)
current: FlowDef | None = None
if self.exists(flow.id):
current = self.read_flow(flow.id, draft=True)
if base_version != current.version:
raise StaleVersion(flow.name, current.version)
if _same_content(current, flow):
return current
current = self.read_flow(flow.id, draft=True)
if base_version != current.version:
raise StaleVersion(flow.name, current.version)
if _same_content(current, flow):
return current
stored = flow.model_copy(
update={"version": (current.version if current else 0) + 1}
)
stored = flow.model_copy(update={"version": current.version + 1})
path = self._draft_file(flow.id)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(stored.model_dump_json(indent=2) + "\n")
self._commit(f"Update draft of flow '{flow.name}'")
self._commit(f"Update draft of flow {named(flow.name, flow.id)}")
return stored
def publish_flow(self, name: str, base_version: int) -> FlowDef:
def publish_flow(self, flow_id: str, base_version: int) -> FlowDef:
"""Promote the working copy to what the engine runs."""
with self._write_lock:
current = self.read_flow(name, draft=True)
current = self.read_flow(flow_id, draft=True)
if base_version != current.version:
raise StaleVersion(name, current.version)
raise StaleVersion(current.name, current.version)
draft = self._draft_file(name)
draft = self._draft_file(flow_id)
if draft.exists():
self._flow_file(name).write_text(
self._flow_file(flow_id).write_text(
current.model_dump_json(indent=2) + "\n"
)
draft.unlink()
drafts = self._draft_nodes_dir(name)
drafts = self._draft_nodes_dir(flow_id)
if drafts.exists():
target = self._flow_dir(name) / "nodes"
target = self._flow_dir(flow_id) / "nodes"
target.mkdir(parents=True, exist_ok=True)
for source in sorted(drafts.glob("*.py")):
shutil.copy2(source, target / source.name)
shutil.rmtree(drafts)
self._commit(f"Publish flow '{name}'")
self._commit(f"Publish flow {named(current.name, flow_id)}")
return current
def discard_draft(self, name: str) -> FlowDef:
"""Throw the unpublished changes away and go back to what is running."""
with self._write_lock:
draft = self._draft_file(name)
if draft.exists():
draft.unlink()
drafts = self._draft_nodes_dir(name)
if drafts.exists():
shutil.rmtree(drafts)
self._commit(f"Discard draft of flow '{name}'")
return self.read_flow(name)
def discard_draft(self, flow_id: str) -> FlowDef:
"""Throw the unpublished changes away and go back to what is running.
def delete_flow(self, name: str) -> None:
with self._write_lock:
directory = self._flow_dir(name)
if not directory.exists():
raise FlowNotFound(name)
shutil.rmtree(directory)
self._commit(f"Delete flow '{name}'")
def duplicate_flow(self, name: str) -> FlowDef:
"""Copy a flow's working copy into a new flow that is only a draft.
Like a new flow, the copy runs nothing until someone publishes it. It is
named ``{name}_copy``, then ``_copy_2`` and on. ``origin`` is dropped,
since the copy is a local flow ``fluksio sync`` must not claim, and
``runtime.json`` stays behind with everything else about running.
What is running comes back a version above the draft, so an editor
still holding the draft is refused rather than matching the next save
made from the published copy.
"""
with self._write_lock:
source = self.read_flow(name, draft=True)
# The directory rather than `exists`: anything left in it would
# otherwise end up in the copy. The copy's id is its key, so that
# has to be free among the keys too.
draft = self._draft_file(flow_id)
if draft.exists():
running = self.read_flow(flow_id)
top = _version(draft)
if running.version <= top:
running.version = top + 1
self._flow_file(flow_id).write_text(
running.model_dump_json(indent=2) + "\n"
)
draft.unlink()
drafts = self._draft_nodes_dir(flow_id)
if drafts.exists():
shutil.rmtree(drafts)
published = self.read_flow(flow_id)
self._commit(f"Discard draft of flow {named(published.name, flow_id)}")
return published
def delete_flow(self, flow_id: str) -> str:
"""Remove a flow's files. Returns the key it had.
The key is buried first, so a delete that stops half-way never leaves
history without a name, and the live document wins while it is there.
"""
with self._write_lock:
directory = self._flow_dir(flow_id)
if not directory.exists():
raise FlowNotFound(flow_id)
key = next(
(k for k, i in self.key_index().items() if i == flow_id), flow_id
)
if self.tombstones is not None:
self.tombstones.bury(flow_id, key)
shutil.rmtree(directory)
self._commit(f"Delete flow {named(key, flow_id)}")
return key
def duplicate_flow(
self, flow_id: str, busy: Callable[[str], bool] | None = None
) -> FlowDef:
"""Copy a flow's working copy into a new flow that is only a draft.
Like a new flow, the copy runs nothing until someone publishes it. It
gets an id of its own and the key ``{name}_copy``, then ``_copy_2`` and
on, past any key *busy* says something still holds. ``origin`` is
dropped, since the copy is a local flow ``fluksio sync`` must not
claim, and ``runtime.json`` stays behind with everything else about
running.
"""
with self._write_lock:
source = self.read_flow(flow_id, draft=True)
keys = self.key_index()
new_name = copy_name(
source.name, lambda n: self._flow_dir(n).exists() or n in keys
source.name, lambda n: n in keys or (busy is not None and busy(n))
)
copy = source.model_copy(
update={
"id": new_name,
"id": self._issue(keys),
"name": new_name,
"title": f"{source.title} (copy)" if source.title else "",
"version": 1,
"origin": None,
}
)
copy = self._claim(copy, create=True)
# Its own messages named with its own key are the copy's now.
self._retarget(copy, f"{source.name}.", f"{new_name}.")
path = self._draft_file(new_name)
path = self._draft_file(copy.id)
path.parent.mkdir(parents=True)
path.write_text(copy.model_dump_json(indent=2) + "\n")
# The bodies the source's editor shows: published, edits over them.
# As drafts, so publishing the copy promotes them like a new flow's.
target = self._draft_nodes_dir(new_name)
for folder in (self._flow_dir(name) / "nodes", self._draft_nodes_dir(name)):
target = self._draft_nodes_dir(copy.id)
for folder in (
self._flow_dir(flow_id) / "nodes",
self._draft_nodes_dir(flow_id),
):
for body in sorted(folder.glob("*.py")):
target.mkdir(exist_ok=True)
shutil.copy2(body, target / body.name)
self._commit(f"Duplicate flow '{name}' as '{new_name}'")
self._commit(
f"Duplicate flow {named(source.name, flow_id)} as "
f"{named(new_name, copy.id)}"
)
return copy
@staticmethod
+130
View File
@@ -0,0 +1,130 @@
"""What is left of a deleted flow: its last key, while its history is kept.
A flow's runs, events and metrics are filed under its id and outlive it. A new
flow's id is opaque, so a tombstone keeps the key it last had, and `names_of`
reads that back for anything showing the history. A flow deleted before
tombstones existed has its old key as its id, which already reads right.
The flow store is the one writer: it buries a flow before removing its files,
asks before issuing an id, and lifts the tombstone of a flow restored under a
pinned id once the restored document is on disk.
"""
from __future__ import annotations
from collections.abc import Iterable
from typing import TYPE_CHECKING
from sqlalchemy import delete, or_
from sqlmodel import Session, col, select
from fluksio.core.db import engine
from fluksio.models import EngineEvent, FlowRun, FlowTombstone, MetricBucket, Run
if TYPE_CHECKING:
from fluksio.flow.store import FlowStore
def referenced(session: Session, flow_id: str) -> bool:
"""Whether any history row names *flow_id*, as its flow or a node's.
A training run's ``for_node`` names the node it trains, in a flow other
than the run's own, so the four ``flow`` columns are not enough.
"""
nodes = f"{flow_id}."
queries = [
select(Run.id).where(
or_(
col(Run.flow) == flow_id,
col(Run.for_node).startswith(nodes, autoescape=True),
)
),
select(FlowRun.id).where(col(FlowRun.flow) == flow_id),
select(MetricBucket.flow).where(
or_(
col(MetricBucket.flow) == flow_id,
col(MetricBucket.node).startswith(nodes, autoescape=True),
)
),
select(EngineEvent.id).where(
or_(
col(EngineEvent.flow) == flow_id,
col(EngineEvent.node).startswith(nodes, autoescape=True),
)
),
]
return any(session.exec(query.limit(1)).first() is not None for query in queries)
class Tombstones:
"""Tombstones and the history behind an id, as the flow store asks."""
def bury(self, flow_id: str, name: str) -> None:
"""Keep *name* for *flow_id*, before its files go.
Written again by a retried delete, which may find another name if the
first attempt never got as far as removing anything.
"""
with Session(engine) as session:
stone = session.get(FlowTombstone, flow_id) or FlowTombstone(
id=flow_id, name=name
)
stone.name = name
session.add(stone)
session.commit()
def lift(self, flow_id: str) -> None:
"""Forget the tombstone of a flow that is live again."""
with Session(engine) as session:
session.execute(
delete(FlowTombstone).where(col(FlowTombstone.id) == flow_id)
)
session.commit()
def recorded(self, flow_id: str) -> str | None:
"""The key the history of a flow that is not live is filed under.
Its tombstone's, or the id itself for history from before tombstones,
whose id was its key. None when nothing names the id at all.
"""
with Session(engine) as session:
stone = session.get(FlowTombstone, flow_id)
if stone is not None:
return stone.name
return flow_id if referenced(session, flow_id) else None
def names_of(ids: Iterable[str], store: FlowStore) -> dict[str, str]:
"""Each flow id's key: the live flow's, else its tombstone's, else the id.
One read of the key index and one query, however many ids a response
carries.
"""
wanted = set(ids)
live = {flow_id: key for key, flow_id in store.key_index().items()}
names = {flow_id: live[flow_id] for flow_id in wanted if flow_id in live}
gone = wanted - names.keys()
if gone:
with Session(engine) as session:
for stone in session.exec(
select(FlowTombstone).where(col(FlowTombstone.id).in_(gone))
):
names[stone.id] = stone.name
return {flow_id: names.get(flow_id, flow_id) for flow_id in wanted}
def name_of(flow_id: str, store: FlowStore) -> str:
"""One flow id's key, as `names_of`."""
return names_of([flow_id], store)[flow_id]
def prune(session: Session) -> int:
"""Drop the tombstones no history row names any more. Returns how many."""
stale = [
stone.id
for stone in session.exec(select(FlowTombstone))
if not referenced(session, stone.id)
]
if stale:
session.execute(delete(FlowTombstone).where(col(FlowTombstone.id).in_(stale)))
return len(stale)