Flows and dashboards are created by POST under an issued or superuser-pinned id; PUT only updates; deleted flows leave a tombstone with their key; the controller stores by id and runs by key, hooks live under /hooks/<flow id>; a key with state or queued work behind it is refused until cleared

This commit is contained in:
2026-11-02 17:41:23 +01:00
parent fae85f5ecb
commit 08a6376836
63 changed files with 2832 additions and 936 deletions
+208 -68
View File
@@ -14,8 +14,9 @@ unpublished — which is what every flow written before this existed looks like.
A flow is two names. Its id is which flow it is: the directory, the argument
every method here takes, and never changed once given. Its key, ``name``, is
the namespace of its messages and unique among flows. A flow from before ids
existed has its key as its id (`fluksio.flow.identity`), and so does every new
one for now. Published and draft documents always share both.
existed has its key as its id (`fluksio.flow.identity`). A new one is given an
opaque id by `create`, or the one a superuser pinned. Published and draft
documents always share both.
"""
from __future__ import annotations
@@ -31,10 +32,13 @@ import threading
from collections.abc import Callable, Iterable
from datetime import UTC, datetime
from pathlib import Path
from typing import Literal
from typing import TYPE_CHECKING, Literal
from fluksio.flow.schemas import FlowDef, _validate_name
if TYPE_CHECKING:
from fluksio.flow.tombstones import Tombstones
logger = logging.getLogger(__name__)
#: Where the sources of nodes shared between flows live.
@@ -97,6 +101,37 @@ class IdentityChange(ValueError):
"""
class RecordedKey(IdentityChange):
"""A pinned id restored under another key than its history is filed under.
The history names the old key's messages, so taking a new key with it is
a rename, not a create.
"""
def __init__(self, obj_id: str, key: str) -> None:
super().__init__(obj_id)
self.id = obj_id
self.key = key
def __str__(self) -> str:
return (
f"'{self.id}' has history filed under the key '{self.key}', so it "
"can only be restored with that key"
)
class IdTaken(ValueError):
"""A pinned id that is in use: an object's, a directory's or another key."""
def __init__(self, obj_id: str, kind: str = "flow") -> None:
super().__init__(obj_id)
self.id = obj_id
self.kind = kind
def __str__(self) -> str:
return f"The id '{self.id}' is already taken by a {self.kind}"
def claim(
obj_id: str,
key: str,
@@ -146,6 +181,11 @@ def read_keys(directories: Iterable[Path], document: str) -> dict[str, str]:
return keys
def named(key: str, obj_id: str) -> str:
"""How a commit message names an object: by key, and by id when it differs."""
return f"'{key}'" if key == obj_id else f"'{key}' ({obj_id})"
def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str:
"""A fresh id for a flow (``f``) or a dashboard (``d``), e.g. ``f_k3q7m2xa``.
@@ -222,6 +262,14 @@ def _updated_at(paths: Iterable[Path]) -> datetime | None:
return datetime.fromtimestamp(max(stamps), UTC) if stamps else None
def _version(path: Path) -> int:
"""The version a stored document carries, 0 for none or an unreadable one."""
try:
return int(json.loads(path.read_text()).get("version", 1))
except (OSError, ValueError, TypeError, AttributeError):
return 0
def _same_content(left: FlowDef, right: FlowDef) -> bool:
"""Equal but for the version counter, which the server owns."""
return left.model_copy(update={"version": 0}) == right.model_copy(
@@ -252,8 +300,11 @@ class FlowStore:
#: a store reached 1898 commits, 5863 loose objects and no pack at all.
_GC_EVERY = 500
def __init__(self, root: Path) -> None:
def __init__(self, root: Path, tombstones: Tombstones | None = None) -> None:
self.root = root
#: Where a deleted flow's key is kept, and what history an id has.
#: None leaves both out: a store without a database, as in tests.
self.tombstones = tombstones
self.root.mkdir(parents=True, exist_ok=True)
# Draft writes are check-and-set, so two clients saving at once must not
# interleave between reading the current version and writing the next.
@@ -679,14 +730,16 @@ class FlowStore:
self._keys = (at, keys)
return keys
def _claim(self, flow: FlowDef) -> FlowDef:
def _claim(self, flow: FlowDef, create: bool = False) -> FlowDef:
"""*flow* with its id, once it is certain to be a save the store allows.
Under `_write_lock`. An id left out is the key, which is how every
flow is created for now.
Under `_write_lock`. A save is to a flow that exists, named by its id
or, from before ids, by its key. Only *create* may make a new one.
"""
flow_id = checked(flow.id or flow.name)
current = self.read_flow(flow_id, draft=True) if self.exists(flow_id) else None
if current is None and not create:
raise FlowNotFound(flow_id)
claim(
flow_id,
flow.name,
@@ -696,6 +749,22 @@ class FlowStore:
)
return flow.model_copy(update={"id": flow_id})
def _issue(self, keys: dict[str, str]) -> str:
"""A fresh flow id. Under `_write_lock`, with *keys* the key index.
Free as a directory, a key and in history: an id a deleted flow's runs
are still filed under would hand them to the new one.
"""
graves = self.tombstones
return new_id(
"f",
lambda c: (
self._flow_dir(c).exists()
or c in keys
or (graves is not None and graves.recorded(c) is not None)
),
)
def read_all(self) -> list[FlowDef]:
"""Every published flow — what the engine runs.
@@ -718,9 +787,14 @@ class FlowStore:
return list(self._read_all)
def write_flow(self, flow: FlowDef) -> bool:
"""Publish a flow directly. Returns False when nothing actually changed."""
"""Publish a flow directly. Returns False when nothing actually changed.
Creates one too, under the id it carries or else its key: what a test
or a seed holding a finished document wants. The API creates only
through `create`.
"""
with self._write_lock:
flow = self._claim(flow)
flow = self._claim(flow, create=True)
path = self._flow_file(flow.id)
content = flow.model_dump_json(indent=2) + "\n"
if path.exists() and path.read_text() == content:
@@ -728,122 +802,188 @@ class FlowStore:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content)
self._commit(f"Update flow '{flow.name}'")
self._commit(f"Update flow {named(flow.name, flow.id)}")
return True
def create(self, flow: FlowDef, pin: str = "") -> FlowDef:
"""A new flow, only a draft until someone publishes it.
Its id is issued here unless *pin* names one, which is a superuser's
to ask for: the Demo, a test, a restore. A pinned id whose history
outlived its flow adopts that history, so it has to take the key the
history is filed under, and its tombstone goes once the document is on
disk.
"""
with self._write_lock:
keys = self.key_index()
if pin:
flow_id = checked(pin)
# The directory rather than `exists`: anything left in it would
# become the new flow's.
if self._flow_dir(flow_id).exists():
raise IdTaken(flow_id)
recorded = (
self.tombstones.recorded(flow_id) if self.tombstones else None
)
if recorded is not None and recorded != flow.name:
raise RecordedKey(flow_id, recorded)
else:
flow_id = self._issue(keys)
flow = self._claim(flow.model_copy(update={"id": flow_id}), create=True)
# Another flow's key as this one's id would make naming either
# ambiguous. Checked after the key, which is the likelier mistake.
if keys.get(flow_id, flow_id) != flow_id:
raise IdTaken(flow_id)
stored = flow.model_copy(update={"version": 1})
path = self._draft_file(flow_id)
path.parent.mkdir(parents=True)
path.write_text(stored.model_dump_json(indent=2) + "\n")
self._commit(f"Create flow {named(flow.name, flow_id)}")
if pin and self.tombstones is not None:
self.tombstones.lift(flow_id)
return stored
def write_draft(self, flow: FlowDef, base_version: int) -> FlowDef:
"""Save unpublished changes, refusing to overwrite someone else's.
``base_version`` is the version the editor last saw. It has to match
what is stored, otherwise another client saved in between and this
write would silently discard their work.
write would silently discard their work. The flow has to exist:
creating one is `create`.
"""
with self._write_lock:
flow = self._claim(flow)
current: FlowDef | None = None
if self.exists(flow.id):
current = self.read_flow(flow.id, draft=True)
if base_version != current.version:
raise StaleVersion(flow.name, current.version)
if _same_content(current, flow):
return current
current = self.read_flow(flow.id, draft=True)
if base_version != current.version:
raise StaleVersion(flow.name, current.version)
if _same_content(current, flow):
return current
stored = flow.model_copy(
update={"version": (current.version if current else 0) + 1}
)
stored = flow.model_copy(update={"version": current.version + 1})
path = self._draft_file(flow.id)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(stored.model_dump_json(indent=2) + "\n")
self._commit(f"Update draft of flow '{flow.name}'")
self._commit(f"Update draft of flow {named(flow.name, flow.id)}")
return stored
def publish_flow(self, name: str, base_version: int) -> FlowDef:
def publish_flow(self, flow_id: str, base_version: int) -> FlowDef:
"""Promote the working copy to what the engine runs."""
with self._write_lock:
current = self.read_flow(name, draft=True)
current = self.read_flow(flow_id, draft=True)
if base_version != current.version:
raise StaleVersion(name, current.version)
raise StaleVersion(current.name, current.version)
draft = self._draft_file(name)
draft = self._draft_file(flow_id)
if draft.exists():
self._flow_file(name).write_text(
self._flow_file(flow_id).write_text(
current.model_dump_json(indent=2) + "\n"
)
draft.unlink()
drafts = self._draft_nodes_dir(name)
drafts = self._draft_nodes_dir(flow_id)
if drafts.exists():
target = self._flow_dir(name) / "nodes"
target = self._flow_dir(flow_id) / "nodes"
target.mkdir(parents=True, exist_ok=True)
for source in sorted(drafts.glob("*.py")):
shutil.copy2(source, target / source.name)
shutil.rmtree(drafts)
self._commit(f"Publish flow '{name}'")
self._commit(f"Publish flow {named(current.name, flow_id)}")
return current
def discard_draft(self, name: str) -> FlowDef:
"""Throw the unpublished changes away and go back to what is running."""
with self._write_lock:
draft = self._draft_file(name)
if draft.exists():
draft.unlink()
drafts = self._draft_nodes_dir(name)
if drafts.exists():
shutil.rmtree(drafts)
self._commit(f"Discard draft of flow '{name}'")
return self.read_flow(name)
def discard_draft(self, flow_id: str) -> FlowDef:
"""Throw the unpublished changes away and go back to what is running.
def delete_flow(self, name: str) -> None:
with self._write_lock:
directory = self._flow_dir(name)
if not directory.exists():
raise FlowNotFound(name)
shutil.rmtree(directory)
self._commit(f"Delete flow '{name}'")
def duplicate_flow(self, name: str) -> FlowDef:
"""Copy a flow's working copy into a new flow that is only a draft.
Like a new flow, the copy runs nothing until someone publishes it. It is
named ``{name}_copy``, then ``_copy_2`` and on. ``origin`` is dropped,
since the copy is a local flow ``fluksio sync`` must not claim, and
``runtime.json`` stays behind with everything else about running.
What is running comes back a version above the draft, so an editor
still holding the draft is refused rather than matching the next save
made from the published copy.
"""
with self._write_lock:
source = self.read_flow(name, draft=True)
# The directory rather than `exists`: anything left in it would
# otherwise end up in the copy. The copy's id is its key, so that
# has to be free among the keys too.
draft = self._draft_file(flow_id)
if draft.exists():
running = self.read_flow(flow_id)
top = _version(draft)
if running.version <= top:
running.version = top + 1
self._flow_file(flow_id).write_text(
running.model_dump_json(indent=2) + "\n"
)
draft.unlink()
drafts = self._draft_nodes_dir(flow_id)
if drafts.exists():
shutil.rmtree(drafts)
published = self.read_flow(flow_id)
self._commit(f"Discard draft of flow {named(published.name, flow_id)}")
return published
def delete_flow(self, flow_id: str) -> str:
"""Remove a flow's files. Returns the key it had.
The key is buried first, so a delete that stops half-way never leaves
history without a name, and the live document wins while it is there.
"""
with self._write_lock:
directory = self._flow_dir(flow_id)
if not directory.exists():
raise FlowNotFound(flow_id)
key = next(
(k for k, i in self.key_index().items() if i == flow_id), flow_id
)
if self.tombstones is not None:
self.tombstones.bury(flow_id, key)
shutil.rmtree(directory)
self._commit(f"Delete flow {named(key, flow_id)}")
return key
def duplicate_flow(
self, flow_id: str, busy: Callable[[str], bool] | None = None
) -> FlowDef:
"""Copy a flow's working copy into a new flow that is only a draft.
Like a new flow, the copy runs nothing until someone publishes it. It
gets an id of its own and the key ``{name}_copy``, then ``_copy_2`` and
on, past any key *busy* says something still holds. ``origin`` is
dropped, since the copy is a local flow ``fluksio sync`` must not
claim, and ``runtime.json`` stays behind with everything else about
running.
"""
with self._write_lock:
source = self.read_flow(flow_id, draft=True)
keys = self.key_index()
new_name = copy_name(
source.name, lambda n: self._flow_dir(n).exists() or n in keys
source.name, lambda n: n in keys or (busy is not None and busy(n))
)
copy = source.model_copy(
update={
"id": new_name,
"id": self._issue(keys),
"name": new_name,
"title": f"{source.title} (copy)" if source.title else "",
"version": 1,
"origin": None,
}
)
copy = self._claim(copy, create=True)
# Its own messages named with its own key are the copy's now.
self._retarget(copy, f"{source.name}.", f"{new_name}.")
path = self._draft_file(new_name)
path = self._draft_file(copy.id)
path.parent.mkdir(parents=True)
path.write_text(copy.model_dump_json(indent=2) + "\n")
# The bodies the source's editor shows: published, edits over them.
# As drafts, so publishing the copy promotes them like a new flow's.
target = self._draft_nodes_dir(new_name)
for folder in (self._flow_dir(name) / "nodes", self._draft_nodes_dir(name)):
target = self._draft_nodes_dir(copy.id)
for folder in (
self._flow_dir(flow_id) / "nodes",
self._draft_nodes_dir(flow_id),
):
for body in sorted(folder.glob("*.py")):
target.mkdir(exist_ok=True)
shutil.copy2(body, target / body.name)
self._commit(f"Duplicate flow '{name}' as '{new_name}'")
self._commit(
f"Duplicate flow {named(source.name, flow_id)} as "
f"{named(new_name, copy.id)}"
)
return copy
@staticmethod