"""Flows on disk, versioned with git. Every flow is a directory: ``flow.json`` for the structure, ``nodes/*.py`` for node logic. The whole tree is a git repository and each saved change is a commit, so a flow's history is readable with ordinary git tooling and two flows never collide in one file. Editing is separated from running. The editor writes ``flow.draft.json`` and ``nodes.draft/*.py``; the engine reads only the published ``flow.json`` and ``nodes/*.py``. Publishing promotes the draft over the published files and removes it, so a flow directory without draft files is simply one with nothing unpublished — which is what every flow written before this existed looks like. A flow is two names. Its id is which flow it is: the directory, the argument every method here takes, and never changed once given. Its key, ``name``, is the namespace of its messages and unique among flows. A flow from before ids existed has its key as its id (`fluksio.flow.identity`). A new one is given an opaque id by `create`, or the one a superuser pinned. Published and draft documents always share both. """ from __future__ import annotations import json import logging import re import secrets import shutil import string import subprocess import threading from collections.abc import Callable, Iterable, Iterator from datetime import UTC, datetime from pathlib import Path from typing import TYPE_CHECKING, Literal from fluksio.flow.schemas import FlowDef, NodeDef, _validate_name if TYPE_CHECKING: from fluksio.flow.tombstones import Tombstones logger = logging.getLogger(__name__) #: Where the sources of nodes shared between flows live. LIB_DIR = "_lib" DEFAULT_SOURCE = '''"""A new node. Return a dict keyed by your output ports.""" def process(): return {} ''' class InvalidName(ValueError): """A name that cannot be a file or directory of the store.""" def __init__(self, name: str) -> None: super().__init__(name) self.name = name def __str__(self) -> str: return ( f"'{self.name}' is not a valid name: use lowercase letters, digits " "and underscores, starting with a letter" ) def checked(name: str) -> str: """*name*, once it is certain to be one path segment inside the store. Every path the stores build starts from a name the API was handed, percent-decoded on the way in, so without this `..` or `_lib` reached the filesystem as written: `delete_flow("..")` removed the data directory. """ try: return _validate_name(name) except ValueError: raise InvalidName(name) from None #: The shape of an id the server issues: a kind, then 40 random bits. ISSUED_ID = re.compile(r"[fd]_[a-z2-7]{8}") class ReservedKey(InvalidName): """A new key shaped like an issued id, which it could one day equal.""" def __str__(self) -> str: return ( f"'{self.name}' has the shape of a generated id, f_ or d_ and eight " "letters or digits, which a new key may not have" ) class IdentityChange(ValueError): """A save that would change which object a document is, or its key. Ids never change, and a key changes only by renaming, which moves its messages along with it. A save that did either would bypass that. """ class RecordedKey(IdentityChange): """A pinned id restored under another key than its history is filed under. The history names the old key's messages, so taking a new key with it is a rename, not a create. """ def __init__(self, obj_id: str, key: str) -> None: super().__init__(obj_id) self.id = obj_id self.key = key def __str__(self) -> str: return ( f"'{self.id}' has history filed under the key '{self.key}', so it " "can only be restored with that key" ) class IdTaken(ValueError): """A pinned id that is in use: an object's, a directory's or another key.""" def __init__(self, obj_id: str, kind: str = "flow") -> None: super().__init__(obj_id) self.id = obj_id self.kind = kind def __str__(self) -> str: return f"The id '{self.id}' is already taken by a {self.kind}" def claim( obj_id: str, key: str, current: tuple[str, str] | None, keys: dict[str, str], taken: type[ValueError], ) -> None: """Refuse a write that changes who an object is, or takes another's key. *current* is the stored ``(id, name)`` at *obj_id*, None for a new object, and *keys* the store's key index. Called under the store's write lock, so two creates of one key cannot both pass. *taken* is what a key already in use raises: `FlowExists`, or the dashboards' own. """ if current is not None: stored_id, stored_key = current if stored_id != obj_id: raise IdentityChange(f"'{obj_id}' holds a document of '{stored_id}'") if stored_key != key: raise IdentityChange( f"'{obj_id}' has the key '{stored_key}', which a save cannot " f"change to '{key}'" ) return # New keys only: one this shape that exists already stays saveable. if ISSUED_ID.fullmatch(key): raise ReservedKey(key) if keys.get(key, obj_id) != obj_id: raise taken(key) def read_keys(directories: Iterable[Path], document: str) -> dict[str, str]: """The key each directory's object holds, to the directory: key to id. Read from the working copy, ``.draft.json`` when there is one, so an object that is only a draft holds its key as much as a published one. """ keys: dict[str, str] = {} for directory in directories: draft = directory / f"{document}.draft.json" path = draft if draft.exists() else directory / f"{document}.json" try: keys.setdefault(json.loads(path.read_text())["name"], directory.name) except (OSError, ValueError, KeyError, TypeError): # Unreadable to the engine too, which reports it when it loads. continue return keys def named(key: str, obj_id: str) -> str: """How a commit message names an object: by key, and by id when it differs.""" return f"'{key}'" if key == obj_id else f"'{key}' ({obj_id})" def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str: """A fresh id for a flow (``f``) or a dashboard (``d``), e.g. ``f_k3q7m2xa``. Random is not unique, so *taken* decides, under the store's write lock: it has to know every live id, every directory left behind and every key, since a key equal to an id makes an argument naming either ambiguous. """ alphabet = string.ascii_lowercase + "234567" while True: candidate = f"{kind}_" + "".join(secrets.choice(alphabet) for _ in range(8)) if not taken(candidate): return candidate class FlowNotFound(KeyError): def __init__(self, name: str) -> None: super().__init__(name) self.name = name def __str__(self) -> str: return f"No flow named '{self.name}'" class FlowExists(ValueError): def __init__(self, name: str) -> None: super().__init__(name) self.name = name def __str__(self) -> str: return f"There is already a flow named '{self.name}'" class LibNotFound(KeyError): def __init__(self, name: str) -> None: super().__init__(name) self.name = name def __str__(self) -> str: return f"No shared node named '{self.name}'" class LibExists(ValueError): def __init__(self, name: str) -> None: super().__init__(name) self.name = name def __str__(self) -> str: return f"There is already a shared node named '{self.name}'" class StaleVersion(ValueError): """Someone else saved this flow since the version being written was read.""" def __init__(self, name: str, current: int) -> None: super().__init__(name) self.name = name self.current = current def __str__(self) -> str: return f"Flow '{self.name}' has changed since you loaded it" def _updated_at(paths: Iterable[Path]) -> datetime | None: """When the newest of these files was written, or ``None`` if none exist. A file's mtime, not the commit that recorded it: every write here commits immediately, so the two are the same instant, and `git log -1 -- ` would walk the history back to the last commit touching that path — 130ms for a document nobody has edited in a week, against a `stat` for all of them. Slowest for exactly the stalest documents is the wrong shape for a list endpoint. """ stamps = [path.stat().st_mtime for path in paths if path.exists()] return datetime.fromtimestamp(max(stamps), UTC) if stamps else None def _version(path: Path) -> int: """The version a stored document carries, 0 for none or an unreadable one.""" try: return int(json.loads(path.read_text()).get("version", 1)) except (OSError, ValueError, TypeError, AttributeError): return 0 def _same_content(left: FlowDef, right: FlowDef) -> bool: """Equal but for the version counter, which the server owns.""" return left.model_copy(update={"version": 0}) == right.model_copy( update={"version": 0} ) def copy_name(name: str, taken: Callable[[str], bool]) -> str: """``{name}_copy``, then ``_copy_2`` and on, from a copy's own original.""" base = re.sub(r"_copy(_\d+)?$", "", name) new_name, count = f"{base}_copy", 1 while taken(new_name): count += 1 new_name = f"{base}_copy_{count}" return new_name class FlowStore: """Reads and writes flows, committing every change.""" #: One warning per process when there is no git to commit with. _warned_no_git = False #: Commits between housekeeping runs. Auto-gc is off (see `_git`), so #: nothing packs unless it is asked for; a commit counter is the trigger #: because the process that needs it never restarts — a seeding session #: commits thousands of times inside one long-lived container, which is how #: a store reached 1898 commits, 5863 loose objects and no pack at all. _GC_EVERY = 500 def __init__(self, root: Path, tombstones: Tombstones | None = None) -> None: self.root = root #: Where a deleted flow's key is kept, and what history an id has. #: None leaves both out: a store without a database, as in tests. self.tombstones = tombstones self.root.mkdir(parents=True, exist_ok=True) # Draft writes are check-and-set, so two clients saving at once must not # interleave between reading the current version and writing the next. self._write_lock = threading.Lock() #: Bumped on every commit, so something derived from every flow at once #: can tell whether it is still current without reading them all again. #: In memory rather than `head()`, which is a git subprocess per call — #: it therefore counts from zero per process and misses an edit made on #: disk behind the API, which no writer here does. self.revision = 0 #: `read_all` memoised against that revision — see the method. self._read_all: list[FlowDef] = [] self._read_all_at = -1 self._read_all_lock = threading.Lock() #: `key_index` and the revision it was read at. self._keys: tuple[int, dict[str, str]] = (-1, {}) if not (self.root / ".git").exists(): self._git("init", "-q") self._commit("Initialise flow store", allow_empty=True) self._migrate() def _migrate(self) -> None: """Rewrite node types and settings stored under a name they no longer have. On 2026-10-15 the node type ``trigger`` became ``debounce`` and the mqtt setting ``publish_queue_size`` became ``send_queue_size``. On 2026-10-16 a node's ``device`` and ``device_policy`` became ``worker`` and ``worker_policy``, so that a device is only ever a paired one. No alias is left for the old names. Each published and draft document is read as plain JSON and only the matching node changes, so a field this version does not model survives, and the old document stays in git. The version goes up by one, so an editor still holding the old document gets a conflict on its next save instead of writing the old name back. A store with nothing to rename is left alone. """ changed = [] paths = [*self.root.glob("*/flow.json"), *self.root.glob("*/flow.draft.json")] for path in sorted(paths): try: data = json.loads(path.read_text()) nodes = [n for n in data["nodes"] if n.get("type") == "trigger"] queues = [ n["params"] for n in data["nodes"] if n.get("type") == "mqtt" and "publish_queue_size" in (n.get("params") or {}) ] placed = [ n for n in data["nodes"] if "device" in n or "device_policy" in n ] except Exception: # Unreadable to the engine too, which reports it when it loads. continue if not nodes and not queues and not placed: continue for node in nodes: node["type"] = "debounce" for params in queues: params["send_queue_size"] = params.pop("publish_queue_size") for node in placed: for old in ("device", "device_policy"): if old in node: node[old.replace("device", "worker")] = node.pop(old) data["version"] = data.get("version", 1) + 1 path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n") changed.append(f"{path.parent.name}/{path.name}") if changed: logger.warning( "Renamed node type 'trigger' to 'debounce', mqtt " "'publish_queue_size' to 'send_queue_size' and a node's " "'device' to 'worker' in %s", ", ".join(changed), ) self._commit( "Rename 'trigger' nodes, mqtt 'publish_queue_size' and a " "node's 'device'" ) # ------------------------------------------------------------------------- # git # ------------------------------------------------------------------------- def _git(self, *args: str) -> subprocess.CompletedProcess[str]: try: return subprocess.run( # `gc.auto=0` on every invocation, because `git commit` otherwise # forks a background `gc --auto` and returns without waiting for # it. `subprocess.run` reaps the commit, not its grandchild, so # the daemon is reparented onto PID 1 — the FastAPI process, # which reaps nothing, and the zombies accumulate one per save. # Packing is asked for explicitly in `_commit` instead. ["git", "-C", str(self.root), "-c", "gc.auto=0", *args], capture_output=True, text=True, check=False, ) except FileNotFoundError: # No git on this machine — a `pip install` on a locked-down host is # where this happens. The flows are files either way, which is what # the store is; what is lost is their history, so say it once and # carry on rather than refusing to start. if not FlowStore._warned_no_git: FlowStore._warned_no_git = True logger.warning( "git is not installed, so flow changes are not versioned. " "Install it to get a commit per save." ) return subprocess.CompletedProcess( args=list(args), returncode=1, stdout="", stderr="git is not installed" ) def _commit(self, message: str, allow_empty: bool = False) -> None: # Before the commit rather than after it: what a reader has to notice # is that the files changed, which is already true whether or not git # is there to record it. self.revision += 1 self._git("add", "-A") result = self._git( "-c", "user.name=fluksio", "-c", "user.email=fluksio@localhost", "commit", "-q", *(["--allow-empty"] if allow_empty else []), "-m", message, ) if result.returncode == 0 or "nothing to commit" in result.stdout: if self.revision % self._GC_EVERY == 0: # In the foreground, so this child is waited on rather than # orphaned. Plain `gc` rather than `gc --auto`, which the flag # above has just disabled. self._git("gc", "--quiet") return if result.stderr == "git is not installed": # Already said once, in `_git`. Repeating it per save is noise. return logger.warning("Could not commit flow change: %s", result.stdout.strip()) # ------------------------------------------------------------------------- # Paths # ------------------------------------------------------------------------- # Every path below starts in one of these three, and each name in it has # been through `checked`. def _flow_dir(self, name: str) -> Path: return self.root / checked(name) def _flow_file(self, name: str) -> Path: return self._flow_dir(name) / "flow.json" def _draft_file(self, name: str) -> Path: return self._flow_dir(name) / "flow.draft.json" def _node_file(self, flow: str, node_id: str) -> Path: return self._flow_dir(flow) / "nodes" / f"{checked(node_id)}.py" def _draft_nodes_dir(self, flow: str) -> Path: return self._flow_dir(flow) / "nodes.draft" def _draft_node_file(self, flow: str, node_id: str) -> Path: return self._draft_nodes_dir(flow) / f"{checked(node_id)}.py" def _runtime_file(self, name: str) -> Path: return self._flow_dir(name) / "runtime.json" def _lib_file(self, name: str) -> Path: return self.root / LIB_DIR / f"{checked(name)}.py" def head(self) -> str: """The commit the store is at, so a result can name the code it ran.""" result = self._git("rev-parse", "HEAD") return result.stdout.strip() if result.returncode == 0 else "" # ------------------------------------------------------------------------- # Module requirements # # What node code may import, versioned with the flows importing it. A file # in the root cannot collide with a flow, which is always a directory. # ------------------------------------------------------------------------- def read_requirements(self) -> str: path = self.root / "requirements.txt" return path.read_text() if path.exists() else "" def write_requirements(self, text: str) -> None: (self.root / "requirements.txt").write_text(text) self._commit("Update module requirements") # ------------------------------------------------------------------------- # Shared node sources # # A node marked reusable moves its code to ``_lib/``; the flows using it # keep their own ports and parameters and point at it by name, so one fix # reaches all of them. ``_lib`` cannot collide with a flow — a flow name has # to start with a letter — and holds no ``flow.json``, so it is invisible to # everything that lists flows. # ------------------------------------------------------------------------- def list_lib(self) -> list[str]: return sorted(path.stem for path in (self.root / LIB_DIR).glob("*.py")) def lib_exists(self, name: str) -> bool: return self._lib_file(name).exists() def read_lib_source(self, name: str) -> str: path = self._lib_file(name) if not path.exists(): raise LibNotFound(name) return path.read_text() def write_lib_source(self, name: str, code: str) -> bool: path = self._lib_file(name) if path.exists() and path.read_text() == code: return False path.parent.mkdir(parents=True, exist_ok=True) path.write_text(code) self._commit(f"Update shared node '{name}'") return True def delete_lib_source(self, name: str) -> None: path = self._lib_file(name) if not path.exists(): raise LibNotFound(name) path.unlink() self._commit(f"Delete shared node '{name}'") def share_node(self, flow: str, node_id: str, lib_name: str) -> None: """Move a node's source into the library and point the node at it.""" with self._write_lock: if self.lib_exists(lib_name): raise LibExists(lib_name) code = self.read_node_source(flow, node_id, draft=True) self._lib_file(lib_name).parent.mkdir(parents=True, exist_ok=True) self._lib_file(lib_name).write_text(code) for path in ( self._node_file(flow, node_id), self._draft_node_file(flow, node_id), ): if path.exists(): path.unlink() self._set_source_ref(flow, node_id, lib_name) self._commit(f"Share node '{flow}.{node_id}' as '{lib_name}'") def unshare_node(self, flow: str, node_id: str) -> None: """Take a private copy of the shared source back into the flow.""" with self._write_lock: definition = self.read_flow(flow, draft=True) node = next((n for n in definition.nodes if n.id == node_id), None) if node is None or not node.source_ref: return code = self.read_lib_source(node.source_ref) # Written as a draft: the running flow keeps the shared code until # this is published, like every other edit. self._draft_node_file(flow, node_id).parent.mkdir( parents=True, exist_ok=True ) self._draft_node_file(flow, node_id).write_text(code) self._set_source_ref(flow, node_id, None) self._commit(f"Unshare node '{flow}.{node_id}'") def detach_lib(self, name: str) -> None: """Give every node running this shared source a copy, then delete it. Published flows and drafts alike, so a running flow keeps the same code and nothing waits on a publish. Each document changed goes up a version, so an editor holding the old one gets a conflict on its next save instead of pointing a node back at a source that is gone. """ with self._write_lock: code = self.read_lib_source(name) for flow in self.list_flows(): for path, node_file in ( (self._flow_file(flow), self._node_file), (self._draft_file(flow), self._draft_node_file), ): if not path.exists(): continue definition = self._load(path) users = [n for n in definition.nodes if n.source_ref == name] if not users: continue for node in users: target = node_file(flow, node.id) target.parent.mkdir(parents=True, exist_ok=True) target.write_text(code) node.source_ref = None definition.version += 1 path.write_text(definition.model_dump_json(indent=2) + "\n") self._lib_file(name).unlink() self._commit(f"Detach and delete shared node '{name}'") def usages(self, lib_name: str, draft: bool = True) -> list[str]: """Every ``.`` pointing at this shared source. Read from the drafts the editor shows, or with ``draft`` off from the published flows the engine runs. """ return sorted( f"{flow.name}.{node.id}" for flow, node in self._lib_nodes(lib_name, draft) ) def lib_users(self, lib_name: str) -> list[str]: """The ids of the published flows running this shared source.""" return sorted({flow.id for flow, _ in self._lib_nodes(lib_name, False)}) def _lib_nodes( self, lib_name: str, draft: bool ) -> Iterator[tuple[FlowDef, NodeDef]]: for flow_id in self.list_flows(): try: definition = self.read_flow(flow_id, draft=draft) except FlowNotFound: continue for node in definition.nodes: if node.source_ref == lib_name: yield definition, node def _set_source_ref(self, flow: str, node_id: str, ref: str | None) -> None: """Point a node at a shared source, in the draft the editor is on.""" definition = self.read_flow(flow, draft=True) definition.nodes = [ node.model_copy(update={"source_ref": ref}) if node.id == node_id else node for node in definition.nodes ] stored = definition.model_copy(update={"version": definition.version + 1}) path = self._draft_file(flow) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(stored.model_dump_json(indent=2) + "\n") # ------------------------------------------------------------------------- # Runtime state # ------------------------------------------------------------------------- def read_enabled(self, name: str) -> bool: """Whether the engine should run this flow. Missing means yes.""" path = self._runtime_file(name) if not path.exists(): return True try: return bool(json.loads(path.read_text()).get("enabled", True)) except (ValueError, OSError): logger.warning("Unreadable runtime state for flow '%s'", name) return True def write_enabled(self, name: str, enabled: bool) -> None: """Stop or start a flow, in a file the editor never writes. This is deliberately not a field on the flow document: that one is autosaved from the canvas, so a stopped flow would start itself again on the next edit. """ path = self._runtime_file(name) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps({"enabled": enabled}, indent=2) + "\n") self._commit(f"{'Start' if enabled else 'Stop'} flow '{name}'") # ------------------------------------------------------------------------- # Flows # ------------------------------------------------------------------------- def list_flows(self) -> list[str]: """Every flow the editor knows, published or not.""" names = {path.parent.name for path in self.root.glob("*/flow.json")} names |= {path.parent.name for path in self.root.glob("*/flow.draft.json")} # `_devices/flow.json` is a device named "flow", not a flow. return sorted(name for name in names if not name.startswith("_")) def exists(self, name: str) -> bool: return self._flow_file(name).exists() or self._draft_file(name).exists() def is_published(self, name: str) -> bool: return self._flow_file(name).exists() def has_draft(self, name: str) -> bool: """Are there unpublished changes — to the structure or to any source?""" if self._draft_file(name).exists(): return True drafts = self._draft_nodes_dir(name) return drafts.exists() and any(drafts.glob("*.py")) def updated_at(self, name: str) -> datetime | None: """When this flow was last written — its document or any node's source. Node code is saved without touching the flow document, so a flow whose last change was to a node body would otherwise look untouched. """ return _updated_at( [self._flow_file(name), self._draft_file(name)] + list((self._flow_dir(name) / "nodes").glob("*.py")) + list(self._draft_nodes_dir(name).glob("*.py")) ) @staticmethod def _load(path: Path) -> FlowDef: """A stored document, with the id one from before ids is known by.""" flow = FlowDef.model_validate_json(path.read_text()) if not flow.id: flow.id = path.parent.name return flow def read_flow(self, name: str, draft: bool = False) -> FlowDef: """The published flow, or with ``draft`` the working copy.""" if draft: path = self._draft_file(name) if path.exists(): return self._load(path) path = self._flow_file(name) if not path.exists(): raise FlowNotFound(name) return self._load(path) def key_index(self) -> dict[str, str]: """Every flow's key, to its id, whether published or only a draft. Read again after any commit, which every write makes — a draft-only create or a delete as much as a save. Shared: read it, never change it. """ at, keys = self._keys if at == self.revision: return keys # Taken before reading, so a commit landing meanwhile reads it again. at = self.revision keys = read_keys((self.root / n for n in self.list_flows()), "flow") self._keys = (at, keys) return keys def _claim(self, flow: FlowDef, create: bool = False) -> FlowDef: """*flow* with its id, once it is certain to be a save the store allows. Under `_write_lock`. A save is to a flow that exists, named by its id or, from before ids, by its key. Only *create* may make a new one. """ flow_id = checked(flow.id or flow.name) current = self.read_flow(flow_id, draft=True) if self.exists(flow_id) else None if current is None and not create: raise FlowNotFound(flow_id) claim( flow_id, flow.name, (current.id, current.name) if current else None, self.key_index(), FlowExists, ) return flow.model_copy(update={"id": flow_id}) def _issue(self, keys: dict[str, str]) -> str: """A fresh flow id. Under `_write_lock`, with *keys* the key index. Free as a directory, a key and in history: an id a deleted flow's runs are still filed under would hand them to the new one. """ graves = self.tombstones return new_id( "f", lambda c: ( self._flow_dir(c).exists() or c in keys or (graves is not None and graves.recorded(c) is not None) ), ) def read_all(self) -> list[FlowDef]: """Every published flow — what the engine runs. Cached against `revision`, because this sits on the publish path: a dashboard slider moving asked every flow's file to be read and validated again, per value. The list is rebuilt on the next commit, and returned as a copy so a caller sorting it cannot disturb the next. """ with self._read_all_lock: if self._read_all_at != self.revision: flows = [] for path in sorted(self.root.glob("*/flow.json")): name = path.parent.name try: flows.append(self.read_flow(name)) except Exception: logger.exception("Skipping unreadable flow '%s'", name) self._read_all = flows self._read_all_at = self.revision return list(self._read_all) def write_flow(self, flow: FlowDef) -> bool: """Publish a flow directly. Returns False when nothing actually changed. Creates one too, under the id it carries or else its key: what a test or a seed holding a finished document wants. The API creates only through `create`. """ with self._write_lock: flow = self._claim(flow, create=True) path = self._flow_file(flow.id) content = flow.model_dump_json(indent=2) + "\n" if path.exists() and path.read_text() == content: return False path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content) self._commit(f"Update flow {named(flow.name, flow.id)}") return True def create(self, flow: FlowDef, pin: str = "") -> FlowDef: """A new flow, only a draft until someone publishes it. Its id is issued here unless *pin* names one, which is a superuser's to ask for: the Demo, a test, a restore. A pinned id whose history outlived its flow adopts that history, so it has to take the key the history is filed under, and its tombstone goes once the document is on disk. """ with self._write_lock: keys = self.key_index() if pin: flow_id = checked(pin) # The directory rather than `exists`: anything left in it would # become the new flow's. if self._flow_dir(flow_id).exists(): raise IdTaken(flow_id) recorded = ( self.tombstones.recorded(flow_id) if self.tombstones else None ) if recorded is not None and recorded != flow.name: raise RecordedKey(flow_id, recorded) else: flow_id = self._issue(keys) flow = self._claim(flow.model_copy(update={"id": flow_id}), create=True) # Another flow's key as this one's id would make naming either # ambiguous. Checked after the key, which is the likelier mistake. if keys.get(flow_id, flow_id) != flow_id: raise IdTaken(flow_id) stored = flow.model_copy(update={"version": 1}) path = self._draft_file(flow_id) path.parent.mkdir(parents=True) path.write_text(stored.model_dump_json(indent=2) + "\n") self._commit(f"Create flow {named(flow.name, flow_id)}") if pin and self.tombstones is not None: self.tombstones.lift(flow_id) return stored def write_draft(self, flow: FlowDef, base_version: int) -> FlowDef: """Save unpublished changes, refusing to overwrite someone else's. ``base_version`` is the version the editor last saw. It has to match what is stored, otherwise another client saved in between and this write would silently discard their work. The flow has to exist: creating one is `create`. """ with self._write_lock: flow = self._claim(flow) current = self.read_flow(flow.id, draft=True) if base_version != current.version: raise StaleVersion(flow.name, current.version) if _same_content(current, flow): return current stored = flow.model_copy(update={"version": current.version + 1}) path = self._draft_file(flow.id) path.write_text(stored.model_dump_json(indent=2) + "\n") self._commit(f"Update draft of flow {named(flow.name, flow.id)}") return stored def publish_flow(self, flow_id: str, base_version: int) -> FlowDef: """Promote the working copy to what the engine runs.""" with self._write_lock: current = self.read_flow(flow_id, draft=True) if base_version != current.version: raise StaleVersion(current.name, current.version) draft = self._draft_file(flow_id) if draft.exists(): self._flow_file(flow_id).write_text( current.model_dump_json(indent=2) + "\n" ) draft.unlink() drafts = self._draft_nodes_dir(flow_id) if drafts.exists(): target = self._flow_dir(flow_id) / "nodes" target.mkdir(parents=True, exist_ok=True) for source in sorted(drafts.glob("*.py")): shutil.copy2(source, target / source.name) shutil.rmtree(drafts) self._commit(f"Publish flow {named(current.name, flow_id)}") return current def discard_draft(self, flow_id: str) -> FlowDef: """Throw the unpublished changes away and go back to what is running. What is running comes back a version above the draft, so an editor still holding the draft is refused rather than matching the next save made from the published copy. """ with self._write_lock: draft = self._draft_file(flow_id) if draft.exists(): running = self.read_flow(flow_id) top = _version(draft) if running.version <= top: running.version = top + 1 self._flow_file(flow_id).write_text( running.model_dump_json(indent=2) + "\n" ) draft.unlink() drafts = self._draft_nodes_dir(flow_id) if drafts.exists(): shutil.rmtree(drafts) published = self.read_flow(flow_id) self._commit(f"Discard draft of flow {named(published.name, flow_id)}") return published def delete_flow(self, flow_id: str) -> str: """Remove a flow's files. Returns the key it had. The key is buried first, so a delete that stops half-way never leaves history without a name, and the live document wins while it is there. """ with self._write_lock: directory = self._flow_dir(flow_id) if not directory.exists(): raise FlowNotFound(flow_id) key = next( (k for k, i in self.key_index().items() if i == flow_id), flow_id ) if self.tombstones is not None: self.tombstones.bury(flow_id, key) shutil.rmtree(directory) self._commit(f"Delete flow {named(key, flow_id)}") return key def duplicate_flow( self, flow_id: str, busy: Callable[[str], bool] | None = None ) -> FlowDef: """Copy a flow's working copy into a new flow that is only a draft. Like a new flow, the copy runs nothing until someone publishes it. It gets an id of its own and the key ``{name}_copy``, then ``_copy_2`` and on, past any key *busy* says something still holds. ``origin`` is dropped, since the copy is a local flow ``fluksio sync`` must not claim, and ``runtime.json`` stays behind with everything else about running. """ with self._write_lock: source = self.read_flow(flow_id, draft=True) keys = self.key_index() new_name = copy_name( source.name, lambda n: n in keys or (busy is not None and busy(n)) ) copy = source.model_copy( update={ "id": self._issue(keys), "name": new_name, "title": f"{source.title} (copy)" if source.title else "", "version": 1, "origin": None, } ) copy = self._claim(copy, create=True) # Its own messages named with its own key are the copy's now. self._retarget(copy, f"{source.name}.", f"{new_name}.") path = self._draft_file(copy.id) path.parent.mkdir(parents=True) path.write_text(copy.model_dump_json(indent=2) + "\n") # The bodies the source's editor shows: published, edits over them. # As drafts, so publishing the copy promotes them like a new flow's. target = self._draft_nodes_dir(copy.id) for folder in ( self._flow_dir(flow_id) / "nodes", self._draft_nodes_dir(flow_id), ): for body in sorted(folder.glob("*.py")): target.mkdir(exist_ok=True) shutil.copy2(body, target / body.name) self._commit( f"Duplicate flow {named(source.name, flow_id)} as " f"{named(new_name, copy.id)}" ) return copy @staticmethod def _retarget(flow: FlowDef, old_prefix: str, new_prefix: str) -> bool: """Point this flow's cross-flow message names at a renamed flow.""" changed = False for node in flow.nodes: for specs in (node.requires, node.provides): for position, spec in enumerate(specs): if spec.name.startswith(old_prefix): tail = spec.name[len(old_prefix) :] specs[position] = spec.model_copy( update={"name": new_prefix + tail} ) changed = True return changed # ------------------------------------------------------------------------- # Node source # ------------------------------------------------------------------------- def read_node_source(self, flow: str, node_id: str, draft: bool = False) -> str: if draft: path = self._draft_node_file(flow, node_id) if path.exists(): return path.read_text() path = self._node_file(flow, node_id) return path.read_text() if path.exists() else DEFAULT_SOURCE def has_node_source(self, flow: str, node_id: str, draft: bool = False) -> bool: """Whether a body was ever stored for this node. `read_node_source` answers a template when there is none, which is what a new node's editor should open with — but it is a placeholder, not code anybody wrote, and telling the two apart is the difference between "someone edited this" and "this was never written". """ if draft and self._draft_node_file(flow, node_id).exists(): return True return self._node_file(flow, node_id).exists() def write_node_source( self, flow: str, node_id: str, code: str, draft: bool = False ) -> bool: if draft: # Compared against the working copy, so re-saving what is already # published does not create a draft file that never gets promoted. if self.read_node_source(flow, node_id, draft=True) == code: return False path = self._draft_node_file(flow, node_id) else: path = self._node_file(flow, node_id) if path.exists() and path.read_text() == code: return False path.parent.mkdir(parents=True, exist_ok=True) path.write_text(code) self._commit(f"Update {'draft of ' if draft else ''}node '{flow}.{node_id}'") return True def delete_node_source(self, flow: str, node_id: str) -> None: path = self._node_file(flow, node_id) if path.exists(): path.unlink() self._commit(f"Delete node '{flow}.{node_id}'")