1070 lines
43 KiB
Python
1070 lines
43 KiB
Python
"""Flows on disk, versioned with git.
|
|
|
|
Every flow is a directory: ``flow.json`` for the structure, ``nodes/*.py`` for
|
|
node logic. The whole tree is a git repository and each saved change is a
|
|
commit, so a flow's history is readable with ordinary git tooling and two
|
|
flows never collide in one file.
|
|
|
|
Editing is separated from running. The editor writes ``flow.draft.json`` and
|
|
``nodes.draft/*.py``; the engine reads only the published ``flow.json`` and
|
|
``nodes/*.py``. Publishing promotes the draft over the published files and
|
|
removes it, so a flow directory without draft files is simply one with nothing
|
|
unpublished — which is what every flow written before this existed looks like.
|
|
|
|
A flow is two names. Its id is which flow it is: the directory, the argument
|
|
every method here takes, and never changed once given. Its key, ``name``, is
|
|
the namespace of its messages and unique among flows. A flow from before ids
|
|
existed has its key as its id (`fluksio.flow.identity`). A new one is given an
|
|
opaque id by `create`, or the one a superuser pinned. Published and draft
|
|
documents always share both.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
import string
|
|
import subprocess
|
|
import threading
|
|
from collections.abc import Callable, Iterable, Iterator
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Literal
|
|
|
|
from fluksio.flow.namespace import FLOW_FIELDS, Rename, rebind
|
|
from fluksio.flow.schemas import FlowDef, NodeDef, _validate_name
|
|
|
|
if TYPE_CHECKING:
|
|
from fluksio.flow.tombstones import Tombstones
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
#: Where the sources of nodes shared between flows live.
|
|
LIB_DIR = "_lib"
|
|
|
|
DEFAULT_SOURCE = '''"""A new node. Return a dict keyed by your output ports."""
|
|
|
|
|
|
def process():
|
|
return {}
|
|
'''
|
|
|
|
|
|
class InvalidName(ValueError):
|
|
"""A name that cannot be a file or directory of the store."""
|
|
|
|
def __init__(self, name: str) -> None:
|
|
super().__init__(name)
|
|
self.name = name
|
|
|
|
def __str__(self) -> str:
|
|
return (
|
|
f"'{self.name}' is not a valid name: use lowercase letters, digits "
|
|
"and underscores, starting with a letter"
|
|
)
|
|
|
|
|
|
def checked(name: str) -> str:
|
|
"""*name*, once it is certain to be one path segment inside the store.
|
|
|
|
Every path the stores build starts from a name the API was handed,
|
|
percent-decoded on the way in, so without this `..` or `_lib` reached the
|
|
filesystem as written: `delete_flow("..")` removed the data directory.
|
|
"""
|
|
try:
|
|
return _validate_name(name)
|
|
except ValueError:
|
|
raise InvalidName(name) from None
|
|
|
|
|
|
#: The shape of an id the server issues: a kind, then 40 random bits.
|
|
ISSUED_ID = re.compile(r"[fd]_[a-z2-7]{8}")
|
|
|
|
|
|
class ReservedKey(InvalidName):
|
|
"""A new key shaped like an issued id, which it could one day equal."""
|
|
|
|
def __str__(self) -> str:
|
|
return (
|
|
f"'{self.name}' has the shape of a generated id, f_ or d_ and eight "
|
|
"letters or digits, which a new key may not have"
|
|
)
|
|
|
|
|
|
class IdentityChange(ValueError):
|
|
"""A save that would change which object a document is, or its key.
|
|
|
|
Ids never change, and a key changes only by renaming, which moves its
|
|
messages along with it. A save that did either would bypass that.
|
|
"""
|
|
|
|
|
|
class RecordedKey(IdentityChange):
|
|
"""A pinned id restored under another key than its history is filed under.
|
|
|
|
The history names the old key's messages, so taking a new key with it is
|
|
a rename, not a create.
|
|
"""
|
|
|
|
def __init__(self, obj_id: str, key: str) -> None:
|
|
super().__init__(obj_id)
|
|
self.id = obj_id
|
|
self.key = key
|
|
|
|
def __str__(self) -> str:
|
|
return (
|
|
f"'{self.id}' has history filed under the key '{self.key}', so it "
|
|
"can only be restored with that key"
|
|
)
|
|
|
|
|
|
class IdTaken(ValueError):
|
|
"""A pinned id that is in use: an object's, a directory's or another key."""
|
|
|
|
def __init__(self, obj_id: str, kind: str = "flow") -> None:
|
|
super().__init__(obj_id)
|
|
self.id = obj_id
|
|
self.kind = kind
|
|
|
|
def __str__(self) -> str:
|
|
return f"The id '{self.id}' is already taken by a {self.kind}"
|
|
|
|
|
|
def claim(
|
|
obj_id: str,
|
|
key: str,
|
|
current: tuple[str, str] | None,
|
|
keys: dict[str, str],
|
|
taken: type[ValueError],
|
|
) -> None:
|
|
"""Refuse a write that changes who an object is, or takes another's key.
|
|
|
|
*current* is the stored ``(id, name)`` at *obj_id*, None for a new object,
|
|
and *keys* the store's key index. Called under the store's write lock, so
|
|
two creates of one key cannot both pass. *taken* is what a key already in
|
|
use raises: `FlowExists`, or the dashboards' own.
|
|
"""
|
|
if current is not None:
|
|
stored_id, stored_key = current
|
|
if stored_id != obj_id:
|
|
raise IdentityChange(f"'{obj_id}' holds a document of '{stored_id}'")
|
|
if stored_key != key:
|
|
raise IdentityChange(
|
|
f"'{obj_id}' has the key '{stored_key}', which a save cannot "
|
|
f"change to '{key}'"
|
|
)
|
|
return
|
|
# New keys only: one this shape that exists already stays saveable.
|
|
if ISSUED_ID.fullmatch(key):
|
|
raise ReservedKey(key)
|
|
if keys.get(key, obj_id) != obj_id:
|
|
raise taken(key)
|
|
|
|
|
|
def read_keys(directories: Iterable[Path], document: str) -> dict[str, str]:
|
|
"""The key each directory's object holds, to the directory: key to id.
|
|
|
|
Read from the working copy, ``<document>.draft.json`` when there is one, so
|
|
an object that is only a draft holds its key as much as a published one.
|
|
"""
|
|
keys: dict[str, str] = {}
|
|
for directory in directories:
|
|
draft = directory / f"{document}.draft.json"
|
|
path = draft if draft.exists() else directory / f"{document}.json"
|
|
try:
|
|
keys.setdefault(json.loads(path.read_text())["name"], directory.name)
|
|
except (OSError, ValueError, KeyError, TypeError):
|
|
# Unreadable to the engine too, which reports it when it loads.
|
|
continue
|
|
return keys
|
|
|
|
|
|
def named(key: str, obj_id: str) -> str:
|
|
"""How a commit message names an object: by key, and by id when it differs."""
|
|
return f"'{key}'" if key == obj_id else f"'{key}' ({obj_id})"
|
|
|
|
|
|
def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str:
|
|
"""A fresh id for a flow (``f``) or a dashboard (``d``), e.g. ``f_k3q7m2xa``.
|
|
|
|
Random is not unique, so *taken* decides, under the store's write lock: it
|
|
has to know every live id, every directory left behind and every key,
|
|
since a key equal to an id makes an argument naming either ambiguous.
|
|
"""
|
|
alphabet = string.ascii_lowercase + "234567"
|
|
while True:
|
|
candidate = f"{kind}_" + "".join(secrets.choice(alphabet) for _ in range(8))
|
|
if not taken(candidate):
|
|
return candidate
|
|
|
|
|
|
class FlowNotFound(KeyError):
|
|
def __init__(self, name: str, published: bool = False) -> None:
|
|
super().__init__(name)
|
|
self.name = name
|
|
self.published = published
|
|
|
|
def __str__(self) -> str:
|
|
what = "published flow" if self.published else "flow"
|
|
return f"No {what} with the id '{self.name}'"
|
|
|
|
|
|
class FlowExists(ValueError):
|
|
def __init__(self, name: str) -> None:
|
|
super().__init__(name)
|
|
self.name = name
|
|
|
|
def __str__(self) -> str:
|
|
return f"There is already a flow named '{self.name}'"
|
|
|
|
|
|
class LibNotFound(KeyError):
|
|
def __init__(self, name: str) -> None:
|
|
super().__init__(name)
|
|
self.name = name
|
|
|
|
def __str__(self) -> str:
|
|
return f"No shared node named '{self.name}'"
|
|
|
|
|
|
class LibExists(ValueError):
|
|
def __init__(self, name: str) -> None:
|
|
super().__init__(name)
|
|
self.name = name
|
|
|
|
def __str__(self) -> str:
|
|
return f"There is already a shared node named '{self.name}'"
|
|
|
|
|
|
class StaleVersion(ValueError):
|
|
"""Someone else saved this flow since the version being written was read."""
|
|
|
|
def __init__(self, name: str, current: int) -> None:
|
|
super().__init__(name)
|
|
self.name = name
|
|
self.current = current
|
|
|
|
def __str__(self) -> str:
|
|
return f"Flow '{self.name}' has changed since you loaded it"
|
|
|
|
|
|
def _updated_at(paths: Iterable[Path]) -> datetime | None:
|
|
"""When the newest of these files was written, or ``None`` if none exist.
|
|
|
|
A file's mtime, not the commit that recorded it: every write here commits
|
|
immediately, so the two are the same instant, and `git log -1 -- <path>`
|
|
would walk the history back to the last commit touching that path — 130ms
|
|
for a document nobody has edited in a week, against a `stat` for all of
|
|
them. Slowest for exactly the stalest documents is the wrong shape for a
|
|
list endpoint.
|
|
"""
|
|
stamps = [path.stat().st_mtime for path in paths if path.exists()]
|
|
return datetime.fromtimestamp(max(stamps), UTC) if stamps else None
|
|
|
|
|
|
def _version(path: Path) -> int:
|
|
"""The version a stored document carries, 0 for none or an unreadable one."""
|
|
try:
|
|
return int(json.loads(path.read_text()).get("version", 1))
|
|
except (OSError, ValueError, TypeError, AttributeError):
|
|
return 0
|
|
|
|
|
|
def _same_content(left: FlowDef, right: FlowDef) -> bool:
|
|
"""Equal but for the version counter, which the server owns."""
|
|
return left.model_copy(update={"version": 0}) == right.model_copy(
|
|
update={"version": 0}
|
|
)
|
|
|
|
|
|
def copy_name(name: str, taken: Callable[[str], bool]) -> str:
|
|
"""``{name}_copy``, then ``_copy_2`` and on, from a copy's own original."""
|
|
base = re.sub(r"_copy(_\d+)?$", "", name)
|
|
new_name, count = f"{base}_copy", 1
|
|
while taken(new_name):
|
|
count += 1
|
|
new_name = f"{base}_copy_{count}"
|
|
return new_name
|
|
|
|
|
|
class FlowStore:
|
|
"""Reads and writes flows, committing every change."""
|
|
|
|
#: One warning per process when there is no git to commit with.
|
|
_warned_no_git = False
|
|
|
|
#: Commits between housekeeping runs. Auto-gc is off (see `_git`), so
|
|
#: nothing packs unless it is asked for; a commit counter is the trigger
|
|
#: because the process that needs it never restarts — a seeding session
|
|
#: commits thousands of times inside one long-lived container, which is how
|
|
#: a store reached 1898 commits, 5863 loose objects and no pack at all.
|
|
_GC_EVERY = 500
|
|
|
|
def __init__(self, root: Path, tombstones: Tombstones | None = None) -> None:
|
|
self.root = root
|
|
#: Where a deleted flow's key is kept, and what history an id has.
|
|
#: None leaves both out: a store without a database, as in tests.
|
|
self.tombstones = tombstones
|
|
self.root.mkdir(parents=True, exist_ok=True)
|
|
# Draft writes are check-and-set, so two clients saving at once must not
|
|
# interleave between reading the current version and writing the next.
|
|
self._write_lock = threading.Lock()
|
|
#: Bumped on every commit, so something derived from every flow at once
|
|
#: can tell whether it is still current without reading them all again.
|
|
#: In memory rather than `head()`, which is a git subprocess per call —
|
|
#: it therefore counts from zero per process and misses an edit made on
|
|
#: disk behind the API, which no writer here does.
|
|
self.revision = 0
|
|
#: `read_all` memoised against that revision — see the method.
|
|
self._read_all: list[FlowDef] = []
|
|
self._read_all_at = -1
|
|
self._read_all_lock = threading.Lock()
|
|
#: `key_index` and the revision it was read at.
|
|
self._keys: tuple[int, dict[str, str]] = (-1, {})
|
|
if not (self.root / ".git").exists():
|
|
self._git("init", "-q")
|
|
self._commit("Initialise flow store", allow_empty=True)
|
|
self._migrate()
|
|
|
|
def _migrate(self) -> None:
|
|
"""Rewrite node types and settings stored under a name they no longer have.
|
|
|
|
On 2026-10-15 the node type ``trigger`` became ``debounce`` and the mqtt
|
|
setting ``publish_queue_size`` became ``send_queue_size``. On 2026-10-16
|
|
a node's ``device`` and ``device_policy`` became ``worker`` and
|
|
``worker_policy``, so that a device is only ever a paired one. No alias
|
|
is left for the old names. Each published and draft document is read as
|
|
plain JSON and only the matching node changes, so a field this version
|
|
does not model survives, and the old document stays in git. The version
|
|
goes up by one, so an editor still holding the old document gets a
|
|
conflict on its next save instead of writing the old name back. A store
|
|
with nothing to rename is left alone.
|
|
"""
|
|
changed = []
|
|
paths = [*self.root.glob("*/flow.json"), *self.root.glob("*/flow.draft.json")]
|
|
for path in sorted(paths):
|
|
try:
|
|
data = json.loads(path.read_text())
|
|
nodes = [n for n in data["nodes"] if n.get("type") == "trigger"]
|
|
queues = [
|
|
n["params"]
|
|
for n in data["nodes"]
|
|
if n.get("type") == "mqtt"
|
|
and "publish_queue_size" in (n.get("params") or {})
|
|
]
|
|
placed = [
|
|
n for n in data["nodes"] if "device" in n or "device_policy" in n
|
|
]
|
|
except Exception:
|
|
# Unreadable to the engine too, which reports it when it loads.
|
|
continue
|
|
if not nodes and not queues and not placed:
|
|
continue
|
|
for node in nodes:
|
|
node["type"] = "debounce"
|
|
for params in queues:
|
|
params["send_queue_size"] = params.pop("publish_queue_size")
|
|
for node in placed:
|
|
for old in ("device", "device_policy"):
|
|
if old in node:
|
|
node[old.replace("device", "worker")] = node.pop(old)
|
|
data["version"] = data.get("version", 1) + 1
|
|
path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n")
|
|
changed.append(f"{path.parent.name}/{path.name}")
|
|
if changed:
|
|
logger.warning(
|
|
"Renamed node type 'trigger' to 'debounce', mqtt "
|
|
"'publish_queue_size' to 'send_queue_size' and a node's "
|
|
"'device' to 'worker' in %s",
|
|
", ".join(changed),
|
|
)
|
|
self._commit(
|
|
"Rename 'trigger' nodes, mqtt 'publish_queue_size' and a "
|
|
"node's 'device'"
|
|
)
|
|
|
|
# -------------------------------------------------------------------------
|
|
# git
|
|
# -------------------------------------------------------------------------
|
|
|
|
def _git(self, *args: str) -> subprocess.CompletedProcess[str]:
|
|
try:
|
|
return subprocess.run(
|
|
# `gc.auto=0` on every invocation, because `git commit` otherwise
|
|
# forks a background `gc --auto` and returns without waiting for
|
|
# it. `subprocess.run` reaps the commit, not its grandchild, so
|
|
# the daemon is reparented onto PID 1 — the FastAPI process,
|
|
# which reaps nothing, and the zombies accumulate one per save.
|
|
# Packing is asked for explicitly in `_commit` instead.
|
|
["git", "-C", str(self.root), "-c", "gc.auto=0", *args],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
except FileNotFoundError:
|
|
# No git on this machine — a `pip install` on a locked-down host is
|
|
# where this happens. The flows are files either way, which is what
|
|
# the store is; what is lost is their history, so say it once and
|
|
# carry on rather than refusing to start.
|
|
if not FlowStore._warned_no_git:
|
|
FlowStore._warned_no_git = True
|
|
logger.warning(
|
|
"git is not installed, so flow changes are not versioned. "
|
|
"Install it to get a commit per save."
|
|
)
|
|
return subprocess.CompletedProcess(
|
|
args=list(args), returncode=1, stdout="", stderr="git is not installed"
|
|
)
|
|
|
|
def _commit(self, message: str, allow_empty: bool = False) -> None:
|
|
# Before the commit rather than after it: what a reader has to notice
|
|
# is that the files changed, which is already true whether or not git
|
|
# is there to record it.
|
|
self.revision += 1
|
|
self._git("add", "-A")
|
|
result = self._git(
|
|
"-c",
|
|
"user.name=fluksio",
|
|
"-c",
|
|
"user.email=fluksio@localhost",
|
|
"commit",
|
|
"-q",
|
|
*(["--allow-empty"] if allow_empty else []),
|
|
"-m",
|
|
message,
|
|
)
|
|
if result.returncode == 0 or "nothing to commit" in result.stdout:
|
|
if self.revision % self._GC_EVERY == 0:
|
|
# In the foreground, so this child is waited on rather than
|
|
# orphaned. Plain `gc` rather than `gc --auto`, which the flag
|
|
# above has just disabled.
|
|
self._git("gc", "--quiet")
|
|
return
|
|
if result.stderr == "git is not installed":
|
|
# Already said once, in `_git`. Repeating it per save is noise.
|
|
return
|
|
logger.warning("Could not commit flow change: %s", result.stdout.strip())
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Paths
|
|
# -------------------------------------------------------------------------
|
|
|
|
# Every path below starts in one of these three, and each name in it has
|
|
# been through `checked`.
|
|
|
|
def _flow_dir(self, name: str) -> Path:
|
|
return self.root / checked(name)
|
|
|
|
def _flow_file(self, name: str) -> Path:
|
|
return self._flow_dir(name) / "flow.json"
|
|
|
|
def _draft_file(self, name: str) -> Path:
|
|
return self._flow_dir(name) / "flow.draft.json"
|
|
|
|
def _node_file(self, flow: str, node_id: str) -> Path:
|
|
return self._flow_dir(flow) / "nodes" / f"{checked(node_id)}.py"
|
|
|
|
def _draft_nodes_dir(self, flow: str) -> Path:
|
|
return self._flow_dir(flow) / "nodes.draft"
|
|
|
|
def _draft_node_file(self, flow: str, node_id: str) -> Path:
|
|
return self._draft_nodes_dir(flow) / f"{checked(node_id)}.py"
|
|
|
|
def _runtime_file(self, name: str) -> Path:
|
|
return self._flow_dir(name) / "runtime.json"
|
|
|
|
def _lib_file(self, name: str) -> Path:
|
|
return self.root / LIB_DIR / f"{checked(name)}.py"
|
|
|
|
def head(self) -> str:
|
|
"""The commit the store is at, so a result can name the code it ran."""
|
|
result = self._git("rev-parse", "HEAD")
|
|
return result.stdout.strip() if result.returncode == 0 else ""
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Module requirements
|
|
#
|
|
# What node code may import, versioned with the flows importing it. A file
|
|
# in the root cannot collide with a flow, which is always a directory.
|
|
# -------------------------------------------------------------------------
|
|
|
|
def read_requirements(self) -> str:
|
|
path = self.root / "requirements.txt"
|
|
return path.read_text() if path.exists() else ""
|
|
|
|
def write_requirements(self, text: str) -> None:
|
|
(self.root / "requirements.txt").write_text(text)
|
|
self._commit("Update module requirements")
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Shared node sources
|
|
#
|
|
# A node marked reusable moves its code to ``_lib/``; the flows using it
|
|
# keep their own ports and parameters and point at it by name, so one fix
|
|
# reaches all of them. ``_lib`` cannot collide with a flow — a flow name has
|
|
# to start with a letter — and holds no ``flow.json``, so it is invisible to
|
|
# everything that lists flows.
|
|
# -------------------------------------------------------------------------
|
|
|
|
def list_lib(self) -> list[str]:
|
|
return sorted(path.stem for path in (self.root / LIB_DIR).glob("*.py"))
|
|
|
|
def lib_exists(self, name: str) -> bool:
|
|
return self._lib_file(name).exists()
|
|
|
|
def read_lib_source(self, name: str) -> str:
|
|
path = self._lib_file(name)
|
|
if not path.exists():
|
|
raise LibNotFound(name)
|
|
return path.read_text()
|
|
|
|
def write_lib_source(self, name: str, code: str) -> bool:
|
|
path = self._lib_file(name)
|
|
if path.exists() and path.read_text() == code:
|
|
return False
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(code)
|
|
self._commit(f"Update shared node '{name}'")
|
|
return True
|
|
|
|
def delete_lib_source(self, name: str) -> None:
|
|
path = self._lib_file(name)
|
|
if not path.exists():
|
|
raise LibNotFound(name)
|
|
path.unlink()
|
|
self._commit(f"Delete shared node '{name}'")
|
|
|
|
def share_node(self, flow: str, node_id: str, lib_name: str) -> None:
|
|
"""Move a node's source into the library and point the node at it."""
|
|
with self._write_lock:
|
|
if self.lib_exists(lib_name):
|
|
raise LibExists(lib_name)
|
|
|
|
code = self.read_node_source(flow, node_id, draft=True)
|
|
self._lib_file(lib_name).parent.mkdir(parents=True, exist_ok=True)
|
|
self._lib_file(lib_name).write_text(code)
|
|
|
|
for path in (
|
|
self._node_file(flow, node_id),
|
|
self._draft_node_file(flow, node_id),
|
|
):
|
|
if path.exists():
|
|
path.unlink()
|
|
|
|
self._set_source_ref(flow, node_id, lib_name)
|
|
self._commit(f"Share node '{flow}.{node_id}' as '{lib_name}'")
|
|
|
|
def unshare_node(self, flow: str, node_id: str) -> None:
|
|
"""Take a private copy of the shared source back into the flow."""
|
|
with self._write_lock:
|
|
definition = self.read_flow(flow, draft=True)
|
|
node = next((n for n in definition.nodes if n.id == node_id), None)
|
|
if node is None or not node.source_ref:
|
|
return
|
|
code = self.read_lib_source(node.source_ref)
|
|
# Written as a draft: the running flow keeps the shared code until
|
|
# this is published, like every other edit.
|
|
self._draft_node_file(flow, node_id).parent.mkdir(
|
|
parents=True, exist_ok=True
|
|
)
|
|
self._draft_node_file(flow, node_id).write_text(code)
|
|
self._set_source_ref(flow, node_id, None)
|
|
self._commit(f"Unshare node '{flow}.{node_id}'")
|
|
|
|
def detach_lib(self, name: str) -> None:
|
|
"""Give every node running this shared source a copy, then delete it.
|
|
|
|
Published flows and drafts alike, so a running flow keeps the same code
|
|
and nothing waits on a publish. Each document changed goes up a version,
|
|
so an editor holding the old one gets a conflict on its next save
|
|
instead of pointing a node back at a source that is gone.
|
|
"""
|
|
with self._write_lock:
|
|
code = self.read_lib_source(name)
|
|
for flow in self.list_flows():
|
|
for path, node_file in (
|
|
(self._flow_file(flow), self._node_file),
|
|
(self._draft_file(flow), self._draft_node_file),
|
|
):
|
|
if not path.exists():
|
|
continue
|
|
definition = self._load(path)
|
|
users = [n for n in definition.nodes if n.source_ref == name]
|
|
if not users:
|
|
continue
|
|
for node in users:
|
|
target = node_file(flow, node.id)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text(code)
|
|
node.source_ref = None
|
|
definition.version += 1
|
|
path.write_text(definition.model_dump_json(indent=2) + "\n")
|
|
self._lib_file(name).unlink()
|
|
self._commit(f"Detach and delete shared node '{name}'")
|
|
|
|
def usages(self, lib_name: str, draft: bool = True) -> list[str]:
|
|
"""Every ``<key>.<node>`` pointing at this shared source.
|
|
|
|
Read from the drafts the editor shows, or with ``draft`` off from the
|
|
published flows the engine runs.
|
|
"""
|
|
return sorted(
|
|
f"{flow.name}.{node.id}" for flow, node in self._lib_nodes(lib_name, draft)
|
|
)
|
|
|
|
def lib_users(self, lib_name: str) -> list[str]:
|
|
"""The ids of the published flows running this shared source."""
|
|
return sorted({flow.id for flow, _ in self._lib_nodes(lib_name, False)})
|
|
|
|
def _lib_nodes(
|
|
self, lib_name: str, draft: bool
|
|
) -> Iterator[tuple[FlowDef, NodeDef]]:
|
|
for flow_id in self.list_flows():
|
|
try:
|
|
definition = self.read_flow(flow_id, draft=draft)
|
|
except FlowNotFound:
|
|
continue
|
|
for node in definition.nodes:
|
|
if node.source_ref == lib_name:
|
|
yield definition, node
|
|
|
|
def _set_source_ref(self, flow: str, node_id: str, ref: str | None) -> None:
|
|
"""Point a node at a shared source, in the draft the editor is on."""
|
|
definition = self.read_flow(flow, draft=True)
|
|
definition.nodes = [
|
|
node.model_copy(update={"source_ref": ref}) if node.id == node_id else node
|
|
for node in definition.nodes
|
|
]
|
|
stored = definition.model_copy(update={"version": definition.version + 1})
|
|
path = self._draft_file(flow)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(stored.model_dump_json(indent=2) + "\n")
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Runtime state
|
|
# -------------------------------------------------------------------------
|
|
|
|
def read_enabled(self, name: str) -> bool:
|
|
"""Whether the engine should run this flow. Missing means yes."""
|
|
path = self._runtime_file(name)
|
|
if not path.exists():
|
|
return True
|
|
try:
|
|
return bool(json.loads(path.read_text()).get("enabled", True))
|
|
except (ValueError, OSError):
|
|
logger.warning("Unreadable runtime state for flow '%s'", name)
|
|
return True
|
|
|
|
def write_enabled(self, name: str, enabled: bool) -> None:
|
|
"""Stop or start a flow, in a file the editor never writes.
|
|
|
|
This is deliberately not a field on the flow document: that one is
|
|
autosaved from the canvas, so a stopped flow would start itself again
|
|
on the next edit.
|
|
"""
|
|
path = self._runtime_file(name)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps({"enabled": enabled}, indent=2) + "\n")
|
|
self._commit(f"{'Start' if enabled else 'Stop'} flow '{name}'")
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Flows
|
|
# -------------------------------------------------------------------------
|
|
|
|
def list_flows(self) -> list[str]:
|
|
"""Every flow the editor knows, published or not."""
|
|
names = {path.parent.name for path in self.root.glob("*/flow.json")}
|
|
names |= {path.parent.name for path in self.root.glob("*/flow.draft.json")}
|
|
# `_devices/flow.json` is a device named "flow", not a flow.
|
|
return sorted(name for name in names if not name.startswith("_"))
|
|
|
|
def exists(self, name: str) -> bool:
|
|
return self._flow_file(name).exists() or self._draft_file(name).exists()
|
|
|
|
def is_published(self, name: str) -> bool:
|
|
return self._flow_file(name).exists()
|
|
|
|
def has_draft(self, name: str) -> bool:
|
|
"""Are there unpublished changes — to the structure or to any source?"""
|
|
if self._draft_file(name).exists():
|
|
return True
|
|
drafts = self._draft_nodes_dir(name)
|
|
return drafts.exists() and any(drafts.glob("*.py"))
|
|
|
|
def updated_at(self, name: str) -> datetime | None:
|
|
"""When this flow was last written — its document or any node's source.
|
|
|
|
Node code is saved without touching the flow document, so a flow whose
|
|
last change was to a node body would otherwise look untouched.
|
|
"""
|
|
return _updated_at(
|
|
[self._flow_file(name), self._draft_file(name)]
|
|
+ list((self._flow_dir(name) / "nodes").glob("*.py"))
|
|
+ list(self._draft_nodes_dir(name).glob("*.py"))
|
|
)
|
|
|
|
@staticmethod
|
|
def _load(path: Path) -> FlowDef:
|
|
"""A stored document, with the id one from before ids is known by."""
|
|
flow = FlowDef.model_validate_json(path.read_text())
|
|
if not flow.id:
|
|
flow.id = path.parent.name
|
|
return flow
|
|
|
|
def read_flow(self, name: str, draft: bool = False) -> FlowDef:
|
|
"""The published flow, or with ``draft`` the working copy."""
|
|
if draft:
|
|
path = self._draft_file(name)
|
|
if path.exists():
|
|
return self._load(path)
|
|
path = self._flow_file(name)
|
|
if not path.exists():
|
|
raise FlowNotFound(name, published=not draft)
|
|
return self._load(path)
|
|
|
|
def key_index(self) -> dict[str, str]:
|
|
"""Every flow's key, to its id, whether published or only a draft.
|
|
|
|
Read again after any commit, which every write makes — a draft-only
|
|
create or a delete as much as a save. Shared: read it, never change it.
|
|
"""
|
|
at, keys = self._keys
|
|
if at == self.revision:
|
|
return keys
|
|
# Taken before reading, so a commit landing meanwhile reads it again.
|
|
at = self.revision
|
|
keys = read_keys((self.root / n for n in self.list_flows()), "flow")
|
|
self._keys = (at, keys)
|
|
return keys
|
|
|
|
def _claim(self, flow: FlowDef, create: bool = False) -> FlowDef:
|
|
"""*flow* with its id, once it is certain to be a save the store allows.
|
|
|
|
Under `_write_lock`. A save is to a flow that exists, named by its id
|
|
or, from before ids, by its key. Only *create* may make a new one.
|
|
"""
|
|
flow_id = checked(flow.id or flow.name)
|
|
current = self.read_flow(flow_id, draft=True) if self.exists(flow_id) else None
|
|
if current is None and not create:
|
|
raise FlowNotFound(flow_id)
|
|
claim(
|
|
flow_id,
|
|
flow.name,
|
|
(current.id, current.name) if current else None,
|
|
self.key_index(),
|
|
FlowExists,
|
|
)
|
|
return flow.model_copy(update={"id": flow_id})
|
|
|
|
def _issue(self, keys: dict[str, str]) -> str:
|
|
"""A fresh flow id. Under `_write_lock`, with *keys* the key index.
|
|
|
|
Free as a directory, a key and in history: an id a deleted flow's runs
|
|
are still filed under would hand them to the new one.
|
|
"""
|
|
graves = self.tombstones
|
|
return new_id(
|
|
"f",
|
|
lambda c: (
|
|
self._flow_dir(c).exists()
|
|
or c in keys
|
|
or (graves is not None and graves.recorded(c) is not None)
|
|
),
|
|
)
|
|
|
|
def read_all(self) -> list[FlowDef]:
|
|
"""Every published flow — what the engine runs.
|
|
|
|
Cached against `revision`, because this sits on the publish path: a
|
|
dashboard slider moving asked every flow's file to be read and
|
|
validated again, per value. The list is rebuilt on the next commit,
|
|
and returned as a copy so a caller sorting it cannot disturb the next.
|
|
"""
|
|
with self._read_all_lock:
|
|
# Taken before scanning, so a commit landing meanwhile scans again.
|
|
at = self.revision
|
|
if self._read_all_at != at:
|
|
flows = []
|
|
for path in sorted(self.root.glob("*/flow.json")):
|
|
name = path.parent.name
|
|
try:
|
|
flows.append(self.read_flow(name))
|
|
except Exception:
|
|
logger.exception("Skipping unreadable flow '%s'", name)
|
|
self._read_all = flows
|
|
self._read_all_at = at
|
|
return list(self._read_all)
|
|
|
|
def write_flow(self, flow: FlowDef) -> bool:
|
|
"""Publish a flow directly. Returns False when nothing actually changed.
|
|
|
|
Creates one too, under the id it carries or else its key: what a test
|
|
or a seed holding a finished document wants. The API creates only
|
|
through `create`.
|
|
"""
|
|
with self._write_lock:
|
|
flow = self._claim(flow, create=True)
|
|
path = self._flow_file(flow.id)
|
|
content = flow.model_dump_json(indent=2) + "\n"
|
|
if path.exists() and path.read_text() == content:
|
|
return False
|
|
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(content)
|
|
self._commit(f"Update flow {named(flow.name, flow.id)}")
|
|
return True
|
|
|
|
def create(self, flow: FlowDef, pin: str = "") -> FlowDef:
|
|
"""A new flow, only a draft until someone publishes it.
|
|
|
|
Its id is issued here unless *pin* names one, which is a superuser's
|
|
to ask for: the Demo, a test, a restore. A pinned id whose history
|
|
outlived its flow adopts that history, so it has to take the key the
|
|
history is filed under, and its tombstone goes once the document is on
|
|
disk.
|
|
"""
|
|
with self._write_lock:
|
|
keys = self.key_index()
|
|
if pin:
|
|
flow_id = checked(pin)
|
|
# The directory rather than `exists`: anything left in it would
|
|
# become the new flow's.
|
|
if self._flow_dir(flow_id).exists():
|
|
raise IdTaken(flow_id)
|
|
recorded = (
|
|
self.tombstones.recorded(flow_id) if self.tombstones else None
|
|
)
|
|
if recorded is not None and recorded != flow.name:
|
|
raise RecordedKey(flow_id, recorded)
|
|
else:
|
|
flow_id = self._issue(keys)
|
|
flow = self._claim(flow.model_copy(update={"id": flow_id}), create=True)
|
|
# Another flow's key as this one's id would make naming either
|
|
# ambiguous. Checked after the key, which is the likelier mistake.
|
|
if keys.get(flow_id, flow_id) != flow_id:
|
|
raise IdTaken(flow_id)
|
|
|
|
stored = flow.model_copy(update={"version": 1})
|
|
path = self._draft_file(flow_id)
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text(stored.model_dump_json(indent=2) + "\n")
|
|
self._commit(f"Create flow {named(flow.name, flow_id)}")
|
|
if pin and self.tombstones is not None:
|
|
self.tombstones.lift(flow_id)
|
|
return stored
|
|
|
|
def write_draft(self, flow: FlowDef, base_version: int) -> FlowDef:
|
|
"""Save unpublished changes, refusing to overwrite someone else's.
|
|
|
|
``base_version`` is the version the editor last saw. It has to match
|
|
what is stored, otherwise another client saved in between and this
|
|
write would silently discard their work. The flow has to exist:
|
|
creating one is `create`.
|
|
"""
|
|
with self._write_lock:
|
|
flow = self._claim(flow)
|
|
current = self.read_flow(flow.id, draft=True)
|
|
if base_version != current.version:
|
|
raise StaleVersion(flow.name, current.version)
|
|
if _same_content(current, flow):
|
|
return current
|
|
|
|
stored = flow.model_copy(update={"version": current.version + 1})
|
|
path = self._draft_file(flow.id)
|
|
path.write_text(stored.model_dump_json(indent=2) + "\n")
|
|
self._commit(f"Update draft of flow {named(flow.name, flow.id)}")
|
|
return stored
|
|
|
|
def publish_flow(self, flow_id: str, base_version: int) -> FlowDef:
|
|
"""Promote the working copy to what the engine runs."""
|
|
with self._write_lock:
|
|
current = self.read_flow(flow_id, draft=True)
|
|
if base_version != current.version:
|
|
raise StaleVersion(current.name, current.version)
|
|
|
|
draft = self._draft_file(flow_id)
|
|
if draft.exists():
|
|
self._flow_file(flow_id).write_text(
|
|
current.model_dump_json(indent=2) + "\n"
|
|
)
|
|
draft.unlink()
|
|
|
|
drafts = self._draft_nodes_dir(flow_id)
|
|
if drafts.exists():
|
|
target = self._flow_dir(flow_id) / "nodes"
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
for source in sorted(drafts.glob("*.py")):
|
|
shutil.copy2(source, target / source.name)
|
|
shutil.rmtree(drafts)
|
|
|
|
self._commit(f"Publish flow {named(current.name, flow_id)}")
|
|
return current
|
|
|
|
def discard_draft(self, flow_id: str) -> FlowDef:
|
|
"""Throw the unpublished changes away and go back to what is running.
|
|
|
|
What is running comes back a version above the draft, so an editor
|
|
still holding the draft is refused rather than matching the next save
|
|
made from the published copy.
|
|
"""
|
|
with self._write_lock:
|
|
draft = self._draft_file(flow_id)
|
|
if draft.exists():
|
|
running = self.read_flow(flow_id)
|
|
top = _version(draft)
|
|
if running.version <= top:
|
|
running.version = top + 1
|
|
self._flow_file(flow_id).write_text(
|
|
running.model_dump_json(indent=2) + "\n"
|
|
)
|
|
draft.unlink()
|
|
drafts = self._draft_nodes_dir(flow_id)
|
|
if drafts.exists():
|
|
shutil.rmtree(drafts)
|
|
published = self.read_flow(flow_id)
|
|
self._commit(f"Discard draft of flow {named(published.name, flow_id)}")
|
|
return published
|
|
|
|
def delete_flow(self, flow_id: str) -> str:
|
|
"""Remove a flow's files. Returns the key it had.
|
|
|
|
The key is buried first, so a delete that stops half-way never leaves
|
|
history without a name, and the live document wins while it is there.
|
|
"""
|
|
with self._write_lock:
|
|
directory = self._flow_dir(flow_id)
|
|
if not directory.exists():
|
|
raise FlowNotFound(flow_id)
|
|
key = next(
|
|
(k for k, i in self.key_index().items() if i == flow_id), flow_id
|
|
)
|
|
if self.tombstones is not None:
|
|
self.tombstones.bury(flow_id, key)
|
|
shutil.rmtree(directory)
|
|
self._commit(f"Delete flow {named(key, flow_id)}")
|
|
return key
|
|
|
|
def replace_documents(self, texts: dict[str, str], message: str) -> bool:
|
|
"""Write flow and dashboard documents whole, *texts* by path relative
|
|
to the store, and commit those paths alone with *message*.
|
|
|
|
A key rename's (`fluksio.flow.rename`): each text carries its version
|
|
already, and whatever else is uncommitted stays so. False when the
|
|
commit failed. A store without git counts as committed.
|
|
"""
|
|
# Here rather than at the top: it reads where dashboards live, and
|
|
# the dashboards module imports this one.
|
|
from fluksio.flow.identity import write_documents
|
|
|
|
with self._write_lock:
|
|
try:
|
|
return write_documents(self.root, texts, message)
|
|
finally:
|
|
# Whatever got written, the caches are to read it again.
|
|
self.revision += 1
|
|
|
|
def duplicate_flow(
|
|
self, flow_id: str, busy: Callable[[str], bool] | None = None
|
|
) -> FlowDef:
|
|
"""Copy a flow's working copy into a new flow that is only a draft.
|
|
|
|
Like a new flow, the copy runs nothing until someone publishes it. It
|
|
gets an id of its own and the key ``{name}_copy``, then ``_copy_2`` and
|
|
on, past any key *busy* says something still holds. ``origin`` is
|
|
dropped, since the copy is a local flow ``fluksio sync`` must not
|
|
claim, and ``runtime.json`` stays behind with everything else about
|
|
running.
|
|
"""
|
|
with self._write_lock:
|
|
source = self.read_flow(flow_id, draft=True)
|
|
keys = self.key_index()
|
|
new_name = copy_name(
|
|
source.name, lambda n: n in keys or (busy is not None and busy(n))
|
|
)
|
|
copy = source.model_copy(
|
|
update={
|
|
"id": self._issue(keys),
|
|
"name": new_name,
|
|
"title": f"{source.title} (copy)" if source.title else "",
|
|
"version": 1,
|
|
"origin": None,
|
|
}
|
|
)
|
|
copy = self._claim(copy, create=True)
|
|
# Its own messages named with its own key are the copy's now.
|
|
document = copy.model_dump(mode="json")
|
|
rebind(document, FLOW_FIELDS, Rename(source.name, new_name).binding)
|
|
copy = FlowDef.model_validate(document)
|
|
path = self._draft_file(copy.id)
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text(copy.model_dump_json(indent=2) + "\n")
|
|
|
|
# The bodies the source's editor shows: published, edits over them.
|
|
# As drafts, so publishing the copy promotes them like a new flow's.
|
|
target = self._draft_nodes_dir(copy.id)
|
|
for folder in (
|
|
self._flow_dir(flow_id) / "nodes",
|
|
self._draft_nodes_dir(flow_id),
|
|
):
|
|
for body in sorted(folder.glob("*.py")):
|
|
target.mkdir(exist_ok=True)
|
|
shutil.copy2(body, target / body.name)
|
|
|
|
self._commit(
|
|
f"Duplicate flow {named(source.name, flow_id)} as "
|
|
f"{named(new_name, copy.id)}"
|
|
)
|
|
return copy
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Node source
|
|
# -------------------------------------------------------------------------
|
|
|
|
def read_node_source(self, flow: str, node_id: str, draft: bool = False) -> str:
|
|
if draft:
|
|
path = self._draft_node_file(flow, node_id)
|
|
if path.exists():
|
|
return path.read_text()
|
|
path = self._node_file(flow, node_id)
|
|
return path.read_text() if path.exists() else DEFAULT_SOURCE
|
|
|
|
def has_node_source(self, flow: str, node_id: str, draft: bool = False) -> bool:
|
|
"""Whether a body was ever stored for this node.
|
|
|
|
`read_node_source` answers a template when there is none, which is what
|
|
a new node's editor should open with — but it is a placeholder, not
|
|
code anybody wrote, and telling the two apart is the difference between
|
|
"someone edited this" and "this was never written".
|
|
"""
|
|
if draft and self._draft_node_file(flow, node_id).exists():
|
|
return True
|
|
return self._node_file(flow, node_id).exists()
|
|
|
|
def write_node_source(
|
|
self, flow: str, node_id: str, code: str, draft: bool = False
|
|
) -> bool:
|
|
if draft:
|
|
# Compared against the working copy, so re-saving what is already
|
|
# published does not create a draft file that never gets promoted.
|
|
if self.read_node_source(flow, node_id, draft=True) == code:
|
|
return False
|
|
path = self._draft_node_file(flow, node_id)
|
|
else:
|
|
path = self._node_file(flow, node_id)
|
|
if path.exists() and path.read_text() == code:
|
|
return False
|
|
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(code)
|
|
self._commit(f"Update {'draft of ' if draft else ''}node '{flow}.{node_id}'")
|
|
return True
|
|
|
|
def delete_node_source(self, flow: str, node_id: str) -> None:
|
|
path = self._node_file(flow, node_id)
|
|
if path.exists():
|
|
path.unlink()
|
|
self._commit(f"Delete node '{flow}.{node_id}'")
|