Flows and dashboards get ids: the store is addressed by id with a key index, the first start gives every document its key as id, fluksio store migrate --dry-run and ls; the rename routes are gone

This commit is contained in:
2026-11-02 16:54:11 +01:00
parent de13c48c90
commit fae85f5ecb
20 changed files with 1287 additions and 420 deletions
+168 -60
View File
@@ -10,6 +10,12 @@ Editing is separated from running. The editor writes ``flow.draft.json`` and
``nodes/*.py``. Publishing promotes the draft over the published files and
removes it, so a flow directory without draft files is simply one with nothing
unpublished — which is what every flow written before this existed looks like.
A flow is two names. Its id is which flow it is: the directory, the argument
every method here takes, and never changed once given. Its key, ``name``, is
the namespace of its messages and unique among flows. A flow from before ids
existed has its key as its id (`fluksio.flow.identity`), and so does every new
one for now. Published and draft documents always share both.
"""
from __future__ import annotations
@@ -17,12 +23,15 @@ from __future__ import annotations
import json
import logging
import re
import secrets
import shutil
import string
import subprocess
import threading
from collections.abc import Callable, Iterable
from datetime import UTC, datetime
from pathlib import Path
from typing import Literal
from fluksio.flow.schemas import FlowDef, _validate_name
@@ -66,6 +75,91 @@ def checked(name: str) -> str:
raise InvalidName(name) from None
#: The shape of an id the server issues: a kind, then 40 random bits.
ISSUED_ID = re.compile(r"[fd]_[a-z2-7]{8}")
class ReservedKey(InvalidName):
"""A new key shaped like an issued id, which it could one day equal."""
def __str__(self) -> str:
return (
f"'{self.name}' has the shape of a generated id, f_ or d_ and eight "
"letters or digits, which a new key may not have"
)
class IdentityChange(ValueError):
"""A save that would change which object a document is, or its key.
Ids never change, and a key changes only by renaming, which moves its
messages along with it. A save that did either would bypass that.
"""
def claim(
obj_id: str,
key: str,
current: tuple[str, str] | None,
keys: dict[str, str],
taken: type[ValueError],
) -> None:
"""Refuse a write that changes who an object is, or takes another's key.
*current* is the stored ``(id, name)`` at *obj_id*, None for a new object,
and *keys* the store's key index. Called under the store's write lock, so
two creates of one key cannot both pass. *taken* is what a key already in
use raises: `FlowExists`, or the dashboards' own.
"""
if current is not None:
stored_id, stored_key = current
if stored_id != obj_id:
raise IdentityChange(f"'{obj_id}' holds a document of '{stored_id}'")
if stored_key != key:
raise IdentityChange(
f"'{obj_id}' has the key '{stored_key}', which a save cannot "
f"change to '{key}'"
)
return
# New keys only: one this shape that exists already stays saveable.
if ISSUED_ID.fullmatch(key):
raise ReservedKey(key)
if keys.get(key, obj_id) != obj_id:
raise taken(key)
def read_keys(directories: Iterable[Path], document: str) -> dict[str, str]:
"""The key each directory's object holds, to the directory: key to id.
Read from the working copy, ``<document>.draft.json`` when there is one, so
an object that is only a draft holds its key as much as a published one.
"""
keys: dict[str, str] = {}
for directory in directories:
draft = directory / f"{document}.draft.json"
path = draft if draft.exists() else directory / f"{document}.json"
try:
keys.setdefault(json.loads(path.read_text())["name"], directory.name)
except (OSError, ValueError, KeyError, TypeError):
# Unreadable to the engine too, which reports it when it loads.
continue
return keys
def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str:
"""A fresh id for a flow (``f``) or a dashboard (``d``), e.g. ``f_k3q7m2xa``.
Random is not unique, so *taken* decides, under the store's write lock: it
has to know every live id, every directory left behind and every key,
since a key equal to an id makes an argument naming either ambiguous.
"""
alphabet = string.ascii_lowercase + "234567"
while True:
candidate = f"{kind}_" + "".join(secrets.choice(alphabet) for _ in range(8))
if not taken(candidate):
return candidate
class FlowNotFound(KeyError):
def __init__(self, name: str) -> None:
super().__init__(name)
@@ -174,6 +268,8 @@ class FlowStore:
self._read_all: list[FlowDef] = []
self._read_all_at = -1
self._read_all_lock = threading.Lock()
#: `key_index` and the revision it was read at.
self._keys: tuple[int, dict[str, str]] = (-1, {})
if not (self.root / ".git").exists():
self._git("init", "-q")
self._commit("Initialise flow store", allow_empty=True)
@@ -441,7 +537,7 @@ class FlowStore:
):
if not path.exists():
continue
definition = FlowDef.model_validate_json(path.read_text())
definition = self._load(path)
users = [n for n in definition.nodes if n.source_ref == name]
if not users:
continue
@@ -549,16 +645,56 @@ class FlowStore:
+ list(self._draft_nodes_dir(name).glob("*.py"))
)
@staticmethod
def _load(path: Path) -> FlowDef:
"""A stored document, with the id one from before ids is known by."""
flow = FlowDef.model_validate_json(path.read_text())
if not flow.id:
flow.id = path.parent.name
return flow
def read_flow(self, name: str, draft: bool = False) -> FlowDef:
"""The published flow, or with ``draft`` the working copy."""
if draft:
path = self._draft_file(name)
if path.exists():
return FlowDef.model_validate_json(path.read_text())
return self._load(path)
path = self._flow_file(name)
if not path.exists():
raise FlowNotFound(name)
return FlowDef.model_validate_json(path.read_text())
return self._load(path)
def key_index(self) -> dict[str, str]:
"""Every flow's key, to its id, whether published or only a draft.
Read again after any commit, which every write makes — a draft-only
create or a delete as much as a save. Shared: read it, never change it.
"""
at, keys = self._keys
if at == self.revision:
return keys
# Taken before reading, so a commit landing meanwhile reads it again.
at = self.revision
keys = read_keys((self.root / n for n in self.list_flows()), "flow")
self._keys = (at, keys)
return keys
def _claim(self, flow: FlowDef) -> FlowDef:
"""*flow* with its id, once it is certain to be a save the store allows.
Under `_write_lock`. An id left out is the key, which is how every
flow is created for now.
"""
flow_id = checked(flow.id or flow.name)
current = self.read_flow(flow_id, draft=True) if self.exists(flow_id) else None
claim(
flow_id,
flow.name,
(current.id, current.name) if current else None,
self.key_index(),
FlowExists,
)
return flow.model_copy(update={"id": flow_id})
def read_all(self) -> list[FlowDef]:
"""Every published flow — what the engine runs.
@@ -583,15 +719,17 @@ class FlowStore:
def write_flow(self, flow: FlowDef) -> bool:
"""Publish a flow directly. Returns False when nothing actually changed."""
path = self._flow_file(flow.name)
content = flow.model_dump_json(indent=2) + "\n"
if path.exists() and path.read_text() == content:
return False
with self._write_lock:
flow = self._claim(flow)
path = self._flow_file(flow.id)
content = flow.model_dump_json(indent=2) + "\n"
if path.exists() and path.read_text() == content:
return False
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content)
self._commit(f"Update flow '{flow.name}'")
return True
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content)
self._commit(f"Update flow '{flow.name}'")
return True
def write_draft(self, flow: FlowDef, base_version: int) -> FlowDef:
"""Save unpublished changes, refusing to overwrite someone else's.
@@ -601,9 +739,10 @@ class FlowStore:
write would silently discard their work.
"""
with self._write_lock:
flow = self._claim(flow)
current: FlowDef | None = None
if self.exists(flow.name):
current = self.read_flow(flow.name, draft=True)
if self.exists(flow.id):
current = self.read_flow(flow.id, draft=True)
if base_version != current.version:
raise StaleVersion(flow.name, current.version)
if _same_content(current, flow):
@@ -612,7 +751,7 @@ class FlowStore:
stored = flow.model_copy(
update={"version": (current.version if current else 0) + 1}
)
path = self._draft_file(flow.name)
path = self._draft_file(flow.id)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(stored.model_dump_json(indent=2) + "\n")
self._commit(f"Update draft of flow '{flow.name}'")
@@ -656,48 +795,12 @@ class FlowStore:
return self.read_flow(name)
def delete_flow(self, name: str) -> None:
directory = self._flow_dir(name)
if not directory.exists():
raise FlowNotFound(name)
shutil.rmtree(directory)
self._commit(f"Delete flow '{name}'")
def rename_flow(self, name: str, new_name: str) -> FlowDef:
"""Rename a flow, carrying its nodes and any references to it.
A flow's name is the namespace of its messages, so other flows reading
``old.temperature`` are rewritten to read ``new.temperature`` — leaving
them pointing at a flow that no longer exists would break them silently.
"""
if not self.exists(name):
raise FlowNotFound(name)
if self.exists(new_name):
raise FlowExists(new_name)
self._flow_dir(name).rename(self._flow_dir(new_name))
# Published and draft alike carry the name, and both have to move.
for path in (self._flow_file(new_name), self._draft_file(new_name)):
if not path.exists():
continue
flow = FlowDef.model_validate_json(path.read_text())
path.write_text(
flow.model_copy(update={"name": new_name}).model_dump_json(indent=2)
+ "\n"
)
for other in self.list_flows():
if other == new_name:
continue
for path in (self._flow_file(other), self._draft_file(other)):
if not path.exists():
continue
flow = FlowDef.model_validate_json(path.read_text())
if self._retarget(flow, f"{name}.", f"{new_name}."):
path.write_text(flow.model_dump_json(indent=2) + "\n")
self._commit(f"Rename flow '{name}' to '{new_name}'")
return self.read_flow(new_name, draft=True)
with self._write_lock:
directory = self._flow_dir(name)
if not directory.exists():
raise FlowNotFound(name)
shutil.rmtree(directory)
self._commit(f"Delete flow '{name}'")
def duplicate_flow(self, name: str) -> FlowDef:
"""Copy a flow's working copy into a new flow that is only a draft.
@@ -710,19 +813,24 @@ class FlowStore:
with self._write_lock:
source = self.read_flow(name, draft=True)
# The directory rather than `exists`: anything left in it would
# otherwise end up in the copy.
new_name = copy_name(name, lambda n: self._flow_dir(n).exists())
# otherwise end up in the copy. The copy's id is its key, so that
# has to be free among the keys too.
keys = self.key_index()
new_name = copy_name(
source.name, lambda n: self._flow_dir(n).exists() or n in keys
)
copy = source.model_copy(
update={
"id": new_name,
"name": new_name,
"title": f"{source.title} (copy)" if source.title else "",
"version": 1,
"origin": None,
}
)
# Its own messages named with its own name are the copy's now.
self._retarget(copy, f"{name}.", f"{new_name}.")
# Its own messages named with its own key are the copy's now.
self._retarget(copy, f"{source.name}.", f"{new_name}.")
path = self._draft_file(new_name)
path.parent.mkdir(parents=True)
path.write_text(copy.model_dump_json(indent=2) + "\n")