Flows and dashboards get ids: the store is addressed by id with a key index, the first start gives every document its key as id, fluksio store migrate --dry-run and ls; the rename routes are gone
This commit is contained in:
+168
-60
@@ -10,6 +10,12 @@ Editing is separated from running. The editor writes ``flow.draft.json`` and
|
||||
``nodes/*.py``. Publishing promotes the draft over the published files and
|
||||
removes it, so a flow directory without draft files is simply one with nothing
|
||||
unpublished — which is what every flow written before this existed looks like.
|
||||
|
||||
A flow is two names. Its id is which flow it is: the directory, the argument
|
||||
every method here takes, and never changed once given. Its key, ``name``, is
|
||||
the namespace of its messages and unique among flows. A flow from before ids
|
||||
existed has its key as its id (`fluksio.flow.identity`), and so does every new
|
||||
one for now. Published and draft documents always share both.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -17,12 +23,15 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import string
|
||||
import subprocess
|
||||
import threading
|
||||
from collections.abc import Callable, Iterable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from fluksio.flow.schemas import FlowDef, _validate_name
|
||||
|
||||
@@ -66,6 +75,91 @@ def checked(name: str) -> str:
|
||||
raise InvalidName(name) from None
|
||||
|
||||
|
||||
#: The shape of an id the server issues: a kind, then 40 random bits.
|
||||
ISSUED_ID = re.compile(r"[fd]_[a-z2-7]{8}")
|
||||
|
||||
|
||||
class ReservedKey(InvalidName):
|
||||
"""A new key shaped like an issued id, which it could one day equal."""
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f"'{self.name}' has the shape of a generated id, f_ or d_ and eight "
|
||||
"letters or digits, which a new key may not have"
|
||||
)
|
||||
|
||||
|
||||
class IdentityChange(ValueError):
|
||||
"""A save that would change which object a document is, or its key.
|
||||
|
||||
Ids never change, and a key changes only by renaming, which moves its
|
||||
messages along with it. A save that did either would bypass that.
|
||||
"""
|
||||
|
||||
|
||||
def claim(
|
||||
obj_id: str,
|
||||
key: str,
|
||||
current: tuple[str, str] | None,
|
||||
keys: dict[str, str],
|
||||
taken: type[ValueError],
|
||||
) -> None:
|
||||
"""Refuse a write that changes who an object is, or takes another's key.
|
||||
|
||||
*current* is the stored ``(id, name)`` at *obj_id*, None for a new object,
|
||||
and *keys* the store's key index. Called under the store's write lock, so
|
||||
two creates of one key cannot both pass. *taken* is what a key already in
|
||||
use raises: `FlowExists`, or the dashboards' own.
|
||||
"""
|
||||
if current is not None:
|
||||
stored_id, stored_key = current
|
||||
if stored_id != obj_id:
|
||||
raise IdentityChange(f"'{obj_id}' holds a document of '{stored_id}'")
|
||||
if stored_key != key:
|
||||
raise IdentityChange(
|
||||
f"'{obj_id}' has the key '{stored_key}', which a save cannot "
|
||||
f"change to '{key}'"
|
||||
)
|
||||
return
|
||||
# New keys only: one this shape that exists already stays saveable.
|
||||
if ISSUED_ID.fullmatch(key):
|
||||
raise ReservedKey(key)
|
||||
if keys.get(key, obj_id) != obj_id:
|
||||
raise taken(key)
|
||||
|
||||
|
||||
def read_keys(directories: Iterable[Path], document: str) -> dict[str, str]:
|
||||
"""The key each directory's object holds, to the directory: key to id.
|
||||
|
||||
Read from the working copy, ``<document>.draft.json`` when there is one, so
|
||||
an object that is only a draft holds its key as much as a published one.
|
||||
"""
|
||||
keys: dict[str, str] = {}
|
||||
for directory in directories:
|
||||
draft = directory / f"{document}.draft.json"
|
||||
path = draft if draft.exists() else directory / f"{document}.json"
|
||||
try:
|
||||
keys.setdefault(json.loads(path.read_text())["name"], directory.name)
|
||||
except (OSError, ValueError, KeyError, TypeError):
|
||||
# Unreadable to the engine too, which reports it when it loads.
|
||||
continue
|
||||
return keys
|
||||
|
||||
|
||||
def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str:
|
||||
"""A fresh id for a flow (``f``) or a dashboard (``d``), e.g. ``f_k3q7m2xa``.
|
||||
|
||||
Random is not unique, so *taken* decides, under the store's write lock: it
|
||||
has to know every live id, every directory left behind and every key,
|
||||
since a key equal to an id makes an argument naming either ambiguous.
|
||||
"""
|
||||
alphabet = string.ascii_lowercase + "234567"
|
||||
while True:
|
||||
candidate = f"{kind}_" + "".join(secrets.choice(alphabet) for _ in range(8))
|
||||
if not taken(candidate):
|
||||
return candidate
|
||||
|
||||
|
||||
class FlowNotFound(KeyError):
|
||||
def __init__(self, name: str) -> None:
|
||||
super().__init__(name)
|
||||
@@ -174,6 +268,8 @@ class FlowStore:
|
||||
self._read_all: list[FlowDef] = []
|
||||
self._read_all_at = -1
|
||||
self._read_all_lock = threading.Lock()
|
||||
#: `key_index` and the revision it was read at.
|
||||
self._keys: tuple[int, dict[str, str]] = (-1, {})
|
||||
if not (self.root / ".git").exists():
|
||||
self._git("init", "-q")
|
||||
self._commit("Initialise flow store", allow_empty=True)
|
||||
@@ -441,7 +537,7 @@ class FlowStore:
|
||||
):
|
||||
if not path.exists():
|
||||
continue
|
||||
definition = FlowDef.model_validate_json(path.read_text())
|
||||
definition = self._load(path)
|
||||
users = [n for n in definition.nodes if n.source_ref == name]
|
||||
if not users:
|
||||
continue
|
||||
@@ -549,16 +645,56 @@ class FlowStore:
|
||||
+ list(self._draft_nodes_dir(name).glob("*.py"))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _load(path: Path) -> FlowDef:
|
||||
"""A stored document, with the id one from before ids is known by."""
|
||||
flow = FlowDef.model_validate_json(path.read_text())
|
||||
if not flow.id:
|
||||
flow.id = path.parent.name
|
||||
return flow
|
||||
|
||||
def read_flow(self, name: str, draft: bool = False) -> FlowDef:
|
||||
"""The published flow, or with ``draft`` the working copy."""
|
||||
if draft:
|
||||
path = self._draft_file(name)
|
||||
if path.exists():
|
||||
return FlowDef.model_validate_json(path.read_text())
|
||||
return self._load(path)
|
||||
path = self._flow_file(name)
|
||||
if not path.exists():
|
||||
raise FlowNotFound(name)
|
||||
return FlowDef.model_validate_json(path.read_text())
|
||||
return self._load(path)
|
||||
|
||||
def key_index(self) -> dict[str, str]:
|
||||
"""Every flow's key, to its id, whether published or only a draft.
|
||||
|
||||
Read again after any commit, which every write makes — a draft-only
|
||||
create or a delete as much as a save. Shared: read it, never change it.
|
||||
"""
|
||||
at, keys = self._keys
|
||||
if at == self.revision:
|
||||
return keys
|
||||
# Taken before reading, so a commit landing meanwhile reads it again.
|
||||
at = self.revision
|
||||
keys = read_keys((self.root / n for n in self.list_flows()), "flow")
|
||||
self._keys = (at, keys)
|
||||
return keys
|
||||
|
||||
def _claim(self, flow: FlowDef) -> FlowDef:
|
||||
"""*flow* with its id, once it is certain to be a save the store allows.
|
||||
|
||||
Under `_write_lock`. An id left out is the key, which is how every
|
||||
flow is created for now.
|
||||
"""
|
||||
flow_id = checked(flow.id or flow.name)
|
||||
current = self.read_flow(flow_id, draft=True) if self.exists(flow_id) else None
|
||||
claim(
|
||||
flow_id,
|
||||
flow.name,
|
||||
(current.id, current.name) if current else None,
|
||||
self.key_index(),
|
||||
FlowExists,
|
||||
)
|
||||
return flow.model_copy(update={"id": flow_id})
|
||||
|
||||
def read_all(self) -> list[FlowDef]:
|
||||
"""Every published flow — what the engine runs.
|
||||
@@ -583,15 +719,17 @@ class FlowStore:
|
||||
|
||||
def write_flow(self, flow: FlowDef) -> bool:
|
||||
"""Publish a flow directly. Returns False when nothing actually changed."""
|
||||
path = self._flow_file(flow.name)
|
||||
content = flow.model_dump_json(indent=2) + "\n"
|
||||
if path.exists() and path.read_text() == content:
|
||||
return False
|
||||
with self._write_lock:
|
||||
flow = self._claim(flow)
|
||||
path = self._flow_file(flow.id)
|
||||
content = flow.model_dump_json(indent=2) + "\n"
|
||||
if path.exists() and path.read_text() == content:
|
||||
return False
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content)
|
||||
self._commit(f"Update flow '{flow.name}'")
|
||||
return True
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content)
|
||||
self._commit(f"Update flow '{flow.name}'")
|
||||
return True
|
||||
|
||||
def write_draft(self, flow: FlowDef, base_version: int) -> FlowDef:
|
||||
"""Save unpublished changes, refusing to overwrite someone else's.
|
||||
@@ -601,9 +739,10 @@ class FlowStore:
|
||||
write would silently discard their work.
|
||||
"""
|
||||
with self._write_lock:
|
||||
flow = self._claim(flow)
|
||||
current: FlowDef | None = None
|
||||
if self.exists(flow.name):
|
||||
current = self.read_flow(flow.name, draft=True)
|
||||
if self.exists(flow.id):
|
||||
current = self.read_flow(flow.id, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(flow.name, current.version)
|
||||
if _same_content(current, flow):
|
||||
@@ -612,7 +751,7 @@ class FlowStore:
|
||||
stored = flow.model_copy(
|
||||
update={"version": (current.version if current else 0) + 1}
|
||||
)
|
||||
path = self._draft_file(flow.name)
|
||||
path = self._draft_file(flow.id)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(stored.model_dump_json(indent=2) + "\n")
|
||||
self._commit(f"Update draft of flow '{flow.name}'")
|
||||
@@ -656,48 +795,12 @@ class FlowStore:
|
||||
return self.read_flow(name)
|
||||
|
||||
def delete_flow(self, name: str) -> None:
|
||||
directory = self._flow_dir(name)
|
||||
if not directory.exists():
|
||||
raise FlowNotFound(name)
|
||||
shutil.rmtree(directory)
|
||||
self._commit(f"Delete flow '{name}'")
|
||||
|
||||
def rename_flow(self, name: str, new_name: str) -> FlowDef:
|
||||
"""Rename a flow, carrying its nodes and any references to it.
|
||||
|
||||
A flow's name is the namespace of its messages, so other flows reading
|
||||
``old.temperature`` are rewritten to read ``new.temperature`` — leaving
|
||||
them pointing at a flow that no longer exists would break them silently.
|
||||
"""
|
||||
if not self.exists(name):
|
||||
raise FlowNotFound(name)
|
||||
if self.exists(new_name):
|
||||
raise FlowExists(new_name)
|
||||
|
||||
self._flow_dir(name).rename(self._flow_dir(new_name))
|
||||
|
||||
# Published and draft alike carry the name, and both have to move.
|
||||
for path in (self._flow_file(new_name), self._draft_file(new_name)):
|
||||
if not path.exists():
|
||||
continue
|
||||
flow = FlowDef.model_validate_json(path.read_text())
|
||||
path.write_text(
|
||||
flow.model_copy(update={"name": new_name}).model_dump_json(indent=2)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
for other in self.list_flows():
|
||||
if other == new_name:
|
||||
continue
|
||||
for path in (self._flow_file(other), self._draft_file(other)):
|
||||
if not path.exists():
|
||||
continue
|
||||
flow = FlowDef.model_validate_json(path.read_text())
|
||||
if self._retarget(flow, f"{name}.", f"{new_name}."):
|
||||
path.write_text(flow.model_dump_json(indent=2) + "\n")
|
||||
|
||||
self._commit(f"Rename flow '{name}' to '{new_name}'")
|
||||
return self.read_flow(new_name, draft=True)
|
||||
with self._write_lock:
|
||||
directory = self._flow_dir(name)
|
||||
if not directory.exists():
|
||||
raise FlowNotFound(name)
|
||||
shutil.rmtree(directory)
|
||||
self._commit(f"Delete flow '{name}'")
|
||||
|
||||
def duplicate_flow(self, name: str) -> FlowDef:
|
||||
"""Copy a flow's working copy into a new flow that is only a draft.
|
||||
@@ -710,19 +813,24 @@ class FlowStore:
|
||||
with self._write_lock:
|
||||
source = self.read_flow(name, draft=True)
|
||||
# The directory rather than `exists`: anything left in it would
|
||||
# otherwise end up in the copy.
|
||||
new_name = copy_name(name, lambda n: self._flow_dir(n).exists())
|
||||
# otherwise end up in the copy. The copy's id is its key, so that
|
||||
# has to be free among the keys too.
|
||||
keys = self.key_index()
|
||||
new_name = copy_name(
|
||||
source.name, lambda n: self._flow_dir(n).exists() or n in keys
|
||||
)
|
||||
|
||||
copy = source.model_copy(
|
||||
update={
|
||||
"id": new_name,
|
||||
"name": new_name,
|
||||
"title": f"{source.title} (copy)" if source.title else "",
|
||||
"version": 1,
|
||||
"origin": None,
|
||||
}
|
||||
)
|
||||
# Its own messages named with its own name are the copy's now.
|
||||
self._retarget(copy, f"{name}.", f"{new_name}.")
|
||||
# Its own messages named with its own key are the copy's now.
|
||||
self._retarget(copy, f"{source.name}.", f"{new_name}.")
|
||||
path = self._draft_file(new_name)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(copy.model_dump_json(indent=2) + "\n")
|
||||
|
||||
Reference in New Issue
Block a user