Flows and dashboards are created by POST under an issued or superuser-pinned id; PUT only updates; deleted flows leave a tombstone with their key; the controller stores by id and runs by key, hooks live under /hooks/<flow id>; a key with state or queued work behind it is refused until cleared
This commit is contained in:
+208
-68
@@ -14,8 +14,9 @@ unpublished — which is what every flow written before this existed looks like.
|
||||
A flow is two names. Its id is which flow it is: the directory, the argument
|
||||
every method here takes, and never changed once given. Its key, ``name``, is
|
||||
the namespace of its messages and unique among flows. A flow from before ids
|
||||
existed has its key as its id (`fluksio.flow.identity`), and so does every new
|
||||
one for now. Published and draft documents always share both.
|
||||
existed has its key as its id (`fluksio.flow.identity`). A new one is given an
|
||||
opaque id by `create`, or the one a superuser pinned. Published and draft
|
||||
documents always share both.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -31,10 +32,13 @@ import threading
|
||||
from collections.abc import Callable, Iterable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
from fluksio.flow.schemas import FlowDef, _validate_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fluksio.flow.tombstones import Tombstones
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Where the sources of nodes shared between flows live.
|
||||
@@ -97,6 +101,37 @@ class IdentityChange(ValueError):
|
||||
"""
|
||||
|
||||
|
||||
class RecordedKey(IdentityChange):
|
||||
"""A pinned id restored under another key than its history is filed under.
|
||||
|
||||
The history names the old key's messages, so taking a new key with it is
|
||||
a rename, not a create.
|
||||
"""
|
||||
|
||||
def __init__(self, obj_id: str, key: str) -> None:
|
||||
super().__init__(obj_id)
|
||||
self.id = obj_id
|
||||
self.key = key
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f"'{self.id}' has history filed under the key '{self.key}', so it "
|
||||
"can only be restored with that key"
|
||||
)
|
||||
|
||||
|
||||
class IdTaken(ValueError):
|
||||
"""A pinned id that is in use: an object's, a directory's or another key."""
|
||||
|
||||
def __init__(self, obj_id: str, kind: str = "flow") -> None:
|
||||
super().__init__(obj_id)
|
||||
self.id = obj_id
|
||||
self.kind = kind
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"The id '{self.id}' is already taken by a {self.kind}"
|
||||
|
||||
|
||||
def claim(
|
||||
obj_id: str,
|
||||
key: str,
|
||||
@@ -146,6 +181,11 @@ def read_keys(directories: Iterable[Path], document: str) -> dict[str, str]:
|
||||
return keys
|
||||
|
||||
|
||||
def named(key: str, obj_id: str) -> str:
|
||||
"""How a commit message names an object: by key, and by id when it differs."""
|
||||
return f"'{key}'" if key == obj_id else f"'{key}' ({obj_id})"
|
||||
|
||||
|
||||
def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str:
|
||||
"""A fresh id for a flow (``f``) or a dashboard (``d``), e.g. ``f_k3q7m2xa``.
|
||||
|
||||
@@ -222,6 +262,14 @@ def _updated_at(paths: Iterable[Path]) -> datetime | None:
|
||||
return datetime.fromtimestamp(max(stamps), UTC) if stamps else None
|
||||
|
||||
|
||||
def _version(path: Path) -> int:
|
||||
"""The version a stored document carries, 0 for none or an unreadable one."""
|
||||
try:
|
||||
return int(json.loads(path.read_text()).get("version", 1))
|
||||
except (OSError, ValueError, TypeError, AttributeError):
|
||||
return 0
|
||||
|
||||
|
||||
def _same_content(left: FlowDef, right: FlowDef) -> bool:
|
||||
"""Equal but for the version counter, which the server owns."""
|
||||
return left.model_copy(update={"version": 0}) == right.model_copy(
|
||||
@@ -252,8 +300,11 @@ class FlowStore:
|
||||
#: a store reached 1898 commits, 5863 loose objects and no pack at all.
|
||||
_GC_EVERY = 500
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
def __init__(self, root: Path, tombstones: Tombstones | None = None) -> None:
|
||||
self.root = root
|
||||
#: Where a deleted flow's key is kept, and what history an id has.
|
||||
#: None leaves both out: a store without a database, as in tests.
|
||||
self.tombstones = tombstones
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
# Draft writes are check-and-set, so two clients saving at once must not
|
||||
# interleave between reading the current version and writing the next.
|
||||
@@ -679,14 +730,16 @@ class FlowStore:
|
||||
self._keys = (at, keys)
|
||||
return keys
|
||||
|
||||
def _claim(self, flow: FlowDef) -> FlowDef:
|
||||
def _claim(self, flow: FlowDef, create: bool = False) -> FlowDef:
|
||||
"""*flow* with its id, once it is certain to be a save the store allows.
|
||||
|
||||
Under `_write_lock`. An id left out is the key, which is how every
|
||||
flow is created for now.
|
||||
Under `_write_lock`. A save is to a flow that exists, named by its id
|
||||
or, from before ids, by its key. Only *create* may make a new one.
|
||||
"""
|
||||
flow_id = checked(flow.id or flow.name)
|
||||
current = self.read_flow(flow_id, draft=True) if self.exists(flow_id) else None
|
||||
if current is None and not create:
|
||||
raise FlowNotFound(flow_id)
|
||||
claim(
|
||||
flow_id,
|
||||
flow.name,
|
||||
@@ -696,6 +749,22 @@ class FlowStore:
|
||||
)
|
||||
return flow.model_copy(update={"id": flow_id})
|
||||
|
||||
def _issue(self, keys: dict[str, str]) -> str:
|
||||
"""A fresh flow id. Under `_write_lock`, with *keys* the key index.
|
||||
|
||||
Free as a directory, a key and in history: an id a deleted flow's runs
|
||||
are still filed under would hand them to the new one.
|
||||
"""
|
||||
graves = self.tombstones
|
||||
return new_id(
|
||||
"f",
|
||||
lambda c: (
|
||||
self._flow_dir(c).exists()
|
||||
or c in keys
|
||||
or (graves is not None and graves.recorded(c) is not None)
|
||||
),
|
||||
)
|
||||
|
||||
def read_all(self) -> list[FlowDef]:
|
||||
"""Every published flow — what the engine runs.
|
||||
|
||||
@@ -718,9 +787,14 @@ class FlowStore:
|
||||
return list(self._read_all)
|
||||
|
||||
def write_flow(self, flow: FlowDef) -> bool:
|
||||
"""Publish a flow directly. Returns False when nothing actually changed."""
|
||||
"""Publish a flow directly. Returns False when nothing actually changed.
|
||||
|
||||
Creates one too, under the id it carries or else its key: what a test
|
||||
or a seed holding a finished document wants. The API creates only
|
||||
through `create`.
|
||||
"""
|
||||
with self._write_lock:
|
||||
flow = self._claim(flow)
|
||||
flow = self._claim(flow, create=True)
|
||||
path = self._flow_file(flow.id)
|
||||
content = flow.model_dump_json(indent=2) + "\n"
|
||||
if path.exists() and path.read_text() == content:
|
||||
@@ -728,122 +802,188 @@ class FlowStore:
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content)
|
||||
self._commit(f"Update flow '{flow.name}'")
|
||||
self._commit(f"Update flow {named(flow.name, flow.id)}")
|
||||
return True
|
||||
|
||||
def create(self, flow: FlowDef, pin: str = "") -> FlowDef:
|
||||
"""A new flow, only a draft until someone publishes it.
|
||||
|
||||
Its id is issued here unless *pin* names one, which is a superuser's
|
||||
to ask for: the Demo, a test, a restore. A pinned id whose history
|
||||
outlived its flow adopts that history, so it has to take the key the
|
||||
history is filed under, and its tombstone goes once the document is on
|
||||
disk.
|
||||
"""
|
||||
with self._write_lock:
|
||||
keys = self.key_index()
|
||||
if pin:
|
||||
flow_id = checked(pin)
|
||||
# The directory rather than `exists`: anything left in it would
|
||||
# become the new flow's.
|
||||
if self._flow_dir(flow_id).exists():
|
||||
raise IdTaken(flow_id)
|
||||
recorded = (
|
||||
self.tombstones.recorded(flow_id) if self.tombstones else None
|
||||
)
|
||||
if recorded is not None and recorded != flow.name:
|
||||
raise RecordedKey(flow_id, recorded)
|
||||
else:
|
||||
flow_id = self._issue(keys)
|
||||
flow = self._claim(flow.model_copy(update={"id": flow_id}), create=True)
|
||||
# Another flow's key as this one's id would make naming either
|
||||
# ambiguous. Checked after the key, which is the likelier mistake.
|
||||
if keys.get(flow_id, flow_id) != flow_id:
|
||||
raise IdTaken(flow_id)
|
||||
|
||||
stored = flow.model_copy(update={"version": 1})
|
||||
path = self._draft_file(flow_id)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(stored.model_dump_json(indent=2) + "\n")
|
||||
self._commit(f"Create flow {named(flow.name, flow_id)}")
|
||||
if pin and self.tombstones is not None:
|
||||
self.tombstones.lift(flow_id)
|
||||
return stored
|
||||
|
||||
def write_draft(self, flow: FlowDef, base_version: int) -> FlowDef:
|
||||
"""Save unpublished changes, refusing to overwrite someone else's.
|
||||
|
||||
``base_version`` is the version the editor last saw. It has to match
|
||||
what is stored, otherwise another client saved in between and this
|
||||
write would silently discard their work.
|
||||
write would silently discard their work. The flow has to exist:
|
||||
creating one is `create`.
|
||||
"""
|
||||
with self._write_lock:
|
||||
flow = self._claim(flow)
|
||||
current: FlowDef | None = None
|
||||
if self.exists(flow.id):
|
||||
current = self.read_flow(flow.id, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(flow.name, current.version)
|
||||
if _same_content(current, flow):
|
||||
return current
|
||||
current = self.read_flow(flow.id, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(flow.name, current.version)
|
||||
if _same_content(current, flow):
|
||||
return current
|
||||
|
||||
stored = flow.model_copy(
|
||||
update={"version": (current.version if current else 0) + 1}
|
||||
)
|
||||
stored = flow.model_copy(update={"version": current.version + 1})
|
||||
path = self._draft_file(flow.id)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(stored.model_dump_json(indent=2) + "\n")
|
||||
self._commit(f"Update draft of flow '{flow.name}'")
|
||||
self._commit(f"Update draft of flow {named(flow.name, flow.id)}")
|
||||
return stored
|
||||
|
||||
def publish_flow(self, name: str, base_version: int) -> FlowDef:
|
||||
def publish_flow(self, flow_id: str, base_version: int) -> FlowDef:
|
||||
"""Promote the working copy to what the engine runs."""
|
||||
with self._write_lock:
|
||||
current = self.read_flow(name, draft=True)
|
||||
current = self.read_flow(flow_id, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(name, current.version)
|
||||
raise StaleVersion(current.name, current.version)
|
||||
|
||||
draft = self._draft_file(name)
|
||||
draft = self._draft_file(flow_id)
|
||||
if draft.exists():
|
||||
self._flow_file(name).write_text(
|
||||
self._flow_file(flow_id).write_text(
|
||||
current.model_dump_json(indent=2) + "\n"
|
||||
)
|
||||
draft.unlink()
|
||||
|
||||
drafts = self._draft_nodes_dir(name)
|
||||
drafts = self._draft_nodes_dir(flow_id)
|
||||
if drafts.exists():
|
||||
target = self._flow_dir(name) / "nodes"
|
||||
target = self._flow_dir(flow_id) / "nodes"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for source in sorted(drafts.glob("*.py")):
|
||||
shutil.copy2(source, target / source.name)
|
||||
shutil.rmtree(drafts)
|
||||
|
||||
self._commit(f"Publish flow '{name}'")
|
||||
self._commit(f"Publish flow {named(current.name, flow_id)}")
|
||||
return current
|
||||
|
||||
def discard_draft(self, name: str) -> FlowDef:
|
||||
"""Throw the unpublished changes away and go back to what is running."""
|
||||
with self._write_lock:
|
||||
draft = self._draft_file(name)
|
||||
if draft.exists():
|
||||
draft.unlink()
|
||||
drafts = self._draft_nodes_dir(name)
|
||||
if drafts.exists():
|
||||
shutil.rmtree(drafts)
|
||||
self._commit(f"Discard draft of flow '{name}'")
|
||||
return self.read_flow(name)
|
||||
def discard_draft(self, flow_id: str) -> FlowDef:
|
||||
"""Throw the unpublished changes away and go back to what is running.
|
||||
|
||||
def delete_flow(self, name: str) -> None:
|
||||
with self._write_lock:
|
||||
directory = self._flow_dir(name)
|
||||
if not directory.exists():
|
||||
raise FlowNotFound(name)
|
||||
shutil.rmtree(directory)
|
||||
self._commit(f"Delete flow '{name}'")
|
||||
|
||||
def duplicate_flow(self, name: str) -> FlowDef:
|
||||
"""Copy a flow's working copy into a new flow that is only a draft.
|
||||
|
||||
Like a new flow, the copy runs nothing until someone publishes it. It is
|
||||
named ``{name}_copy``, then ``_copy_2`` and on. ``origin`` is dropped,
|
||||
since the copy is a local flow ``fluksio sync`` must not claim, and
|
||||
``runtime.json`` stays behind with everything else about running.
|
||||
What is running comes back a version above the draft, so an editor
|
||||
still holding the draft is refused rather than matching the next save
|
||||
made from the published copy.
|
||||
"""
|
||||
with self._write_lock:
|
||||
source = self.read_flow(name, draft=True)
|
||||
# The directory rather than `exists`: anything left in it would
|
||||
# otherwise end up in the copy. The copy's id is its key, so that
|
||||
# has to be free among the keys too.
|
||||
draft = self._draft_file(flow_id)
|
||||
if draft.exists():
|
||||
running = self.read_flow(flow_id)
|
||||
top = _version(draft)
|
||||
if running.version <= top:
|
||||
running.version = top + 1
|
||||
self._flow_file(flow_id).write_text(
|
||||
running.model_dump_json(indent=2) + "\n"
|
||||
)
|
||||
draft.unlink()
|
||||
drafts = self._draft_nodes_dir(flow_id)
|
||||
if drafts.exists():
|
||||
shutil.rmtree(drafts)
|
||||
published = self.read_flow(flow_id)
|
||||
self._commit(f"Discard draft of flow {named(published.name, flow_id)}")
|
||||
return published
|
||||
|
||||
def delete_flow(self, flow_id: str) -> str:
|
||||
"""Remove a flow's files. Returns the key it had.
|
||||
|
||||
The key is buried first, so a delete that stops half-way never leaves
|
||||
history without a name, and the live document wins while it is there.
|
||||
"""
|
||||
with self._write_lock:
|
||||
directory = self._flow_dir(flow_id)
|
||||
if not directory.exists():
|
||||
raise FlowNotFound(flow_id)
|
||||
key = next(
|
||||
(k for k, i in self.key_index().items() if i == flow_id), flow_id
|
||||
)
|
||||
if self.tombstones is not None:
|
||||
self.tombstones.bury(flow_id, key)
|
||||
shutil.rmtree(directory)
|
||||
self._commit(f"Delete flow {named(key, flow_id)}")
|
||||
return key
|
||||
|
||||
def duplicate_flow(
|
||||
self, flow_id: str, busy: Callable[[str], bool] | None = None
|
||||
) -> FlowDef:
|
||||
"""Copy a flow's working copy into a new flow that is only a draft.
|
||||
|
||||
Like a new flow, the copy runs nothing until someone publishes it. It
|
||||
gets an id of its own and the key ``{name}_copy``, then ``_copy_2`` and
|
||||
on, past any key *busy* says something still holds. ``origin`` is
|
||||
dropped, since the copy is a local flow ``fluksio sync`` must not
|
||||
claim, and ``runtime.json`` stays behind with everything else about
|
||||
running.
|
||||
"""
|
||||
with self._write_lock:
|
||||
source = self.read_flow(flow_id, draft=True)
|
||||
keys = self.key_index()
|
||||
new_name = copy_name(
|
||||
source.name, lambda n: self._flow_dir(n).exists() or n in keys
|
||||
source.name, lambda n: n in keys or (busy is not None and busy(n))
|
||||
)
|
||||
|
||||
copy = source.model_copy(
|
||||
update={
|
||||
"id": new_name,
|
||||
"id": self._issue(keys),
|
||||
"name": new_name,
|
||||
"title": f"{source.title} (copy)" if source.title else "",
|
||||
"version": 1,
|
||||
"origin": None,
|
||||
}
|
||||
)
|
||||
copy = self._claim(copy, create=True)
|
||||
# Its own messages named with its own key are the copy's now.
|
||||
self._retarget(copy, f"{source.name}.", f"{new_name}.")
|
||||
path = self._draft_file(new_name)
|
||||
path = self._draft_file(copy.id)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(copy.model_dump_json(indent=2) + "\n")
|
||||
|
||||
# The bodies the source's editor shows: published, edits over them.
|
||||
# As drafts, so publishing the copy promotes them like a new flow's.
|
||||
target = self._draft_nodes_dir(new_name)
|
||||
for folder in (self._flow_dir(name) / "nodes", self._draft_nodes_dir(name)):
|
||||
target = self._draft_nodes_dir(copy.id)
|
||||
for folder in (
|
||||
self._flow_dir(flow_id) / "nodes",
|
||||
self._draft_nodes_dir(flow_id),
|
||||
):
|
||||
for body in sorted(folder.glob("*.py")):
|
||||
target.mkdir(exist_ok=True)
|
||||
shutil.copy2(body, target / body.name)
|
||||
|
||||
self._commit(f"Duplicate flow '{name}' as '{new_name}'")
|
||||
self._commit(
|
||||
f"Duplicate flow {named(source.name, flow_id)} as "
|
||||
f"{named(new_name, copy.id)}"
|
||||
)
|
||||
return copy
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user