Rename journal, steps and recovery: rename.json (RENAME_JOURNAL_FILE) records each step, documents are written whole and committed alone (FlowStore.replace_documents, identity.write_documents) and rolled back before the journal goes forward; history per run scoped by flow id, alert ledger, alerts.json, state and queue steps idempotent and checkpointed; audit row once per op; recover_documents before assign_ids and recover_runtime before the engine is built, retried until done; repeated requests get the stored result, stale ones 409; plan refuses rename_in_progress; alerts.json written atomically; completeness scanner in the test utilities

This commit is contained in:
2026-11-02 21:30:01 +01:00
parent fa2a213631
commit 2d65118d82
9 changed files with 1420 additions and 25 deletions
+39 -7
View File
@@ -210,7 +210,7 @@ def assign_ids(root: Path) -> list[str]:
_replace(path, json.dumps(stamped, indent=2, ensure_ascii=False) + "\n")
paths = sorted(changed)
committed = _commit(root, paths)
committed = _commit(root, paths, COMMIT_MESSAGE)
logger.warning(
"Gave each flow and dashboard without an id its key as one, in %s",
", ".join(paths),
@@ -220,18 +220,42 @@ def assign_ids(root: Path) -> list[str]:
return paths
def write_documents(root: Path, texts: dict[str, str], message: str) -> bool:
"""Write each of *texts*, by path relative to *root*, whose file holds
something else, then commit those paths alone with *message*.
What a key rename writes and undoes (`fluksio.flow.rename`). Whatever else
is uncommitted stays so. False when the commit failed (`_commit`).
"""
for rel, text in texts.items():
path = root / rel
if not path.exists() or path.read_text() != text:
_replace(path, text)
return _commit(root, sorted(texts), message)
def _replace(path: Path, text: str) -> None:
"""Write *path* whole or not at all: a crash leaves the old file or the new."""
"""Write *path* whole or not at all: a crash leaves the old file or the new.
The directory is synced too, without which the replace itself may not
survive a power cut.
"""
temp = path.with_name(f".{path.name}.tmp")
with temp.open("w") as handle:
handle.write(text)
handle.flush()
os.fsync(handle.fileno())
os.replace(temp, path)
directory = os.open(path.parent, os.O_RDONLY)
try:
os.fsync(directory)
finally:
os.close(directory)
def _commit(root: Path, paths: list[str]) -> bool:
"""Commit *paths* alone, leaving whatever else is uncommitted as it was.
def _commit(root: Path, paths: list[str], message: str) -> bool:
"""Commit *paths* alone with *message*, leaving whatever else is
uncommitted as it was.
True once nothing is left to commit: done now, done by the run this one
finishes, or no git to do it with, in which case the store is just files,
@@ -262,7 +286,7 @@ def _commit(root: Path, paths: list[str]) -> bool:
"-q",
"--only",
"-m",
COMMIT_MESSAGE,
message,
"--",
*paths,
)
@@ -270,11 +294,19 @@ def _commit(root: Path, paths: list[str]) -> bool:
return True
if result.returncode != 0:
logger.warning(
"Could not commit the ids, so the next start tries again: %s",
"Could not commit '%s': %s",
message,
(result.stderr or result.stdout).strip(),
)
return False
return True
__all__ = ["COMMIT_MESSAGE", "IdPlan", "IdsRefused", "assign_ids", "plan_ids"]
__all__ = [
"COMMIT_MESSAGE",
"IdPlan",
"IdsRefused",
"assign_ids",
"plan_ids",
"write_documents",
]
+563 -5
View File
@@ -4,24 +4,31 @@ A rename moves everything named under a flow's key, in documents, history,
live state and queued work, so nothing may write through the API meanwhile.
`RenameGate` is the door that keeps it out. `plan_rename` says beforehand what
a rename would change and what refuses it, and `rewrite_documents` is its
document part. ``docs/private/id_refac_rename.md`` has the whole protocol.
document part. The `Journal` records each step before the next, so a rename a
crash interrupted is undone or finished on the next start (`recover_documents`,
`recover_runtime`). ``docs/private/id_refac_rename.md`` has the whole protocol.
"""
from __future__ import annotations
import asyncio
import contextlib
import hashlib
import json
import logging
import re
import threading
import time
from collections import Counter
from collections.abc import Callable, Iterable, Iterator
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Literal, cast
import orjson
from pydantic import BaseModel
from sqlalchemy import String
from pydantic import BaseModel, Field
from sqlalchemy import String, literal, update
from sqlmodel import Session, col, func, select
from starlette.types import Scope
@@ -36,13 +43,25 @@ from fluksio.flow.dashboards import (
DashboardStore,
results_name,
)
from fluksio.flow.events import EventBus
from fluksio.flow.executor import wait_out
from fluksio.flow.identity import _replace, write_documents
from fluksio.flow.messages import flow_of
from fluksio.flow.namespace import FLOW_FIELDS, NAME, OUTPUTS, Rename, names, rebind
from fluksio.flow.queue import WorkQueue
from fluksio.flow.schemas import FlowDef, _validate_name
from fluksio.flow.state import MemoryState
from fluksio.flow.state import MemoryState, StateBackend
from fluksio.flow.store import ISSUED_ID, FlowStore, InvalidName, ReservedKey
from fluksio.models import AlertSent, Run, RunArtifact, RunMetric, RunNode
from fluksio.models import (
AlertSent,
EngineEvent,
Run,
RunArtifact,
RunMetric,
RunNode,
)
logger = logging.getLogger(__name__)
Kind = Literal["flow", "dashboard"]
@@ -452,6 +471,12 @@ def plan_rename(
if new_name == old:
refuse("same_name", f"The {kind} is already called '{old}'")
return plan
journal = read_journal()
if journal is not None and not journal.final:
refuse(
"rename_in_progress",
f"The rename of '{journal.old}' to '{journal.new}' is finishing",
)
rename = Rename(old, new_name)
rewrites = rewrite_documents(store, dashboards, kind, obj_id, rename)
@@ -689,3 +714,536 @@ def _fingerprint(plan: RenamePlan) -> str:
)
body = json.dumps([seen, plan.name, plan.new_name]).encode()
return f"sha256:{hashlib.sha256(body).hexdigest()}"
# -----------------------------------------------------------------------------
# Journal
# -----------------------------------------------------------------------------
#: Where a rename stands. ``prepared`` holds the documents' texts before any
#: is written, and a crash there is undone. ``forward`` follows the scoped
#: commit, and from there on the rename is only finished, never undone.
Phase = Literal["prepared", "forward", "done", "rolled_back"]
#: Waits between attempts at a rename's remaining steps, in seconds: the
#: first, doubled after each failure up to the last.
RETRY_S = (2.0, 60.0)
def fault(point: str) -> None:
"""Called at each named point of a rename's steps, and does nothing.
A test replaces it to stop a rename there, as a crash would. Each point
follows the work it names and precedes the journal saying so:
``commit`` once the documents are committed, ``sql:run`` after each
run's history, ``state:keys`` after each batch of state keys,
``queue:<structure>`` after each queue structure (`WorkQueue.rename`),
and ``alert_ledger``, ``alerts_file``, ``state``, ``queue`` and
``audit`` after each step. ``done`` follows the journal saying so. A
crash between two document writes is `identity._replace`'s to fake.
"""
class Affected(BaseModel):
"""What a rename changes, by id."""
#: Flows whose documents change: the renamed one and its dependents.
flows: list[str] = []
#: Of those, the ones the controller built, which it builds again.
rebuilt: list[str] = []
dashboards: list[str] = []
class JournaledDocument(BaseModel):
"""One document variant's whole text before and after the rename."""
kind: Kind
id: str
#: Relative to the store.
path: str
before: str
after: str
class SqlProgress(BaseModel):
done: bool = False
#: The last run whose history moved, in id order. Saved at most once a
#: second; a run moved twice is moved once, since only old names match.
after_run: str = ""
class Steps(BaseModel):
"""The steps after the documents, each set once it is done."""
sql: SqlProgress = Field(default_factory=SqlProgress)
alert_ledger: bool = False
alerts_file: bool = False
state: bool = False
queue: bool = False
audit: bool = False
class RenamedDocument(BaseModel):
"""A document's versions after a rename. Both variants have the same."""
kind: Kind
id: str
version: int
#: None for a document without a draft.
draft_version: int | None = None
class RenameResult(BaseModel):
"""What a finished rename answers, again for a repeated request."""
op: str
kind: Kind
id: str
old_name: str
new_name: str
documents: list[RenamedDocument]
class Journal(BaseModel):
"""A rename's record on disk (`settings.RENAME_JOURNAL_FILE`), written
before each step, so whatever stops it is finished or undone from here.
The last one stays, which is how a repeated request is recognised."""
format: int = 1
#: The request's id.
op: str
kind: Kind
id: str
old: str
new: str
actor: str = ""
started_at: float
phase: Phase = "prepared"
fingerprint: str = ""
#: False with memory backends: a restart then has no state or queued
#: work to move, since it lost them with the process.
durable: bool = True
affected: Affected = Field(default_factory=Affected)
files: list[JournaledDocument] = []
steps: Steps = Field(default_factory=Steps)
result: RenameResult | None = None
#: Why the last attempt at the steps failed.
error: str = ""
@property
def final(self) -> bool:
return self.phase in ("done", "rolled_back")
@property
def rename(self) -> Rename:
return Rename(self.old, self.new)
def save(self) -> None:
"""Whole or not at all, and on disk before this returns."""
path = settings.RENAME_JOURNAL_FILE
path.parent.mkdir(parents=True, exist_ok=True)
_replace(path, self.model_dump_json(indent=2) + "\n")
def read_journal() -> Journal | None:
"""The last rename's journal, None when there never was one."""
try:
text = settings.RENAME_JOURNAL_FILE.read_text()
except FileNotFoundError:
return None
return Journal.model_validate_json(text)
class RenameConflict(Exception):
"""A rename request that cannot go ahead as asked, with the status and
code it is answered with."""
def __init__(self, status: int, code: str, detail: str) -> None:
super().__init__(detail)
self.status = status
self.code = code
self.detail = detail
def repeated(
request_id: str,
name: str,
version: int,
expected_name: str,
expected_version: int,
) -> RenameResult | None:
"""The stored result of a rename asked for again, or None for a request
that may go ahead. *name* and *version* are the object's now, the
expected ones what the request was planned against.
A matching key is no proof of the same operation: a client that retries
a finished rename under a new request id expected the old key, and is
told its request is stale.
:raises RenameConflict: 503 ``completing`` for this request still
running, 503 ``rename_in_progress`` for another one, 409 ``stale``
for a request planned against another key or version.
"""
journal = read_journal()
if journal is not None and journal.op == request_id:
if journal.phase == "done" and journal.result is not None:
return journal.result
if not journal.final:
raise RenameConflict(503, "completing", "This rename is finishing")
if journal is not None and not journal.final:
raise RenameConflict(
503,
"rename_in_progress",
f"The rename of '{journal.old}' to '{journal.new}' is finishing",
)
if (expected_name, expected_version) != (name, version):
raise RenameConflict(
409,
"stale",
f"It is '{name}' at version {version} now, not '{expected_name}' "
f"at version {expected_version}",
)
return None
def begin(
op: str,
kind: Kind,
obj_id: str,
rename: Rename,
rewrites: list[Rewrite],
*,
actor: str = "",
fingerprint: str = "",
durable: bool = True,
rebuilt: Iterable[str] = (),
) -> Journal:
"""Write the journal of a rename about to change *rewrites*: its first
write, after which it is finished or undone but no longer called off."""
journal = Journal(
op=op,
kind=kind,
id=obj_id,
old=rename.old,
new=rename.new,
actor=actor,
started_at=time.time(),
fingerprint=fingerprint,
durable=durable,
affected=Affected(
flows=[r.id for r in rewrites if r.kind == "flow"],
rebuilt=[*rebuilt],
dashboards=[r.id for r in rewrites if r.kind == "dashboard"],
),
files=[
JournaledDocument(
kind=r.kind, id=r.id, path=path, before=before, after=after
)
for r in rewrites
for path, before, after in r.files
],
)
journal.save()
return journal
# -----------------------------------------------------------------------------
# Steps
# -----------------------------------------------------------------------------
def _message(journal: Journal, revert: bool = False) -> str:
what = f"{journal.kind} '{journal.old}' to '{journal.new}'"
if revert:
return f"Revert rename of {what}"
return f"Rename {what} ({journal.id})"
def apply_documents(journal: Journal, store: FlowStore) -> None:
"""Replace every document with its new text and commit them alone, then
go forward. Raises when that fails, and `roll_back` undoes it."""
texts = {doc.path: doc.after for doc in journal.files}
if not store.replace_documents(texts, _message(journal)):
raise RuntimeError("Could not commit the renamed documents")
fault("commit")
journal.phase = "forward"
journal.save()
def roll_back(journal: Journal, store: FlowStore) -> None:
"""Put back the documents' texts from before, and commit that. Only
before the journal went forward; nothing else was written by then."""
store.replace_documents(
{doc.path: doc.before for doc in journal.files}, _message(journal, True)
)
journal.phase = "rolled_back"
journal.save()
def run_steps(
journal: Journal, state: StateBackend | None, queue: WorkQueue | None
) -> None:
"""Every step after the documents that is not done yet, in order: the
history, the alert ledger and channels, live state and queued work.
Each can be run again, since only what still has an old name moves, so
the journal only says which are finished. Without *state* or *queue*
their steps are passed over: a memory backend's restart has nothing left
to move. The engine must be held meanwhile.
"""
if journal.kind != "flow":
return
rename, steps = journal.rename, journal.steps
def finished(step: str) -> None:
fault(step)
setattr(steps, step, True)
journal.save()
try:
if not steps.sql.done:
_rename_history(journal)
if not steps.alert_ledger:
_rename_ledger(rename)
finished("alert_ledger")
if not steps.alerts_file:
_rename_channels(rename)
finished("alerts_file")
if not steps.state:
if state is not None:
state.rename_keys(rename, lambda what: fault(f"state:{what}"))
finished("state")
if not steps.queue:
if queue is not None:
queue.rename(rename, lambda what: fault(f"queue:{what}"))
finished("queue")
except Exception as exc:
journal.error = f"{type(exc).__name__}: {exc}"
with contextlib.suppress(OSError):
journal.save()
raise
def finish(journal: Journal) -> RenameResult:
"""Write the audit row, once, and mark the journal done with the result
a repeated request is answered with."""
if not journal.steps.audit:
_audit(journal)
fault("audit")
journal.steps.audit = True
journal.save()
journal.result = _result(journal)
journal.phase = "done"
journal.error = ""
journal.save()
fault("done")
return journal.result
def _rename_history(journal: Journal) -> None:
"""Message names in the history of the affected flows, a run at a time.
Scoped by the runs' flow ids, so a deleted flow that once had the same
key keeps its history as it was. Renamed outputs are no longer what
their cache key promised, so the key goes, and the outputs and where
they were restored from stay.
"""
rename, progress = journal.rename, journal.steps.sql
old, new = f"{rename.old}.", f"{rename.new}."
with Session(db_engine) as session:
runs = session.exec(
select(Run.id)
.where(
col(Run.flow_id).in_(journal.affected.flows),
col(Run.id) > progress.after_run,
)
.order_by(col(Run.id))
).all()
saved = time.monotonic()
for run_id in runs:
with Session(db_engine) as session:
for model in (RunMetric, RunArtifact):
name = col(model.name)
session.exec(
update(model)
.where(
col(model.run_id) == run_id,
func.substr(name, 1, len(old)) == old,
)
.values(name=literal(new).concat(func.substr(name, len(old) + 1)))
)
for row in session.exec(
select(RunNode).where(
col(RunNode.run_id) == run_id,
col(RunNode.outputs).contains(f'"{old}', autoescape=True),
)
).all():
outputs = json.loads(row.outputs or "{}")
moved = {rename.binding(key): value for key, value in outputs.items()}
if moved != outputs:
# Canonical, as `runs._cacheable` writes it.
row.outputs = json.dumps(
moved, sort_keys=True, separators=(",", ":")
)
row.cache_key = ""
session.add(row)
session.commit()
fault("sql:run")
if time.monotonic() - saved >= 1.0:
progress.after_run = run_id
journal.save()
saved = time.monotonic()
progress.done = True
journal.save()
def _rename_ledger(rename: Rename) -> None:
"""The keys alerts were sent under, so a cooldown carries over."""
with Session(db_engine) as session:
for row in session.exec(select(AlertSent)).all():
key = rename.dedup_key(row.key)
if key != row.key:
row.key = key
session.add(row)
session.commit()
def _rename_channels(rename: Rename) -> None:
"""The message a dashboard alert channel publishes to, in `alerts.json`."""
path = settings.ALERTS_FILE
try:
config = AlertsConfig.model_validate_json(path.read_text())
except (OSError, ValueError):
# None, or one the engine reads as empty: no channel names anything.
return
moved = 0
for channel in config.channels:
message = channel.config.get("message")
if channel.kind == "dashboard" and isinstance(message, str):
channel.config["message"] = rename.binding(message)
moved += channel.config["message"] != message
if moved:
_replace(path, config.model_dump_json(indent=2))
def _audit(journal: Journal) -> None:
"""Who renamed what, straight into the database rather than through the
bus, and only if it is not there already: a rename finished twice, by a
crash and the recovery, is recorded once."""
what = "" if journal.kind == "flow" else "dashboard "
detail = f"renamed {what}'{journal.old}' to '{journal.new}' [op {journal.op}]"
with Session(db_engine) as session:
seen = session.exec(
select(EngineEvent.id).where(
col(EngineEvent.type) == "audit", col(EngineEvent.detail) == detail
)
).first()
if seen is None:
session.add(
EngineEvent(
ts=datetime.now(UTC),
type="audit",
flow_id=journal.id if journal.kind == "flow" else "",
detail=detail,
actor=journal.actor,
)
)
session.commit()
def _result(journal: Journal) -> RenameResult:
documents: dict[tuple[str, str], RenamedDocument] = {}
for doc in journal.files:
version = int(orjson.loads(doc.after)["version"])
entry = documents.setdefault(
(doc.kind, doc.id),
RenamedDocument(kind=doc.kind, id=doc.id, version=version),
)
if doc.path.endswith(".draft.json"):
entry.draft_version = version
return RenameResult(
op=journal.op,
kind=journal.kind,
id=journal.id,
old_name=journal.old,
new_name=journal.new,
documents=[*documents.values()],
)
# -----------------------------------------------------------------------------
# Recovery
# -----------------------------------------------------------------------------
def recover_documents(root: Path) -> None:
"""Settle the documents of a rename a crash interrupted. For startup,
before anything reads the store at *root*, ids being assigned included.
Before the journal went forward the old texts go back, otherwise the new
ones are made sure of, and only those paths are committed: a published
document and its draft left with different keys would refuse to start.
Plain files, no `FlowStore`.
"""
journal = read_journal()
if journal is None or journal.final:
return
if journal.phase == "prepared":
logger.warning(
"Undoing the rename of %s '%s' to '%s', which was interrupted "
"before it changed anything but its documents",
journal.kind,
journal.old,
journal.new,
)
texts = {doc.path: doc.before for doc in journal.files}
write_documents(root, texts, _message(journal, True))
journal.phase = "rolled_back"
journal.save()
return
write_documents(
root, {doc.path: doc.after for doc in journal.files}, _message(journal)
)
def recover_runtime(state: StateBackend, queue: WorkQueue) -> None:
"""Finish a rename a crash interrupted once its documents are settled
(`recover_documents`): the steps left, the audit row and the journal's
end. For startup, before anything reads alerts, devices, modules,
metrics, the controller or runs. Raises when a step fails, to be tried
again (`retry_until_done`)."""
journal = read_journal()
if journal is None or journal.phase != "forward":
return
logger.warning(
"Finishing the rename of %s '%s' to '%s'",
journal.kind,
journal.old,
journal.new,
)
if journal.durable:
run_steps(journal, state, queue)
else:
run_steps(journal, None, None)
finish(journal)
async def retry_until_done(attempt: Callable[[], None], events: EventBus) -> None:
"""Run *attempt* on a thread until it succeeds, however long that takes:
a rename half done is not to be served. Each failure is logged and
published as ``engine_degraded``, and the wait before the next doubles."""
wait, longest = RETRY_S
while True:
try:
await asyncio.to_thread(attempt)
return
except Exception as exc:
logger.exception("A rename could not finish; trying again in %.0f s", wait)
events.publish(
{
"type": "engine_degraded",
"detail": f"A rename could not finish: {exc}",
}
)
await asyncio.sleep(wait)
wait = min(wait * 2, longest)
+19
View File
@@ -943,6 +943,25 @@ class FlowStore:
self._commit(f"Delete flow {named(key, flow_id)}")
return key
def replace_documents(self, texts: dict[str, str], message: str) -> bool:
"""Write flow and dashboard documents whole, *texts* by path relative
to the store, and commit those paths alone with *message*.
A key rename's (`fluksio.flow.rename`): each text carries its version
already, and whatever else is uncommitted stays so. False when the
commit failed. A store without git counts as committed.
"""
# Here rather than at the top: it reads where dashboards live, and
# the dashboards module imports this one.
from fluksio.flow.identity import write_documents
with self._write_lock:
try:
return write_documents(self.root, texts, message)
finally:
# Whatever got written, the caches are to read it again.
self.revision += 1
def duplicate_flow(
self, flow_id: str, busy: Callable[[str], bool] | None = None
) -> FlowDef: