Rename journal, steps and recovery: rename.json (RENAME_JOURNAL_FILE) records each step, documents are written whole and committed alone (FlowStore.replace_documents, identity.write_documents) and rolled back before the journal goes forward; history per run scoped by flow id, alert ledger, alerts.json, state and queue steps idempotent and checkpointed; audit row once per op; recover_documents before assign_ids and recover_runtime before the engine is built, retried until done; repeated requests get the stored result, stale ones 409; plan refuses rename_in_progress; alerts.json written atomically; completeness scanner in the test utilities

This commit is contained in:
2026-11-02 21:30:01 +01:00
parent fa2a213631
commit 2d65118d82
9 changed files with 1420 additions and 25 deletions
+39 -7
View File
@@ -210,7 +210,7 @@ def assign_ids(root: Path) -> list[str]:
_replace(path, json.dumps(stamped, indent=2, ensure_ascii=False) + "\n")
paths = sorted(changed)
committed = _commit(root, paths)
committed = _commit(root, paths, COMMIT_MESSAGE)
logger.warning(
"Gave each flow and dashboard without an id its key as one, in %s",
", ".join(paths),
@@ -220,18 +220,42 @@ def assign_ids(root: Path) -> list[str]:
return paths
def write_documents(root: Path, texts: dict[str, str], message: str) -> bool:
"""Write each of *texts*, by path relative to *root*, whose file holds
something else, then commit those paths alone with *message*.
What a key rename writes and undoes (`fluksio.flow.rename`). Whatever else
is uncommitted stays so. False when the commit failed (`_commit`).
"""
for rel, text in texts.items():
path = root / rel
if not path.exists() or path.read_text() != text:
_replace(path, text)
return _commit(root, sorted(texts), message)
def _replace(path: Path, text: str) -> None:
"""Write *path* whole or not at all: a crash leaves the old file or the new."""
"""Write *path* whole or not at all: a crash leaves the old file or the new.
The directory is synced too, without which the replace itself may not
survive a power cut.
"""
temp = path.with_name(f".{path.name}.tmp")
with temp.open("w") as handle:
handle.write(text)
handle.flush()
os.fsync(handle.fileno())
os.replace(temp, path)
directory = os.open(path.parent, os.O_RDONLY)
try:
os.fsync(directory)
finally:
os.close(directory)
def _commit(root: Path, paths: list[str]) -> bool:
"""Commit *paths* alone, leaving whatever else is uncommitted as it was.
def _commit(root: Path, paths: list[str], message: str) -> bool:
"""Commit *paths* alone with *message*, leaving whatever else is
uncommitted as it was.
True once nothing is left to commit: done now, done by the run this one
finishes, or no git to do it with, in which case the store is just files,
@@ -262,7 +286,7 @@ def _commit(root: Path, paths: list[str]) -> bool:
"-q",
"--only",
"-m",
COMMIT_MESSAGE,
message,
"--",
*paths,
)
@@ -270,11 +294,19 @@ def _commit(root: Path, paths: list[str]) -> bool:
return True
if result.returncode != 0:
logger.warning(
"Could not commit the ids, so the next start tries again: %s",
"Could not commit '%s': %s",
message,
(result.stderr or result.stdout).strip(),
)
return False
return True
__all__ = ["COMMIT_MESSAGE", "IdPlan", "IdsRefused", "assign_ids", "plan_ids"]
__all__ = [
"COMMIT_MESSAGE",
"IdPlan",
"IdsRefused",
"assign_ids",
"plan_ids",
"write_documents",
]