Rename journal, steps and recovery: rename.json (RENAME_JOURNAL_FILE) records each step, documents are written whole and committed alone (FlowStore.replace_documents, identity.write_documents) and rolled back before the journal goes forward; history per run scoped by flow id, alert ledger, alerts.json, state and queue steps idempotent and checkpointed; audit row once per op; recover_documents before assign_ids and recover_runtime before the engine is built, retried until done; repeated requests get the stored result, stale ones 409; plan refuses rename_in_progress; alerts.json written atomically; completeness scanner in the test utilities
This commit is contained in:
@@ -210,7 +210,7 @@ def assign_ids(root: Path) -> list[str]:
|
||||
_replace(path, json.dumps(stamped, indent=2, ensure_ascii=False) + "\n")
|
||||
|
||||
paths = sorted(changed)
|
||||
committed = _commit(root, paths)
|
||||
committed = _commit(root, paths, COMMIT_MESSAGE)
|
||||
logger.warning(
|
||||
"Gave each flow and dashboard without an id its key as one, in %s",
|
||||
", ".join(paths),
|
||||
@@ -220,18 +220,42 @@ def assign_ids(root: Path) -> list[str]:
|
||||
return paths
|
||||
|
||||
|
||||
def write_documents(root: Path, texts: dict[str, str], message: str) -> bool:
|
||||
"""Write each of *texts*, by path relative to *root*, whose file holds
|
||||
something else, then commit those paths alone with *message*.
|
||||
|
||||
What a key rename writes and undoes (`fluksio.flow.rename`). Whatever else
|
||||
is uncommitted stays so. False when the commit failed (`_commit`).
|
||||
"""
|
||||
for rel, text in texts.items():
|
||||
path = root / rel
|
||||
if not path.exists() or path.read_text() != text:
|
||||
_replace(path, text)
|
||||
return _commit(root, sorted(texts), message)
|
||||
|
||||
|
||||
def _replace(path: Path, text: str) -> None:
|
||||
"""Write *path* whole or not at all: a crash leaves the old file or the new."""
|
||||
"""Write *path* whole or not at all: a crash leaves the old file or the new.
|
||||
|
||||
The directory is synced too, without which the replace itself may not
|
||||
survive a power cut.
|
||||
"""
|
||||
temp = path.with_name(f".{path.name}.tmp")
|
||||
with temp.open("w") as handle:
|
||||
handle.write(text)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temp, path)
|
||||
directory = os.open(path.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
|
||||
|
||||
def _commit(root: Path, paths: list[str]) -> bool:
|
||||
"""Commit *paths* alone, leaving whatever else is uncommitted as it was.
|
||||
def _commit(root: Path, paths: list[str], message: str) -> bool:
|
||||
"""Commit *paths* alone with *message*, leaving whatever else is
|
||||
uncommitted as it was.
|
||||
|
||||
True once nothing is left to commit: done now, done by the run this one
|
||||
finishes, or no git to do it with, in which case the store is just files,
|
||||
@@ -262,7 +286,7 @@ def _commit(root: Path, paths: list[str]) -> bool:
|
||||
"-q",
|
||||
"--only",
|
||||
"-m",
|
||||
COMMIT_MESSAGE,
|
||||
message,
|
||||
"--",
|
||||
*paths,
|
||||
)
|
||||
@@ -270,11 +294,19 @@ def _commit(root: Path, paths: list[str]) -> bool:
|
||||
return True
|
||||
if result.returncode != 0:
|
||||
logger.warning(
|
||||
"Could not commit the ids, so the next start tries again: %s",
|
||||
"Could not commit '%s': %s",
|
||||
message,
|
||||
(result.stderr or result.stdout).strip(),
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
__all__ = ["COMMIT_MESSAGE", "IdPlan", "IdsRefused", "assign_ids", "plan_ids"]
|
||||
__all__ = [
|
||||
"COMMIT_MESSAGE",
|
||||
"IdPlan",
|
||||
"IdsRefused",
|
||||
"assign_ids",
|
||||
"plan_ids",
|
||||
"write_documents",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user