Rename journal, steps and recovery: rename.json (RENAME_JOURNAL_FILE) records each step, documents are written whole and committed alone (FlowStore.replace_documents, identity.write_documents) and rolled back before the journal goes forward; history per run scoped by flow id, alert ledger, alerts.json, state and queue steps idempotent and checkpointed; audit row once per op; recover_documents before assign_ids and recover_runtime before the engine is built, retried until done; repeated requests get the stored result, stale ones 409; plan refuses rename_in_progress; alerts.json written atomically; completeness scanner in the test utilities
This commit is contained in:
@@ -15,6 +15,7 @@ from fluksio.api.deps import (
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.flow import telegram, webpush
|
||||
from fluksio.flow.alerts import Alert, AlertsConfig
|
||||
from fluksio.flow.identity import _replace
|
||||
from fluksio.flow.secrets import SECRET_REF, SecretNotFound, resolve_params
|
||||
from fluksio.models import Message
|
||||
|
||||
@@ -48,10 +49,11 @@ def read_config() -> AlertsConfig:
|
||||
|
||||
|
||||
def write_config(config: AlertsConfig) -> None:
|
||||
"""Blocking."""
|
||||
"""Blocking. Whole or not at all: a file cut short would read as no
|
||||
configuration, which is alerting turned off."""
|
||||
path = _path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(config.model_dump_json(indent=2))
|
||||
_replace(path, config.model_dump_json(indent=2))
|
||||
|
||||
|
||||
@router.get("/config", response_model=AlertsConfig, dependencies=SUPERUSER)
|
||||
|
||||
@@ -23,6 +23,7 @@ DERIVED_PATHS = {
|
||||
"FLOWS_DIR": "flows",
|
||||
"SECRETS_FILE": "secrets.enc",
|
||||
"ALERTS_FILE": "alerts.json",
|
||||
"RENAME_JOURNAL_FILE": "rename.json",
|
||||
"WEBPUSH_FILE": "webpush.json",
|
||||
"PANELS_FILE": "panels.json",
|
||||
"DEVICES_FILE": "devices.json",
|
||||
@@ -85,6 +86,11 @@ class Settings(BaseSettings):
|
||||
# Which failures reach which channel. Beside the flows, not in them:
|
||||
# alerting is the deployment's concern, not any one flow's.
|
||||
ALERTS_FILE: Path = Path("flow-data/alerts.json")
|
||||
# The last key rename, step by step, so one a crash interrupted is
|
||||
# finished or undone on the next start. Outside the flows, whose git
|
||||
# history it must not enter, and kept once done so a repeated request is
|
||||
# recognised.
|
||||
RENAME_JOURNAL_FILE: Path = Path("flow-data/rename.json")
|
||||
# This instance's web push keypair and the browsers subscribed to it.
|
||||
# Beside the alerts it serves; deleting it makes every device subscribe
|
||||
# again.
|
||||
|
||||
@@ -210,7 +210,7 @@ def assign_ids(root: Path) -> list[str]:
|
||||
_replace(path, json.dumps(stamped, indent=2, ensure_ascii=False) + "\n")
|
||||
|
||||
paths = sorted(changed)
|
||||
committed = _commit(root, paths)
|
||||
committed = _commit(root, paths, COMMIT_MESSAGE)
|
||||
logger.warning(
|
||||
"Gave each flow and dashboard without an id its key as one, in %s",
|
||||
", ".join(paths),
|
||||
@@ -220,18 +220,42 @@ def assign_ids(root: Path) -> list[str]:
|
||||
return paths
|
||||
|
||||
|
||||
def write_documents(root: Path, texts: dict[str, str], message: str) -> bool:
|
||||
"""Write each of *texts*, by path relative to *root*, whose file holds
|
||||
something else, then commit those paths alone with *message*.
|
||||
|
||||
What a key rename writes and undoes (`fluksio.flow.rename`). Whatever else
|
||||
is uncommitted stays so. False when the commit failed (`_commit`).
|
||||
"""
|
||||
for rel, text in texts.items():
|
||||
path = root / rel
|
||||
if not path.exists() or path.read_text() != text:
|
||||
_replace(path, text)
|
||||
return _commit(root, sorted(texts), message)
|
||||
|
||||
|
||||
def _replace(path: Path, text: str) -> None:
|
||||
"""Write *path* whole or not at all: a crash leaves the old file or the new."""
|
||||
"""Write *path* whole or not at all: a crash leaves the old file or the new.
|
||||
|
||||
The directory is synced too, without which the replace itself may not
|
||||
survive a power cut.
|
||||
"""
|
||||
temp = path.with_name(f".{path.name}.tmp")
|
||||
with temp.open("w") as handle:
|
||||
handle.write(text)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temp, path)
|
||||
directory = os.open(path.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
|
||||
|
||||
def _commit(root: Path, paths: list[str]) -> bool:
|
||||
"""Commit *paths* alone, leaving whatever else is uncommitted as it was.
|
||||
def _commit(root: Path, paths: list[str], message: str) -> bool:
|
||||
"""Commit *paths* alone with *message*, leaving whatever else is
|
||||
uncommitted as it was.
|
||||
|
||||
True once nothing is left to commit: done now, done by the run this one
|
||||
finishes, or no git to do it with, in which case the store is just files,
|
||||
@@ -262,7 +286,7 @@ def _commit(root: Path, paths: list[str]) -> bool:
|
||||
"-q",
|
||||
"--only",
|
||||
"-m",
|
||||
COMMIT_MESSAGE,
|
||||
message,
|
||||
"--",
|
||||
*paths,
|
||||
)
|
||||
@@ -270,11 +294,19 @@ def _commit(root: Path, paths: list[str]) -> bool:
|
||||
return True
|
||||
if result.returncode != 0:
|
||||
logger.warning(
|
||||
"Could not commit the ids, so the next start tries again: %s",
|
||||
"Could not commit '%s': %s",
|
||||
message,
|
||||
(result.stderr or result.stdout).strip(),
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
__all__ = ["COMMIT_MESSAGE", "IdPlan", "IdsRefused", "assign_ids", "plan_ids"]
|
||||
__all__ = [
|
||||
"COMMIT_MESSAGE",
|
||||
"IdPlan",
|
||||
"IdsRefused",
|
||||
"assign_ids",
|
||||
"plan_ids",
|
||||
"write_documents",
|
||||
]
|
||||
|
||||
@@ -4,24 +4,31 @@ A rename moves everything named under a flow's key, in documents, history,
|
||||
live state and queued work, so nothing may write through the API meanwhile.
|
||||
`RenameGate` is the door that keeps it out. `plan_rename` says beforehand what
|
||||
a rename would change and what refuses it, and `rewrite_documents` is its
|
||||
document part. ``docs/private/id_refac_rename.md`` has the whole protocol.
|
||||
document part. The `Journal` records each step before the next, so a rename a
|
||||
crash interrupted is undone or finished on the next start (`recover_documents`,
|
||||
`recover_runtime`). ``docs/private/id_refac_rename.md`` has the whole protocol.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import orjson
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import String
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import String, literal, update
|
||||
from sqlmodel import Session, col, func, select
|
||||
from starlette.types import Scope
|
||||
|
||||
@@ -36,13 +43,25 @@ from fluksio.flow.dashboards import (
|
||||
DashboardStore,
|
||||
results_name,
|
||||
)
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.executor import wait_out
|
||||
from fluksio.flow.identity import _replace, write_documents
|
||||
from fluksio.flow.messages import flow_of
|
||||
from fluksio.flow.namespace import FLOW_FIELDS, NAME, OUTPUTS, Rename, names, rebind
|
||||
from fluksio.flow.queue import WorkQueue
|
||||
from fluksio.flow.schemas import FlowDef, _validate_name
|
||||
from fluksio.flow.state import MemoryState
|
||||
from fluksio.flow.state import MemoryState, StateBackend
|
||||
from fluksio.flow.store import ISSUED_ID, FlowStore, InvalidName, ReservedKey
|
||||
from fluksio.models import AlertSent, Run, RunArtifact, RunMetric, RunNode
|
||||
from fluksio.models import (
|
||||
AlertSent,
|
||||
EngineEvent,
|
||||
Run,
|
||||
RunArtifact,
|
||||
RunMetric,
|
||||
RunNode,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
Kind = Literal["flow", "dashboard"]
|
||||
|
||||
@@ -452,6 +471,12 @@ def plan_rename(
|
||||
if new_name == old:
|
||||
refuse("same_name", f"The {kind} is already called '{old}'")
|
||||
return plan
|
||||
journal = read_journal()
|
||||
if journal is not None and not journal.final:
|
||||
refuse(
|
||||
"rename_in_progress",
|
||||
f"The rename of '{journal.old}' to '{journal.new}' is finishing",
|
||||
)
|
||||
|
||||
rename = Rename(old, new_name)
|
||||
rewrites = rewrite_documents(store, dashboards, kind, obj_id, rename)
|
||||
@@ -689,3 +714,536 @@ def _fingerprint(plan: RenamePlan) -> str:
|
||||
)
|
||||
body = json.dumps([seen, plan.name, plan.new_name]).encode()
|
||||
return f"sha256:{hashlib.sha256(body).hexdigest()}"
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Journal
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
#: Where a rename stands. ``prepared`` holds the documents' texts before any
|
||||
#: is written, and a crash there is undone. ``forward`` follows the scoped
|
||||
#: commit, and from there on the rename is only finished, never undone.
|
||||
Phase = Literal["prepared", "forward", "done", "rolled_back"]
|
||||
|
||||
#: Waits between attempts at a rename's remaining steps, in seconds: the
|
||||
#: first, doubled after each failure up to the last.
|
||||
RETRY_S = (2.0, 60.0)
|
||||
|
||||
|
||||
def fault(point: str) -> None:
|
||||
"""Called at each named point of a rename's steps, and does nothing.
|
||||
|
||||
A test replaces it to stop a rename there, as a crash would. Each point
|
||||
follows the work it names and precedes the journal saying so:
|
||||
``commit`` once the documents are committed, ``sql:run`` after each
|
||||
run's history, ``state:keys`` after each batch of state keys,
|
||||
``queue:<structure>`` after each queue structure (`WorkQueue.rename`),
|
||||
and ``alert_ledger``, ``alerts_file``, ``state``, ``queue`` and
|
||||
``audit`` after each step. ``done`` follows the journal saying so. A
|
||||
crash between two document writes is `identity._replace`'s to fake.
|
||||
"""
|
||||
|
||||
|
||||
class Affected(BaseModel):
|
||||
"""What a rename changes, by id."""
|
||||
|
||||
#: Flows whose documents change: the renamed one and its dependents.
|
||||
flows: list[str] = []
|
||||
#: Of those, the ones the controller built, which it builds again.
|
||||
rebuilt: list[str] = []
|
||||
dashboards: list[str] = []
|
||||
|
||||
|
||||
class JournaledDocument(BaseModel):
|
||||
"""One document variant's whole text before and after the rename."""
|
||||
|
||||
kind: Kind
|
||||
id: str
|
||||
#: Relative to the store.
|
||||
path: str
|
||||
before: str
|
||||
after: str
|
||||
|
||||
|
||||
class SqlProgress(BaseModel):
|
||||
done: bool = False
|
||||
#: The last run whose history moved, in id order. Saved at most once a
|
||||
#: second; a run moved twice is moved once, since only old names match.
|
||||
after_run: str = ""
|
||||
|
||||
|
||||
class Steps(BaseModel):
|
||||
"""The steps after the documents, each set once it is done."""
|
||||
|
||||
sql: SqlProgress = Field(default_factory=SqlProgress)
|
||||
alert_ledger: bool = False
|
||||
alerts_file: bool = False
|
||||
state: bool = False
|
||||
queue: bool = False
|
||||
audit: bool = False
|
||||
|
||||
|
||||
class RenamedDocument(BaseModel):
|
||||
"""A document's versions after a rename. Both variants have the same."""
|
||||
|
||||
kind: Kind
|
||||
id: str
|
||||
version: int
|
||||
#: None for a document without a draft.
|
||||
draft_version: int | None = None
|
||||
|
||||
|
||||
class RenameResult(BaseModel):
|
||||
"""What a finished rename answers, again for a repeated request."""
|
||||
|
||||
op: str
|
||||
kind: Kind
|
||||
id: str
|
||||
old_name: str
|
||||
new_name: str
|
||||
documents: list[RenamedDocument]
|
||||
|
||||
|
||||
class Journal(BaseModel):
|
||||
"""A rename's record on disk (`settings.RENAME_JOURNAL_FILE`), written
|
||||
before each step, so whatever stops it is finished or undone from here.
|
||||
The last one stays, which is how a repeated request is recognised."""
|
||||
|
||||
format: int = 1
|
||||
#: The request's id.
|
||||
op: str
|
||||
kind: Kind
|
||||
id: str
|
||||
old: str
|
||||
new: str
|
||||
actor: str = ""
|
||||
started_at: float
|
||||
phase: Phase = "prepared"
|
||||
fingerprint: str = ""
|
||||
#: False with memory backends: a restart then has no state or queued
|
||||
#: work to move, since it lost them with the process.
|
||||
durable: bool = True
|
||||
affected: Affected = Field(default_factory=Affected)
|
||||
files: list[JournaledDocument] = []
|
||||
steps: Steps = Field(default_factory=Steps)
|
||||
result: RenameResult | None = None
|
||||
#: Why the last attempt at the steps failed.
|
||||
error: str = ""
|
||||
|
||||
@property
|
||||
def final(self) -> bool:
|
||||
return self.phase in ("done", "rolled_back")
|
||||
|
||||
@property
|
||||
def rename(self) -> Rename:
|
||||
return Rename(self.old, self.new)
|
||||
|
||||
def save(self) -> None:
|
||||
"""Whole or not at all, and on disk before this returns."""
|
||||
path = settings.RENAME_JOURNAL_FILE
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
_replace(path, self.model_dump_json(indent=2) + "\n")
|
||||
|
||||
|
||||
def read_journal() -> Journal | None:
|
||||
"""The last rename's journal, None when there never was one."""
|
||||
try:
|
||||
text = settings.RENAME_JOURNAL_FILE.read_text()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
return Journal.model_validate_json(text)
|
||||
|
||||
|
||||
class RenameConflict(Exception):
|
||||
"""A rename request that cannot go ahead as asked, with the status and
|
||||
code it is answered with."""
|
||||
|
||||
def __init__(self, status: int, code: str, detail: str) -> None:
|
||||
super().__init__(detail)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.detail = detail
|
||||
|
||||
|
||||
def repeated(
|
||||
request_id: str,
|
||||
name: str,
|
||||
version: int,
|
||||
expected_name: str,
|
||||
expected_version: int,
|
||||
) -> RenameResult | None:
|
||||
"""The stored result of a rename asked for again, or None for a request
|
||||
that may go ahead. *name* and *version* are the object's now, the
|
||||
expected ones what the request was planned against.
|
||||
|
||||
A matching key is no proof of the same operation: a client that retries
|
||||
a finished rename under a new request id expected the old key, and is
|
||||
told its request is stale.
|
||||
|
||||
:raises RenameConflict: 503 ``completing`` for this request still
|
||||
running, 503 ``rename_in_progress`` for another one, 409 ``stale``
|
||||
for a request planned against another key or version.
|
||||
"""
|
||||
journal = read_journal()
|
||||
if journal is not None and journal.op == request_id:
|
||||
if journal.phase == "done" and journal.result is not None:
|
||||
return journal.result
|
||||
if not journal.final:
|
||||
raise RenameConflict(503, "completing", "This rename is finishing")
|
||||
if journal is not None and not journal.final:
|
||||
raise RenameConflict(
|
||||
503,
|
||||
"rename_in_progress",
|
||||
f"The rename of '{journal.old}' to '{journal.new}' is finishing",
|
||||
)
|
||||
if (expected_name, expected_version) != (name, version):
|
||||
raise RenameConflict(
|
||||
409,
|
||||
"stale",
|
||||
f"It is '{name}' at version {version} now, not '{expected_name}' "
|
||||
f"at version {expected_version}",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def begin(
|
||||
op: str,
|
||||
kind: Kind,
|
||||
obj_id: str,
|
||||
rename: Rename,
|
||||
rewrites: list[Rewrite],
|
||||
*,
|
||||
actor: str = "",
|
||||
fingerprint: str = "",
|
||||
durable: bool = True,
|
||||
rebuilt: Iterable[str] = (),
|
||||
) -> Journal:
|
||||
"""Write the journal of a rename about to change *rewrites*: its first
|
||||
write, after which it is finished or undone but no longer called off."""
|
||||
journal = Journal(
|
||||
op=op,
|
||||
kind=kind,
|
||||
id=obj_id,
|
||||
old=rename.old,
|
||||
new=rename.new,
|
||||
actor=actor,
|
||||
started_at=time.time(),
|
||||
fingerprint=fingerprint,
|
||||
durable=durable,
|
||||
affected=Affected(
|
||||
flows=[r.id for r in rewrites if r.kind == "flow"],
|
||||
rebuilt=[*rebuilt],
|
||||
dashboards=[r.id for r in rewrites if r.kind == "dashboard"],
|
||||
),
|
||||
files=[
|
||||
JournaledDocument(
|
||||
kind=r.kind, id=r.id, path=path, before=before, after=after
|
||||
)
|
||||
for r in rewrites
|
||||
for path, before, after in r.files
|
||||
],
|
||||
)
|
||||
journal.save()
|
||||
return journal
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Steps
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _message(journal: Journal, revert: bool = False) -> str:
|
||||
what = f"{journal.kind} '{journal.old}' to '{journal.new}'"
|
||||
if revert:
|
||||
return f"Revert rename of {what}"
|
||||
return f"Rename {what} ({journal.id})"
|
||||
|
||||
|
||||
def apply_documents(journal: Journal, store: FlowStore) -> None:
|
||||
"""Replace every document with its new text and commit them alone, then
|
||||
go forward. Raises when that fails, and `roll_back` undoes it."""
|
||||
texts = {doc.path: doc.after for doc in journal.files}
|
||||
if not store.replace_documents(texts, _message(journal)):
|
||||
raise RuntimeError("Could not commit the renamed documents")
|
||||
fault("commit")
|
||||
journal.phase = "forward"
|
||||
journal.save()
|
||||
|
||||
|
||||
def roll_back(journal: Journal, store: FlowStore) -> None:
|
||||
"""Put back the documents' texts from before, and commit that. Only
|
||||
before the journal went forward; nothing else was written by then."""
|
||||
store.replace_documents(
|
||||
{doc.path: doc.before for doc in journal.files}, _message(journal, True)
|
||||
)
|
||||
journal.phase = "rolled_back"
|
||||
journal.save()
|
||||
|
||||
|
||||
def run_steps(
|
||||
journal: Journal, state: StateBackend | None, queue: WorkQueue | None
|
||||
) -> None:
|
||||
"""Every step after the documents that is not done yet, in order: the
|
||||
history, the alert ledger and channels, live state and queued work.
|
||||
|
||||
Each can be run again, since only what still has an old name moves, so
|
||||
the journal only says which are finished. Without *state* or *queue*
|
||||
their steps are passed over: a memory backend's restart has nothing left
|
||||
to move. The engine must be held meanwhile.
|
||||
"""
|
||||
if journal.kind != "flow":
|
||||
return
|
||||
rename, steps = journal.rename, journal.steps
|
||||
|
||||
def finished(step: str) -> None:
|
||||
fault(step)
|
||||
setattr(steps, step, True)
|
||||
journal.save()
|
||||
|
||||
try:
|
||||
if not steps.sql.done:
|
||||
_rename_history(journal)
|
||||
if not steps.alert_ledger:
|
||||
_rename_ledger(rename)
|
||||
finished("alert_ledger")
|
||||
if not steps.alerts_file:
|
||||
_rename_channels(rename)
|
||||
finished("alerts_file")
|
||||
if not steps.state:
|
||||
if state is not None:
|
||||
state.rename_keys(rename, lambda what: fault(f"state:{what}"))
|
||||
finished("state")
|
||||
if not steps.queue:
|
||||
if queue is not None:
|
||||
queue.rename(rename, lambda what: fault(f"queue:{what}"))
|
||||
finished("queue")
|
||||
except Exception as exc:
|
||||
journal.error = f"{type(exc).__name__}: {exc}"
|
||||
with contextlib.suppress(OSError):
|
||||
journal.save()
|
||||
raise
|
||||
|
||||
|
||||
def finish(journal: Journal) -> RenameResult:
|
||||
"""Write the audit row, once, and mark the journal done with the result
|
||||
a repeated request is answered with."""
|
||||
if not journal.steps.audit:
|
||||
_audit(journal)
|
||||
fault("audit")
|
||||
journal.steps.audit = True
|
||||
journal.save()
|
||||
journal.result = _result(journal)
|
||||
journal.phase = "done"
|
||||
journal.error = ""
|
||||
journal.save()
|
||||
fault("done")
|
||||
return journal.result
|
||||
|
||||
|
||||
def _rename_history(journal: Journal) -> None:
|
||||
"""Message names in the history of the affected flows, a run at a time.
|
||||
|
||||
Scoped by the runs' flow ids, so a deleted flow that once had the same
|
||||
key keeps its history as it was. Renamed outputs are no longer what
|
||||
their cache key promised, so the key goes, and the outputs and where
|
||||
they were restored from stay.
|
||||
"""
|
||||
rename, progress = journal.rename, journal.steps.sql
|
||||
old, new = f"{rename.old}.", f"{rename.new}."
|
||||
with Session(db_engine) as session:
|
||||
runs = session.exec(
|
||||
select(Run.id)
|
||||
.where(
|
||||
col(Run.flow_id).in_(journal.affected.flows),
|
||||
col(Run.id) > progress.after_run,
|
||||
)
|
||||
.order_by(col(Run.id))
|
||||
).all()
|
||||
saved = time.monotonic()
|
||||
for run_id in runs:
|
||||
with Session(db_engine) as session:
|
||||
for model in (RunMetric, RunArtifact):
|
||||
name = col(model.name)
|
||||
session.exec(
|
||||
update(model)
|
||||
.where(
|
||||
col(model.run_id) == run_id,
|
||||
func.substr(name, 1, len(old)) == old,
|
||||
)
|
||||
.values(name=literal(new).concat(func.substr(name, len(old) + 1)))
|
||||
)
|
||||
for row in session.exec(
|
||||
select(RunNode).where(
|
||||
col(RunNode.run_id) == run_id,
|
||||
col(RunNode.outputs).contains(f'"{old}', autoescape=True),
|
||||
)
|
||||
).all():
|
||||
outputs = json.loads(row.outputs or "{}")
|
||||
moved = {rename.binding(key): value for key, value in outputs.items()}
|
||||
if moved != outputs:
|
||||
# Canonical, as `runs._cacheable` writes it.
|
||||
row.outputs = json.dumps(
|
||||
moved, sort_keys=True, separators=(",", ":")
|
||||
)
|
||||
row.cache_key = ""
|
||||
session.add(row)
|
||||
session.commit()
|
||||
fault("sql:run")
|
||||
if time.monotonic() - saved >= 1.0:
|
||||
progress.after_run = run_id
|
||||
journal.save()
|
||||
saved = time.monotonic()
|
||||
progress.done = True
|
||||
journal.save()
|
||||
|
||||
|
||||
def _rename_ledger(rename: Rename) -> None:
|
||||
"""The keys alerts were sent under, so a cooldown carries over."""
|
||||
with Session(db_engine) as session:
|
||||
for row in session.exec(select(AlertSent)).all():
|
||||
key = rename.dedup_key(row.key)
|
||||
if key != row.key:
|
||||
row.key = key
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
|
||||
def _rename_channels(rename: Rename) -> None:
|
||||
"""The message a dashboard alert channel publishes to, in `alerts.json`."""
|
||||
path = settings.ALERTS_FILE
|
||||
try:
|
||||
config = AlertsConfig.model_validate_json(path.read_text())
|
||||
except (OSError, ValueError):
|
||||
# None, or one the engine reads as empty: no channel names anything.
|
||||
return
|
||||
moved = 0
|
||||
for channel in config.channels:
|
||||
message = channel.config.get("message")
|
||||
if channel.kind == "dashboard" and isinstance(message, str):
|
||||
channel.config["message"] = rename.binding(message)
|
||||
moved += channel.config["message"] != message
|
||||
if moved:
|
||||
_replace(path, config.model_dump_json(indent=2))
|
||||
|
||||
|
||||
def _audit(journal: Journal) -> None:
|
||||
"""Who renamed what, straight into the database rather than through the
|
||||
bus, and only if it is not there already: a rename finished twice, by a
|
||||
crash and the recovery, is recorded once."""
|
||||
what = "" if journal.kind == "flow" else "dashboard "
|
||||
detail = f"renamed {what}'{journal.old}' to '{journal.new}' [op {journal.op}]"
|
||||
with Session(db_engine) as session:
|
||||
seen = session.exec(
|
||||
select(EngineEvent.id).where(
|
||||
col(EngineEvent.type) == "audit", col(EngineEvent.detail) == detail
|
||||
)
|
||||
).first()
|
||||
if seen is None:
|
||||
session.add(
|
||||
EngineEvent(
|
||||
ts=datetime.now(UTC),
|
||||
type="audit",
|
||||
flow_id=journal.id if journal.kind == "flow" else "",
|
||||
detail=detail,
|
||||
actor=journal.actor,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
|
||||
def _result(journal: Journal) -> RenameResult:
|
||||
documents: dict[tuple[str, str], RenamedDocument] = {}
|
||||
for doc in journal.files:
|
||||
version = int(orjson.loads(doc.after)["version"])
|
||||
entry = documents.setdefault(
|
||||
(doc.kind, doc.id),
|
||||
RenamedDocument(kind=doc.kind, id=doc.id, version=version),
|
||||
)
|
||||
if doc.path.endswith(".draft.json"):
|
||||
entry.draft_version = version
|
||||
return RenameResult(
|
||||
op=journal.op,
|
||||
kind=journal.kind,
|
||||
id=journal.id,
|
||||
old_name=journal.old,
|
||||
new_name=journal.new,
|
||||
documents=[*documents.values()],
|
||||
)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Recovery
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def recover_documents(root: Path) -> None:
|
||||
"""Settle the documents of a rename a crash interrupted. For startup,
|
||||
before anything reads the store at *root*, ids being assigned included.
|
||||
|
||||
Before the journal went forward the old texts go back, otherwise the new
|
||||
ones are made sure of, and only those paths are committed: a published
|
||||
document and its draft left with different keys would refuse to start.
|
||||
Plain files, no `FlowStore`.
|
||||
"""
|
||||
journal = read_journal()
|
||||
if journal is None or journal.final:
|
||||
return
|
||||
if journal.phase == "prepared":
|
||||
logger.warning(
|
||||
"Undoing the rename of %s '%s' to '%s', which was interrupted "
|
||||
"before it changed anything but its documents",
|
||||
journal.kind,
|
||||
journal.old,
|
||||
journal.new,
|
||||
)
|
||||
texts = {doc.path: doc.before for doc in journal.files}
|
||||
write_documents(root, texts, _message(journal, True))
|
||||
journal.phase = "rolled_back"
|
||||
journal.save()
|
||||
return
|
||||
write_documents(
|
||||
root, {doc.path: doc.after for doc in journal.files}, _message(journal)
|
||||
)
|
||||
|
||||
|
||||
def recover_runtime(state: StateBackend, queue: WorkQueue) -> None:
|
||||
"""Finish a rename a crash interrupted once its documents are settled
|
||||
(`recover_documents`): the steps left, the audit row and the journal's
|
||||
end. For startup, before anything reads alerts, devices, modules,
|
||||
metrics, the controller or runs. Raises when a step fails, to be tried
|
||||
again (`retry_until_done`)."""
|
||||
journal = read_journal()
|
||||
if journal is None or journal.phase != "forward":
|
||||
return
|
||||
logger.warning(
|
||||
"Finishing the rename of %s '%s' to '%s'",
|
||||
journal.kind,
|
||||
journal.old,
|
||||
journal.new,
|
||||
)
|
||||
if journal.durable:
|
||||
run_steps(journal, state, queue)
|
||||
else:
|
||||
run_steps(journal, None, None)
|
||||
finish(journal)
|
||||
|
||||
|
||||
async def retry_until_done(attempt: Callable[[], None], events: EventBus) -> None:
|
||||
"""Run *attempt* on a thread until it succeeds, however long that takes:
|
||||
a rename half done is not to be served. Each failure is logged and
|
||||
published as ``engine_degraded``, and the wait before the next doubles."""
|
||||
wait, longest = RETRY_S
|
||||
while True:
|
||||
try:
|
||||
await asyncio.to_thread(attempt)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.exception("A rename could not finish; trying again in %.0f s", wait)
|
||||
events.publish(
|
||||
{
|
||||
"type": "engine_degraded",
|
||||
"detail": f"A rename could not finish: {exc}",
|
||||
}
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
wait = min(wait * 2, longest)
|
||||
|
||||
@@ -943,6 +943,25 @@ class FlowStore:
|
||||
self._commit(f"Delete flow {named(key, flow_id)}")
|
||||
return key
|
||||
|
||||
def replace_documents(self, texts: dict[str, str], message: str) -> bool:
|
||||
"""Write flow and dashboard documents whole, *texts* by path relative
|
||||
to the store, and commit those paths alone with *message*.
|
||||
|
||||
A key rename's (`fluksio.flow.rename`): each text carries its version
|
||||
already, and whatever else is uncommitted stays so. False when the
|
||||
commit failed. A store without git counts as committed.
|
||||
"""
|
||||
# Here rather than at the top: it reads where dashboards live, and
|
||||
# the dashboards module imports this one.
|
||||
from fluksio.flow.identity import write_documents
|
||||
|
||||
with self._write_lock:
|
||||
try:
|
||||
return write_documents(self.root, texts, message)
|
||||
finally:
|
||||
# Whatever got written, the caches are to read it again.
|
||||
self.revision += 1
|
||||
|
||||
def duplicate_flow(
|
||||
self, flow_id: str, busy: Callable[[str], bool] | None = None
|
||||
) -> FlowDef:
|
||||
|
||||
+28
-11
@@ -49,7 +49,12 @@ from fluksio.flow.plugins import load_plugins
|
||||
from fluksio.flow.provision import load_provisioners
|
||||
from fluksio.flow.queue import MemoryWorkQueue, RedisWorkQueue, WorkQueue
|
||||
from fluksio.flow.remote import RemoteWorkerHub
|
||||
from fluksio.flow.rename import RenameGate
|
||||
from fluksio.flow.rename import (
|
||||
RenameGate,
|
||||
recover_documents,
|
||||
recover_runtime,
|
||||
retry_until_done,
|
||||
)
|
||||
from fluksio.flow.resources import (
|
||||
ResourceAccountant,
|
||||
fair_share_env,
|
||||
@@ -295,15 +300,10 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
# Connectors register their node types before any flow is built with them.
|
||||
load_plugins()
|
||||
|
||||
alerts = AlertManager(
|
||||
event_bus, config=read_alerts_config(), ledger=AlertLedger()
|
||||
)
|
||||
execution = ExecutionService(
|
||||
queue=_work_queue(),
|
||||
max_workers=settings.FLOW_MAX_WORKERS,
|
||||
events=event_bus,
|
||||
max_cascades=settings.FLOW_MAX_CASCADES,
|
||||
)
|
||||
# A rename the last process left half done settles its documents
|
||||
# first: a published document and its draft written apart would
|
||||
# have different keys, which assigning ids refuses.
|
||||
await run_in_threadpool(recover_documents, settings.FLOWS_DIR)
|
||||
# Every flow and dashboard has an id before anything reads one: the
|
||||
# store's own `_migrate`, the device migrations below and every cache
|
||||
# after them write documents back with whatever id they read.
|
||||
@@ -313,6 +313,23 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
# no flow holds any more is passed on as it is, which is the id of a
|
||||
# flow from before ids.
|
||||
event_bus.flow_id_of = lambda key: store.key_index().get(key, key)
|
||||
state = _state_backend()
|
||||
queue = _work_queue()
|
||||
# Then the rest of it, before anything reads alerts, devices, modules,
|
||||
# metrics, flows or runs. Until it is finished nothing is served.
|
||||
await retry_until_done(lambda: recover_runtime(state, queue), event_bus)
|
||||
# Work that arrived while a rename held the queue, which a crash
|
||||
# left behind.
|
||||
await run_in_threadpool(queue.release_held)
|
||||
alerts = AlertManager(
|
||||
event_bus, config=read_alerts_config(), ledger=AlertLedger()
|
||||
)
|
||||
execution = ExecutionService(
|
||||
queue=queue,
|
||||
max_workers=settings.FLOW_MAX_WORKERS,
|
||||
events=event_bus,
|
||||
max_cascades=settings.FLOW_MAX_CASCADES,
|
||||
)
|
||||
# The packages node code imports. In the background, because a first
|
||||
# install of anything substantial takes minutes and `uv` is given five
|
||||
# of them twice over — held here, the container never answers its
|
||||
@@ -376,7 +393,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
app.state.worker_hub = worker_hub
|
||||
controller = FlowController(
|
||||
store=store,
|
||||
state=_state_backend(),
|
||||
state=state,
|
||||
events=event_bus,
|
||||
max_workers=settings.FLOW_MAX_WORKERS,
|
||||
fastapi_app=app,
|
||||
|
||||
Reference in New Issue
Block a user