"""What is left of a deleted flow: its last key, while its history is kept. A flow's runs, events and metrics are filed under its id and outlive it. A new flow's id is opaque, so a tombstone keeps the key it last had, and `names_of` reads that back for anything showing the history. A flow deleted before tombstones existed has its old key as its id, which already reads right. The flow store is the one writer: it buries a flow before removing its files, asks before issuing an id, and lifts the tombstone of a flow restored under a pinned id once the restored document is on disk. """ from __future__ import annotations from collections.abc import Iterable from typing import TYPE_CHECKING from sqlalchemy import delete, or_ from sqlmodel import Session, col, select from fluksio.core.db import engine from fluksio.models import EngineEvent, FlowRun, FlowTombstone, MetricBucket, Run if TYPE_CHECKING: from fluksio.flow.store import FlowStore def referenced(session: Session, flow_id: str) -> bool: """Whether any history row names *flow_id*, as its flow or a node's. A training run's ``for_node`` names the node it trains, in a flow other than the run's own, so the four ``flow`` columns are not enough. """ nodes = f"{flow_id}." queries = [ select(Run.id).where( or_( col(Run.flow) == flow_id, col(Run.for_node).startswith(nodes, autoescape=True), ) ), select(FlowRun.id).where(col(FlowRun.flow) == flow_id), select(MetricBucket.flow).where( or_( col(MetricBucket.flow) == flow_id, col(MetricBucket.node).startswith(nodes, autoescape=True), ) ), select(EngineEvent.id).where( or_( col(EngineEvent.flow) == flow_id, col(EngineEvent.node).startswith(nodes, autoescape=True), ) ), ] return any(session.exec(query.limit(1)).first() is not None for query in queries) class Tombstones: """Tombstones and the history behind an id, as the flow store asks.""" def bury(self, flow_id: str, name: str) -> None: """Keep *name* for *flow_id*, before its files go. Written again by a retried delete, which may find another name if the first attempt never got as far as removing anything. """ with Session(engine) as session: stone = session.get(FlowTombstone, flow_id) or FlowTombstone( id=flow_id, name=name ) stone.name = name session.add(stone) session.commit() def lift(self, flow_id: str) -> None: """Forget the tombstone of a flow that is live again.""" with Session(engine) as session: session.execute( delete(FlowTombstone).where(col(FlowTombstone.id) == flow_id) ) session.commit() def recorded(self, flow_id: str) -> str | None: """The key the history of a flow that is not live is filed under. Its tombstone's, or the id itself for history from before tombstones, whose id was its key. None when nothing names the id at all. """ with Session(engine) as session: stone = session.get(FlowTombstone, flow_id) if stone is not None: return stone.name return flow_id if referenced(session, flow_id) else None def names_of(ids: Iterable[str], store: FlowStore) -> dict[str, str]: """Each flow id's key: the live flow's, else its tombstone's, else the id. One read of the key index and one query, however many ids a response carries. """ wanted = set(ids) live = {flow_id: key for key, flow_id in store.key_index().items()} names = {flow_id: live[flow_id] for flow_id in wanted if flow_id in live} gone = wanted - names.keys() if gone: with Session(engine) as session: for stone in session.exec( select(FlowTombstone).where(col(FlowTombstone.id).in_(gone)) ): names[stone.id] = stone.name return {flow_id: names.get(flow_id, flow_id) for flow_id in wanted} def name_of(flow_id: str, store: FlowStore) -> str: """One flow id's key, as `names_of`.""" return names_of([flow_id], store)[flow_id] def prune(session: Session) -> int: """Drop the tombstones no history row names any more. Returns how many.""" stale = [ stone.id for stone in session.exec(select(FlowTombstone)) if not referenced(session, stone.id) ] if stale: session.execute(delete(FlowTombstone).where(col(FlowTombstone.id).in_(stale))) return len(stale)