"""What is left of a deleted flow: its last key, while its history is kept. A flow's runs, events and metrics are filed under its id and outlive it. A new flow's id is opaque, so a tombstone keeps the key it last had, and `names_of` reads that back for anything showing the history. A flow deleted before tombstones existed has its old key as its id, which already reads right. History names a node ``.``. The API shows and takes it as ``.``, the way the engine names it, and `shown_nodes` and `stored_node` translate between the two. The flow store is the one writer: it buries a flow before removing its files, asks before issuing an id, and lifts the tombstone of a flow restored under a pinned id once the restored document is on disk. """ from __future__ import annotations from collections.abc import Iterable from typing import TYPE_CHECKING from sqlalchemy import delete, or_ from sqlmodel import Session, col, select from fluksio.core.db import engine from fluksio.models import EngineEvent, FlowRun, FlowTombstone, MetricBucket, Run if TYPE_CHECKING: from fluksio.flow.store import FlowStore def referenced(session: Session, flow_id: str) -> bool: """Whether any history row names *flow_id*, as its flow or a node's. A training run's ``for_node`` names the node it trains, in a flow other than the run's own, so the four ``flow`` columns are not enough. """ nodes = f"{flow_id}." queries = [ select(Run.id).where( or_( col(Run.flow_id) == flow_id, col(Run.for_node).startswith(nodes, autoescape=True), ) ), select(FlowRun.id).where(col(FlowRun.flow_id) == flow_id), select(MetricBucket.flow_id).where( or_( col(MetricBucket.flow_id) == flow_id, col(MetricBucket.node).startswith(nodes, autoescape=True), ) ), select(EngineEvent.id).where( or_( col(EngineEvent.flow_id) == flow_id, col(EngineEvent.node).startswith(nodes, autoescape=True), ) ), ] return any(session.exec(query.limit(1)).first() is not None for query in queries) class Tombstones: """Tombstones and the history behind an id, as the flow store asks.""" def bury(self, flow_id: str, name: str) -> None: """Keep *name* for *flow_id*, before its files go. Written again by a retried delete, which may find another name if the first attempt never got as far as removing anything. """ with Session(engine) as session: stone = session.get(FlowTombstone, flow_id) or FlowTombstone( id=flow_id, name=name ) stone.name = name session.add(stone) session.commit() def lift(self, flow_id: str) -> None: """Forget the tombstone of a flow that is live again.""" with Session(engine) as session: session.execute( delete(FlowTombstone).where(col(FlowTombstone.id) == flow_id) ) session.commit() def recorded(self, flow_id: str) -> str | None: """The key the history of a flow that is not live is filed under. Its tombstone's, or the id itself for history from before tombstones, whose id was its key. None when nothing names the id at all. """ with Session(engine) as session: stone = session.get(FlowTombstone, flow_id) if stone is not None: return stone.name return flow_id if referenced(session, flow_id) else None def names_of(ids: Iterable[str], store: FlowStore) -> dict[str, str]: """Each flow id's key: the live flow's, else its tombstone's, else the id. One read of the key index and one query, however many ids a response carries. """ wanted = set(ids) live = {flow_id: key for key, flow_id in store.key_index().items()} names = {flow_id: live[flow_id] for flow_id in wanted if flow_id in live} gone = wanted - names.keys() if gone: with Session(engine) as session: for stone in session.exec( select(FlowTombstone).where(col(FlowTombstone.id).in_(gone)) ): names[stone.id] = stone.name return {flow_id: names.get(flow_id, flow_id) for flow_id in wanted} def name_of(flow_id: str, store: FlowStore) -> str: """One flow id's key, as `names_of`.""" return names_of([flow_id], store)[flow_id] def id_of(key: str, store: FlowStore) -> str: """The flow id behind *key*: the live flow's, else that of the one deleted flow that last held it, else *key* itself, as for history from before ids. """ live = store.key_index().get(key) if live is not None: return live with Session(engine) as session: graves = session.exec( select(FlowTombstone.id).where(col(FlowTombstone.name) == key).limit(2) ).all() return graves[0] if len(graves) == 1 else key def shown_nodes(nodes: Iterable[str], store: FlowStore) -> dict[str, str]: """Each stored ``.`` as ``.``, with one lookup.""" split = {node: node.partition(".") for node in nodes} names = names_of({flow for flow, dot, _ in split.values() if dot}, store) return { node: f"{names[flow]}.{rest}" if dot else node for node, (flow, dot, rest) in split.items() } def stored_node(node: str, store: FlowStore, flow_id: str = "") -> str: """A ``.`` as history stores it, ``.``. With *flow_id*, a key that is that flow's own resolves to it, which is how a node of a deleted or renamed flow is still found by the name it shows. """ key, dot, rest = node.partition(".") if not dot: return node if flow_id and name_of(flow_id, store) == key: return f"{flow_id}.{rest}" return f"{id_of(key, store)}.{rest}" def prune(session: Session) -> int: """Drop the tombstones no history row names any more. Returns how many.""" stale = [ stone.id for stone in session.exec(select(FlowTombstone)) if not referenced(session, stone.id) ] if stale: session.execute(delete(FlowTombstone).where(col(FlowTombstone.id).in_(stale))) return len(stale)