173 lines
6.2 KiB
Python
173 lines
6.2 KiB
Python
"""What is left of a deleted flow: its last key, while its history is kept.
|
|
|
|
A flow's runs, events and metrics are filed under its id and outlive it. A new
|
|
flow's id is opaque, so a tombstone keeps the key it last had, and `names_of`
|
|
reads that back for anything showing the history. A flow deleted before
|
|
tombstones existed has its old key as its id, which already reads right.
|
|
|
|
History names a node ``<flow id>.<node>``. The API shows and takes it as
|
|
``<key>.<node>``, the way the engine names it, and `shown_nodes` and
|
|
`stored_node` translate between the two.
|
|
|
|
The flow store is the one writer: it buries a flow before removing its files,
|
|
asks before issuing an id, and lifts the tombstone of a flow restored under a
|
|
pinned id once the restored document is on disk.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterable
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import delete, or_
|
|
from sqlmodel import Session, col, select
|
|
|
|
from fluksio.core.db import engine
|
|
from fluksio.models import EngineEvent, FlowRun, FlowTombstone, MetricBucket, Run
|
|
|
|
if TYPE_CHECKING:
|
|
from fluksio.flow.store import FlowStore
|
|
|
|
|
|
def referenced(session: Session, flow_id: str) -> bool:
|
|
"""Whether any history row names *flow_id*, as its flow or a node's.
|
|
|
|
A training run's ``for_node`` names the node it trains, in a flow other
|
|
than the run's own, so the four ``flow`` columns are not enough.
|
|
"""
|
|
nodes = f"{flow_id}."
|
|
queries = [
|
|
select(Run.id).where(
|
|
or_(
|
|
col(Run.flow_id) == flow_id,
|
|
col(Run.for_node).startswith(nodes, autoescape=True),
|
|
)
|
|
),
|
|
select(FlowRun.id).where(col(FlowRun.flow_id) == flow_id),
|
|
select(MetricBucket.flow_id).where(
|
|
or_(
|
|
col(MetricBucket.flow_id) == flow_id,
|
|
col(MetricBucket.node).startswith(nodes, autoescape=True),
|
|
)
|
|
),
|
|
select(EngineEvent.id).where(
|
|
or_(
|
|
col(EngineEvent.flow_id) == flow_id,
|
|
col(EngineEvent.node).startswith(nodes, autoescape=True),
|
|
)
|
|
),
|
|
]
|
|
return any(session.exec(query.limit(1)).first() is not None for query in queries)
|
|
|
|
|
|
class Tombstones:
|
|
"""Tombstones and the history behind an id, as the flow store asks."""
|
|
|
|
def bury(self, flow_id: str, name: str) -> None:
|
|
"""Keep *name* for *flow_id*, before its files go.
|
|
|
|
Written again by a retried delete, which may find another name if the
|
|
first attempt never got as far as removing anything.
|
|
"""
|
|
with Session(engine) as session:
|
|
stone = session.get(FlowTombstone, flow_id) or FlowTombstone(
|
|
id=flow_id, name=name
|
|
)
|
|
stone.name = name
|
|
session.add(stone)
|
|
session.commit()
|
|
|
|
def lift(self, flow_id: str) -> None:
|
|
"""Forget the tombstone of a flow that is live again."""
|
|
with Session(engine) as session:
|
|
session.execute(
|
|
delete(FlowTombstone).where(col(FlowTombstone.id) == flow_id)
|
|
)
|
|
session.commit()
|
|
|
|
def recorded(self, flow_id: str) -> str | None:
|
|
"""The key the history of a flow that is not live is filed under.
|
|
|
|
Its tombstone's, or the id itself for history from before tombstones,
|
|
whose id was its key. None when nothing names the id at all.
|
|
"""
|
|
with Session(engine) as session:
|
|
stone = session.get(FlowTombstone, flow_id)
|
|
if stone is not None:
|
|
return stone.name
|
|
return flow_id if referenced(session, flow_id) else None
|
|
|
|
|
|
def names_of(ids: Iterable[str], store: FlowStore) -> dict[str, str]:
|
|
"""Each flow id's key: the live flow's, else its tombstone's, else the id.
|
|
|
|
One read of the key index and one query, however many ids a response
|
|
carries.
|
|
"""
|
|
wanted = set(ids)
|
|
live = {flow_id: key for key, flow_id in store.key_index().items()}
|
|
names = {flow_id: live[flow_id] for flow_id in wanted if flow_id in live}
|
|
gone = wanted - names.keys()
|
|
if gone:
|
|
with Session(engine) as session:
|
|
for stone in session.exec(
|
|
select(FlowTombstone).where(col(FlowTombstone.id).in_(gone))
|
|
):
|
|
names[stone.id] = stone.name
|
|
return {flow_id: names.get(flow_id, flow_id) for flow_id in wanted}
|
|
|
|
|
|
def name_of(flow_id: str, store: FlowStore) -> str:
|
|
"""One flow id's key, as `names_of`."""
|
|
return names_of([flow_id], store)[flow_id]
|
|
|
|
|
|
def id_of(key: str, store: FlowStore) -> str:
|
|
"""The flow id behind *key*: the live flow's, else that of the one deleted
|
|
flow that last held it, else *key* itself, as for history from before ids.
|
|
"""
|
|
live = store.key_index().get(key)
|
|
if live is not None:
|
|
return live
|
|
with Session(engine) as session:
|
|
graves = session.exec(
|
|
select(FlowTombstone.id).where(col(FlowTombstone.name) == key).limit(2)
|
|
).all()
|
|
return graves[0] if len(graves) == 1 else key
|
|
|
|
|
|
def shown_nodes(nodes: Iterable[str], store: FlowStore) -> dict[str, str]:
|
|
"""Each stored ``<flow id>.<node>`` as ``<key>.<node>``, with one lookup."""
|
|
split = {node: node.partition(".") for node in nodes}
|
|
names = names_of({flow for flow, dot, _ in split.values() if dot}, store)
|
|
return {
|
|
node: f"{names[flow]}.{rest}" if dot else node
|
|
for node, (flow, dot, rest) in split.items()
|
|
}
|
|
|
|
|
|
def stored_node(node: str, store: FlowStore, flow_id: str = "") -> str:
|
|
"""A ``<key>.<node>`` as history stores it, ``<flow id>.<node>``.
|
|
|
|
With *flow_id*, a key that is that flow's own resolves to it, which is how
|
|
a node of a deleted or renamed flow is still found by the name it shows.
|
|
"""
|
|
key, dot, rest = node.partition(".")
|
|
if not dot:
|
|
return node
|
|
if flow_id and name_of(flow_id, store) == key:
|
|
return f"{flow_id}.{rest}"
|
|
return f"{id_of(key, store)}.{rest}"
|
|
|
|
|
|
def prune(session: Session) -> int:
|
|
"""Drop the tombstones no history row names any more. Returns how many."""
|
|
stale = [
|
|
stone.id
|
|
for stone in session.exec(select(FlowTombstone))
|
|
if not referenced(session, stone.id)
|
|
]
|
|
if stale:
|
|
session.execute(delete(FlowTombstone).where(col(FlowTombstone.id).in_(stale)))
|
|
return len(stale)
|