131 lines
4.6 KiB
Python
131 lines
4.6 KiB
Python
"""What is left of a deleted flow: its last key, while its history is kept.
|
|
|
|
A flow's runs, events and metrics are filed under its id and outlive it. A new
|
|
flow's id is opaque, so a tombstone keeps the key it last had, and `names_of`
|
|
reads that back for anything showing the history. A flow deleted before
|
|
tombstones existed has its old key as its id, which already reads right.
|
|
|
|
The flow store is the one writer: it buries a flow before removing its files,
|
|
asks before issuing an id, and lifts the tombstone of a flow restored under a
|
|
pinned id once the restored document is on disk.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterable
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import delete, or_
|
|
from sqlmodel import Session, col, select
|
|
|
|
from fluksio.core.db import engine
|
|
from fluksio.models import EngineEvent, FlowRun, FlowTombstone, MetricBucket, Run
|
|
|
|
if TYPE_CHECKING:
|
|
from fluksio.flow.store import FlowStore
|
|
|
|
|
|
def referenced(session: Session, flow_id: str) -> bool:
|
|
"""Whether any history row names *flow_id*, as its flow or a node's.
|
|
|
|
A training run's ``for_node`` names the node it trains, in a flow other
|
|
than the run's own, so the four ``flow`` columns are not enough.
|
|
"""
|
|
nodes = f"{flow_id}."
|
|
queries = [
|
|
select(Run.id).where(
|
|
or_(
|
|
col(Run.flow) == flow_id,
|
|
col(Run.for_node).startswith(nodes, autoescape=True),
|
|
)
|
|
),
|
|
select(FlowRun.id).where(col(FlowRun.flow) == flow_id),
|
|
select(MetricBucket.flow).where(
|
|
or_(
|
|
col(MetricBucket.flow) == flow_id,
|
|
col(MetricBucket.node).startswith(nodes, autoescape=True),
|
|
)
|
|
),
|
|
select(EngineEvent.id).where(
|
|
or_(
|
|
col(EngineEvent.flow) == flow_id,
|
|
col(EngineEvent.node).startswith(nodes, autoescape=True),
|
|
)
|
|
),
|
|
]
|
|
return any(session.exec(query.limit(1)).first() is not None for query in queries)
|
|
|
|
|
|
class Tombstones:
|
|
"""Tombstones and the history behind an id, as the flow store asks."""
|
|
|
|
def bury(self, flow_id: str, name: str) -> None:
|
|
"""Keep *name* for *flow_id*, before its files go.
|
|
|
|
Written again by a retried delete, which may find another name if the
|
|
first attempt never got as far as removing anything.
|
|
"""
|
|
with Session(engine) as session:
|
|
stone = session.get(FlowTombstone, flow_id) or FlowTombstone(
|
|
id=flow_id, name=name
|
|
)
|
|
stone.name = name
|
|
session.add(stone)
|
|
session.commit()
|
|
|
|
def lift(self, flow_id: str) -> None:
|
|
"""Forget the tombstone of a flow that is live again."""
|
|
with Session(engine) as session:
|
|
session.execute(
|
|
delete(FlowTombstone).where(col(FlowTombstone.id) == flow_id)
|
|
)
|
|
session.commit()
|
|
|
|
def recorded(self, flow_id: str) -> str | None:
|
|
"""The key the history of a flow that is not live is filed under.
|
|
|
|
Its tombstone's, or the id itself for history from before tombstones,
|
|
whose id was its key. None when nothing names the id at all.
|
|
"""
|
|
with Session(engine) as session:
|
|
stone = session.get(FlowTombstone, flow_id)
|
|
if stone is not None:
|
|
return stone.name
|
|
return flow_id if referenced(session, flow_id) else None
|
|
|
|
|
|
def names_of(ids: Iterable[str], store: FlowStore) -> dict[str, str]:
|
|
"""Each flow id's key: the live flow's, else its tombstone's, else the id.
|
|
|
|
One read of the key index and one query, however many ids a response
|
|
carries.
|
|
"""
|
|
wanted = set(ids)
|
|
live = {flow_id: key for key, flow_id in store.key_index().items()}
|
|
names = {flow_id: live[flow_id] for flow_id in wanted if flow_id in live}
|
|
gone = wanted - names.keys()
|
|
if gone:
|
|
with Session(engine) as session:
|
|
for stone in session.exec(
|
|
select(FlowTombstone).where(col(FlowTombstone.id).in_(gone))
|
|
):
|
|
names[stone.id] = stone.name
|
|
return {flow_id: names.get(flow_id, flow_id) for flow_id in wanted}
|
|
|
|
|
|
def name_of(flow_id: str, store: FlowStore) -> str:
|
|
"""One flow id's key, as `names_of`."""
|
|
return names_of([flow_id], store)[flow_id]
|
|
|
|
|
|
def prune(session: Session) -> int:
|
|
"""Drop the tombstones no history row names any more. Returns how many."""
|
|
stale = [
|
|
stone.id
|
|
for stone in session.exec(select(FlowTombstone))
|
|
if not referenced(session, stone.id)
|
|
]
|
|
if stale:
|
|
session.execute(delete(FlowTombstone).where(col(FlowTombstone.id).in_(stale)))
|
|
return len(stale)
|