Un-key stage-cache rows written before the current key shape
CACHE_SHAPE_SINCE marks the last change that moved every cache key. The run service's periodic sweep clears cache_key on rows of runs created before it, which nothing can match any more. Their outputs stay.
This commit is contained in:
@@ -52,7 +52,7 @@ from fluksio.core.db import engine as db_engine
|
||||
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
|
||||
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
|
||||
from fluksio.flow.messages import qualify
|
||||
from fluksio.flow.pipeline import CacheHit, NodeOutcome, Pipeline
|
||||
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
|
||||
from fluksio.flow.queue import WorkItem, WorkQueue
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
from fluksio.flow.state import MemoryState, StateBackend
|
||||
@@ -759,6 +759,26 @@ def _cacheable(outcome: NodeOutcome) -> str | None:
|
||||
return text if len(text) <= OUTPUT_CAP else None
|
||||
|
||||
|
||||
def prune_stale_cache() -> int:
|
||||
"""Un-key the stage-cache rows written before the current key shape.
|
||||
|
||||
Such a row can match no key a node computes any more, so it is a cache
|
||||
entry in name only. The row and its outputs stay: they are the record of
|
||||
what that run's node returned. Returns how many rows were un-keyed.
|
||||
"""
|
||||
with Session(db_engine) as session:
|
||||
older = select(Run.id).where(col(Run.created_at) < CACHE_SHAPE_SINCE)
|
||||
result = session.exec(
|
||||
update(RunNode)
|
||||
.where(col(RunNode.cache_key) != "", col(RunNode.run_id).in_(older))
|
||||
.values(cache_key="")
|
||||
)
|
||||
session.commit()
|
||||
if result.rowcount:
|
||||
logger.info("Un-keyed %d stage-cache rows of an older shape", result.rowcount)
|
||||
return int(result.rowcount)
|
||||
|
||||
|
||||
class RunCache:
|
||||
"""What earlier runs already worked out, looked up by cache key.
|
||||
|
||||
@@ -1353,6 +1373,7 @@ class RunService:
|
||||
if time.monotonic() - last_sweep >= SWEEP_INTERVAL_S:
|
||||
last_sweep = time.monotonic()
|
||||
self._sweep_abandoned(now)
|
||||
prune_stale_cache()
|
||||
except Exception:
|
||||
logger.exception("Could not refresh run leases")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user