Un-key stage-cache rows written before the current key shape

CACHE_SHAPE_SINCE marks the last change that moved every cache key. The
run service's periodic sweep clears cache_key on rows of runs created
before it, which nothing can match any more. Their outputs stay.
This commit is contained in:
2026-10-15 10:54:22 +02:00
parent 6fa44bacf3
commit a95be36dba
5 changed files with 69 additions and 4 deletions
+22 -1
View File
@@ -52,7 +52,7 @@ from fluksio.core.db import engine as db_engine
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
from fluksio.flow.messages import qualify
from fluksio.flow.pipeline import CacheHit, NodeOutcome, Pipeline
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
from fluksio.flow.queue import WorkItem, WorkQueue
from fluksio.flow.schemas import FlowDef, NodeDef
from fluksio.flow.state import MemoryState, StateBackend
@@ -759,6 +759,26 @@ def _cacheable(outcome: NodeOutcome) -> str | None:
return text if len(text) <= OUTPUT_CAP else None
def prune_stale_cache() -> int:
"""Un-key the stage-cache rows written before the current key shape.
Such a row can match no key a node computes any more, so it is a cache
entry in name only. The row and its outputs stay: they are the record of
what that run's node returned. Returns how many rows were un-keyed.
"""
with Session(db_engine) as session:
older = select(Run.id).where(col(Run.created_at) < CACHE_SHAPE_SINCE)
result = session.exec(
update(RunNode)
.where(col(RunNode.cache_key) != "", col(RunNode.run_id).in_(older))
.values(cache_key="")
)
session.commit()
if result.rowcount:
logger.info("Un-keyed %d stage-cache rows of an older shape", result.rowcount)
return int(result.rowcount)
class RunCache:
"""What earlier runs already worked out, looked up by cache key.
@@ -1353,6 +1373,7 @@ class RunService:
if time.monotonic() - last_sweep >= SWEEP_INTERVAL_S:
last_sweep = time.monotonic()
self._sweep_abandoned(now)
prune_stale_cache()
except Exception:
logger.exception("Could not refresh run leases")