Un-key stage-cache rows written before the current key shape
CACHE_SHAPE_SINCE marks the last change that moved every cache key. The run service's periodic sweep clears cache_key on rows of runs created before it, which nothing can match any more. Their outputs stay.
This commit is contained in:
@@ -19,7 +19,7 @@ from fluksio.core.db import engine as db_engine
|
||||
from fluksio.flow.artifacts import ArtifactStore
|
||||
from fluksio.flow.controller import FlowController
|
||||
from fluksio.flow.messages import DType, MessageSpec
|
||||
from fluksio.flow.pipeline import NodeOutcome
|
||||
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, NodeOutcome
|
||||
from fluksio.flow.runs import (
|
||||
LEASE_STALE_S,
|
||||
OUTPUT_CAP,
|
||||
@@ -28,6 +28,7 @@ from fluksio.flow.runs import (
|
||||
RunService,
|
||||
_cacheable,
|
||||
new_run_id,
|
||||
prune_stale_cache,
|
||||
resolve_references,
|
||||
seed_values,
|
||||
)
|
||||
@@ -106,6 +107,38 @@ def test_what_may_be_stored_as_a_cache_entry():
|
||||
assert _cacheable(ok.model_copy(update={"output_values": big})) is None
|
||||
|
||||
|
||||
def test_a_row_keyed_before_the_current_shape_is_unkeyed():
|
||||
"""It matches nothing any more, but it is still the run's record."""
|
||||
outputs = json.dumps({"study.loss": 1.0})
|
||||
with Session(db_engine) as session:
|
||||
for run_id, made in (
|
||||
("shape-old", CACHE_SHAPE_SINCE - timedelta(days=1)),
|
||||
("shape-new", CACHE_SHAPE_SINCE + timedelta(seconds=1)),
|
||||
):
|
||||
session.add(Run(id=run_id, flow="study", status="ok", created_at=made))
|
||||
session.add(
|
||||
RunNode(
|
||||
run_id=run_id,
|
||||
node="study.a",
|
||||
cache_key=run_id,
|
||||
outputs=outputs,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
try:
|
||||
assert prune_stale_cache() == 1
|
||||
with Session(db_engine) as session:
|
||||
old = session.get(RunNode, ("shape-old", "study.a"))
|
||||
new = session.get(RunNode, ("shape-new", "study.a"))
|
||||
assert old is not None and old.cache_key == "" and old.outputs == outputs
|
||||
assert new is not None and new.cache_key == "shape-new"
|
||||
finally:
|
||||
with Session(db_engine) as session:
|
||||
for run_id in ("shape-old", "shape-new"):
|
||||
session.delete(session.get(Run, run_id))
|
||||
session.commit()
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Naming an artifact from outside the process that made it
|
||||
#
|
||||
|
||||
Reference in New Issue
Block a user