DELETE /artifacts/{digest}: superuser only, 409 while referenced
The keep-set the hourly sweep uses is now referenced_artifacts, shared by the sweep and the route. The volatile ring is not touched.
This commit is contained in:
@@ -160,6 +160,19 @@ class ArtifactStore:
|
||||
self.put_file(source)
|
||||
return True
|
||||
|
||||
def delete(self, digest: str) -> bool:
|
||||
"""Remove one artifact's bytes. Returns whether this store had them.
|
||||
|
||||
The ring is not looked in: its frames fall out on their own.
|
||||
"""
|
||||
if not valid_digest(digest):
|
||||
return False
|
||||
try:
|
||||
self._path(digest).unlink()
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def read(self, digest: str) -> Iterator[bytes]:
|
||||
target = self.path(digest)
|
||||
if target is None:
|
||||
|
||||
@@ -382,13 +382,12 @@ def sweep_artifacts(
|
||||
) -> int:
|
||||
"""Remove artifact bytes nothing refers to any more. Returns how many went.
|
||||
|
||||
Three things refer to an artifact: a run that recorded it, a message
|
||||
currently holding it, and a flow input's ``initial``, published or draft,
|
||||
which a flow that has never started holds nowhere else. The second is what
|
||||
makes a stream of media affordable — a camera publishing a frame a second
|
||||
replaces the reference each time, so yesterday's frames are unreferenced by
|
||||
definition and the store does not grow without bound. What a node *returns*
|
||||
is recorded against its run and kept, what it emits along the way is not.
|
||||
What still refers to one is `referenced_artifacts`. A message holding one is
|
||||
what makes a stream of media affordable — a camera publishing a frame a
|
||||
second replaces the reference each time, so yesterday's frames are
|
||||
unreferenced by definition and the store does not grow without bound. What
|
||||
a node *returns* is recorded against its run and kept, what it emits along
|
||||
the way is not.
|
||||
|
||||
Skipped entirely while anything is running: a node that stores a checkpoint
|
||||
an hour before it returns has neither a row nor a message naming it yet,
|
||||
@@ -401,6 +400,23 @@ def sweep_artifacts(
|
||||
if active is not None:
|
||||
logger.debug("Artifact sweep skipped: run %s is in flight", active)
|
||||
return 0
|
||||
|
||||
removed = store.collect(referenced_artifacts(state, flows), grace_s)
|
||||
if removed:
|
||||
logger.info("Artifact sweep removed %d unreferenced artifacts", removed)
|
||||
return removed
|
||||
|
||||
|
||||
def referenced_artifacts(
|
||||
state: StateBackend, flows: FlowStore | None = None
|
||||
) -> set[str]:
|
||||
"""Every artifact digest something still refers to.
|
||||
|
||||
Three things do: a run that recorded it, a message currently holding it,
|
||||
and a flow input's ``initial``, published or draft, which a flow that has
|
||||
never started holds nowhere else.
|
||||
"""
|
||||
with Session(db_engine) as session:
|
||||
keep = set(session.exec(select(col(RunArtifact.digest)).distinct()).all())
|
||||
|
||||
if flows is not None:
|
||||
@@ -417,11 +433,7 @@ def sweep_artifacts(
|
||||
for start in range(0, len(keys), _SWEEP_BATCH):
|
||||
for value in state.get_multi(keys[start : start + _SWEEP_BATCH]).values():
|
||||
_references_in(value, keep)
|
||||
|
||||
removed = store.collect(keep, grace_s)
|
||||
if removed:
|
||||
logger.info("Artifact sweep removed %d unreferenced artifacts", removed)
|
||||
return removed
|
||||
return keep
|
||||
|
||||
|
||||
def seed_values(
|
||||
|
||||
Reference in New Issue
Block a user