stroblme
7ea53ccf17
The keep-set the hourly sweep uses is now referenced_artifacts, shared by the sweep and the route. The volatile ring is not touched.
192 lines
7.3 KiB
Python
192 lines
7.3 KiB
Python
"""Artifacts over HTTP: the one way bytes get in and out of the store.
|
|
|
|
A node on this host could reach the directory itself, but a node on a remote
|
|
worker cannot — and having one path rather than two is what keeps a flow's
|
|
code the same wherever it runs.
|
|
"""
|
|
|
|
import re
|
|
import tempfile
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
|
|
from fastapi.responses import FileResponse
|
|
from jwt.exceptions import InvalidTokenError
|
|
from pydantic import BaseModel
|
|
from sqlmodel import Session
|
|
from starlette.concurrency import run_in_threadpool
|
|
|
|
from fluksio.api.deps import (
|
|
FlowControllerDep,
|
|
get_current_active_superuser,
|
|
user_from_token,
|
|
worker_claims,
|
|
)
|
|
from fluksio.core.config import settings
|
|
from fluksio.core.db import engine
|
|
from fluksio.flow.artifacts import ArtifactStore
|
|
from fluksio.flow.runs import referenced_artifacts
|
|
|
|
#: What may be echoed back as a response content type. The caller holds the
|
|
#: reference and passes its media type, so this guards a header rather than
|
|
#: trusting one — anything else is served as bytes.
|
|
_MEDIA_TYPE = re.compile(r"^[\w.+-]+/[\w.+-]+$")
|
|
|
|
#: What a browser shows without running anything, so it is served as it is.
|
|
#: Every other type is served sandboxed: HTML, SVG and XML run script when
|
|
#: opened as a page, and so may a type nobody listed here.
|
|
_INERT = re.compile(r"^(audio|video)/|^image/(?!svg)|^application/(pdf|json)$")
|
|
|
|
|
|
def artifact_caller(request: Request) -> str:
|
|
"""Who may move artifacts: a signed-in person, or an attached worker.
|
|
|
|
A worker's node stores its checkpoints through this endpoint, so its own
|
|
credential has to open it — and only it. The token is no use anywhere else
|
|
in the API, which is why this check is here rather than in the shared
|
|
dependency every other route uses.
|
|
"""
|
|
header = request.headers.get("Authorization", "")
|
|
token = header[7:] if header.lower().startswith("bearer ") else ""
|
|
if not token:
|
|
raise HTTPException(status_code=401, detail="Not authenticated")
|
|
try:
|
|
claims = worker_claims(token)
|
|
except InvalidTokenError:
|
|
pass
|
|
else:
|
|
return f"worker:{claims.get('sub')}"
|
|
with Session(engine) as session:
|
|
# With the request, so a credential that is scoped by route — a wall
|
|
# panel's — is judged against this one rather than waved through.
|
|
user = user_from_token(session, token, request)
|
|
if user is None:
|
|
raise HTTPException(status_code=401, detail="Not authenticated")
|
|
return user.email
|
|
|
|
|
|
router = APIRouter(
|
|
prefix="/artifacts", tags=["artifacts"], dependencies=[Depends(artifact_caller)]
|
|
)
|
|
|
|
|
|
class ArtifactRef(BaseModel):
|
|
digest: str
|
|
size: int
|
|
media_type: str
|
|
name: str = ""
|
|
|
|
|
|
def _store(request: Request) -> ArtifactStore:
|
|
store: ArtifactStore | None = getattr(request.app.state, "artifact_store", None)
|
|
if store is None:
|
|
raise HTTPException(status_code=503, detail="The artifact store is not ready")
|
|
return store
|
|
|
|
|
|
@router.put("", response_model=ArtifactRef)
|
|
async def put_artifact(
|
|
request: Request,
|
|
name: str = Query(default=""),
|
|
media_type: str = Query(default=""),
|
|
volatile: bool = Query(default=False),
|
|
) -> Any:
|
|
"""Store the request body and answer with the reference to it.
|
|
|
|
Spooled to disk as it arrives rather than buffered: a video segment is as
|
|
legitimate a body here as a checkpoint, and neither should have to fit in
|
|
memory twice. Capped, because nothing else here was: any account, and any
|
|
worker credential, could otherwise fill the data volume.
|
|
|
|
``volatile`` puts it in the ring instead of the store — a frame a screen is
|
|
watching now, which the oldest of falls out of memory rather than being
|
|
kept. A worker on another host publishing a camera sends this.
|
|
"""
|
|
store = _store(request)
|
|
cap = settings.MAX_ARTIFACT_BYTES
|
|
declared = request.headers.get("content-length")
|
|
if cap and declared and declared.isdigit() and int(declared) > cap:
|
|
raise HTTPException(
|
|
status_code=413, detail=f"An artifact may be at most {cap} bytes"
|
|
)
|
|
handle = tempfile.NamedTemporaryFile(dir=store.root, delete=False)
|
|
written = 0
|
|
try:
|
|
with handle:
|
|
async for chunk in request.stream():
|
|
written += len(chunk)
|
|
# A chunked body declares no length, so the stream is what
|
|
# actually holds the limit.
|
|
if cap and written > cap:
|
|
raise HTTPException(
|
|
status_code=413,
|
|
detail=f"An artifact may be at most {cap} bytes",
|
|
)
|
|
# Off the event loop: this is a write syscall per chunk, for
|
|
# as long as the upload lasts.
|
|
await run_in_threadpool(handle.write, chunk)
|
|
return await run_in_threadpool(
|
|
store.put_file, Path(handle.name), media_type, name, volatile
|
|
)
|
|
finally:
|
|
Path(handle.name).unlink(missing_ok=True)
|
|
|
|
|
|
@router.get("/{digest}")
|
|
def get_artifact(
|
|
digest: str, request: Request, media_type: str = Query(default="")
|
|
) -> Any:
|
|
"""Serve one artifact back.
|
|
|
|
The caller passes the media type off the reference it holds, which is what
|
|
lets a browser play a clip rather than download it; the store itself keeps
|
|
only bytes. Ranged requests are answered because an audio or video element
|
|
scrubbing through a file asks for them.
|
|
"""
|
|
store = _store(request)
|
|
path = store.path(digest)
|
|
if path is None:
|
|
raise HTTPException(status_code=404, detail="No such artifact")
|
|
if not _MEDIA_TYPE.match(media_type):
|
|
media_type = "application/octet-stream"
|
|
# The digest is the content, so it is also the perfect validator.
|
|
headers = {"ETag": f'"{digest}"'}
|
|
if not _INERT.match(media_type.lower()):
|
|
# The caller names the type and any member can upload, so an artifact
|
|
# asked for as text/html would otherwise run its script on this origin,
|
|
# beside the session, for whoever follows a link to it.
|
|
headers["Content-Security-Policy"] = "sandbox"
|
|
headers["X-Content-Type-Options"] = "nosniff"
|
|
return FileResponse(path, media_type=media_type, headers=headers)
|
|
|
|
|
|
@router.delete(
|
|
"/{digest}",
|
|
status_code=204,
|
|
dependencies=[Depends(get_current_active_superuser)],
|
|
)
|
|
def delete_artifact(
|
|
digest: str, request: Request, controller: FlowControllerDep
|
|
) -> Response:
|
|
"""Remove one artifact's bytes from the store.
|
|
|
|
A superuser's call, because the store records no uploader and identical
|
|
content from two callers is one file, so nobody else can be said to own
|
|
it. Refused while anything still names the digest, the same references the
|
|
hourly sweep keeps, so a reference that exists never answers 404. The
|
|
volatile ring is left alone: its frames fall out on their own.
|
|
"""
|
|
store = _store(request)
|
|
if digest in referenced_artifacts(controller.state, controller.store):
|
|
raise HTTPException(
|
|
status_code=409,
|
|
detail=(
|
|
"The artifact is still referenced by a run, a current message or "
|
|
"a flow input's initial value"
|
|
),
|
|
)
|
|
if not store.delete(digest):
|
|
raise HTTPException(status_code=404, detail="No such artifact")
|
|
return Response(status_code=204)
|