diff --git a/backend/fluksio/api/routes/flows.py b/backend/fluksio/api/routes/flows.py index 5519c6d..29cc3ee 100644 --- a/backend/fluksio/api/routes/flows.py +++ b/backend/fluksio/api/routes/flows.py @@ -16,7 +16,6 @@ from fastapi import ( from fastapi.concurrency import run_in_threadpool from jwt.exceptions import InvalidTokenError from pydantic import BaseModel -from sqlalchemy import delete from sqlmodel import Session, col, select from fluksio.api.deps import ( @@ -59,7 +58,7 @@ from fluksio.flow.store import ( LibNotFound, StaleVersion, ) -from fluksio.models import Flavor, Message, Run, RunArtifact, RunMetric, RunNode +from fluksio.models import Flavor, Message, Run router = APIRouter( prefix="/flows", tags=["flows"], dependencies=[Depends(get_current_user)] @@ -242,25 +241,6 @@ def _audit(action: str, flow: str, user: CurrentUser) -> None: ) -def _forget_runs(session: Session, flow: str) -> None: - """A deleted flow's runs, and everything hanging off them. - - Here rather than in ``FlowController.forget_flow`` because renaming a flow - calls that too, and a rename must keep its experiment history. - - Only the run tables: ``flow_run``, ``metric_minute`` and ``engine_event`` - are the observability rollups, deliberately kept as a record of what ran - and already pruned at OBS_RETENTION_DAYS. - """ - # A subquery, not a materialised list of ids: a demo can hold thousands. - runs = select(col(Run.id)).where(col(Run.flow) == flow) - session.execute(delete(RunNode).where(col(RunNode.run_id).in_(runs))) - session.execute(delete(RunMetric).where(col(RunMetric.run_id).in_(runs))) - session.execute(delete(RunArtifact).where(col(RunArtifact.run_id).in_(runs))) - session.execute(delete(Run).where(col(Run.flow) == flow)) - session.commit() - - def _source_ref(definition: FlowDef, node_id: str) -> str | None: """The library source this node runs, if it is a shared one.""" node = next((n for n in definition.nodes if n.id == node_id), None) @@ -483,7 +463,7 @@ async def discard_draft(name: str, controller: FlowControllerDep) -> Any: async def delete_flow( name: str, controller: FlowControllerDep, user: CurrentUser, session: SessionDep ) -> Any: - """Delete a flow and everything in it.""" + """Delete a flow and everything in it, except the record of what it ran.""" live = session.exec( select(Run.id).where( col(Run.flow) == name, col(Run.status).in_(("running", "queued")) @@ -503,8 +483,10 @@ async def delete_flow( raise HTTPException(status_code=404, detail=f"No flow named '{name}'") _audit("deleted", name, user) # Its files are gone; its values and queued work would otherwise linger. + # Its runs stay: a run record is a record of what ran, and surviving the + # flow it belonged to is the point of keeping one. `DELETE /runs?flow=` + # is what clears them. await run_in_threadpool(controller.forget_flow, name) - await run_in_threadpool(_forget_runs, session, name) await controller.reload_flow(name) return Message(message=f"Deleted flow '{name}'") diff --git a/backend/fluksio/api/routes/runs.py b/backend/fluksio/api/routes/runs.py index a49cab8..e0ef98e 100644 --- a/backend/fluksio/api/routes/runs.py +++ b/backend/fluksio/api/routes/runs.py @@ -28,7 +28,7 @@ from fluksio.flow.events import event_bus from fluksio.flow.messages import requalify from fluksio.flow.runs import RunRejected, RunService, new_run_id from fluksio.flow.store import FlowNotFound -from fluksio.models import Run, RunArtifact, RunMetric, RunNode +from fluksio.models import Message, Run, RunArtifact, RunMetric, RunNode router = APIRouter( prefix="/runs", tags=["runs"], dependencies=[Depends(get_current_user)] @@ -703,27 +703,88 @@ async def retry_run(run_id: str, request: Request, user: CurrentUser) -> Any: raise HTTPException(status_code=status, detail=detail) from exc -@router.delete("/{run_id}", status_code=204) -def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response: - """Forget a run and everything hanging off it. +def _drop_runs(session: Session, chosen: Any) -> None: + """A selection of runs and everything hanging off them. - The same four statements ``_forget_runs`` uses when a flow goes: the run - tables carry a plain string ``run_id`` and no foreign key, so nothing - cascades on its own. ``flow_run``, ``metric_minute`` and ``engine_event`` - stay — they are the observability record and are pruned on their own window. - - A live run is refused rather than raced: the driver writes its nodes back - when it finishes, and those rows would arrive for a run that no longer - exists. Cancel it first. + The run tables carry a plain string ``run_id`` and no foreign key, so + nothing cascades on its own and these four statements are the whole of it. + ``flow_run``, ``metric_minute`` and ``engine_event`` stay — they are the + observability record and are pruned on their own window. Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage - cache, so a later run loses hits this one would have served. And a node - restored from this run points here through ``cached_from`` — ``_series`` + cache, so a later run loses hits these would have served. And a node + restored from one of them points here through ``cached_from`` — ``_series`` already reads a missing source as an empty curve, which is what ``NO_CURVE`` explains on the screen. The artifact bytes need no help: ``sweep_artifacts`` keeps whatever a ``run_artifact`` row or a live message still names, so dropping the rows is enough and the hourly sweep reclaims the blobs. """ + session.execute(delete(RunNode).where(col(RunNode.run_id).in_(chosen))) + session.execute(delete(RunMetric).where(col(RunMetric.run_id).in_(chosen))) + session.execute(delete(RunArtifact).where(col(RunArtifact.run_id).in_(chosen))) + session.execute(delete(Run).where(col(Run.id).in_(chosen))) + + +def _live_run(session: Session, flow: str) -> str | None: + """A run of this flow the driver is still writing to, if there is one.""" + return session.exec( + select(Run.id).where( + col(Run.flow) == flow, col(Run.status).in_(("running", "queued")) + ) + ).first() + + +@router.delete("", response_model=Message) +def delete_runs( + session: SessionDep, + user: CurrentUser, + flow: Annotated[str, Query(min_length=1)], +) -> Any: + """Every run of one flow at once. + + The counterpart to the list's ``flow`` filter, and what a reseed needs: + deleting a flow leaves its runs standing on purpose, so this is the only + way to clear a history without a call per run. ``flow`` is required — + there is no "delete every run of everything" here, by design. + """ + with writing(session): + live = _live_run(session, flow) + if live is not None: + raise HTTPException( + status_code=409, + detail=( + f"Flow '{flow}' has a run in progress ({live}). Cancel it, or " + "wait for it to finish, before deleting its runs." + ), + ) + # Counted before the delete, and a subquery rather than a materialised + # list of ids on the way in: a demo can hold thousands. + chosen = select(col(Run.id)).where(col(Run.flow) == flow) + gone = session.exec( + select(func.count()).select_from(Run).where(col(Run.flow) == flow) + ).one() + _drop_runs(session, chosen) + session.commit() + event_bus.publish( + { + "type": "audit", + "action": f"deleted {gone} run(s)", + "flow": flow, + "user": user.email, + "ts": time.time(), + } + ) + return Message(message=f"Deleted {gone} run(s) of '{flow}'") + + +@router.delete("/{run_id}", status_code=204) +def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response: + """Forget a run and everything hanging off it. + + A live run is refused rather than raced: the driver writes its nodes back + when it finishes, and those rows would arrive for a run that no longer + exists. Cancel it first. + """ # Reads the run, then decides whether to delete it — so it takes the write # lock up front rather than upgrading and losing to whichever flush # committed in between. @@ -740,10 +801,7 @@ def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response: ), ) flow = run.flow - session.execute(delete(RunNode).where(col(RunNode.run_id) == run_id)) - session.execute(delete(RunMetric).where(col(RunMetric.run_id) == run_id)) - session.execute(delete(RunArtifact).where(col(RunArtifact.run_id) == run_id)) - session.execute(delete(Run).where(col(Run.id) == run_id)) + _drop_runs(session, [run_id]) session.commit() event_bus.publish( { @@ -792,8 +850,8 @@ def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]: for node_row in restored: source_flow = sources.get(node_row.cached_from) if source_flow is None: - # The run it came from is gone — deleted with its flow. The - # outputs are still on this run; the curve is not recoverable. + # The run it came from was deleted. The outputs are still on + # this run; the curve is not recoverable. continue key = ( node_row.cached_from, diff --git a/backend/tests/api/routes/test_flows.py b/backend/tests/api/routes/test_flows.py index 89e874a..548c273 100644 --- a/backend/tests/api/routes/test_flows.py +++ b/backend/tests/api/routes/test_flows.py @@ -299,10 +299,17 @@ def test_delete_flow( client.get(f"{PREFIX}/demo", headers=superuser_token_headers).status_code == 404 ) - # Deleting the flow takes its runs with it, so a reseeded demo starts clean. + # The run outlives the flow: a run record is a record of what ran, and + # `DELETE /runs?flow=` is the only thing that clears one. db.expire_all() for model in (Run, RunNode, RunMetric, RunArtifact): - assert db.exec(select(func.count()).select_from(model)).one() == 0 + assert db.exec(select(func.count()).select_from(model)).one() == 1 + assert ( + client.get( + f"{settings.API_V1_STR}/runs/run-1", headers=superuser_token_headers + ).status_code + == 200 + ) def test_delete_flow_is_refused_while_a_run_is_live( diff --git a/backend/tests/api/routes/test_runs.py b/backend/tests/api/routes/test_runs.py index b5a8549..8178f7f 100644 --- a/backend/tests/api/routes/test_runs.py +++ b/backend/tests/api/routes/test_runs.py @@ -774,6 +774,49 @@ def test_a_running_run_is_refused_rather_than_raced(client, superuser_token_head session.commit() +def test_deleting_a_flows_runs_clears_that_flow_only(client, superuser_token_headers): + """The counterpart to the list's `flow` filter — what a reseed needs. + + Deleting the flow itself leaves the history standing, so this is the tool + that clears one. It refuses while a run of that flow is still going, for + the same reason deleting a single run does. + """ + made = datetime.now(UTC) + url = f"{settings.API_V1_STR}/runs" + with Session(db_engine) as session: + session.add(Run(id="bulk-1", flow="swept", status="ok", created_at=made)) + session.add(RunNode(run_id="bulk-1", node="swept.a", status="ok")) + session.add(RunMetric(run_id="bulk-1", name="swept.loss", step=0, value=1.0)) + session.add(Run(id="bulk-2", flow="swept", status="running", created_at=made)) + session.add(Run(id="keep-1", flow="other", status="ok", created_at=made)) + session.commit() + + busy = client.delete(url, params={"flow": "swept"}, headers=superuser_token_headers) + assert busy.status_code == 409 + assert "bulk-2" in busy.json()["detail"] + + with Session(db_engine) as session: + session.get(Run, "bulk-2").status = "cancelled" + session.commit() + + answer = client.delete( + url, params={"flow": "swept"}, headers=superuser_token_headers + ) + assert answer.status_code == 200 + assert "2" in answer.json()["message"] + + with Session(db_engine) as session: + assert session.exec(select(Run).where(col(Run.flow) == "swept")).all() == [] + for table in (RunNode, RunMetric): + assert ( + session.exec(select(table).where(col(table.run_id) == "bulk-1")).all() + == [] + ) + assert session.get(Run, "keep-1") is not None + session.delete(session.get(Run, "keep-1")) + session.commit() + + # ----------------------------------------------------------------------------- # A cached node's curve # @@ -847,7 +890,7 @@ def test_a_cached_node_answers_with_the_curve_it_was_restored_from( def test_a_curve_whose_run_is_gone_is_empty_rather_than_an_error( client, superuser_token_headers ): - """Deleting a flow deletes its runs; what pointed at one is left holding it.""" + """Delete the run a cache hit points at and the reusing run holds nothing.""" with Session(db_engine) as session: session.delete(session.get(Run, "src-1")) session.commit() diff --git a/docs/code/api.md b/docs/code/api.md index 825a63c..cecbb54 100644 --- a/docs/code/api.md +++ b/docs/code/api.md @@ -46,7 +46,7 @@ Agents authenticate differently; see [Agents over MCP](agents.md). | `POST` | `/flows/{name}/publish` | put the draft live | | `POST` | `/flows/{name}/discard-draft` | throw the draft away | | `POST` | `/flows/{name}/rename` | rename it | -| `DELETE` | `/flows/{name}` | delete it | +| `DELETE` | `/flows/{name}` | delete it. Its runs stay — a run is the record of what ran, and only `DELETE /runs` clears one. Refused with 409 while a run of it is `running` or `queued` | | `GET` | `/flows/node-types` | every node type and its parameter schema | | `GET` | `/flows/graph` | every flow as one graph — what Home draws | @@ -104,12 +104,14 @@ published to. Flows own the namespace; everything else is a client of it. | `POST` | `/runs/flows/{name}` | queue one run — `{"params": {...}, "seed": 7, "draft": false, "no_cache": false}`. `"cause"` says where it came from — `api` (the default), `cli` or `sdk` | | `POST` | `/runs/flows/{name}/sweep` | queue many, sharing a `group_id` | | `GET` | `/runs` | the queryable history: `?flow=`, `?status=`, `?group=`, `?digest=`, `?since=`, `?before=`, `?limit=`, `?offset=` | +| `DELETE` | `/runs?flow=` | every run of one flow, and everything hanging off them. `flow` is required. Refused with 409 while one of them is still going | | `GET` | `/runs/overview` | one row per flow that has runs, with how many are running or queued | | `GET` | `/runs/export/metrics?…&name=&stride=&format=` | every selected run's series as one long table: `run, name, step, ts, value` | | `GET` | `/runs/export/runs?…¶ms=&metrics=&format=` | one row per run: its inputs as columns, its final numbers, its status and provenance | | `GET` | `/runs/metrics/names?…` | every metric name the selected runs recorded, distinct; takes the export's own filters | | `GET` | `/runs/{id}` | one run in full: params, result, per-node record, artifacts | | `POST` | `/runs/{id}/cancel` | stop it | +| `DELETE` | `/runs/{id}` | forget it, with its nodes, series and artifact rows. Cancel a live one first | | `POST` | `/runs/{id}/retry` | run the same thing again, as a new run naming this one | | `GET` | `/runs/{id}/metrics?name=&stride=` | one metric's series, in step order; every series of the run without `name` | | `GET` | `/runs/series/compare?ids=a,b,c&metric=&x=` | that metric across several runs. `x` is what to plot against: nothing or `step`, `time` (seconds since each run's own first reading), or another metric's name, joined on the step the two share | diff --git a/docs/concepts/runs.md b/docs/concepts/runs.md index cf5a4ca..fb43e21 100644 --- a/docs/concepts/runs.md +++ b/docs/concepts/runs.md @@ -233,8 +233,9 @@ execution that is not happening this time, so its series is not rewritten either. The run it was restored from is recorded instead, and that is where the curve is read back from: asking the reusing run for its metrics answers with the same points, under its own flow's names. The one way to be left with a -result and no curve is for that earlier run to have been deleted, which -deleting its flow does. +result and no curve is for that earlier run to have been deleted. Deleting a +flow does not do that — its runs stay — so this only happens when the run +itself was deleted. ## Objects that cannot be serialized @@ -346,6 +347,14 @@ the artifacts it made, its metrics and its result. **Retry** submits it again as a run of its own, keeping the flow, the inputs, the seed and the sweep it belonged to. +A run outlives the flow it ran. Deleting a flow leaves its history standing — +being the record of what ran is the point of keeping one — so the Runs screen +still lists it, and opening it says **flow deleted** with Retry disabled, +because there is nothing left to run it against. Clearing that history is a +decision of its own: a run at a time on the screen, or +`DELETE /api/v1/runs?flow=` for a whole flow's worth, which is what a +reseed wants. + ### Taking it into a dataframe An analysis wants a table rather than a screen, and there are two it usually @@ -407,9 +416,8 @@ sweep, or specific runs. It re-reads on its own and whenever a run finishes. ### When a run draws nothing A node restored from the [stage cache](#stage-caching) has its curve read back -from the run that recorded it. Delete that run, which deleting its flow does, -and the reusing run is left with a result and an empty curve, and the chart says so -rather than looking broken. +from the run that recorded it. Delete that run and the reusing run is left with +a result and an empty curve, and the chart says so rather than looking broken. ## What this costs, compared diff --git a/frontend/src/client/core/OpenAPI.ts b/frontend/src/client/core/OpenAPI.ts index 327a9ad..979d72b 100644 --- a/frontend/src/client/core/OpenAPI.ts +++ b/frontend/src/client/core/OpenAPI.ts @@ -48,7 +48,7 @@ export const OpenAPI: OpenAPIConfig = { PASSWORD: undefined, TOKEN: undefined, USERNAME: undefined, - VERSION: '0.1.4+dev', + VERSION: '0.1.5+dev', WITH_CREDENTIALS: false, interceptors: { request: new Interceptors(), diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index c51d483..31a17bf 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsReadWebpushKeyResponse, AlertsAddWebpushSubscriptionData, AlertsAddWebpushSubscriptionResponse, AlertsRemoveWebpushSubscriptionData, AlertsRemoveWebpushSubscriptionResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudDisconnectResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsReadRunsData, RunsReadRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadRunData, RunsReadRunResponse, RunsDeleteRunData, RunsDeleteRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsRetryRunData, RunsRetryRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SearchReadSearchIndexResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; +import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsReadWebpushKeyResponse, AlertsAddWebpushSubscriptionData, AlertsAddWebpushSubscriptionResponse, AlertsRemoveWebpushSubscriptionData, AlertsRemoveWebpushSubscriptionResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudDisconnectResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsReadRunsData, RunsReadRunsResponse, RunsDeleteRunsData, RunsDeleteRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadMetricNamesData, RunsReadMetricNamesResponse, RunsReadRunData, RunsReadRunResponse, RunsDeleteRunData, RunsDeleteRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsRetryRunData, RunsRetryRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SearchReadSearchIndexResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; export class AlertsService { /** @@ -680,7 +680,7 @@ export class FlowsService { /** * Delete Flow - * Delete a flow and everything in it. + * Delete a flow and everything in it, except the record of what it ran. * @param data The data for the request. * @param data.name * @returns Message Successful Response @@ -1981,6 +1981,32 @@ export class RunsService { }); } + /** + * Delete Runs + * Every run of one flow at once. + * + * The counterpart to the list's ``flow`` filter, and what a reseed needs: + * deleting a flow leaves its runs standing on purpose, so this is the only + * way to clear a history without a call per run. ``flow`` is required — + * there is no "delete every run of everything" here, by design. + * @param data The data for the request. + * @param data.flow + * @returns Message Successful Response + * @throws ApiError + */ + public static deleteRuns(data: RunsDeleteRunsData): CancelablePromise { + return __request(OpenAPI, { + method: 'DELETE', + url: '/api/v1/runs', + query: { + flow: data.flow + }, + errors: { + 422: 'Validation Error' + } + }); + } + /** * Read Overview * One row per flow that has ever run, busiest-recent first. @@ -2085,6 +2111,44 @@ export class RunsService { }); } + /** + * Read Metric Names + * Every metric name the selected runs recorded. + * + * A name is flow-qualified — a node of `train` writing `train_loss` records + * `train.train_loss` — so this is the answer to "what would match". Exact + * over the whole selection: the client used to read the newest run that had + * measured anything and take its names for the vocabulary, which missed a + * name only an older run ever wrote. Declared before `/{run_id}`, or that + * route would take "metrics" for an id. + * @param data The data for the request. + * @param data.flow + * @param data.status + * @param data.group + * @param data.ids + * @param data.since + * @param data.until + * @returns string Successful Response + * @throws ApiError + */ + public static readMetricNames(data: RunsReadMetricNamesData = {}): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/runs/metrics/names', + query: { + flow: data.flow, + status: data.status, + group: data.group, + ids: data.ids, + since: data.since, + until: data.until + }, + errors: { + 422: 'Validation Error' + } + }); + } + /** * Read Run * One run in full: what it was asked, what each node did, what it made. @@ -2110,22 +2174,9 @@ export class RunsService { * Delete Run * Forget a run and everything hanging off it. * - * The same four statements ``_forget_runs`` uses when a flow goes: the run - * tables carry a plain string ``run_id`` and no foreign key, so nothing - * cascades on its own. ``flow_run``, ``metric_minute`` and ``engine_event`` - * stay — they are the observability record and are pruned on their own window. - * * A live run is refused rather than raced: the driver writes its nodes back * when it finishes, and those rows would arrive for a run that no longer * exists. Cancel it first. - * - * Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage - * cache, so a later run loses hits this one would have served. And a node - * restored from this run points here through ``cached_from`` — ``_series`` - * already reads a missing source as an empty curve, which is what ``NO_CURVE`` - * explains on the screen. The artifact bytes need no help: ``sweep_artifacts`` - * keeps whatever a ``run_artifact`` row or a live message still names, so - * dropping the rows is enough and the hourly sweep reclaims the blobs. * @param data The data for the request. * @param data.runId * @returns void Successful Response diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 9b55317..6bb2e4c 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -1830,6 +1830,12 @@ export type RunsReadRunsData = { export type RunsReadRunsResponse = (Array); +export type RunsDeleteRunsData = { + flow: string; +}; + +export type RunsDeleteRunsResponse = (Message); + export type RunsReadOverviewResponse = (Array); export type RunsExportMetricsData = { @@ -1860,6 +1866,17 @@ export type RunsExportRunsData = { export type RunsExportRunsResponse = (unknown); +export type RunsReadMetricNamesData = { + flow?: (string | null); + group?: (string | null); + ids?: string; + since?: (string | null); + status?: (string | null); + until?: (string | null); +}; + +export type RunsReadMetricNamesResponse = (Array<(string)>); + export type RunsReadRunData = { runId: string; }; diff --git a/frontend/src/components/Runs/RunDetail.tsx b/frontend/src/components/Runs/RunDetail.tsx index 5d60f14..5c626e9 100644 --- a/frontend/src/components/Runs/RunDetail.tsx +++ b/frontend/src/components/Runs/RunDetail.tsx @@ -23,6 +23,7 @@ import { OpenInDashboard } from "./OpenInDashboard" import { CARD, downloadArtifact, + FLOW_GONE, isLive, NO_CURVE, paramText, @@ -42,7 +43,7 @@ export function RunDetail({ id }: { id: string }) { const cancel = useCancelRun() const retry = useRetryRun() const names = useMetricNames(id) - const declared = useFlowInputs(run?.flow) + const { declared, gone } = useFlowInputs(run?.flow) const [metric, setMetric] = useState("") if (isPending || !run) return @@ -96,6 +97,14 @@ export function RunDetail({ id }: { id: string }) { ran the working copy )} + {gone && ( + + flow deleted + + )}
@@ -115,7 +124,8 @@ export function RunDetail({ id }: { id: string }) { size="sm" className="h-8" onClick={() => retry.mutate(run.id)} - disabled={retry.isPending} + // There is no flow left to run it against; the route would 404. + disabled={retry.isPending || gone} data-testid="retry-run" > @@ -126,6 +136,7 @@ export function RunDetail({ id }: { id: string }) { {reason &&

{reason}

} + {gone &&

{FLOW_GONE}

}
{ago(String(run.created_at ?? ""))} @@ -157,7 +168,9 @@ export function RunDetail({ id }: { id: string }) {

Inputs

{fed.length === 0 ? (

- This flow declares no inputs, so there was nothing to choose. + {gone + ? "The run carried no parameters of its own, and its flow is gone, so what it took from the flow cannot be read back." + : "This flow declares no inputs, so there was nothing to choose."}

) : (
diff --git a/frontend/src/components/Runs/RunsScreen.tsx b/frontend/src/components/Runs/RunsScreen.tsx index 544ed29..019af8d 100644 --- a/frontend/src/components/Runs/RunsScreen.tsx +++ b/frontend/src/components/Runs/RunsScreen.tsx @@ -721,7 +721,7 @@ function ParamDiff({ rows }: { rows: RunRow[] }) { // ponytail: only when the picks share one flow — spanning flows would mean a // declaration lookup per flow, and a sweep comparison never does. const one = rows.every((run) => run.flow === rows[0].flow) - const declared = useFlowInputs(one ? rows[0].flow : undefined) + const { declared } = useFlowInputs(one ? rows[0].flow : undefined) const varying = varyingKeys(rows) // Not a parameter, but it is part of what produced the number, and in a // sweep it is often the only thing that moved. diff --git a/frontend/src/components/Runs/queries.ts b/frontend/src/components/Runs/queries.ts index 833f719..82e5e87 100644 --- a/frontend/src/components/Runs/queries.ts +++ b/frontend/src/components/Runs/queries.ts @@ -2,7 +2,7 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" import { useNavigate } from "@tanstack/react-router" import { useMemo } from "react" -import { OpenAPI, RunsService } from "@/client" +import { ApiError, OpenAPI, RunsService } from "@/client" import { flowQueryOptions } from "@/components/Flow/queries" import { apiToken } from "@/lib/portal" @@ -176,8 +176,12 @@ export async function downloadArtifact(digest: string, name: string) { URL.revokeObjectURL(url) } +/** The server saying a thing is not there, rather than failing to answer. */ +const isMissing = (error: unknown) => + error instanceof ApiError && error.status === 404 + /** - * What a flow declares it can be given, by input name. + * What a flow declares it can be given, by input name — and whether it is gone. * * A batch flow's inputs *are* its parameters — a run supplies values for the * ones it names and takes the flow's own for the rest — so this is what turns @@ -185,15 +189,21 @@ export async function downloadArtifact(digest: string, name: string) { * * The declarations are the flow's *current* ones, while a run carries the * `flow_version` it was submitted against. An input added since is shown on an - * older run as a default it never actually received. + * older run as a default it never actually received. And the flow may not be + * there at all: deleting one leaves its runs standing, so `gone` is an ordinary + * answer here rather than a fault, and the screen says so instead of drawing a + * run that declares nothing. */ export function useFlowInputs(flow: string | undefined) { - const { data } = useQuery({ + const { data, error } = useQuery({ ...flowQueryOptions(flow ?? ""), enabled: Boolean(flow), + // A flow that is gone stays gone; asking three more times only delays it. + retry: (count: number, failure: unknown) => + !isMissing(failure) && count < 3, }) const inputs = data?.definition.inputs - return useMemo( + const declared = useMemo( () => new Map( (inputs ?? []) @@ -202,6 +212,7 @@ export function useFlowInputs(flow: string | undefined) { ), [inputs], ) + return { declared, gone: isMissing(error) } } /** @@ -267,13 +278,17 @@ export async function cancelThenDelete(runId: string) { * Why a finished run can have nothing to draw. * * A cache hit replays no emissions, so a restored node's curve is read back - * from the run that recorded it. Deleting that run — deleting its flow does — - * takes the curve with it, and then a reused run has a result and nothing to - * draw. Said out loud rather than left as an empty chart, which reads as a fault. + * from the run that recorded it. Deleting that run takes the curve with it, and + * then a reused run has a result and nothing to draw. Said out loud rather than + * left as an empty chart, which reads as a fault. */ export const NO_CURVE = "No curve was recorded. A node restored from the cache is read back from the run that produced it, so this draws nothing once that run has been deleted — its outputs are still on the result." +/** Why a run can name a flow that is not there any more. */ +export const FLOW_GONE = + "The flow this ran has been deleted. The run is kept as the record of what ran, but there is nothing left to retry it against." + /** A run id, short enough for a table cell. The tail is the random half. */ export const shortId = (id: string) => id.slice(-8)