Keep a run's lease while its pipeline builds; allow cancelling it then

A claimed run only joined the leased set once its pipeline was built, so a
build waiting for a busy worker pool got the run swept as abandoned. Cancel
now also stops such a run, and a terminal status being overwritten is logged.

backend/tests/api/routes/test_runs.py

@@ -7,6 +7,7 @@ The pipeline half — what a hit restores and what a key is made of — is in
 import base64
 import csv
 import json
+import threading
 from datetime import UTC, datetime, timedelta
 
 import pytest
@@ -19,6 +20,7 @@ from fluksio.flow.controller import FlowController
 from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.pipeline import NodeOutcome
 from fluksio.flow.runs import (
+    LEASE_STALE_S,
     OUTPUT_CAP,
     RunCache,
     RunRejected,
@@ -778,6 +780,77 @@ def test_runs_left_queued_are_woken_by_the_next_engine():
             session.commit()
 
 
+class _BusyWorkers(_OneFlow):
+    """Builds a run's pipeline the way a full worker pool does: eventually."""
+
+    events = None
+
+    def __init__(self, flow):
+        super().__init__(flow)
+        self.building = threading.Event()
+        self.free = threading.Event()
+
+    def build_run_pipeline(self, flow, **_kwargs):
+        self.building.set()
+        self.free.wait(10)
+        return object()  # never run: the test cancels it first
+
+
+def test_a_run_waiting_for_a_worker_keeps_its_lease_and_can_be_cancelled(
+    monkeypatch,
+):
+    """Only a run whose pipeline was built had its lease kept.
+
+    Building one loads every python node in a worker, so a run claimed while
+    all of them were busy was swept as `abandoned` and finished `ok` minutes
+    later, and a cancel in that window was refused.
+    """
+    controller = _BusyWorkers(FlowDef(name="study", mode="batch"))
+    service = RunService(controller=controller, queue=_Collect())
+    run = service.submit("study", {})
+    driver = threading.Thread(target=service._drive, args=(run.id,))
+    driver.start()
+    try:
+        assert controller.building.wait(5)
+        # As if it had waited past the cutoff, then one pass of the keeper:
+        # its lease is renewed before the sweep looks at it.
+        with Session(db_engine) as session:
+            session.get(Run, run.id).lease_at = datetime.now(UTC) - timedelta(
+                seconds=2 * LEASE_STALE_S
+            )
+            session.commit()
+        monkeypatch.setattr("fluksio.flow.runs.LEASE_INTERVAL_S", 0)
+        monkeypatch.setattr("fluksio.flow.runs.SWEEP_INTERVAL_S", 0)
+        sweep = service._sweep_abandoned
+
+        def sweep_once(now):
+            sweep(now)
+            service._stop.set()
+
+        monkeypatch.setattr(service, "_sweep_abandoned", sweep_once)
+        service._keep_leases()
+        with Session(db_engine) as session:
+            assert session.get(Run, run.id).status == "running"
+
+        assert service.cancel(run.id)
+        with Session(db_engine) as session:
+            assert session.get(Run, run.id).status == "cancelled"
+
+        # A worker frees up: the build returns and the run must not start.
+        controller.free.set()
+        driver.join(5)
+        with Session(db_engine) as session:
+            stored = session.get(Run, run.id)
+            assert stored.status == "cancelled"
+            assert stored.status_reason == "Cancelled before it started"
+    finally:
+        controller.free.set()
+        driver.join(5)
+        with Session(db_engine) as session:
+            session.delete(session.get(Run, run.id))
+            session.commit()
+
+
 def test_a_key_nobody_used_submits_normally(
     client, superuser_token_headers, monkeypatch
 ):