Keep a run's lease while its pipeline builds; allow cancelling it then

A claimed run only joined the leased set once its pipeline was built, so a
build waiting for a busy worker pool got the run swept as abandoned. Cancel
now also stops such a run, and a terminal status being overwritten is logged.

backend/fluksio/flow/runs.py

@@ -809,8 +809,9 @@ class RunService:
         self._consumer: threading.Thread | None = None
         self._keeper: threading.Thread | None = None
         # Runs this process is driving, and the pipeline each is running, so a
-        # cancel has something to hold on to.
-        self._active: dict[str, Pipeline] = {}
+        # cancel has something to hold on to. None while it is still being
+        # built — the run is claimed, so its lease is ours to keep all the same.
+        self._active: dict[str, Pipeline | None] = {}
         self._cancelled: set[str] = set()
         self._lock = threading.Lock()
 
@@ -1105,15 +1106,21 @@ class RunService:
     def cancel(self, run_id: str) -> bool:
         """Stop a run: kill what it is executing, schedule nothing further."""
         with self._lock:
-            pipeline = self._active.get(run_id)
-            if pipeline is None:
+            if run_id not in self._active:
                 # Not running here — if it is still queued, refusing to start
                 # is all the cancelling it needs.
-                cancelled = self._finish_queued(run_id)
+                cancelled = self._finish_unstarted(run_id, "queued")
                 if cancelled:
                     self._cancelled.add(run_id)
                 return cancelled
             self._cancelled.add(run_id)
+            pipeline = self._active[run_id]
+            if pipeline is None:
+                # Claimed, and its build still waiting for a worker: the driver
+                # checks the mark before it starts anything, and the row says
+                # so now rather than once a worker frees up.
+                self._finish_unstarted(run_id, "running")
+                return True
         # The gate first, so nothing new is submitted while the running nodes
         # are being killed; a gated node is never handed to the executor, so
         # the graph drains instead of going further.
@@ -1165,11 +1172,12 @@ class RunService:
             run = session.get(Run, run_id)
             return run.flow if run else None
 
-    def _finish_queued(self, run_id: str) -> bool:
+    def _finish_unstarted(self, run_id: str, status: str) -> bool:
+        """Cancel a run nothing of has executed, if it is still in ``status``."""
         with Session(db_engine) as session:
             result = session.exec(
                 update(Run)
-                .where(col(Run.id) == run_id, col(Run.status) == "queued")
+                .where(col(Run.id) == run_id, col(Run.status) == status)
                 .values(
                     status="cancelled",
                     finished_at=datetime.now(UTC),
@@ -1382,6 +1390,11 @@ class RunService:
             ),
         )
         try:
+            with self._lock:
+                # Leased from the claim, not from when its pipeline exists:
+                # building one loads every python node in a worker, and waits
+                # for one while all of them are busy — for longer than a lease.
+                self._active[run_id] = None
             flow = self.controller.store.read_flow(run.flow, draft=run.draft)
             # Read again, now that it is this run's turn: a sweep queues every
             # run at once, and the code on disk is free to move in the hours
@@ -1406,9 +1419,12 @@ class RunService:
                 run_cache=None if run.no_cache else self._cache,
             )
             with self._lock:
-                self._active[run_id] = pipeline
                 if run_id in self._cancelled:
-                    pipeline.pause(flow.name)
+                    # Cancelled while it was being built, so nothing of it
+                    # ran; the finally below records it.
+                    status, reason = "cancelled", "Cancelled before it started"
+                    return
+                self._active[run_id] = pipeline
             self._publish(run, "run_started")
             pipeline.run(seed_values(flow, run.params, run.seed))
             result = collect_result(flow, state)
@@ -1529,6 +1545,16 @@ class RunService:
     ) -> None:
         try:
             with Session(db_engine) as session:
+                was = session.exec(
+                    select(Run.status).where(col(Run.id) == run_id)
+                ).first()
+                if was not in (None, "running", status):
+                    # Swept as abandoned, say, while it was in fact still going.
+                    # What it ended as is the truth, but a client may already
+                    # have taken the first answer as final.
+                    logger.warning(
+                        "Run %s was %s and has now ended %s", run_id, was, status
+                    )
                 session.exec(
                     update(Run)
                     .where(col(Run.id) == run_id)