Let a sweep run more than four at a time, and name the run a failure was in

Concurrent runs sat at 4 whatever FLOW_MAX_CASCADES said: that setting bounds
cascades, and the run drivers read a hardcoded MAX_PARALLEL nobody could reach.
FLOW_MAX_RUNS is the knob they read now, --max-runs/--max-cascades/--max-workers
are the same three as flags on serve, and the engine says which numbers it
started with — which is the only way to tell that a settings file was read.

Events keep the run they happened in. The payload always carried it and the
persist path dropped it, so reading one run's failures meant filtering the
engine-wide list; a batch run's id reaches those events now too, since a run
has no journaled item to name itself by.

Also: a provisioner's 0 means "no deadline" rather than "cancel on the next
reconcile", and a command that reaches no engine says how to start one.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015sbYeYaVgYQqm1sbx7wPdL

backend/fluksio/flow/controller.py

@@ -1612,6 +1612,7 @@ class FlowController:
             observer=observer,
             emission_observer=emission_observer,
             run_cache=run_cache,
+            run_id=run.run_id if run is not None else "",
         )
         pipeline.history_limits = self.history_limits
         return pipeline

backend/fluksio/flow/metrics.py

@@ -155,7 +155,11 @@ class MetricsCollector:
         """Fold one event in. Synchronous: this is arithmetic on dicts."""
         kind = str(event.get("type") or "")
         ts = float(event.get("ts") or time.time())
-        run = self._runs.get(str(event.get("run") or ""))
+        # Two different things: the id as the event carries it, which is what a
+        # recorded row keeps, and the live cascade it belongs to — which a batch
+        # run has none of, since nothing here started one for it.
+        run_id = str(event.get("run") or "")[:64]
+        run = self._runs.get(run_id)
 
         if kind == "node_executed":
             bucket = self._bucket(event)
@@ -199,6 +203,7 @@ class MetricsCollector:
                     detail=(f"{error}\n{traceback}" if traceback else error)[
                         :DETAIL_CAP
                     ],
+                    run=run_id,
                 )
             )
             return
@@ -223,6 +228,7 @@ class MetricsCollector:
                         flow=str(event.get("flow") or ""),
                         node=str(event.get("node") or ""),
                         detail=_detail(event) or "Reported itself down.",
+                        run=run_id,
                     )
                 )
             return
@@ -247,6 +253,7 @@ class MetricsCollector:
                     flow=str(event.get("flow") or ""),
                     node=str(event.get("node") or event.get("task") or ""),
                     detail=_detail(event),
+                    run=run_id,
                 )
             )
 

backend/fluksio/flow/pipeline.py

@@ -237,6 +237,7 @@ class Pipeline:
         "observer",
         "emission_observer",
         "run_cache",
+        "run_id",
     )
 
     def __init__(
@@ -252,6 +253,7 @@ class Pipeline:
         observer: Callable[[NodeOutcome], None] | None = None,
         emission_observer: Callable[[str, dict[str, Any]], None] | None = None,
         run_cache: RunCacheLookup | None = None,
+        run_id: str = "",
     ) -> None:
         self._nodes = nodes or []
         # Stopped flows are stored and survive a restart; paused ones are a
@@ -286,6 +288,11 @@ class Pipeline:
         # none: a cascade is about what just happened, not about what a node
         # once returned for the same inputs.
         self.run_cache = run_cache
+        # The batch run this pipeline belongs to, if any. A live cascade names
+        # the journaled item it came from instead, which is what the events
+        # below carry; a run has no such item, so without this its failures
+        # would be recorded belonging to nothing.
+        self.run_id = run_id
         # How deep to keep each message's series; a chart asking for more
         # than the default puts its message in here. Swapped, never mutated.
         self.history_limits: dict[str, int] = {}
@@ -853,7 +860,7 @@ class Pipeline:
                 "flow": node.flow,
                 "node": node.id,
                 "error": error,
-                "run": entry_id,
+                "run": entry_id or self.run_id,
                 "ts": time.time(),
             }
         )
@@ -916,7 +923,7 @@ class Pipeline:
                 "node": node.id,
                 "outputs": len(outputs or {}),
                 "duration_ms": 0.0,
-                "run": entry_id,
+                "run": entry_id or self.run_id,
                 "ts": time.time(),
             }
         )
@@ -1007,7 +1014,7 @@ class Pipeline:
                     # which is a different thing to show than one that emitted.
                     "outputs": len(result or {}),
                     "duration_ms": duration_ms,
-                    "run": entry_id,
+                    "run": entry_id or self.run_id,
                     "ts": time.time(),
                 }
             )

backend/fluksio/flow/provision.py

@@ -125,7 +125,12 @@ class SlurmProvisioner:
         profiles: list[SlurmProfile],
         ssh_key: str = "",
         artifact_url: str = "",
+        #: How long a machine sits idle before it gives itself back; 0 keeps it
+        #: for as long as the job runs, which is what a queue paid for in hours
+        #: of wall time wants.
         max_idle_s: float = 300.0,
+        #: How long a submitted job may take to attach before it is cancelled;
+        #: 0 waits for as long as the scheduler makes it wait.
         provision_timeout_s: float = 900.0,
         events: EventBus | None = None,
     ) -> None:
@@ -255,7 +260,10 @@ class SlurmProvisioner:
                 if job.worker and job.worker in attached:
                     # It arrived. Asking again is somebody else's decision.
                     del self._outstanding[name]
-                elif time.monotonic() - job.since > self.provision_timeout_s:
+                elif (
+                    self.provision_timeout_s > 0
+                    and time.monotonic() - job.since > self.provision_timeout_s
+                ):
                     del self._outstanding[name]
                     if job.job_id:
                         expired.append(job.job_id)

backend/fluksio/flow/runs.py

@@ -68,9 +68,12 @@ LEASE_INTERVAL_S = 20.0
 LEASE_STALE_S = 90.0
 #: How often stale leases are looked for.
 SWEEP_INTERVAL_S = 30.0
-#: Runs driven at once. Node bodies are bounded by the worker pool anyway;
-#: this only bounds how many graphs are in flight.
+#: Runs driven at once, unless `FLOW_MAX_RUNS` says otherwise. Node bodies are
+#: bounded by the worker pool anyway; this only bounds how many graphs are in
+#: flight.
 MAX_PARALLEL = 4
+#: Runs claimed per poll, at least. Raising the pool raises this with it, so a
+#: queue of ready runs fills the drivers in one pass rather than four a second.
 CLAIM_COUNT = 4
 CLAIM_BLOCK_MS = 1000
 ERROR_CAP = 2000
@@ -706,7 +709,10 @@ class RunService:
         # the isolation it wants, minus surviving the process.
         self._state_factory = state_factory or (lambda _ns: MemoryState())
         self.engine_name = f"{socket.gethostname()}-{os.getpid()}"[:64]
-        self._pool = ThreadPoolExecutor(max_workers=parallel, thread_name_prefix="run")
+        self.parallel = max(1, parallel)
+        self._pool = ThreadPoolExecutor(
+            max_workers=self.parallel, thread_name_prefix="run"
+        )
         self._stop = threading.Event()
         self._consumer: threading.Thread | None = None
         self._keeper: threading.Thread | None = None
@@ -882,7 +888,9 @@ class RunService:
                 # Runs put back to wait for a worker come due here. The claim
                 # below blocks for a second, so this is about once a second.
                 self.queue.move_due(time.time())
-                items = self.queue.claim(CLAIM_COUNT, CLAIM_BLOCK_MS)
+                items = self.queue.claim(
+                    max(CLAIM_COUNT, self.parallel), CLAIM_BLOCK_MS
+                )
                 failures = 0
             except Exception as exc:
                 failures += 1