Runs report while they run, and the reader frames lines properly

A training loop has numbers worth keeping thousands of steps before it has a
result. Node code now imports fluksio and calls log_metric/progress, which
sends a line back without ending the call; the engine writes those to
run_metric in batches from the run's own driver rather than folding them off
the event bus, which drops what it cannot keep up with.

Two things fall out. Each event resets the worker deadline, so a node's
timeout measures silence rather than duration — which is what lets a two-hour
training keep a liveness contract instead of racing it. And the worker pool's
_running is now keyed by (run, node), so cancelling one config of a sweep
kills that training and leaves the rest alone.

Fixes a latent framing bug: read_line returned whatever a read had taken,
which was fine while a worker only ever sent one line per request and
unparseable as soon as it sent several. It now keeps the remainder.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AD8SfVhzXBG2nAfFcVh3iD

backend/tests/flow/test_workers.py

@@ -175,3 +175,108 @@ def test_a_pool_can_stop_while_a_node_is_running(pool):
 
     with pytest.raises(Exception, match="shutting down"):
         run(pool, "def process(params):\n    return {'out': 1}\n")
+
+
+# -----------------------------------------------------------------------------
+# Reporting from inside a node that has not returned yet
+# -----------------------------------------------------------------------------
+
+
+def test_a_node_reports_metrics_while_it_is_still_running(pool):
+    seen = []
+    result = pool.run(
+        "demo",
+        "train",
+        "import fluksio\n"
+        "def process(params):\n"
+        "    for step in range(3):\n"
+        "        fluksio.log_metric('loss', 1.0 / (step + 1), step)\n"
+        "    fluksio.progress(0.5, 'halfway')\n"
+        "    return {'out': 1}\n",
+        {},
+        {},
+        "demo.train",
+        timeout=5,
+        run_id="r1",
+        on_event=seen.append,
+    )
+
+    assert result == {"out": 1}
+    metrics = [event for event in seen if event["event"] == "metric"]
+    assert [(m["name"], m["step"]) for m in metrics] == [
+        ("loss", 0),
+        ("loss", 1),
+        ("loss", 2),
+    ]
+    assert metrics[0]["value"] == 1.0
+    # Every event says which call it belongs to, so a sweep can tell them apart.
+    assert {m["call_id"] for m in metrics} == {"r1:demo.train"}
+    assert [event["event"] for event in seen if event["event"] == "progress"] == [
+        "progress"
+    ]
+
+
+def test_events_hold_off_the_timeout_but_silence_does_not(pool):
+    # The deadline measures silence: a node reporting every 0.05s stays alive
+    # well past a 0.3s timeout, which is what a two-hour training needs.
+    result = pool.run(
+        "demo",
+        "slow",
+        "import time, fluksio\n"
+        "def process(params):\n"
+        "    for step in range(12):\n"
+        "        time.sleep(0.05)\n"
+        "        fluksio.log_metric('beat', step, step)\n"
+        "    return {'done': True}\n",
+        {},
+        {},
+        "demo.slow",
+        timeout=0.3,
+        run_id="r2",
+        on_event=lambda _event: None,
+    )
+    assert result == {"done": True}
+
+    with pytest.raises(NodeTimeout):
+        pool.run(
+            "demo",
+            "quiet",
+            "import time\ndef process(params):\n    time.sleep(2)\n    return {}\n",
+            {},
+            {},
+            "demo.quiet",
+            timeout=0.3,
+        )
+
+
+def test_cancelling_one_run_leaves_the_same_node_in_another_alone(pool):
+    # Keyed by (run, node): cancelling a config of a sweep must not kill the
+    # rest of it. With one slot the second run is not executing, so the check
+    # is that the pool refuses to find it rather than killing the wrong worker.
+    started = threading.Event()
+
+    def hold():
+        try:
+            pool.run(
+                "demo",
+                "hold",
+                "import time\ndef process(params):\n    time.sleep(5)\n    return {}\n",
+                {},
+                {},
+                "demo.hold",
+                timeout=10,
+                run_id="run-a",
+            )
+        except Exception:
+            pass
+        finally:
+            started.set()
+
+    thread = threading.Thread(target=hold, daemon=True)
+    thread.start()
+    time.sleep(0.5)
+
+    assert pool.cancel("demo.hold", run_id="run-b") is False
+    assert pool.cancel("demo.hold", run_id="run-a") is True
+    started.wait(timeout=5)
+    thread.join(timeout=5)