Refuse what a node cannot publish, and stop timing out work that is fine

Four things the python SDK turned up, each fixed where every client sees it.

A key no port declares is now an error rather than a silent drop, on the
return, the yield and the emit alike — the contract the docs already stated.
The SDK reads literal yields at sync time, so a typo fails before anything
runs, and an emission of one fails the call rather than being logged where
nobody looks.

NaN and infinity are refused at the port. JSON cannot spell either, so one
that travelled came back as a 500, a socket frame that stopped the canvas, or
a metric batch the database dropped whole.

An artifact input takes `@run:<id>.<output>` or a bare digest, resolved on the
engine — so the CLI, the run dialog and a python caller mean the same thing,
and a sweep can pass one at all.

Node timeouts are off by default. The clock measured silence, which a training
node is full of, and remote workers had already stopped enforcing it — their
heartbeat reset it. Now a heartbeat proves the agent rather than the node,
ninety seconds of nothing fails the call either way, and the engine touches
work it is still running so a long node is not redelivered at sixty seconds.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019V5bsYGNxcgPs4xXmTPx69

backend/tests/flow/test_messages.py

@@ -114,3 +114,41 @@ def test_qualify_scopes_bare_names_only():
     assert qualify("heating", "solar.power") == "solar.power"
     assert qualify("heating", "") == ""
     assert flow_of("heating.temp") == "heating"
+
+
+# -----------------------------------------------------------------------------
+# NaN and infinity
+#
+# JSON cannot spell either, so one travelling through a port would come back as
+# a response nobody can parse, a socket frame that stops a canvas, or a row the
+# database rejects — a long way from the node that produced it.
+# -----------------------------------------------------------------------------
+
+
+def test_a_float_port_refuses_nan_and_infinity():
+    spec = MessageSpec(name="score", dtype=DType.FLOAT)
+    spec.check(0.5)
+    for value in (float("nan"), float("inf"), float("-inf")):
+        with pytest.raises(TypeError, match="score"):
+            spec.check(value)
+
+
+def test_a_json_port_refuses_a_nan_nested_in_it():
+    spec = MessageSpec(name="report", dtype=DType.JSON)
+    spec.check({"groups": [{"mean": 1.0}]})
+    with pytest.raises(TypeError, match="JSON"):
+        spec.check({"groups": [{"mean": float("nan")}]})
+
+
+def test_a_series_refuses_a_nan_point():
+    spec = MessageSpec(name="curve", dtype=DType.SERIES)
+    lines = [{"label": "loss", "points": [[1.0, float("nan")]]}]
+    with pytest.raises(TypeError):
+        spec.check({"lines": lines})
+
+
+def test_a_value_that_refers_to_itself_does_not_hang_the_check():
+    spec = MessageSpec(name="report", dtype=DType.JSON)
+    loop: dict = {}
+    loop["self"] = loop
+    spec.check(loop)

backend/tests/flow/test_queue.py

@@ -1,7 +1,9 @@
 """The work queue, and what the execution service does with it."""
 
+import threading
 import time
 
+from fluksio.flow import executor
 from fluksio.flow.executor import ExecutionService
 from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.nodes import Node
@@ -264,3 +266,72 @@ def test_a_replayed_item_does_not_repeat_a_side_effect():
     service._run_item(item)
 
     assert calls == [5.0]
+
+
+# -----------------------------------------------------------------------------
+# Telling the queue that a long node is working, not lost
+#
+# What marks an item abandoned is nobody touching it. A node with no timeout
+# may run for hours, so the engine holding it says so on a timer — and an
+# engine that died says nothing, which is the distinction the reaper needs.
+# -----------------------------------------------------------------------------
+
+
+class RecordingQueue(MemoryWorkQueue):
+    """A memory queue that writes down what it was asked to hold on to."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.touched: list[list[str]] = []
+
+    def touch(self, entry_ids: list[str]) -> None:
+        self.touched.append(list(entry_ids))
+
+
+def test_work_in_flight_is_touched_until_it_finishes(monkeypatch):
+    monkeypatch.setattr(executor, "TOUCH_INTERVAL_S", 0.0)
+    monkeypatch.setattr(executor, "DELAYED_INTERVAL_S", 0.05)
+    running = threading.Event()
+    release = threading.Event()
+
+    def slow(reading, params):
+        running.set()
+        release.wait(5)
+        return {"doubled": reading * 2}
+
+    source = Node(
+        f=lambda params: None,
+        provides=[MessageSpec(name="reading", dtype=DType.FLOAT)],
+        name="source",
+    )
+    consumer = Node(
+        f=slow,
+        requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
+        provides=[MessageSpec(name="doubled", dtype=DType.FLOAT)],
+        name="consumer",
+    )
+    source.assign_flow("f", "source")
+    consumer.assign_flow("f", "consumer")
+
+    queue = RecordingQueue()
+    pipeline = Pipeline(
+        nodes=[source, consumer], state=MemoryState(), work_queue=queue
+    )
+    service = ExecutionService(queue)
+    service.bind(pipeline)
+    service.start()
+    try:
+        source.inject({"reading": 3.0})
+        assert running.wait(5)
+        # Give the timer a couple of passes while the node is still in there.
+        time.sleep(0.2)
+        held = [ids for ids in queue.touched if ids]
+        assert held, "a running item was never touched"
+
+        release.set()
+        time.sleep(0.3)
+        # Once it is done it is acknowledged, so there is nothing to hold.
+        assert queue.touched[-1] == []
+    finally:
+        release.set()
+        service.stop()

backend/tests/flow/test_remote.py

@@ -9,10 +9,12 @@ what happened rather than a wait that never ends.
 
 import asyncio
 import threading
+import time
 from collections.abc import Iterator
 
 import pytest
 
+from fluksio.flow import remote
 from fluksio.flow.remote import NoWorker, RemoteWorker, RemoteWorkerHub
 from fluksio.flow.workers import NodeTimeout, RemoteError
 
@@ -220,3 +222,76 @@ def test_cancelling_a_run_reaches_only_that_run(loop):
     assert hub.cancel_run("run-a") == 1
     cancels = [frame for frame in socket.sent if frame.get("op") == "cancel"]
     assert [frame["call_id"] for frame in cancels] == ["run-a:flow.node"]
+
+
+# -----------------------------------------------------------------------------
+# Liveness, and what a heartbeat is evidence of
+#
+# The agent beats every ten seconds while it is executing. That says the agent
+# is alive; it says nothing about the node, which is why it bounds the silence
+# deadline and not the node's own timeout.
+# -----------------------------------------------------------------------------
+
+
+def test_heartbeats_do_not_stave_off_a_nodes_own_timeout(loop):
+    hub = RemoteWorkerHub()
+    worker, socket = attach(hub, loop)
+    caught: list[Exception] = []
+
+    def call() -> None:
+        try:
+            hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=0.5)
+        except Exception as exc:
+            caught.append(exc)
+
+    thread = call_in_thread(call)
+    assert socket.arrived.wait(5)
+    call_id = socket.sent[0]["call_id"]
+    # Beating faster than the deadline. A node that reports its progress is
+    # held off; one whose agent is merely alive is not.
+    for _ in range(10):
+        worker.deliver({"call_id": call_id, "event": "heartbeat"})
+        time.sleep(0.1)
+
+    thread.join(timeout=5)
+    assert isinstance(caught[0], NodeTimeout)
+
+
+def test_with_no_timeout_total_silence_is_still_bounded(loop, monkeypatch):
+    monkeypatch.setattr(remote, "SILENCE_S", 0.3)
+    hub = RemoteWorkerHub()
+    attach(hub, loop)
+    caught: list[Exception] = []
+
+    def call() -> None:
+        try:
+            hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=0)
+        except Exception as exc:
+            caught.append(exc)
+
+    call_in_thread(call).join(timeout=5)
+    assert isinstance(caught[0], RemoteError)
+    assert not isinstance(caught[0], NodeTimeout)
+    assert "presumed gone" in str(caught[0])
+
+
+def test_with_no_timeout_a_beating_worker_is_left_to_finish(loop, monkeypatch):
+    monkeypatch.setattr(remote, "SILENCE_S", 0.3)
+    hub = RemoteWorkerHub()
+    worker, socket = attach(hub, loop)
+    result: dict = {}
+
+    thread = call_in_thread(
+        lambda: result.update(
+            value=hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=0)
+        )
+    )
+    assert socket.arrived.wait(5)
+    call_id = socket.sent[0]["call_id"]
+    for _ in range(6):
+        worker.deliver({"call_id": call_id, "event": "heartbeat"})
+        time.sleep(0.1)
+    worker.deliver({"call_id": call_id, "ok": True, "result": {"done": True}})
+
+    thread.join(timeout=5)
+    assert result["value"] == {"done": True}

backend/tests/flow/test_runs.py

@@ -8,6 +8,7 @@ parameters a caller sends are refused before anything runs if they are wrong.
 
 import pytest
 
+from fluksio.flow.events import EventBus
 from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.nodes import Node
 from fluksio.flow.pipeline import NodeOutcome, Pipeline, run_cache_key
@@ -241,17 +242,24 @@ def test_emissions_are_checked_against_the_port_they_name():
     assert "loss" in seen[0].error
 
 
-def test_an_emission_that_nothing_declares_is_ignored():
+def test_an_emission_that_nothing_declares_names_what_was_emitted():
+    """A mistyped metric name is how a training curve goes missing."""
+    events = []
+
     def stray(params):
         yield {"undeclared": 1.0}
         return {"final_loss": 2.0}
 
+    bus = EventBus()
+    bus.publish = events.append  # type: ignore[method-assign]
     node = make_node("train", "study", stray, provides=[spec("final_loss")])
     state = MemoryState()
-    Pipeline(nodes=[node], state=state).run()
+    Pipeline(nodes=[node], state=state, events=bus).run()
 
+    (error,) = [e for e in events if e["type"] == "node_error"]
+    assert "undeclared" in error["error"]
+    assert "final_loss" in error["error"]
     assert "study.undeclared" not in state
-    assert state["study.final_loss"] == 2.0
 
 
 def test_emissions_reach_the_run_as_a_series_with_a_step_each():
@@ -435,3 +443,4 @@ def test_a_node_with_no_fingerprint_is_never_looked_up():
     assert calls == [1]
     assert cache.asked == []
     assert seen[0].cache_key == ""
+

backend/tests/flow/test_workers.py

@@ -373,3 +373,98 @@ def test_a_node_saves_and_loads_an_artifact(tmp_path):
         assert loaded == {"size": 2048}
     finally:
         pool.stop()
+
+
+# -----------------------------------------------------------------------------
+# No timeout at all
+#
+# The default: silence is a node working, not a node stuck. What still ends a
+# call is the worker dying, which arrives as its pipe closing rather than as a
+# deadline.
+# -----------------------------------------------------------------------------
+
+
+def test_a_node_with_no_timeout_runs_past_what_the_default_would_have_killed(pool):
+    assert pool.run(
+        "demo",
+        "patient",
+        "import time\n\n\ndef process():\n    time.sleep(2)\n    return {'out': 1}\n",
+        {},
+        "demo.patient",
+        timeout=0,
+    ) == {"out": 1}
+
+
+def test_a_worker_that_dies_still_fails_promptly_with_no_timeout(pool):
+    started = time.monotonic()
+    with pytest.raises(Exception, match="worker died"):
+        pool.run(
+            "demo",
+            "doomed",
+            "import os\n\n\ndef process():\n    os._exit(1)\n",
+            {},
+            "demo.doomed",
+            timeout=0,
+        )
+    # Not waiting out a poll interval: the pipe closing is what wakes the read.
+    assert time.monotonic() - started < 5
+
+    assert run(pool, "def process():\n    return {'out': 3}\n") == {"out": 3}
+
+
+def test_a_node_with_no_timeout_can_still_be_cancelled(pool):
+    def stop_it() -> None:
+        for _ in range(100):
+            if pool.cancel("demo.slow"):
+                return
+            time.sleep(0.05)
+
+    stopper = threading.Thread(target=stop_it)
+    stopper.start()
+    try:
+        with pytest.raises(NodeCancelled):
+            pool.run(
+                "demo",
+                "slow",
+                "import time\n\n\ndef process():\n    time.sleep(30)\n",
+                {},
+                "demo.slow",
+                timeout=0,
+            )
+    finally:
+        stopper.join()
+
+
+def test_an_emission_on_an_undeclared_port_fails_the_call(pool):
+    """The engine's sink raises, and that has to reach the node's author.
+
+    A yield is held one behind — the last one is the return value when there is
+    no explicit return — so the mistake surfaces on the loop's second pass
+    rather than its first. Which is what a training loop does in a moment, and
+    a long way short of the hours it used to cost.
+    """
+    from fluksio.flow.nodes.base import NodeOutputError
+
+    def refuse(event):
+        raise NodeOutputError("'demo.gen' produced 'lss', which no port declares")
+
+    started = time.monotonic()
+    with pytest.raises(NodeOutputError, match="lss"):
+        pool.run(
+            "demo",
+            "gen",
+            "import time\n\n\ndef process():\n"
+            "    for _ in range(3):\n"
+            "        yield {'lss': 1.0}\n"
+            "    time.sleep(30)\n"
+            "    return {'out': 1}\n",
+            {},
+            "demo.gen",
+            timeout=0,
+            on_event=refuse,
+        )
+    # It did not wait out the node: the failure stopped the call.
+    assert time.monotonic() - started < 10
+
+    # The worker was retired rather than left mid-generator, so the slot works.
+    assert run(pool, "def process():\n    return {'out': 4}\n") == {"out": 4}