Refuse what a node cannot publish, and stop timing out work that is fine

Four things the python SDK turned up, each fixed where every client sees it.

A key no port declares is now an error rather than a silent drop, on the
return, the yield and the emit alike — the contract the docs already stated.
The SDK reads literal yields at sync time, so a typo fails before anything
runs, and an emission of one fails the call rather than being logged where
nobody looks.

NaN and infinity are refused at the port. JSON cannot spell either, so one
that travelled came back as a 500, a socket frame that stopped the canvas, or
a metric batch the database dropped whole.

An artifact input takes `@run:<id>.<output>` or a bare digest, resolved on the
engine — so the CLI, the run dialog and a python caller mean the same thing,
and a sweep can pass one at all.

Node timeouts are off by default. The clock measured silence, which a training
node is full of, and remote workers had already stopped enforcing it — their
heartbeat reset it. Now a heartbeat proves the agent rather than the node,
ninety seconds of nothing fails the call either way, and the engine touches
work it is still running so a long node is not redelivered at sixty seconds.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019V5bsYGNxcgPs4xXmTPx69

backend/tests/api/routes/test_runs.py

@@ -5,14 +5,26 @@ The pipeline half — what a hit restores and what a key is made of — is in
 """
 
 import json
+from datetime import UTC, datetime
 
+import pytest
 from sqlmodel import Session
 
 from fluksio.core.db import engine as db_engine
 from fluksio.flow.artifacts import ArtifactStore
+from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.pipeline import NodeOutcome
-from fluksio.flow.runs import OUTPUT_CAP, RunCache, _cacheable
-from fluksio.models import RunNode
+from fluksio.flow.runs import (
+    OUTPUT_CAP,
+    RunCache,
+    RunRejected,
+    _cacheable,
+    new_run_id,
+    resolve_references,
+    seed_values,
+)
+from fluksio.flow.schemas import FlowDef, FlowInput, NodeDef
+from fluksio.models import Run, RunArtifact, RunNode
 
 
 def test_a_run_cache_finds_what_an_earlier_run_recorded(tmp_path):
@@ -75,3 +87,110 @@ def test_what_may_be_stored_as_a_cache_entry():
     assert _cacheable(ok.model_copy(update={"cache_key": ""})) is None
     big = {"study.data": "x" * (OUTPUT_CAP + 1)}
     assert _cacheable(ok.model_copy(update={"output_values": big})) is None
+
+
+# -----------------------------------------------------------------------------
+# Naming an artifact from outside the process that made it
+#
+# A python caller passes the reference it holds. A shell holds nothing, so the
+# same input also takes `@run:<id>.<output>` or a bare digest, resolved here
+# rather than in each client.
+# -----------------------------------------------------------------------------
+
+
+def artifact_flow() -> FlowDef:
+    """A flow taking a dataset somebody else's run produced."""
+    dataset = MessageSpec(name="dataset", dtype=DType.ARTIFACT)
+    return FlowDef(
+        name="study",
+        mode="batch",
+        inputs=[FlowInput(spec=dataset)],
+        nodes=[NodeDef(id="train", requires=[dataset])],
+    )
+
+
+@pytest.fixture
+def made_artifact():
+    """A finished run with one artifact, as a later run would find it."""
+    digest = "sha256:" + "a1" * 32
+    reference = {
+        "digest": digest,
+        "size": 12,
+        "media_type": "text/csv",
+        "name": "cities.csv",
+    }
+    run_id = new_run_id()
+    with Session(db_engine) as session:
+        session.add(
+            Run(
+                id=run_id,
+                flow="prepare",
+                status="ok",
+                result={"dataset": reference},
+                created_at=datetime.now(UTC),
+            )
+        )
+        session.add(
+            RunArtifact(
+                run_id=run_id,
+                name="prepare.dataset",
+                node="load",
+                digest=digest,
+                size=12,
+            )
+        )
+        session.commit()
+    yield run_id, reference
+    with Session(db_engine) as session:
+        session.delete(session.get(RunArtifact, (run_id, "prepare.dataset")))
+        session.delete(session.get(Run, run_id))
+        session.commit()
+
+
+def test_a_run_reference_resolves_to_what_that_run_produced(made_artifact):
+    run_id, reference = made_artifact
+    resolved = resolve_references(artifact_flow(), {"dataset": f"@run:{run_id}.dataset"})
+
+    # The producer's own reference, file name and all — not one rebuilt from
+    # the row, which carries the message name instead.
+    assert resolved["dataset"] == reference
+
+
+def test_a_bare_digest_resolves_to_the_bytes_under_it(made_artifact):
+    _run_id, reference = made_artifact
+    resolved = resolve_references(artifact_flow(), {"dataset": reference["digest"]})
+
+    assert resolved["dataset"]["digest"] == reference["digest"]
+    assert resolved["dataset"]["size"] == 12
+
+
+def test_a_resolved_reference_passes_the_input_check(made_artifact):
+    run_id, _reference = made_artifact
+    flow = artifact_flow()
+    resolved = resolve_references(flow, {"dataset": f"@run:{run_id}.dataset"})
+
+    assert "study.dataset" in seed_values(flow, resolved)
+
+
+def test_an_output_a_run_never_made_says_what_it_did(made_artifact):
+    run_id, _reference = made_artifact
+    with pytest.raises(RunRejected, match="prepare.dataset"):
+        resolve_references(artifact_flow(), {"dataset": f"@run:{run_id}.weights"})
+
+
+def test_a_reference_to_no_run_at_all_is_refused():
+    with pytest.raises(RunRejected, match="no run"):
+        resolve_references(artifact_flow(), {"dataset": "@run:nothing.dataset"})
+
+
+def test_an_unknown_digest_is_refused():
+    with pytest.raises(RunRejected, match="nothing here"):
+        resolve_references(artifact_flow(), {"dataset": "sha256:" + "b2" * 32})
+
+
+def test_a_reference_passed_whole_is_left_alone(made_artifact):
+    """A python caller already has the object, and hands it over as one."""
+    _run_id, reference = made_artifact
+    assert resolve_references(artifact_flow(), {"dataset": reference}) == {
+        "dataset": reference
+    }

backend/tests/flow/test_messages.py

@@ -114,3 +114,41 @@ def test_qualify_scopes_bare_names_only():
     assert qualify("heating", "solar.power") == "solar.power"
     assert qualify("heating", "") == ""
     assert flow_of("heating.temp") == "heating"
+
+
+# -----------------------------------------------------------------------------
+# NaN and infinity
+#
+# JSON cannot spell either, so one travelling through a port would come back as
+# a response nobody can parse, a socket frame that stops a canvas, or a row the
+# database rejects — a long way from the node that produced it.
+# -----------------------------------------------------------------------------
+
+
+def test_a_float_port_refuses_nan_and_infinity():
+    spec = MessageSpec(name="score", dtype=DType.FLOAT)
+    spec.check(0.5)
+    for value in (float("nan"), float("inf"), float("-inf")):
+        with pytest.raises(TypeError, match="score"):
+            spec.check(value)
+
+
+def test_a_json_port_refuses_a_nan_nested_in_it():
+    spec = MessageSpec(name="report", dtype=DType.JSON)
+    spec.check({"groups": [{"mean": 1.0}]})
+    with pytest.raises(TypeError, match="JSON"):
+        spec.check({"groups": [{"mean": float("nan")}]})
+
+
+def test_a_series_refuses_a_nan_point():
+    spec = MessageSpec(name="curve", dtype=DType.SERIES)
+    lines = [{"label": "loss", "points": [[1.0, float("nan")]]}]
+    with pytest.raises(TypeError):
+        spec.check({"lines": lines})
+
+
+def test_a_value_that_refers_to_itself_does_not_hang_the_check():
+    spec = MessageSpec(name="report", dtype=DType.JSON)
+    loop: dict = {}
+    loop["self"] = loop
+    spec.check(loop)

backend/tests/flow/test_queue.py

@@ -1,7 +1,9 @@
 """The work queue, and what the execution service does with it."""
 
+import threading
 import time
 
+from fluksio.flow import executor
 from fluksio.flow.executor import ExecutionService
 from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.nodes import Node
@@ -264,3 +266,72 @@ def test_a_replayed_item_does_not_repeat_a_side_effect():
     service._run_item(item)
 
     assert calls == [5.0]
+
+
+# -----------------------------------------------------------------------------
+# Telling the queue that a long node is working, not lost
+#
+# What marks an item abandoned is nobody touching it. A node with no timeout
+# may run for hours, so the engine holding it says so on a timer — and an
+# engine that died says nothing, which is the distinction the reaper needs.
+# -----------------------------------------------------------------------------
+
+
+class RecordingQueue(MemoryWorkQueue):
+    """A memory queue that writes down what it was asked to hold on to."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.touched: list[list[str]] = []
+
+    def touch(self, entry_ids: list[str]) -> None:
+        self.touched.append(list(entry_ids))
+
+
+def test_work_in_flight_is_touched_until_it_finishes(monkeypatch):
+    monkeypatch.setattr(executor, "TOUCH_INTERVAL_S", 0.0)
+    monkeypatch.setattr(executor, "DELAYED_INTERVAL_S", 0.05)
+    running = threading.Event()
+    release = threading.Event()
+
+    def slow(reading, params):
+        running.set()
+        release.wait(5)
+        return {"doubled": reading * 2}
+
+    source = Node(
+        f=lambda params: None,
+        provides=[MessageSpec(name="reading", dtype=DType.FLOAT)],
+        name="source",
+    )
+    consumer = Node(
+        f=slow,
+        requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
+        provides=[MessageSpec(name="doubled", dtype=DType.FLOAT)],
+        name="consumer",
+    )
+    source.assign_flow("f", "source")
+    consumer.assign_flow("f", "consumer")
+
+    queue = RecordingQueue()
+    pipeline = Pipeline(
+        nodes=[source, consumer], state=MemoryState(), work_queue=queue
+    )
+    service = ExecutionService(queue)
+    service.bind(pipeline)
+    service.start()
+    try:
+        source.inject({"reading": 3.0})
+        assert running.wait(5)
+        # Give the timer a couple of passes while the node is still in there.
+        time.sleep(0.2)
+        held = [ids for ids in queue.touched if ids]
+        assert held, "a running item was never touched"
+
+        release.set()
+        time.sleep(0.3)
+        # Once it is done it is acknowledged, so there is nothing to hold.
+        assert queue.touched[-1] == []
+    finally:
+        release.set()
+        service.stop()

backend/tests/flow/test_remote.py

@@ -9,10 +9,12 @@ what happened rather than a wait that never ends.
 
 import asyncio
 import threading
+import time
 from collections.abc import Iterator
 
 import pytest
 
+from fluksio.flow import remote
 from fluksio.flow.remote import NoWorker, RemoteWorker, RemoteWorkerHub
 from fluksio.flow.workers import NodeTimeout, RemoteError
 
@@ -220,3 +222,76 @@ def test_cancelling_a_run_reaches_only_that_run(loop):
     assert hub.cancel_run("run-a") == 1
     cancels = [frame for frame in socket.sent if frame.get("op") == "cancel"]
     assert [frame["call_id"] for frame in cancels] == ["run-a:flow.node"]
+
+
+# -----------------------------------------------------------------------------
+# Liveness, and what a heartbeat is evidence of
+#
+# The agent beats every ten seconds while it is executing. That says the agent
+# is alive; it says nothing about the node, which is why it bounds the silence
+# deadline and not the node's own timeout.
+# -----------------------------------------------------------------------------
+
+
+def test_heartbeats_do_not_stave_off_a_nodes_own_timeout(loop):
+    hub = RemoteWorkerHub()
+    worker, socket = attach(hub, loop)
+    caught: list[Exception] = []
+
+    def call() -> None:
+        try:
+            hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=0.5)
+        except Exception as exc:
+            caught.append(exc)
+
+    thread = call_in_thread(call)
+    assert socket.arrived.wait(5)
+    call_id = socket.sent[0]["call_id"]
+    # Beating faster than the deadline. A node that reports its progress is
+    # held off; one whose agent is merely alive is not.
+    for _ in range(10):
+        worker.deliver({"call_id": call_id, "event": "heartbeat"})
+        time.sleep(0.1)
+
+    thread.join(timeout=5)
+    assert isinstance(caught[0], NodeTimeout)
+
+
+def test_with_no_timeout_total_silence_is_still_bounded(loop, monkeypatch):
+    monkeypatch.setattr(remote, "SILENCE_S", 0.3)
+    hub = RemoteWorkerHub()
+    attach(hub, loop)
+    caught: list[Exception] = []
+
+    def call() -> None:
+        try:
+            hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=0)
+        except Exception as exc:
+            caught.append(exc)
+
+    call_in_thread(call).join(timeout=5)
+    assert isinstance(caught[0], RemoteError)
+    assert not isinstance(caught[0], NodeTimeout)
+    assert "presumed gone" in str(caught[0])
+
+
+def test_with_no_timeout_a_beating_worker_is_left_to_finish(loop, monkeypatch):
+    monkeypatch.setattr(remote, "SILENCE_S", 0.3)
+    hub = RemoteWorkerHub()
+    worker, socket = attach(hub, loop)
+    result: dict = {}
+
+    thread = call_in_thread(
+        lambda: result.update(
+            value=hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=0)
+        )
+    )
+    assert socket.arrived.wait(5)
+    call_id = socket.sent[0]["call_id"]
+    for _ in range(6):
+        worker.deliver({"call_id": call_id, "event": "heartbeat"})
+        time.sleep(0.1)
+    worker.deliver({"call_id": call_id, "ok": True, "result": {"done": True}})
+
+    thread.join(timeout=5)
+    assert result["value"] == {"done": True}

backend/tests/flow/test_runs.py

@@ -8,6 +8,7 @@ parameters a caller sends are refused before anything runs if they are wrong.
 
 import pytest
 
+from fluksio.flow.events import EventBus
 from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.nodes import Node
 from fluksio.flow.pipeline import NodeOutcome, Pipeline, run_cache_key
@@ -241,17 +242,24 @@ def test_emissions_are_checked_against_the_port_they_name():
     assert "loss" in seen[0].error
 
 
-def test_an_emission_that_nothing_declares_is_ignored():
+def test_an_emission_that_nothing_declares_names_what_was_emitted():
+    """A mistyped metric name is how a training curve goes missing."""
+    events = []
+
     def stray(params):
         yield {"undeclared": 1.0}
         return {"final_loss": 2.0}
 
+    bus = EventBus()
+    bus.publish = events.append  # type: ignore[method-assign]
     node = make_node("train", "study", stray, provides=[spec("final_loss")])
     state = MemoryState()
-    Pipeline(nodes=[node], state=state).run()
+    Pipeline(nodes=[node], state=state, events=bus).run()
 
+    (error,) = [e for e in events if e["type"] == "node_error"]
+    assert "undeclared" in error["error"]
+    assert "final_loss" in error["error"]
     assert "study.undeclared" not in state
-    assert state["study.final_loss"] == 2.0
 
 
 def test_emissions_reach_the_run_as_a_series_with_a_step_each():
@@ -435,3 +443,4 @@ def test_a_node_with_no_fingerprint_is_never_looked_up():
     assert calls == [1]
     assert cache.asked == []
     assert seen[0].cache_key == ""
+

backend/tests/flow/test_workers.py

@@ -373,3 +373,98 @@ def test_a_node_saves_and_loads_an_artifact(tmp_path):
         assert loaded == {"size": 2048}
     finally:
         pool.stop()
+
+
+# -----------------------------------------------------------------------------
+# No timeout at all
+#
+# The default: silence is a node working, not a node stuck. What still ends a
+# call is the worker dying, which arrives as its pipe closing rather than as a
+# deadline.
+# -----------------------------------------------------------------------------
+
+
+def test_a_node_with_no_timeout_runs_past_what_the_default_would_have_killed(pool):
+    assert pool.run(
+        "demo",
+        "patient",
+        "import time\n\n\ndef process():\n    time.sleep(2)\n    return {'out': 1}\n",
+        {},
+        "demo.patient",
+        timeout=0,
+    ) == {"out": 1}
+
+
+def test_a_worker_that_dies_still_fails_promptly_with_no_timeout(pool):
+    started = time.monotonic()
+    with pytest.raises(Exception, match="worker died"):
+        pool.run(
+            "demo",
+            "doomed",
+            "import os\n\n\ndef process():\n    os._exit(1)\n",
+            {},
+            "demo.doomed",
+            timeout=0,
+        )
+    # Not waiting out a poll interval: the pipe closing is what wakes the read.
+    assert time.monotonic() - started < 5
+
+    assert run(pool, "def process():\n    return {'out': 3}\n") == {"out": 3}
+
+
+def test_a_node_with_no_timeout_can_still_be_cancelled(pool):
+    def stop_it() -> None:
+        for _ in range(100):
+            if pool.cancel("demo.slow"):
+                return
+            time.sleep(0.05)
+
+    stopper = threading.Thread(target=stop_it)
+    stopper.start()
+    try:
+        with pytest.raises(NodeCancelled):
+            pool.run(
+                "demo",
+                "slow",
+                "import time\n\n\ndef process():\n    time.sleep(30)\n",
+                {},
+                "demo.slow",
+                timeout=0,
+            )
+    finally:
+        stopper.join()
+
+
+def test_an_emission_on_an_undeclared_port_fails_the_call(pool):
+    """The engine's sink raises, and that has to reach the node's author.
+
+    A yield is held one behind — the last one is the return value when there is
+    no explicit return — so the mistake surfaces on the loop's second pass
+    rather than its first. Which is what a training loop does in a moment, and
+    a long way short of the hours it used to cost.
+    """
+    from fluksio.flow.nodes.base import NodeOutputError
+
+    def refuse(event):
+        raise NodeOutputError("'demo.gen' produced 'lss', which no port declares")
+
+    started = time.monotonic()
+    with pytest.raises(NodeOutputError, match="lss"):
+        pool.run(
+            "demo",
+            "gen",
+            "import time\n\n\ndef process():\n"
+            "    for _ in range(3):\n"
+            "        yield {'lss': 1.0}\n"
+            "    time.sleep(30)\n"
+            "    return {'out': 1}\n",
+            {},
+            "demo.gen",
+            timeout=0,
+            on_event=refuse,
+        )
+    # It did not wait out the node: the failure stopped the call.
+    assert time.monotonic() - started < 10
+
+    # The worker was retired rather than left mid-generator, so the slot works.
+    assert run(pool, "def process():\n    return {'out': 4}\n") == {"out": 4}

backend/tests/sdk/test_build.py

@@ -236,3 +236,83 @@ def test_the_decorators_leave_the_function_alone():
     assert prepare(limit=2)["rows"] == 2
     assert [step["loss"] for step in fit(None, 0.5, epochs=2)] == [1.0, 0.5]
     assert evaluate(None) == 0.5
+
+
+# -----------------------------------------------------------------------------
+# What a function emits against what it declares
+#
+# The engine refuses an undeclared key at the first yield, which is right but
+# late — a sweep can be an hour in. A literal one is a typo, and a typo is
+# readable from the source.
+# -----------------------------------------------------------------------------
+
+
+def trains_with_a_typo(steps=3):
+    for _ in range(steps):
+        yield {"lss": 0.5}
+    return {"final_loss": 0.5}
+
+
+def trains(steps=3):
+    for _ in range(steps):
+        yield {"loss": 0.5}
+    return {"final_loss": 0.5}
+
+
+def emits_a_typo(steps=3):
+    import fluksio
+
+    fluksio.emit(lss=0.5)
+    return {"final_loss": 0.5}
+
+
+def yields_a_name_it_computes(steps=3):
+    for index in range(steps):
+        yield {f"loss_{index}": 0.5}
+    return {"final_loss": 0.5}
+
+
+def trains_beside_a_helper(steps=3):
+    def every_pair():
+        yield {"internal": 1}
+
+    for _ in range(steps):
+        yield {"loss": 0.5}
+    return {"final_loss": 0.5}
+
+
+def test_a_yielded_key_no_port_declares_is_refused():
+    with pytest.raises(SyncError, match="lss"):
+        node(provides=[Port("loss", "float"), Port("final_loss", "float")])(
+            trains_with_a_typo
+        )
+
+
+def test_an_emitted_key_no_port_declares_is_refused():
+    with pytest.raises(SyncError, match="lss"):
+        node(provides=[Port("final_loss", "float")])(emits_a_typo)
+
+
+def test_declared_keys_pass():
+    assert node(provides=[Port("loss", "float"), Port("final_loss", "float")])(trains)
+
+
+def test_a_key_the_code_computes_is_left_to_the_engine():
+    """Only literals are readable here; the rest is checked where it runs."""
+    assert node(provides=[Port("final_loss", "float")])(yields_a_name_it_computes)
+
+
+def test_a_helper_defined_inside_the_node_is_not_the_nodes_ports():
+    assert node(provides=[Port("loss", "float"), Port("final_loss", "float")])(
+        trains_beside_a_helper
+    )
+
+
+def test_a_negative_timeout_is_refused():
+    with pytest.raises(SyncError, match="0 or more"):
+        node(requires=["a"], timeout=-1)(one_default)
+
+
+def test_a_zero_timeout_means_no_limit():
+    decorated = node(requires=["a"], timeout=0)(one_default)
+    assert decorated.__fluksio__.timeout == 0