Refuse what a node cannot publish, and stop timing out work that is fine

Four things the python SDK turned up, each fixed where every client sees it.

A key no port declares is now an error rather than a silent drop, on the
return, the yield and the emit alike — the contract the docs already stated.
The SDK reads literal yields at sync time, so a typo fails before anything
runs, and an emission of one fails the call rather than being logged where
nobody looks.

NaN and infinity are refused at the port. JSON cannot spell either, so one
that travelled came back as a 500, a socket frame that stopped the canvas, or
a metric batch the database dropped whole.

An artifact input takes `@run:<id>.<output>` or a bare digest, resolved on the
engine — so the CLI, the run dialog and a python caller mean the same thing,
and a sweep can pass one at all.

Node timeouts are off by default. The clock measured silence, which a training
node is full of, and remote workers had already stopped enforcing it — their
heartbeat reset it. Now a heartbeat proves the agent rather than the node,
ninety seconds of nothing fails the call either way, and the engine touches
work it is still running so a long node is not redelivered at sixty seconds.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019V5bsYGNxcgPs4xXmTPx69
This commit is contained in:
2026-08-25 07:30:14 +02:00
co-authored by Claude Opus 5
parent c33fa404a4
commit 93374a310e
32 changed files with 968 additions and 67 deletions
+75
View File
@@ -9,10 +9,12 @@ what happened rather than a wait that never ends.
import asyncio
import threading
import time
from collections.abc import Iterator
import pytest
from fluksio.flow import remote
from fluksio.flow.remote import NoWorker, RemoteWorker, RemoteWorkerHub
from fluksio.flow.workers import NodeTimeout, RemoteError
@@ -220,3 +222,76 @@ def test_cancelling_a_run_reaches_only_that_run(loop):
assert hub.cancel_run("run-a") == 1
cancels = [frame for frame in socket.sent if frame.get("op") == "cancel"]
assert [frame["call_id"] for frame in cancels] == ["run-a:flow.node"]
# -----------------------------------------------------------------------------
# Liveness, and what a heartbeat is evidence of
#
# The agent beats every ten seconds while it is executing. That says the agent
# is alive; it says nothing about the node, which is why it bounds the silence
# deadline and not the node's own timeout.
# -----------------------------------------------------------------------------
def test_heartbeats_do_not_stave_off_a_nodes_own_timeout(loop):
hub = RemoteWorkerHub()
worker, socket = attach(hub, loop)
caught: list[Exception] = []
def call() -> None:
try:
hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=0.5)
except Exception as exc:
caught.append(exc)
thread = call_in_thread(call)
assert socket.arrived.wait(5)
call_id = socket.sent[0]["call_id"]
# Beating faster than the deadline. A node that reports its progress is
# held off; one whose agent is merely alive is not.
for _ in range(10):
worker.deliver({"call_id": call_id, "event": "heartbeat"})
time.sleep(0.1)
thread.join(timeout=5)
assert isinstance(caught[0], NodeTimeout)
def test_with_no_timeout_total_silence_is_still_bounded(loop, monkeypatch):
monkeypatch.setattr(remote, "SILENCE_S", 0.3)
hub = RemoteWorkerHub()
attach(hub, loop)
caught: list[Exception] = []
def call() -> None:
try:
hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=0)
except Exception as exc:
caught.append(exc)
call_in_thread(call).join(timeout=5)
assert isinstance(caught[0], RemoteError)
assert not isinstance(caught[0], NodeTimeout)
assert "presumed gone" in str(caught[0])
def test_with_no_timeout_a_beating_worker_is_left_to_finish(loop, monkeypatch):
monkeypatch.setattr(remote, "SILENCE_S", 0.3)
hub = RemoteWorkerHub()
worker, socket = attach(hub, loop)
result: dict = {}
thread = call_in_thread(
lambda: result.update(
value=hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=0)
)
)
assert socket.arrived.wait(5)
call_id = socket.sent[0]["call_id"]
for _ in range(6):
worker.deliver({"call_id": call_id, "event": "heartbeat"})
time.sleep(0.1)
worker.deliver({"call_id": call_id, "ok": True, "result": {"done": True}})
thread.join(timeout=5)
assert result["value"] == {"done": True}