Let a worker say what machine it is

A worker reported its labels and nothing about the machine behind them, so the
engine could route a node to a GPU box but not tell whether that box had a GPU
free. Inventory — cores, GPUs, memory — now arrives with the hello frame, and
the run frame carries back what the engine allocated for that call.

Which is protocol 2 on both ends. GPUs are never probed: asking a vendor tool
would make the one dependency two, so a GPU is what the batch job says it was
given or what --gpus says. A worker that reports nothing still attaches and is
scheduled by its label alone.

Two things a job scheduler needs: --max-idle stops a worker started for one job
rather than letting it hold its allocation to the walltime, and a refusal is now
fatal instead of a reconnect loop that reads as a hang in a job's log.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01A6HeySA27EkGANZN95QySW
This commit is contained in:
2026-08-27 08:29:35 +02:00
co-authored by Claude Opus 5
parent 0ffcabfdb9
commit 1a9753fa9d
7 changed files with 359 additions and 36 deletions
+70 -1
View File
@@ -15,7 +15,12 @@ from collections.abc import Iterator
import pytest
from fluksio.flow import remote
from fluksio.flow.remote import NoWorker, RemoteWorker, RemoteWorkerHub
from fluksio.flow.remote import (
NoWorker,
RemoteWorker,
RemoteWorkerHub,
WorkerInventory,
)
from fluksio.flow.workers import NodeTimeout, RemoteError
@@ -295,3 +300,67 @@ def test_with_no_timeout_a_beating_worker_is_left_to_finish(loop, monkeypatch):
thread.join(timeout=5)
assert result["value"] == {"done": True}
def test_an_allocation_rides_along_with_the_call(loop):
"""Thread caps and devices reach the worker on the frame that needs them.
The worker starts a process per call, so this is the only moment it can
apply them: a library reads them when it is imported and never again.
"""
hub = RemoteWorkerHub()
worker, socket = attach(hub, loop)
thread = call_in_thread(
lambda: hub.run(
"gpu",
"flow",
"node",
"src",
{},
"flow.node",
timeout=5,
env={"OMP_NUM_THREADS": "4", "CUDA_VISIBLE_DEVICES": "0"},
)
)
assert socket.arrived.wait(5)
assert socket.sent[0]["env"] == {
"OMP_NUM_THREADS": "4",
"CUDA_VISIBLE_DEVICES": "0",
}
worker.deliver({"call_id": socket.sent[0]["call_id"], "ok": True, "result": None})
thread.join(timeout=5)
def test_a_call_with_nothing_allocated_carries_no_env(loop):
hub = RemoteWorkerHub()
worker, socket = attach(hub, loop)
thread = call_in_thread(
lambda: hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=5)
)
assert socket.arrived.wait(5)
assert "env" not in socket.sent[0]
worker.deliver({"call_id": socket.sent[0]["call_id"], "ok": True, "result": None})
thread.join(timeout=5)
def test_a_worker_that_reports_nothing_is_the_machine_it_used_to_be():
"""Inventory is read leniently: absent is a default, junk is a default.
A worker saying nothing has to keep being scheduled by its label, and one
that learns to report something new must not need this engine taught about
it first.
"""
assert WorkerInventory.from_hello({}) == WorkerInventory(cpus=1, gpus=0)
assert WorkerInventory.from_hello(None) == WorkerInventory()
read = WorkerInventory.from_hello(
{"cpus": "8", "gpus": 2, "ram_mb": 64000, "tpus": 4}
)
assert (read.cpus, read.gpus, read.ram_mb) == (8, 2, 64000)
junk = WorkerInventory.from_hello({"cpus": "many", "gpus": None, "ram_mb": 0})
assert (junk.cpus, junk.gpus, junk.ram_mb) == (1, 0, None)