Let a worker say what machine it is
A worker reported its labels and nothing about the machine behind them, so the engine could route a node to a GPU box but not tell whether that box had a GPU free. Inventory — cores, GPUs, memory — now arrives with the hello frame, and the run frame carries back what the engine allocated for that call. Which is protocol 2 on both ends. GPUs are never probed: asking a vendor tool would make the one dependency two, so a GPU is what the batch job says it was given or what --gpus says. A worker that reports nothing still attaches and is scheduled by its label alone. Two things a job scheduler needs: --max-idle stops a worker started for one job rather than letting it hold its allocation to the walltime, and a refusal is now fatal instead of a reconnect loop that reads as a hang in a job's log. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01A6HeySA27EkGANZN95QySW
This commit is contained in:
@@ -15,7 +15,12 @@ from collections.abc import Iterator
|
||||
import pytest
|
||||
|
||||
from fluksio.flow import remote
|
||||
from fluksio.flow.remote import NoWorker, RemoteWorker, RemoteWorkerHub
|
||||
from fluksio.flow.remote import (
|
||||
NoWorker,
|
||||
RemoteWorker,
|
||||
RemoteWorkerHub,
|
||||
WorkerInventory,
|
||||
)
|
||||
from fluksio.flow.workers import NodeTimeout, RemoteError
|
||||
|
||||
|
||||
@@ -295,3 +300,67 @@ def test_with_no_timeout_a_beating_worker_is_left_to_finish(loop, monkeypatch):
|
||||
|
||||
thread.join(timeout=5)
|
||||
assert result["value"] == {"done": True}
|
||||
|
||||
|
||||
def test_an_allocation_rides_along_with_the_call(loop):
|
||||
"""Thread caps and devices reach the worker on the frame that needs them.
|
||||
|
||||
The worker starts a process per call, so this is the only moment it can
|
||||
apply them: a library reads them when it is imported and never again.
|
||||
"""
|
||||
hub = RemoteWorkerHub()
|
||||
worker, socket = attach(hub, loop)
|
||||
|
||||
thread = call_in_thread(
|
||||
lambda: hub.run(
|
||||
"gpu",
|
||||
"flow",
|
||||
"node",
|
||||
"src",
|
||||
{},
|
||||
"flow.node",
|
||||
timeout=5,
|
||||
env={"OMP_NUM_THREADS": "4", "CUDA_VISIBLE_DEVICES": "0"},
|
||||
)
|
||||
)
|
||||
assert socket.arrived.wait(5)
|
||||
assert socket.sent[0]["env"] == {
|
||||
"OMP_NUM_THREADS": "4",
|
||||
"CUDA_VISIBLE_DEVICES": "0",
|
||||
}
|
||||
|
||||
worker.deliver({"call_id": socket.sent[0]["call_id"], "ok": True, "result": None})
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
def test_a_call_with_nothing_allocated_carries_no_env(loop):
|
||||
hub = RemoteWorkerHub()
|
||||
worker, socket = attach(hub, loop)
|
||||
|
||||
thread = call_in_thread(
|
||||
lambda: hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=5)
|
||||
)
|
||||
assert socket.arrived.wait(5)
|
||||
assert "env" not in socket.sent[0]
|
||||
|
||||
worker.deliver({"call_id": socket.sent[0]["call_id"], "ok": True, "result": None})
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
def test_a_worker_that_reports_nothing_is_the_machine_it_used_to_be():
|
||||
"""Inventory is read leniently: absent is a default, junk is a default.
|
||||
|
||||
A worker saying nothing has to keep being scheduled by its label, and one
|
||||
that learns to report something new must not need this engine taught about
|
||||
it first.
|
||||
"""
|
||||
assert WorkerInventory.from_hello({}) == WorkerInventory(cpus=1, gpus=0)
|
||||
assert WorkerInventory.from_hello(None) == WorkerInventory()
|
||||
|
||||
read = WorkerInventory.from_hello(
|
||||
{"cpus": "8", "gpus": 2, "ram_mb": 64000, "tpus": 4}
|
||||
)
|
||||
assert (read.cpus, read.gpus, read.ram_mb) == (8, 2, 64000)
|
||||
|
||||
junk = WorkerInventory.from_hello({"cpus": "many", "gpus": None, "ram_mb": 0})
|
||||
assert (junk.cpus, junk.gpus, junk.ram_mb) == (1, 0, None)
|
||||
|
||||
Reference in New Issue
Block a user