Declare this machine's GPUs from serve, and refuse a bad limit as a flag

GPU count is not detected, so FLOW_GPUS was 0 on a fresh install and a node
asking for one was silently clamped to zero and ran concurrently with every
other. Setting the variable serialised them, but it was an environment
variable only — `serve` had --max-runs and --max-workers and no --gpus.
The clamp warning now names the flag when nothing here declares a card.

The same flags are written into the environment before the settings are
built, so a value they refused died in a pydantic import naming no flag.
They are checked where they are typed instead.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019Hra4ndWMCLU5F3KjUuVAc
This commit is contained in:
2026-08-29 13:54:33 +02:00
co-authored by Claude Opus 5
parent 53b49e5f68
commit 8bd30db016
4 changed files with 62 additions and 5 deletions
+28 -3
View File
@@ -21,6 +21,7 @@ import os
import secrets import secrets
import socket import socket
import sys import sys
from collections.abc import Callable
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -228,9 +229,26 @@ CONCURRENCY_FLAGS = {
"max_workers": "FLOW_MAX_WORKERS", "max_workers": "FLOW_MAX_WORKERS",
"max_cascades": "FLOW_MAX_CASCADES", "max_cascades": "FLOW_MAX_CASCADES",
"max_runs": "FLOW_MAX_RUNS", "max_runs": "FLOW_MAX_RUNS",
"gpus": "FLOW_GPUS",
} }
def _at_least(minimum: int) -> Callable[[str], int]:
"""A flag's value, checked here rather than by the settings.
These are written into the environment before the settings are built, so a
number they refuse dies inside a pydantic import with no flag named in it.
"""
def parse(text: str) -> int:
value = int(text)
if value < minimum:
raise argparse.ArgumentTypeError(f"is {value}, needs at least {minimum}")
return value
return parse
#: What `serve` listens on when nobody says. Taken often enough — another #: What `serve` listens on when nobody says. Taken often enough — another
#: engine, another framework's dev server — that dying on it is the first #: engine, another framework's dev server — that dying on it is the first
#: thing a zero-config start would hit. #: thing a zero-config start would hit.
@@ -426,25 +444,32 @@ def _parser() -> argparse.ArgumentParser:
) )
serve.add_argument( serve.add_argument(
"--max-runs", "--max-runs",
type=int, type=_at_least(1),
default=None, default=None,
metavar="N", metavar="N",
help="batch runs driven at once (default 4, FLOW_MAX_RUNS)", help="batch runs driven at once (default 4, FLOW_MAX_RUNS)",
) )
serve.add_argument( serve.add_argument(
"--max-cascades", "--max-cascades",
type=int, type=_at_least(1),
default=None, default=None,
metavar="N", metavar="N",
help="cascades in flight at once (default 4, FLOW_MAX_CASCADES)", help="cascades in flight at once (default 4, FLOW_MAX_CASCADES)",
) )
serve.add_argument( serve.add_argument(
"--max-workers", "--max-workers",
type=int, type=_at_least(1),
default=None, default=None,
metavar="N", metavar="N",
help="python worker processes (default 4, FLOW_MAX_WORKERS)", help="python worker processes (default 4, FLOW_MAX_WORKERS)",
) )
serve.add_argument(
"--gpus",
type=_at_least(0),
default=None,
metavar="N",
help="GPUs on this machine a node may be given (default 0, FLOW_GPUS)",
)
serve.set_defaults(func=cmd_serve) serve.set_defaults(func=cmd_serve)
enroll = subparsers.add_parser( enroll = subparsers.add_parser(
+11 -1
View File
@@ -215,9 +215,18 @@ class Placer:
said = [shape[2] for shape in capable if shape[2]] said = [shape[2] for shape in capable if shape[2]]
ram = min(ram, max(said)) if said and ram else ram ram = min(ram, max(said)) if said and ram else ram
if (cpus, gpus, ram) != (wanted.cpus, wanted.gpus, wanted.ram or 0): if (cpus, gpus, ram) != (wanted.cpus, wanted.gpus, wanted.ram or 0):
# Cards are not detected, so a machine that has one still reports
# none until it is told — which reads as "no GPU here" to a node
# that then runs unserialised beside every other one.
hint = (
"; no machine here declares a GPU — `fluksio serve --gpus N` "
"(or FLOW_GPUS) says how many this one has"
if wanted.gpus and not gpus
else ""
)
logger.warning( logger.warning(
"%s asked for %d cpu(s), %d gpu(s) and %s MB; " "%s asked for %d cpu(s), %d gpu(s) and %s MB; "
"the largest machine here can give %d, %d and %s", "the largest machine here can give %d, %d and %s%s",
node or "a node", node or "a node",
wanted.cpus, wanted.cpus,
wanted.gpus, wanted.gpus,
@@ -225,6 +234,7 @@ class Placer:
cpus, cpus,
gpus, gpus,
ram or "no stated", ram or "no stated",
hint,
) )
return cpus, gpus, ram return cpus, gpus, ram
+5 -1
View File
@@ -129,7 +129,7 @@ def test_a_preferred_label_falls_back_here_and_is_still_accounted(loop):
assert placer.local.snapshot()["cpus"]["free"] == 1 assert placer.local.snapshot()["cpus"]["free"] == 1
def test_asking_for_more_than_anything_has_gets_what_there_is(loop): def test_asking_for_more_than_anything_has_gets_what_there_is(loop, caplog):
"""A flow written on a cluster still has to run on a laptop.""" """A flow written on a cluster still has to run on a laptop."""
placer = placer_over(cpus=2) placer = placer_over(cpus=2)
@@ -138,6 +138,10 @@ def test_asking_for_more_than_anything_has_gets_what_there_is(loop):
assert allocation.cpus == 2 assert allocation.cpus == 2
assert allocation.gpus == () assert allocation.gpus == ()
# Cards are declared, not detected, so a machine that has one reads as
# having none until it is told — and the warning is where that is noticed.
assert "fluksio serve --gpus" in caplog.text
def test_what_is_clamped_to_is_a_machine_that_exists(loop): def test_what_is_clamped_to_is_a_machine_that_exists(loop):
"""Each dimension taken separately can describe a machine nobody has. """Each dimension taken separately can describe a machine nobody has.
+18
View File
@@ -520,6 +520,24 @@ def test_two_files_of_one_name_are_refused(tmp_path) -> None:
_import(*_module_of(second), second) _import(*_module_of(second), second)
def test_a_serve_limit_is_refused_as_a_flag_not_as_a_traceback(capsys) -> None:
"""These are written into the environment before the settings are built."""
import pytest
from fluksio.cli import _parser
parser = _parser()
assert parser.parse_args(["serve", "--max-workers", "2"]).max_workers == 2
# A machine may genuinely have no card, so zero is a number here.
assert parser.parse_args(["serve", "--gpus", "0"]).gpus == 0
assert parser.parse_args(["serve"]).gpus is None
for flag, value in (("--max-workers", "0"), ("--gpus", "-1")):
with pytest.raises(SystemExit):
parser.parse_args(["serve", flag, value])
assert "at least" in capsys.readouterr().err
def test_run_and_sweep_take_what_to_sync() -> None: def test_run_and_sweep_take_what_to_sync() -> None:
from fluksio.cli import _parser from fluksio.cli import _parser