Declare this machine's GPUs from serve, and refuse a bad limit as a flag
GPU count is not detected, so FLOW_GPUS was 0 on a fresh install and a node asking for one was silently clamped to zero and ran concurrently with every other. Setting the variable serialised them, but it was an environment variable only — `serve` had --max-runs and --max-workers and no --gpus. The clamp warning now names the flag when nothing here declares a card. The same flags are written into the environment before the settings are built, so a value they refused died in a pydantic import naming no flag. They are checked where they are typed instead. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019Hra4ndWMCLU5F3KjUuVAc
This commit is contained in:
+28
-3
@@ -21,6 +21,7 @@ import os
|
|||||||
import secrets
|
import secrets
|
||||||
import socket
|
import socket
|
||||||
import sys
|
import sys
|
||||||
|
from collections.abc import Callable
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -228,9 +229,26 @@ CONCURRENCY_FLAGS = {
|
|||||||
"max_workers": "FLOW_MAX_WORKERS",
|
"max_workers": "FLOW_MAX_WORKERS",
|
||||||
"max_cascades": "FLOW_MAX_CASCADES",
|
"max_cascades": "FLOW_MAX_CASCADES",
|
||||||
"max_runs": "FLOW_MAX_RUNS",
|
"max_runs": "FLOW_MAX_RUNS",
|
||||||
|
"gpus": "FLOW_GPUS",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _at_least(minimum: int) -> Callable[[str], int]:
|
||||||
|
"""A flag's value, checked here rather than by the settings.
|
||||||
|
|
||||||
|
These are written into the environment before the settings are built, so a
|
||||||
|
number they refuse dies inside a pydantic import with no flag named in it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def parse(text: str) -> int:
|
||||||
|
value = int(text)
|
||||||
|
if value < minimum:
|
||||||
|
raise argparse.ArgumentTypeError(f"is {value}, needs at least {minimum}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
return parse
|
||||||
|
|
||||||
|
|
||||||
#: What `serve` listens on when nobody says. Taken often enough — another
|
#: What `serve` listens on when nobody says. Taken often enough — another
|
||||||
#: engine, another framework's dev server — that dying on it is the first
|
#: engine, another framework's dev server — that dying on it is the first
|
||||||
#: thing a zero-config start would hit.
|
#: thing a zero-config start would hit.
|
||||||
@@ -426,25 +444,32 @@ def _parser() -> argparse.ArgumentParser:
|
|||||||
)
|
)
|
||||||
serve.add_argument(
|
serve.add_argument(
|
||||||
"--max-runs",
|
"--max-runs",
|
||||||
type=int,
|
type=_at_least(1),
|
||||||
default=None,
|
default=None,
|
||||||
metavar="N",
|
metavar="N",
|
||||||
help="batch runs driven at once (default 4, FLOW_MAX_RUNS)",
|
help="batch runs driven at once (default 4, FLOW_MAX_RUNS)",
|
||||||
)
|
)
|
||||||
serve.add_argument(
|
serve.add_argument(
|
||||||
"--max-cascades",
|
"--max-cascades",
|
||||||
type=int,
|
type=_at_least(1),
|
||||||
default=None,
|
default=None,
|
||||||
metavar="N",
|
metavar="N",
|
||||||
help="cascades in flight at once (default 4, FLOW_MAX_CASCADES)",
|
help="cascades in flight at once (default 4, FLOW_MAX_CASCADES)",
|
||||||
)
|
)
|
||||||
serve.add_argument(
|
serve.add_argument(
|
||||||
"--max-workers",
|
"--max-workers",
|
||||||
type=int,
|
type=_at_least(1),
|
||||||
default=None,
|
default=None,
|
||||||
metavar="N",
|
metavar="N",
|
||||||
help="python worker processes (default 4, FLOW_MAX_WORKERS)",
|
help="python worker processes (default 4, FLOW_MAX_WORKERS)",
|
||||||
)
|
)
|
||||||
|
serve.add_argument(
|
||||||
|
"--gpus",
|
||||||
|
type=_at_least(0),
|
||||||
|
default=None,
|
||||||
|
metavar="N",
|
||||||
|
help="GPUs on this machine a node may be given (default 0, FLOW_GPUS)",
|
||||||
|
)
|
||||||
serve.set_defaults(func=cmd_serve)
|
serve.set_defaults(func=cmd_serve)
|
||||||
|
|
||||||
enroll = subparsers.add_parser(
|
enroll = subparsers.add_parser(
|
||||||
|
|||||||
@@ -215,9 +215,18 @@ class Placer:
|
|||||||
said = [shape[2] for shape in capable if shape[2]]
|
said = [shape[2] for shape in capable if shape[2]]
|
||||||
ram = min(ram, max(said)) if said and ram else ram
|
ram = min(ram, max(said)) if said and ram else ram
|
||||||
if (cpus, gpus, ram) != (wanted.cpus, wanted.gpus, wanted.ram or 0):
|
if (cpus, gpus, ram) != (wanted.cpus, wanted.gpus, wanted.ram or 0):
|
||||||
|
# Cards are not detected, so a machine that has one still reports
|
||||||
|
# none until it is told — which reads as "no GPU here" to a node
|
||||||
|
# that then runs unserialised beside every other one.
|
||||||
|
hint = (
|
||||||
|
"; no machine here declares a GPU — `fluksio serve --gpus N` "
|
||||||
|
"(or FLOW_GPUS) says how many this one has"
|
||||||
|
if wanted.gpus and not gpus
|
||||||
|
else ""
|
||||||
|
)
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"%s asked for %d cpu(s), %d gpu(s) and %s MB; "
|
"%s asked for %d cpu(s), %d gpu(s) and %s MB; "
|
||||||
"the largest machine here can give %d, %d and %s",
|
"the largest machine here can give %d, %d and %s%s",
|
||||||
node or "a node",
|
node or "a node",
|
||||||
wanted.cpus,
|
wanted.cpus,
|
||||||
wanted.gpus,
|
wanted.gpus,
|
||||||
@@ -225,6 +234,7 @@ class Placer:
|
|||||||
cpus,
|
cpus,
|
||||||
gpus,
|
gpus,
|
||||||
ram or "no stated",
|
ram or "no stated",
|
||||||
|
hint,
|
||||||
)
|
)
|
||||||
return cpus, gpus, ram
|
return cpus, gpus, ram
|
||||||
|
|
||||||
|
|||||||
@@ -129,7 +129,7 @@ def test_a_preferred_label_falls_back_here_and_is_still_accounted(loop):
|
|||||||
assert placer.local.snapshot()["cpus"]["free"] == 1
|
assert placer.local.snapshot()["cpus"]["free"] == 1
|
||||||
|
|
||||||
|
|
||||||
def test_asking_for_more_than_anything_has_gets_what_there_is(loop):
|
def test_asking_for_more_than_anything_has_gets_what_there_is(loop, caplog):
|
||||||
"""A flow written on a cluster still has to run on a laptop."""
|
"""A flow written on a cluster still has to run on a laptop."""
|
||||||
placer = placer_over(cpus=2)
|
placer = placer_over(cpus=2)
|
||||||
|
|
||||||
@@ -138,6 +138,10 @@ def test_asking_for_more_than_anything_has_gets_what_there_is(loop):
|
|||||||
assert allocation.cpus == 2
|
assert allocation.cpus == 2
|
||||||
assert allocation.gpus == ()
|
assert allocation.gpus == ()
|
||||||
|
|
||||||
|
# Cards are declared, not detected, so a machine that has one reads as
|
||||||
|
# having none until it is told — and the warning is where that is noticed.
|
||||||
|
assert "fluksio serve --gpus" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_what_is_clamped_to_is_a_machine_that_exists(loop):
|
def test_what_is_clamped_to_is_a_machine_that_exists(loop):
|
||||||
"""Each dimension taken separately can describe a machine nobody has.
|
"""Each dimension taken separately can describe a machine nobody has.
|
||||||
|
|||||||
@@ -520,6 +520,24 @@ def test_two_files_of_one_name_are_refused(tmp_path) -> None:
|
|||||||
_import(*_module_of(second), second)
|
_import(*_module_of(second), second)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_serve_limit_is_refused_as_a_flag_not_as_a_traceback(capsys) -> None:
|
||||||
|
"""These are written into the environment before the settings are built."""
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from fluksio.cli import _parser
|
||||||
|
|
||||||
|
parser = _parser()
|
||||||
|
assert parser.parse_args(["serve", "--max-workers", "2"]).max_workers == 2
|
||||||
|
# A machine may genuinely have no card, so zero is a number here.
|
||||||
|
assert parser.parse_args(["serve", "--gpus", "0"]).gpus == 0
|
||||||
|
assert parser.parse_args(["serve"]).gpus is None
|
||||||
|
|
||||||
|
for flag, value in (("--max-workers", "0"), ("--gpus", "-1")):
|
||||||
|
with pytest.raises(SystemExit):
|
||||||
|
parser.parse_args(["serve", flag, value])
|
||||||
|
assert "at least" in capsys.readouterr().err
|
||||||
|
|
||||||
|
|
||||||
def test_run_and_sweep_take_what_to_sync() -> None:
|
def test_run_and_sweep_take_what_to_sync() -> None:
|
||||||
from fluksio.cli import _parser
|
from fluksio.cli import _parser
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user