GPUs follow the engine's own CUDA_VISIBLE_DEVICES

Its entries are the cards present, and a local child's allocation is
named through them, so under =2,3 card 0 is device 2. Remote workers
keep mapping against their own mask.
This commit is contained in:
2026-10-15 09:53:38 +02:00
parent 290b9fa705
commit f2a823218a
5 changed files with 69 additions and 10 deletions
+27 -1
View File
@@ -33,6 +33,7 @@ def test_the_cards_are_counted_from_the_devices_the_driver_makes(monkeypatch):
"""
from fluksio.flow import resources
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
monkeypatch.setattr(
resources.glob,
"glob",
@@ -150,12 +151,37 @@ def test_the_share_becomes_the_thread_limit():
assert "CUDA_VISIBLE_DEVICES" not in env
def test_a_gpu_node_is_told_which_card_is_its():
def test_a_gpu_node_is_told_which_card_is_its(monkeypatch):
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
env = derive_env(Resources(cpus=1, gpus=2), Allocation(cpus=1, gpus=(1, 3)))
assert env["CUDA_VISIBLE_DEVICES"] == "1,3"
@pytest.mark.parametrize(
("mask", "cards", "named"),
[(None, 2, "0"), ("2,3", 2, "2"), ("GPU-a1b2, MIG-c3d4", 2, "GPU-a1b2")],
)
def test_the_engines_own_mask_is_the_cards_it_has(monkeypatch, mask, cards, named):
"""Under ``CUDA_VISIBLE_DEVICES=2,3`` card 0 is device 2, not device 0.
A child's mask replaces the engine's, so an unmapped 0 would be a card
the engine was never given. A remote worker maps against its own mask.
"""
from fluksio.flow import resources
monkeypatch.setattr(resources.glob, "glob", lambda pattern: ["/dev/nvidia0"] * 2)
if mask is None:
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
else:
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
wanted, allocation = Resources(cpus=1, gpus=1), Allocation(cpus=1, gpus=(0,))
assert machine_gpus() == cards
assert derive_env(wanted, allocation)["CUDA_VISIBLE_DEVICES"] == named
assert derive_env(wanted, allocation, local=False)["CUDA_VISIBLE_DEVICES"] == "0"
def test_what_the_node_asked_for_wins():
"""A declaration outranks what the allocation would imply — it is deliberate."""
wanted = Resources(