GPUs follow the engine's own CUDA_VISIBLE_DEVICES
Its entries are the cards present, and a local child's allocation is named through them, so under =2,3 card 0 is device 2. Remote workers keep mapping against their own mask.
This commit is contained in:
@@ -33,6 +33,7 @@ def test_the_cards_are_counted_from_the_devices_the_driver_makes(monkeypatch):
|
||||
"""
|
||||
from fluksio.flow import resources
|
||||
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
monkeypatch.setattr(
|
||||
resources.glob,
|
||||
"glob",
|
||||
@@ -150,12 +151,37 @@ def test_the_share_becomes_the_thread_limit():
|
||||
assert "CUDA_VISIBLE_DEVICES" not in env
|
||||
|
||||
|
||||
def test_a_gpu_node_is_told_which_card_is_its():
|
||||
def test_a_gpu_node_is_told_which_card_is_its(monkeypatch):
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
env = derive_env(Resources(cpus=1, gpus=2), Allocation(cpus=1, gpus=(1, 3)))
|
||||
|
||||
assert env["CUDA_VISIBLE_DEVICES"] == "1,3"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mask", "cards", "named"),
|
||||
[(None, 2, "0"), ("2,3", 2, "2"), ("GPU-a1b2, MIG-c3d4", 2, "GPU-a1b2")],
|
||||
)
|
||||
def test_the_engines_own_mask_is_the_cards_it_has(monkeypatch, mask, cards, named):
|
||||
"""Under ``CUDA_VISIBLE_DEVICES=2,3`` card 0 is device 2, not device 0.
|
||||
|
||||
A child's mask replaces the engine's, so an unmapped 0 would be a card
|
||||
the engine was never given. A remote worker maps against its own mask.
|
||||
"""
|
||||
from fluksio.flow import resources
|
||||
|
||||
monkeypatch.setattr(resources.glob, "glob", lambda pattern: ["/dev/nvidia0"] * 2)
|
||||
if mask is None:
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
|
||||
wanted, allocation = Resources(cpus=1, gpus=1), Allocation(cpus=1, gpus=(0,))
|
||||
|
||||
assert machine_gpus() == cards
|
||||
assert derive_env(wanted, allocation)["CUDA_VISIBLE_DEVICES"] == named
|
||||
assert derive_env(wanted, allocation, local=False)["CUDA_VISIBLE_DEVICES"] == "0"
|
||||
|
||||
|
||||
def test_what_the_node_asked_for_wins():
|
||||
"""A declaration outranks what the allocation would imply — it is deliberate."""
|
||||
wanted = Resources(
|
||||
|
||||
Reference in New Issue
Block a user