GPUs follow the engine's own CUDA_VISIBLE_DEVICES
Its entries are the cards present, and a local child's allocation is named through them, so under =2,3 card 0 is device 2. Remote workers keep mapping against their own mask.
This commit is contained in:
@@ -979,7 +979,7 @@ class FlowController:
|
||||
with placer.claim( # type: ignore[union-attr]
|
||||
wanted, device=device, policy=policy, node=node_id, run=run_id
|
||||
) as (target, allocation):
|
||||
env = derive_env(wanted, allocation)
|
||||
env = derive_env(wanted, allocation, local=target.worker is None)
|
||||
if target.worker is None:
|
||||
return pool.for_env(env).run( # type: ignore[union-attr]
|
||||
owner,
|
||||
|
||||
@@ -96,15 +96,32 @@ def warn_oversubscribed(limits: dict[str, int]) -> None:
|
||||
)
|
||||
|
||||
|
||||
def visible_gpus() -> list[str] | None:
|
||||
"""The cards the engine's own ``CUDA_VISIBLE_DEVICES`` lists, if it is set.
|
||||
|
||||
Indices, UUIDs or MIG instances, kept as written: a child is handed the
|
||||
same names, so whatever the driver took here it takes there.
|
||||
"""
|
||||
mask = os.environ.get("CUDA_VISIBLE_DEVICES")
|
||||
if mask is None:
|
||||
return None
|
||||
return [part.strip() for part in mask.split(",") if part.strip()]
|
||||
|
||||
|
||||
def machine_gpus() -> int:
|
||||
"""Cards a node may be given here, when nobody said.
|
||||
|
||||
The device nodes the NVIDIA driver creates, counted — no vendor tool, so
|
||||
The engine's own ``CUDA_VISIBLE_DEVICES`` where it has one, since a card
|
||||
the process can see but was not given is not its to hand out. Otherwise
|
||||
the device nodes the NVIDIA driver creates, counted — no vendor tool, so
|
||||
the one dependency does not become two. It answers for the case that
|
||||
actually bit people (a box with cards, an engine told nothing, every GPU
|
||||
node clamped to zero and running at once) and for nothing else: another
|
||||
vendor, or a card this process cannot see, is still `--gpus N`.
|
||||
"""
|
||||
visible = visible_gpus()
|
||||
if visible is not None:
|
||||
return len(visible)
|
||||
return len(glob.glob("/dev/nvidia[0-9]*"))
|
||||
|
||||
|
||||
@@ -229,7 +246,9 @@ class ResourceAccountant:
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def derive_env(wanted: Resources, allocation: Allocation) -> dict[str, str]:
|
||||
def derive_env(
|
||||
wanted: Resources, allocation: Allocation, local: bool = True
|
||||
) -> dict[str, str]:
|
||||
"""The environment a worker running this node is started with.
|
||||
|
||||
Precedence is engine environment, then what the allocation implies, then
|
||||
@@ -240,10 +259,19 @@ def derive_env(wanted: Resources, allocation: Allocation) -> dict[str, str]:
|
||||
contents depend on the version installed, so writing it here would silently
|
||||
replace whatever the author had put there. It travels in ``resources.env``,
|
||||
where it is visible.
|
||||
|
||||
``local`` is whether the allocation is this machine's. A child's
|
||||
``CUDA_VISIBLE_DEVICES`` replaces the engine's rather than narrowing it, so
|
||||
here a card is named the way the engine's own list names it: under ``=2,3``
|
||||
card 0 is device 2. A remote worker does the same against its own list.
|
||||
"""
|
||||
env = {var: str(allocation.cpus) for var in THREAD_VARS}
|
||||
if wanted.gpus:
|
||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(str(index) for index in allocation.gpus)
|
||||
names = (visible_gpus() if local else None) or []
|
||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(
|
||||
names[index] if index < len(names) else str(index)
|
||||
for index in allocation.gpus
|
||||
)
|
||||
env.update(wanted.env)
|
||||
return env
|
||||
|
||||
|
||||
Reference in New Issue
Block a user