Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GPUs follow the engine's own CUDA_VISIBLE_DEVICES

Browse Source 
Its entries are the cards present, and a local child's allocation is
named through them, so under =2,3 card 0 is device 2. Remote workers
keep mapping against their own mask.
This commit is contained in:
stroblme
2026-10-15 09:53:38 +02:00
parent 290b9fa705
commit f2a823218a
5 changed files with 69 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/fluksio/flow/controller.py
+1 -1
View File
@@ -979,7 +979,7 @@ class FlowController:
with placer.claim( # type: ignore[union-attr]
wanted, device=device, policy=policy, node=node_id, run=run_id
) as (target, allocation):
env = derive_env(wanted, allocation)
env = derive_env(wanted, allocation, local=target.worker is None)
if target.worker is None:
return pool.for_env(env).run( # type: ignore[union-attr]
owner,
backend/fluksio/flow/resources.py
+31 -3
View File
@@ -96,15 +96,32 @@ def warn_oversubscribed(limits: dict[str, int]) -> None:
)
def visible_gpus() -> list[str] | None:
"""The cards the engine's own ``CUDA_VISIBLE_DEVICES`` lists, if it is set.
Indices, UUIDs or MIG instances, kept as written: a child is handed the
same names, so whatever the driver took here it takes there.
"""
mask = os.environ.get("CUDA_VISIBLE_DEVICES")
if mask is None:
return None
return [part.strip() for part in mask.split(",") if part.strip()]
def machine_gpus() -> int:
"""Cards a node may be given here, when nobody said.
The device nodes the NVIDIA driver creates, counted — no vendor tool, so
The engine's own ``CUDA_VISIBLE_DEVICES`` where it has one, since a card
the process can see but was not given is not its to hand out. Otherwise
the device nodes the NVIDIA driver creates, counted — no vendor tool, so
the one dependency does not become two. It answers for the case that
actually bit people (a box with cards, an engine told nothing, every GPU
node clamped to zero and running at once) and for nothing else: another
vendor, or a card this process cannot see, is still `--gpus N`.
"""
visible = visible_gpus()
if visible is not None:
return len(visible)
return len(glob.glob("/dev/nvidia[0-9]*"))
@@ -229,7 +246,9 @@ class ResourceAccountant:
# -----------------------------------------------------------------------------
def derive_env(wanted: Resources, allocation: Allocation) -> dict[str, str]:
def derive_env(
wanted: Resources, allocation: Allocation, local: bool = True
) -> dict[str, str]:
"""The environment a worker running this node is started with.
Precedence is engine environment, then what the allocation implies, then
@@ -240,10 +259,19 @@ def derive_env(wanted: Resources, allocation: Allocation) -> dict[str, str]:
contents depend on the version installed, so writing it here would silently
replace whatever the author had put there. It travels in ``resources.env``,
where it is visible.
``local`` is whether the allocation is this machine's. A child's
``CUDA_VISIBLE_DEVICES`` replaces the engine's rather than narrowing it, so
here a card is named the way the engine's own list names it: under ``=2,3``
card 0 is device 2. A remote worker does the same against its own list.
"""
env = {var: str(allocation.cpus) for var in THREAD_VARS}
if wanted.gpus:
env["CUDA_VISIBLE_DEVICES"] = ",".join(str(index) for index in allocation.gpus)
names = (visible_gpus() if local else None) or []
env["CUDA_VISIBLE_DEVICES"] = ",".join(
names[index] if index < len(names) else str(index)
for index in allocation.gpus
)
env.update(wanted.env)
return env
backend/tests/flow/test_resources.py
+27 -1
View File
@@ -33,6 +33,7 @@ def test_the_cards_are_counted_from_the_devices_the_driver_makes(monkeypatch):
"""
from fluksio.flow import resources
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
monkeypatch.setattr(
resources.glob,
"glob",
@@ -150,12 +151,37 @@ def test_the_share_becomes_the_thread_limit():
assert "CUDA_VISIBLE_DEVICES" not in env
def test_a_gpu_node_is_told_which_card_is_its():
def test_a_gpu_node_is_told_which_card_is_its(monkeypatch):
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
env = derive_env(Resources(cpus=1, gpus=2), Allocation(cpus=1, gpus=(1, 3)))
assert env["CUDA_VISIBLE_DEVICES"] == "1,3"
@pytest.mark.parametrize(
("mask", "cards", "named"),
[(None, 2, "0"), ("2,3", 2, "2"), ("GPU-a1b2, MIG-c3d4", 2, "GPU-a1b2")],
)
def test_the_engines_own_mask_is_the_cards_it_has(monkeypatch, mask, cards, named):
"""Under ``CUDA_VISIBLE_DEVICES=2,3`` card 0 is device 2, not device 0.
A child's mask replaces the engine's, so an unmapped 0 would be a card
the engine was never given. A remote worker maps against its own mask.
"""
from fluksio.flow import resources
monkeypatch.setattr(resources.glob, "glob", lambda pattern: ["/dev/nvidia0"] * 2)
if mask is None:
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
else:
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask)
wanted, allocation = Resources(cpus=1, gpus=1), Allocation(cpus=1, gpus=(0,))
assert machine_gpus() == cards
assert derive_env(wanted, allocation)["CUDA_VISIBLE_DEVICES"] == named
assert derive_env(wanted, allocation, local=False)["CUDA_VISIBLE_DEVICES"] == "0"
def test_what_the_node_asked_for_wins():
"""A declaration outranks what the allocation would imply — it is deliberate."""
wanted = Resources(
docs/code/cli.md
+3 -2
View File
@@ -65,8 +65,9 @@ that instance's running engine; plain mode refuses to start a duplicate.
| `--worker-max-rss MB` | `0` (disabled) | Retire a worker above this resident memory after its call |
Resource defaults can also come from the corresponding `FLOW_*` settings;
see [Configuration](../reference/configuration.md). GPU detection uses
NVIDIA device nodes. Set `--gpus` when your hardware is not detected.
see [Configuration](../reference/configuration.md). GPU detection counts the
entries of `CUDA_VISIBLE_DEVICES` when it is set, otherwise NVIDIA device
nodes. Set `--gpus` when your hardware is not detected.
Use local disk for the data directory on a cluster: SQLite's write-ahead log
does not support NFS.
docs/reference/configuration.md
+7 -3
View File
@@ -126,7 +126,7 @@ Set `TZ` in `.env` for schedules that should follow local time, such as
| `FLOW_MAX_RUNS` | `4` | concurrent batch runs |
| `FLOW_NODE_TIMEOUT` | `0` | maximum node silence in seconds, unless the node overrides it; `0` disables |
| `FLOW_CPUS` | `0` | CPU budget; `0` detects cores and reserves two for the engine |
| `FLOW_GPUS` | `0` | local GPU count; `0` detects NVIDIA device nodes |
| `FLOW_GPUS` | `0` | local GPU count; `0` counts the engine's `CUDA_VISIBLE_DEVICES`, or NVIDIA device nodes without it |
| `FLOW_RAM` | `0` | memory budget in MB; `0` detects host/container memory and reserves 1 GiB |
| `FLOW_WORKER_MAX_RSS_MB` | `0` | retire a local worker above this memory use after a call; `0` disables |
| `FLOW_NODE_STOP_TIMEOUT` | `5` | seconds a node gets to stop during a rebuild before the rebuild moves on |
@@ -139,14 +139,18 @@ Set `TZ` in `.env` for schedules that should follow local time, such as
The three concurrency limits must be at least `1`. They also have CLI flags:
`--max-workers`, `--max-cascades` and `--max-runs`. When `FLOW_MAX_WORKERS` or
`FLOW_MAX_CASCADES` is more than four times the machine's core count, the
engine logs a warning at startup and uses the value anyway. Resource flags are `--gpus`,
`--ram` and `--worker-max-rss`.
engine logs a warning at startup and uses the value anyway. Resource flags are
`--gpus`, `--ram` and `--worker-max-rss`.
Nodes with declared resources are scheduled against the available budget.
Undeclared nodes use the shared pool and receive thread caps. Attached workers
report their own inventory; the engine's resource settings describe only its
own machine. See [Remote workers](../code/workers.md) and the **Workers** screen.
When the engine runs with `CUDA_VISIBLE_DEVICES`, a node's card is named the
way that list names it, so under `2,3` the first card is device `2`. UUID and
MIG entries are passed on as written.
## Artifacts
| Variable | Default | Purpose |