A node's worker label is worker, so a device is only a paired one
- NodeDef `device`/`device_policy` become `worker`/`worker_policy`, the issue `ignored_device` becomes `ignored_worker`, and the SDK's `@node` takes `worker=` and `worker_policy=` - FlowStore._migrate renames the keys in published and draft documents once, bumping their version; a queued run's `needs` reads the old key - placement passes the label as `label`, client regenerated - docs: worker placement wording, MCP issuer without DOMAIN, device rows in the CLI data directory and the API table, device links in compose comments Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f8011d2a8a
commit
a9d453c9d9
@@ -1,7 +1,7 @@
|
||||
"""Which machine a node runs on, and the waiting when none of them has room.
|
||||
|
||||
The engine used to answer this twice, in two ways that could not see each
|
||||
other. A node with a ``device`` went to a worker carrying that label, chosen by
|
||||
other. A node with a ``worker`` went to a worker carrying that label, chosen by
|
||||
which of them had least in flight — a machine's *name*, never its size. A node
|
||||
with ``resources`` was accounted against the engine's own cores, and only the
|
||||
engine's: declaring both meant the second answer silently won and nothing was
|
||||
@@ -63,15 +63,15 @@ class Target:
|
||||
#: None is the engine itself, which runs nodes in its own worker pool.
|
||||
worker: RemoteWorker | None = None
|
||||
|
||||
def carries(self, device: str) -> bool:
|
||||
"""Whether a node asking for this device may run here.
|
||||
def carries(self, label: str) -> bool:
|
||||
"""Whether a node asking for this label may run here.
|
||||
|
||||
The engine carries no label: a label names a machine somebody attached
|
||||
for the purpose, and the engine is where the node would have run anyway.
|
||||
"""
|
||||
if self.worker is None:
|
||||
return False
|
||||
return device == self.name or device in self.worker.labels
|
||||
return label == self.name or label in self.worker.labels
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -138,7 +138,7 @@ class Placer:
|
||||
cpus: int,
|
||||
gpus: int,
|
||||
ram_mb: int,
|
||||
device: str | None,
|
||||
label: str | None,
|
||||
policy: str,
|
||||
) -> list[Target]:
|
||||
"""The machines that could run this node, best first."""
|
||||
@@ -146,8 +146,8 @@ class Placer:
|
||||
remote = [target for target in targets if target.worker is not None]
|
||||
engine = [target for target in targets if target.worker is None]
|
||||
|
||||
if device:
|
||||
labelled = [target for target in remote if target.carries(device)]
|
||||
if label:
|
||||
labelled = [target for target in remote if target.carries(label)]
|
||||
fitting = self._fitting(labelled, cpus, gpus, ram_mb)
|
||||
if fitting or policy != "prefer":
|
||||
# `require` waits for the labelled machine even when the engine
|
||||
@@ -174,7 +174,7 @@ class Placer:
|
||||
|
||||
# -- what can actually be granted ------------------------------------------
|
||||
|
||||
def _shapes(self, device: str | None, policy: str) -> list[tuple[int, int, int]]:
|
||||
def _shapes(self, label: str | None, policy: str) -> list[tuple[int, int, int]]:
|
||||
"""Every machine's size, as something to measure a request against.
|
||||
|
||||
A provisioner's job shapes count: a machine it can start on demand is
|
||||
@@ -182,18 +182,18 @@ class Placer:
|
||||
"""
|
||||
shapes = []
|
||||
for target in self.targets():
|
||||
if device and not target.carries(device):
|
||||
if label and not target.carries(label):
|
||||
# A `prefer` node may end up here, so the engine still counts.
|
||||
if policy == "require" or target.worker is not None:
|
||||
continue
|
||||
books = target.accountant
|
||||
shapes.append((books.cpus, books.gpus, books.ram_mb or 0))
|
||||
for provisioner in self.provisioners:
|
||||
shapes.extend(provisioner.shapes(device))
|
||||
shapes.extend(provisioner.shapes(label))
|
||||
return shapes or [(self.local.cpus, self.local.gpus, self.local.ram_mb or 0)]
|
||||
|
||||
def _clamp(
|
||||
self, wanted: Resources, node: str, device: str | None, policy: str
|
||||
self, wanted: Resources, node: str, label: str | None, policy: str
|
||||
) -> tuple[int, int, int]:
|
||||
"""What can be granted somewhere, which may be less than was asked.
|
||||
|
||||
@@ -206,7 +206,7 @@ class Placer:
|
||||
measured only against machines that carry that many.
|
||||
"""
|
||||
ram = wanted.ram or 0
|
||||
shapes = self._shapes(device, policy)
|
||||
shapes = self._shapes(label, policy)
|
||||
gpus = min(wanted.gpus, max(shape[1] for shape in shapes))
|
||||
capable = [shape for shape in shapes if shape[1] >= gpus]
|
||||
cpus = min(wanted.cpus, max(shape[0] for shape in capable))
|
||||
@@ -246,7 +246,7 @@ class Placer:
|
||||
self,
|
||||
wanted: Resources,
|
||||
*,
|
||||
device: str | None = None,
|
||||
label: str | None = None,
|
||||
policy: str = "require",
|
||||
node: str = "",
|
||||
run: str = "",
|
||||
@@ -259,8 +259,8 @@ class Placer:
|
||||
|
||||
with self._condition:
|
||||
while chosen is None:
|
||||
cpus, gpus, ram = self._clamp(wanted, node, device, policy)
|
||||
for target in self._candidates(cpus, gpus, ram, device, policy):
|
||||
cpus, gpus, ram = self._clamp(wanted, node, label, policy)
|
||||
for target in self._candidates(cpus, gpus, ram, label, policy):
|
||||
got = target.accountant.try_take(cpus, gpus, ram)
|
||||
if got is not None:
|
||||
chosen, allocation = target, got
|
||||
@@ -269,9 +269,9 @@ class Placer:
|
||||
break
|
||||
if not ticket:
|
||||
ticket = self._announce(
|
||||
node, run, waited_from, self._reason(cpus, gpus, device)
|
||||
node, run, waited_from, self._reason(cpus, gpus, label)
|
||||
)
|
||||
self._provision(cpus, gpus, ram, device)
|
||||
self._provision(cpus, gpus, ram, label)
|
||||
self._condition.wait(timeout=WAKE_S)
|
||||
if ticket:
|
||||
self._waiting.pop(ticket, None)
|
||||
@@ -291,21 +291,21 @@ class Placer:
|
||||
# timeout, a cancellation, or the worker going away mid-call.
|
||||
chosen.accountant.give_back(allocation)
|
||||
|
||||
def _reason(self, cpus: int, gpus: int, device: str | None) -> str:
|
||||
def _reason(self, cpus: int, gpus: int, label: str | None) -> str:
|
||||
parts = []
|
||||
if gpus:
|
||||
parts.append(f"{gpus} gpu(s)")
|
||||
parts.append(f"{cpus} cpu(s)")
|
||||
wants = " and ".join(parts)
|
||||
if device:
|
||||
return f"waiting for a worker labelled '{device}' with {wants} free"
|
||||
if label:
|
||||
return f"waiting for a worker labelled '{label}' with {wants} free"
|
||||
return f"waiting for {wants}"
|
||||
|
||||
def _provision(self, cpus: int, gpus: int, ram_mb: int, device: str | None) -> None:
|
||||
def _provision(self, cpus: int, gpus: int, ram_mb: int, label: str | None) -> None:
|
||||
"""Ask for a machine, if something here can start one."""
|
||||
for provisioner in self.provisioners:
|
||||
if provisioner.covers(cpus, gpus, ram_mb, device):
|
||||
provisioner.provision(cpus, gpus, ram_mb, device)
|
||||
if provisioner.covers(cpus, gpus, ram_mb, label):
|
||||
provisioner.provision(cpus, gpus, ram_mb, label)
|
||||
return
|
||||
|
||||
def provision_for(self, needs: dict[str, Any] | None) -> None:
|
||||
@@ -321,7 +321,7 @@ class Placer:
|
||||
int(needs.get("cpus") or 1),
|
||||
int(needs.get("gpus") or 0),
|
||||
int(needs.get("ram_mb") or 0),
|
||||
needs.get("device") or None,
|
||||
needs.get("worker") or None,
|
||||
)
|
||||
|
||||
def _announce(self, node: str, run: str, since: float, reason: str) -> int:
|
||||
@@ -362,15 +362,15 @@ class Placer:
|
||||
cpus = int(needs.get("cpus") or 1)
|
||||
gpus = int(needs.get("gpus") or 0)
|
||||
ram = int(needs.get("ram_mb") or 0)
|
||||
device = needs.get("device") or None
|
||||
label = needs.get("worker") or None
|
||||
for target in self.targets():
|
||||
if device and not target.carries(device):
|
||||
if label and not target.carries(label):
|
||||
continue
|
||||
if target.accountant.fits(cpus, gpus, ram):
|
||||
# Something here could take it. Busy is queueing, not waiting.
|
||||
return None
|
||||
if not any(
|
||||
provisioner.covers(cpus, gpus, ram, device)
|
||||
provisioner.covers(cpus, gpus, ram, label)
|
||||
for provisioner in self.provisioners
|
||||
):
|
||||
# Nothing attached fits and nothing can be started, so the node
|
||||
|
||||
Reference in New Issue
Block a user