Schedule a node across every machine, not just this one
The engine answered "where does this node run" twice, in two ways that could not see each other: a device sent it to a worker carrying that label, and resources were counted against the engine's own cores. Declaring both meant the second answer won and nothing was counted at all — which the data-science getting-started page and the worked example both do. One question now, in flow/placement.py: of every machine attached, which could grant what this node asked for, and which of those has it free. The books move onto each machine — one accountant per worker, built from the inventory it reported — and the waiting moves above them, where one condition variable can be woken by a release anywhere or by a worker attaching. Locks go one way: placer, then a machine's books, never back. So a node asking for a card now finds the box that has one, rather than being clamped down to none and run here. When nothing can grant the ask at all it is still cut down and run — a flow written on a cluster has to work on a laptop — but the ceiling is one real machine now, since taking the largest of each dimension separately can describe a machine nobody has. Two things fixed on the way. A device on a connector node held every batch run of its flow forever, waiting for a worker that could never run an entry point. And `prefer` falling back to the engine skipped the books, so the fallback held nothing. The bench flow's node has taken a `params` argument that with_settings has not forwarded for some time, so the benchmark could not run at all: 62 ms median submit-to-result with this, against the 61 ms on record. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01A6HeySA27EkGANZN95QySW
This commit is contained in:
Claude Opus 5
@@ -62,6 +62,7 @@ from fluksio.flow.pipeline import (
|
||||
ValidationIssue,
|
||||
ValueSource,
|
||||
)
|
||||
from fluksio.flow.placement import Placer
|
||||
from fluksio.flow.remote import RemoteWorkerHub
|
||||
from fluksio.flow.resources import ResourceAccountant, derive_env
|
||||
from fluksio.flow.schemas import (
|
||||
@@ -413,6 +414,7 @@ class FlowController:
|
||||
workers: PythonWorkerPool | None = None,
|
||||
remote: RemoteWorkerHub | None = None,
|
||||
resources: ResourceAccountant | None = None,
|
||||
placer: Placer | None = None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
# Without a pool, python nodes are compiled and run in this process —
|
||||
@@ -420,9 +422,11 @@ class FlowController:
|
||||
self.workers = workers
|
||||
# Workers on other hosts. A node without a device never touches it.
|
||||
self.remote = remote
|
||||
# What the machine has, for the nodes that say what they need. Without
|
||||
# one, a declaration is recorded and nothing is held against it.
|
||||
# This machine's own books. The pool sizes its fair share off them.
|
||||
self.resources = resources
|
||||
# Every machine there is, for the nodes that say what they need.
|
||||
# Without one, a declaration is recorded and nothing is held against it.
|
||||
self.placer = placer
|
||||
self.state = state if state is not None else MemoryState()
|
||||
self.events = events
|
||||
self.max_workers = max_workers
|
||||
@@ -865,9 +869,18 @@ class FlowController:
|
||||
# Building
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def _allocated(
|
||||
def _runs_elsewhere(self, node_def: NodeDef) -> bool:
|
||||
"""Whether this node asks for more than this machine could ever give."""
|
||||
if node_def.resources is None or self.placer is None:
|
||||
return False
|
||||
wanted = node_def.resources
|
||||
return not self.placer.local.fits(wanted.cpus, wanted.gpus, wanted.ram or 0)
|
||||
|
||||
def _placed(
|
||||
self,
|
||||
wanted: Resources,
|
||||
device: str | None,
|
||||
policy: str,
|
||||
owner: str,
|
||||
local: str,
|
||||
code: str,
|
||||
@@ -877,32 +890,40 @@ class FlowController:
|
||||
run_id: str,
|
||||
on_event: Callable[[dict[str, Any]], None],
|
||||
) -> Callable[..., Any]:
|
||||
"""A call that holds its share of the machine while it runs.
|
||||
"""A call that picks a machine and holds its share while it runs.
|
||||
|
||||
Which machine is decided per call rather than when the flow was built,
|
||||
so a worker that attaches later is used without anything being rebuilt.
|
||||
|
||||
The order is load-bearing: the resources are claimed first, and only
|
||||
then is a worker slot taken. The other way round, a node holding a slot
|
||||
could sit waiting for cores that a node holding the cores cannot get a
|
||||
slot to release.
|
||||
|
||||
The worker comes from the pool whose environment this allocation
|
||||
derives, so what the node is told about its share is what the library
|
||||
inside it reads at import — the only moment those variables are read.
|
||||
What the allocation implies is handed to the process the node runs in,
|
||||
here or on the worker, because a library reads those variables when it
|
||||
is imported and never again.
|
||||
"""
|
||||
if self.workers is None or self.resources is None:
|
||||
return self.workers.proxy( # type: ignore[union-attr]
|
||||
owner,
|
||||
local,
|
||||
code,
|
||||
node_id=node_id,
|
||||
timeout=timeout,
|
||||
run_id=run_id,
|
||||
on_event=on_event,
|
||||
)
|
||||
accountant, pool = self.resources, self.workers
|
||||
placer, pool = self.placer, self.workers
|
||||
|
||||
def call(**kwargs: Any) -> Any:
|
||||
with accountant.claim(wanted, node=node_id, run=run_id) as allocation:
|
||||
return pool.for_env(derive_env(wanted, allocation)).run(
|
||||
with placer.claim(
|
||||
wanted, device=device, policy=policy, node=node_id, run=run_id
|
||||
) as (target, allocation):
|
||||
env = derive_env(wanted, allocation)
|
||||
if target.worker is None:
|
||||
return pool.for_env(env).run(
|
||||
owner,
|
||||
local,
|
||||
code,
|
||||
kwargs,
|
||||
node_id,
|
||||
timeout,
|
||||
run_id=run_id,
|
||||
on_event=on_event,
|
||||
)
|
||||
return self.remote.run_on( # type: ignore[union-attr]
|
||||
target.worker,
|
||||
owner,
|
||||
local,
|
||||
code,
|
||||
@@ -911,6 +932,7 @@ class FlowController:
|
||||
timeout,
|
||||
run_id=run_id,
|
||||
on_event=on_event,
|
||||
env=env,
|
||||
)
|
||||
|
||||
return call
|
||||
@@ -1003,6 +1025,11 @@ class FlowController:
|
||||
problem = self.remote.compile(
|
||||
node_def.device or "", owner, local, code
|
||||
)
|
||||
elif self._runs_elsewhere(node_def):
|
||||
# Asks for more than this machine has, so it will run on
|
||||
# one that has it. Same reason as a device: checking the
|
||||
# import here would fail a node that is fine there.
|
||||
problem = None
|
||||
else:
|
||||
problem = self.workers.compile(owner, local, code)
|
||||
if problem:
|
||||
@@ -1016,18 +1043,15 @@ class FlowController:
|
||||
if node_def.timeout is not None
|
||||
else settings.FLOW_NODE_TIMEOUT
|
||||
)
|
||||
function = self.workers.proxy(
|
||||
owner,
|
||||
local,
|
||||
code,
|
||||
node_id=node_id,
|
||||
timeout=timeout,
|
||||
run_id=run.run_id if run else "",
|
||||
on_event=emissions.handle,
|
||||
)
|
||||
if node_def.resources is not None and not node_def.device:
|
||||
function = self._allocated(
|
||||
if node_def.resources is not None and self.placer is not None:
|
||||
# Says how much of a machine it takes, so which machine
|
||||
# and how much of it are one decision — including when
|
||||
# it also names a device, which used to mean the two
|
||||
# answers disagreed and nothing was accounted at all.
|
||||
function = self._placed(
|
||||
node_def.resources,
|
||||
node_def.device,
|
||||
node_def.device_policy,
|
||||
owner,
|
||||
local,
|
||||
code,
|
||||
@@ -1036,11 +1060,9 @@ class FlowController:
|
||||
run_id=run.run_id if run else "",
|
||||
on_event=emissions.handle,
|
||||
)
|
||||
if node_def.device and self.remote is not None:
|
||||
# A node with a device runs on a worker carrying that
|
||||
# label. Which worker is decided per call, so one that
|
||||
# attaches after this flow was built is used without
|
||||
# anything being rebuilt.
|
||||
elif node_def.device and self.remote is not None:
|
||||
# A device and nothing about size: the label alone
|
||||
# decides, least busy first, as it always has.
|
||||
function = self.remote.proxy(
|
||||
node_def.device,
|
||||
owner,
|
||||
@@ -1051,9 +1073,30 @@ class FlowController:
|
||||
run_id=run.run_id if run else "",
|
||||
on_event=emissions.handle,
|
||||
fallback=(
|
||||
function if node_def.device_policy == "prefer" else None
|
||||
self.workers.proxy(
|
||||
owner,
|
||||
local,
|
||||
code,
|
||||
node_id=node_id,
|
||||
timeout=timeout,
|
||||
run_id=run.run_id if run else "",
|
||||
on_event=emissions.handle,
|
||||
)
|
||||
if node_def.device_policy == "prefer"
|
||||
else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
# Declares nothing: the shared pool, at no extra cost.
|
||||
function = self.workers.proxy(
|
||||
owner,
|
||||
local,
|
||||
code,
|
||||
node_id=node_id,
|
||||
timeout=timeout,
|
||||
run_id=run.run_id if run else "",
|
||||
on_event=emissions.handle,
|
||||
)
|
||||
# Outermost, so it sees the result whichever of the three
|
||||
# above answered the call.
|
||||
function = emissions.wrap(function)
|
||||
|
||||
Reference in New Issue
Block a user