Bound warm workers' host memory and account local RAM
Docs / docs (push) Successful in 19s
Playwright Tests / test-playwright (1, 2) (push) Failing after 12s
Playwright Tests / test-playwright (2, 2) (push) Failing after 9s
pre-commit / pre-commit (push) Failing after 1m56s
Test Backend / test-backend (push) Failing after 2m50s
Compose Smoke Test / test-compose (push) Failing after 9s
Playwright Tests / merge-reports (push) Failing after 2m16s

A worker over FLOW_WORKER_MAX_RSS_MB (--worker-max-rss) when its node
returns is retired instead of kept warm, in the shared pool and in every
declared one; 0, the default, keeps them all. The engine now accounts its
own memory (FLOW_RAM / --ram, else the box or the cgroup limit less 1 GB),
so a node's `ram` declaration holds locally. A declared environment the
shared pool already has is that pool rather than a duplicate set of warm
workers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-27 16:11:13 +02:00
co-authored by Claude Opus 5.5
parent cc4091166e
commit dafdcb76c4
10 changed files with 143 additions and 8 deletions
+22 -1
View File
@@ -153,6 +153,17 @@ class _Worker:
def alive(self) -> bool:
return self.proc.poll() is None
def rss_mb(self) -> int:
"""Resident memory in megabytes, or 0 where it cannot be asked."""
# ponytail: /proc is Linux-only; elsewhere every worker reads as small
# and the ceiling never retires one.
try:
with open(f"/proc/{self.proc.pid}/statm") as statm:
pages = int(statm.read().split()[1])
except (OSError, ValueError, IndexError):
return 0
return pages * os.sysconf("SC_PAGE_SIZE") // 2**20
def send(self, request: dict[str, Any]) -> None:
assert self.proc.stdin is not None
# Node inputs may be keyed by something other than a string, which
@@ -221,10 +232,16 @@ class PythonWorkerPool:
size: int = 4,
events: EventBus | None = None,
env: dict[str, str] | None = None,
max_rss_mb: int = 0,
) -> None:
self.python = python
self.size = size
self.events = events
# A worker holding more than this when its call returns is retired
# rather than kept warm; 0 keeps them all. A library like JAX keeps what
# it allocated for as long as the process lives, and warm is the point
# of a pool, so nothing else would ever give that memory back.
self.max_rss_mb = max_rss_mb
# Put into every worker's environment past the denylist — where the
# artifact store is, which node code needs and cannot guess.
self.env = env or {}
@@ -271,8 +288,10 @@ class PythonWorkerPool:
How many of them may run at once is the accountant's business, not this
pool's: the slots here only bound one environment's own concurrency.
An environment this pool's workers already have is this pool, rather
than a second set of warm workers holding the same imports.
"""
if not extra:
if extra.items() <= self.env.items():
return self
key = frozenset(extra.items())
with self._lock:
@@ -283,6 +302,7 @@ class PythonWorkerPool:
size=self.size,
events=self.events,
env={**self.env, **extra},
max_rss_mb=self.max_rss_mb,
)
child._generation = self._generation
child.start()
@@ -379,6 +399,7 @@ class PythonWorkerPool:
worker.alive()
and worker.generation == self._generation
and not worker.cancelled
and not (self.max_rss_mb and worker.rss_mb() > self.max_rss_mb)
)
if reusable:
self._idle.put(worker)