Warn at startup when worker or cascade limits are far past the cores
More than four times os.cpu_count() logs one warning; the value is kept. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx
This commit is contained in:
@@ -65,12 +65,37 @@ ENGINE_RAM_RESERVE_MB = 1024
|
||||
#: Where a container finds its own memory limit (cgroup v2).
|
||||
CGROUP_MEMORY_MAX = Path("/sys/fs/cgroup/memory.max")
|
||||
|
||||
#: How many times the core count a concurrency limit may reach before startup
|
||||
#: warns. Above one on purpose: a node waiting on the network holds no core.
|
||||
OVERSUBSCRIBE_FACTOR = 4
|
||||
|
||||
|
||||
def machine_cpus() -> int:
|
||||
"""Cores a node may be given here, when nobody said."""
|
||||
return max(1, (os.cpu_count() or 1) - ENGINE_RESERVE)
|
||||
|
||||
|
||||
def warn_oversubscribed(limits: dict[str, int]) -> None:
|
||||
"""Warn, never clamp, when a concurrency limit is far past the cores.
|
||||
|
||||
A high number can be right for nodes that wait on devices, but one far
|
||||
past the machine is more often a zero too many.
|
||||
"""
|
||||
cores = os.cpu_count() or 1
|
||||
high = [
|
||||
f"{name}={value}"
|
||||
for name, value in limits.items()
|
||||
if value > OVERSUBSCRIBE_FACTOR * cores
|
||||
]
|
||||
if high:
|
||||
logger.warning(
|
||||
"Concurrency above %dx this machine's %d cores: %s",
|
||||
OVERSUBSCRIBE_FACTOR,
|
||||
cores,
|
||||
", ".join(high),
|
||||
)
|
||||
|
||||
|
||||
def machine_gpus() -> int:
|
||||
"""Cards a node may be given here, when nobody said.
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ from fluksio.flow.resources import (
|
||||
fair_share_env,
|
||||
machine_gpus,
|
||||
machine_ram_mb,
|
||||
warn_oversubscribed,
|
||||
)
|
||||
from fluksio.flow.runs import RUN_STATE_TTL, RunService, sweep_artifacts
|
||||
from fluksio.flow.secrets import init_secrets
|
||||
@@ -372,6 +373,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
accountant.ram_mb,
|
||||
settings.FLOW_WORKER_MAX_RSS_MB,
|
||||
)
|
||||
warn_oversubscribed(
|
||||
{
|
||||
"FLOW_MAX_WORKERS": settings.FLOW_MAX_WORKERS,
|
||||
"FLOW_MAX_CASCADES": settings.FLOW_MAX_CASCADES,
|
||||
}
|
||||
)
|
||||
watchdog = LoopWatchdog(event_bus)
|
||||
app.state.watchdog = watchdog
|
||||
|
||||
|
||||
Reference in New Issue
Block a user