Remote workers: a GPU box dials in and runs the nodes bound to it

The engine runs where the automations are and the GPU is somewhere else,
usually behind a different network — so the worker connects out and the engine
answers over the socket it was given. Nothing has to expose Redis, and the
same connection works through the tunnel the hosted access will use.

What travels is the protocol the local pool already speaks, so a node cannot
tell which kind of worker it is on. A node declares device: gpu and
device_policy, the label is resolved per call (a worker attaching later needs
no rebuild), and a run whose labels nothing carries waits in the queue saying
what it waits for rather than failing — submit from the couch, the GPU box
picks it up when it is switched on.

Two things had to move with it. Compiling now happens on the machine that will
run the node: a node importing torch is correct on the GPU box and a missing
module on the engine, so checking it here failed nodes that were fine. And the
artifact endpoint accepts a worker's own credential, because storing a
checkpoint is exactly what that credential is for — and only that.

Verified against the real split: the training ran on this host (its checkpoint
names the machine and a numpy the engine does not have), streamed 40 metric
points back mid-run, and the evaluate node read the checkpoint on the engine.
Cancel kills the remote training; pulling the worker fails the run in six
seconds instead of waiting out its ten-minute timeout.
This commit is contained in:
2026-08-18 17:52:42 +02:00
parent d6c6df48ba
commit b6dc854d04
16 changed files with 1401 additions and 11 deletions
+291
View File
@@ -0,0 +1,291 @@
"""The agent that runs Fluksio nodes on a machine the engine cannot reach.
Copy this file and ``worker_main.py`` onto the box with the GPU, point it at
the engine, and it dials in::
pip install websockets
python fluksio_worker.py --url wss://api.example.com/api/v1/workers/attach \\
--token "$FLUKSIO_WORKER_TOKEN" --labels gpu --python /opt/venv/bin/python
It connects *out*, so the engine needs no route back and nothing has to expose
Redis. What it then does is what the engine's own worker pool does: hold a few
subprocesses running ``worker_main.py``, hand each call to one, and pass back
everything that comes out — including the metrics a training loop reports
while it is still running.
Deliberately one file with one dependency. Nothing of the app is imported
here; a worker host installs Python, ``websockets``, and whatever the nodes
themselves need.
"""
from __future__ import annotations
import argparse
import asyncio
import contextlib
import json
import logging
import os
import signal
import subprocess
import sys
from pathlib import Path
from typing import Any
try:
import websockets
except ImportError: # pragma: no cover - the one dependency, named plainly
print(
"This needs the 'websockets' package: pip install websockets", file=sys.stderr
)
raise SystemExit(1) from None
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("fluksio-worker")
PROTOCOL = 1
#: Sent while a call is running, so the engine can tell working from wedged.
HEARTBEAT_S = 10.0
#: Reconnection backs off to this and no further.
MAX_BACKOFF_S = 30.0
WORKER_MAIN = Path(__file__).with_name("worker_main.py")
class Subprocess:
"""One user-code process and the framing of one call over its pipes."""
def __init__(self, python: str, env: dict[str, str]) -> None:
self.proc = subprocess.Popen(
[python, str(WORKER_MAIN)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
close_fds=True,
env=env,
)
self._buffer = bytearray()
def alive(self) -> bool:
return self.proc.poll() is None
def send(self, request: dict[str, Any]) -> None:
assert self.proc.stdin is not None
self.proc.stdin.write((json.dumps(request) + "\n").encode())
self.proc.stdin.flush()
def read_line(self) -> str:
"""One line, blocking. Empty when the process is gone.
Read at the file-descriptor level so a report arriving mid-call is
passed on the moment it is written rather than when a buffer fills.
"""
assert self.proc.stdout is not None
fd = self.proc.stdout.fileno()
while True:
end = self._buffer.find(b"\n")
if end >= 0:
line = bytes(self._buffer[: end + 1])
del self._buffer[: end + 1]
return line.decode(errors="replace")
try:
chunk = os.read(fd, 65536)
except OSError:
chunk = b""
if not chunk:
self._buffer.clear()
return ""
self._buffer += chunk
def kill(self) -> None:
with contextlib.suppress(OSError):
self.proc.send_signal(signal.SIGKILL)
with contextlib.suppress(subprocess.TimeoutExpired):
self.proc.wait(timeout=5)
class Agent:
"""Holds the connection, and one subprocess per call in flight."""
def __init__(self, args: argparse.Namespace) -> None:
self.args = args
self.env = dict(os.environ)
self.env["FLUKSIO_ARTIFACT_URL"] = args.artifact_url or _artifacts_from(
args.url
)
self.env["FLUKSIO_ARTIFACT_TOKEN"] = args.token
self.running: dict[str, Subprocess] = {}
async def serve_forever(self) -> None:
backoff = 1.0
while True:
try:
await self._session()
backoff = 1.0
except Exception as exc:
log.warning("disconnected: %s — retrying in %.0fs", exc, backoff)
await asyncio.sleep(backoff)
backoff = min(MAX_BACKOFF_S, backoff * 2)
async def _session(self) -> None:
url = f"{self.args.url}?token={self.args.token}"
async with websockets.connect(url, max_size=None, ping_interval=20) as socket:
await socket.send(
json.dumps(
{
"op": "hello",
"protocol": PROTOCOL,
"name": self.args.name,
"labels": self.args.labels,
"python": self.args.python,
"max_parallel": self.args.parallel,
"venv_digest": _venv_digest(self.args.python),
}
)
)
welcome = json.loads(await socket.recv())
if welcome.get("op") != "welcome":
raise RuntimeError(str(welcome.get("reason") or "refused"))
log.info(
"attached to %s as '%s' with labels %s",
self.args.url,
welcome.get("name"),
self.args.labels,
)
async for raw in socket:
message = json.loads(raw)
op = message.get("op")
if op in ("run", "compile"):
# Compiling is loading the source, which is the same trip
# through a subprocess a call is — and has to happen here
# rather than on the engine, because "does this import"
# is a question about *this* machine's packages.
log.info("%s %s", op, message.get("call_id"))
task = asyncio.create_task(self._run(socket, message))
# Without this a failure in the task is only noticed when
# it is garbage collected, which reads as a call that
# vanished.
task.add_done_callback(_report_failure)
elif op == "cancel":
self._cancel(str(message.get("call_id") or ""))
async def _run(self, socket: Any, request: dict[str, Any]) -> None:
"""Execute one call in a subprocess, streaming what it says back."""
call_id = str(request.get("call_id") or "")
loop = asyncio.get_running_loop()
worker = Subprocess(self.args.python, self.env)
self.running[call_id] = worker
async def beat() -> None:
while True:
await asyncio.sleep(HEARTBEAT_S)
with contextlib.suppress(Exception):
await socket.send(
json.dumps({"call_id": call_id, "event": "heartbeat"})
)
heartbeat = asyncio.create_task(beat())
try:
await loop.run_in_executor(None, worker.send, request)
while True:
line = await loop.run_in_executor(None, worker.read_line)
if not line:
await socket.send(
json.dumps(
{
"call_id": call_id,
"ok": False,
"error": {
"type": "NodeCancelled"
if worker.proc.returncode
else "RemoteError",
"message": "the node process stopped",
"short": "the node process stopped",
"traceback": "",
},
}
)
)
return
await socket.send(line.strip())
# Anything without an `event` is the answer; the call is over.
if not json.loads(line).get("event"):
return
finally:
heartbeat.cancel()
self.running.pop(call_id, None)
worker.kill()
def _cancel(self, call_id: str) -> None:
worker = self.running.get(call_id)
if worker is not None:
log.info("cancelling %s", call_id)
worker.kill()
def _report_failure(task: asyncio.Task[Any]) -> None:
if not task.cancelled() and task.exception() is not None:
log.exception("call failed", exc_info=task.exception())
def _artifacts_from(url: str) -> str:
"""The artifact endpoint beside the socket, so one URL configures both."""
base = url.replace("wss://", "https://").replace("ws://", "http://")
return base.rsplit("/workers/attach", 1)[0] + "/artifacts"
def _venv_digest(python: str) -> str:
"""What is installed here, so the engine can say when it has drifted."""
try:
listing = subprocess.run(
[python, "-m", "pip", "freeze"],
capture_output=True,
text=True,
timeout=60,
check=False,
).stdout
except (OSError, subprocess.SubprocessError):
return ""
import hashlib
return hashlib.sha256(listing.encode()).hexdigest()[:16]
def main() -> int:
parser = argparse.ArgumentParser(description="Run Fluksio nodes on this machine.")
parser.add_argument("--url", required=True, help="wss://…/api/v1/workers/attach")
parser.add_argument(
"--token",
default=os.environ.get("FLUKSIO_WORKER_TOKEN", ""),
help="issued by POST /api/v1/workers/tokens",
)
parser.add_argument("--name", default=os.uname().nodename)
parser.add_argument(
"--labels",
default="",
help="comma-separated, e.g. gpu,cuda12 — what a node's device matches",
)
parser.add_argument(
"--python",
default=sys.executable,
help="the interpreter node code runs on; point it at the venv with torch",
)
parser.add_argument("--parallel", type=int, default=1)
parser.add_argument("--artifact-url", default="")
args = parser.parse_args()
if not args.token:
parser.error("a token is required (--token or FLUKSIO_WORKER_TOKEN)")
args.labels = [part.strip() for part in args.labels.split(",") if part.strip()]
if not WORKER_MAIN.exists():
parser.error(f"{WORKER_MAIN} is missing — copy it beside this file")
agent = Agent(args)
try:
asyncio.run(agent.serve_forever())
except KeyboardInterrupt:
log.info("stopping")
return 0
if __name__ == "__main__":
raise SystemExit(main())