Service node: a server kept running on a worker (openpanel V2)

- `service` node type: command, port, worker label, health path, env and
  weights (an artifact digest the worker fetches as {weights}). Its flow
  starts it; it restarts with backoff, reads degraded while starting or
  without a worker and down after three starts that never got healthy, and
  publishes http://<worker>:<port> once /health answers 200, "" when it ends.
- Worker protocol 3: a `serve` call, every call in its own process group,
  killed when the engine ends the call, the socket drops or the agent gets
  SIGTERM. `--service-host` overrides the address the engine sees.
- Services and compiles take no `--parallel` slot, and a flow's detail is
  built off the event loop, so a save no longer stalls the engine.
This commit is contained in:
Melvin Strobl
2026-10-19 18:43:04 +02:00
parent 80e0201612
commit 85dad39973
16 changed files with 757 additions and 34 deletions
+14 -9
View File
@@ -220,6 +220,11 @@ def _endpoints(controller: FlowController, definition: FlowDef) -> list[Endpoint
def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail:
"""The flow and how its nodes are doing. Blocking: call it off the loop.
A draft is built to report on it, which asks a worker to compile its
nodes, and that waits on the socket the loop serves.
"""
name = definition.name
running = {
"enabled": controller.is_enabled(name),
@@ -534,7 +539,7 @@ async def save_flow(
status_code=409,
detail={"message": str(exc), "current_version": exc.current},
)
return _detail(controller, stored)
return await run_in_threadpool(_detail, controller, stored)
@router.post("/{name}/publish", response_model=FlowDetail)
@@ -561,7 +566,7 @@ async def publish_flow(
)
_audit("published", name, user)
await controller.reload_flow(name)
return _detail(controller, published)
return await run_in_threadpool(_detail, controller, published)
@router.post("/{name}/discard-draft", response_model=FlowDetail)
@@ -581,7 +586,7 @@ async def discard_draft(name: str, controller: FlowControllerDep) -> Any:
),
)
published = await run_in_threadpool(controller.store.discard_draft, name)
return _detail(controller, published)
return await run_in_threadpool(_detail, controller, published)
@router.delete("/{name}", response_model=Message)
@@ -644,7 +649,7 @@ async def rename_flow(
# run, so this is cleanup rather than a migration.
await run_in_threadpool(controller.forget_flow, name)
await controller.reload()
return _detail(controller, renamed)
return await run_in_threadpool(_detail, controller, renamed)
@router.post("/{name}/duplicate", response_model=FlowDetail)
@@ -662,7 +667,7 @@ async def duplicate_flow(
copy = await run_in_threadpool(controller.store.duplicate_flow, name)
except FlowNotFound:
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
return _detail(controller, copy)
return await run_in_threadpool(_detail, controller, copy)
# -----------------------------------------------------------------------------
@@ -810,7 +815,7 @@ async def share_node(
)
except LibExists as exc:
raise HTTPException(status_code=409, detail=str(exc))
return _detail(controller, _read_flow(controller, name))
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
@router.post("/{name}/nodes/{node_id}/unshare", response_model=FlowDetail)
@@ -825,7 +830,7 @@ async def unshare_node(
await run_in_threadpool(controller.store.unshare_node, name, node_id)
except LibNotFound as exc:
raise HTTPException(status_code=404, detail=str(exc))
return _detail(controller, _read_flow(controller, name))
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
# -----------------------------------------------------------------------------
@@ -841,7 +846,7 @@ async def start_flow(
_read_flow(controller, name)
await controller.set_enabled(name, True)
_audit("started", name, user)
return _detail(controller, _read_flow(controller, name))
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
@router.post("/{name}/stop", response_model=FlowDetail)
@@ -850,7 +855,7 @@ async def stop_flow(name: str, controller: FlowControllerDep, user: CurrentUser)
_read_flow(controller, name)
await controller.set_enabled(name, False)
_audit("stopped", name, user)
return _detail(controller, _read_flow(controller, name))
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
@router.post("/{name}/pause", response_model=Message)
+4
View File
@@ -337,6 +337,10 @@ async def _attached_session(websocket: WebSocket, claims: dict[str, Any]) -> Non
"version": hello.get("version"),
},
inventory=WorkerInventory.from_hello(hello.get("inventory")),
host=str(
hello.get("service_host")
or (websocket.client.host if websocket.client else "")
),
)
hub.attach(worker)
_behind.pop(name, None)
+8
View File
@@ -56,6 +56,7 @@ from fluksio.flow.nodes import (
Node,
NtfyNode,
RbeNode,
ServiceNode,
SwitchNode,
WebSocketNode,
WrenNode,
@@ -425,6 +426,13 @@ NODE_TYPES: dict[str, NodeType] = {
runs_code=True,
params_schema=_schema_of(ExecNode),
),
"service": NodeType(
title="Service",
description="Keep a server running on a worker, and publish its url.",
cls=ServiceNode,
runs_code=True,
params_schema=_schema_of(ServiceNode),
),
"file": NodeType(
title="File",
description="Read a file into the flow, or write one out of it.",
+2
View File
@@ -17,6 +17,7 @@ from fluksio.flow.nodes.logic import ChangeNode, JoinNode, RbeNode, SwitchNode
from fluksio.flow.nodes.mlp import MLPNode
from fluksio.flow.nodes.mqtt import MqttNode
from fluksio.flow.nodes.ntfy import NtfyNode
from fluksio.flow.nodes.service import ServiceNode
from fluksio.flow.nodes.websocket import WebSocketNode
from fluksio.flow.nodes.wren import WrenNode
@@ -37,6 +38,7 @@ __all__ = [
"NtfyNode",
"RESERVED_SETTINGS",
"RbeNode",
"ServiceNode",
"SwitchNode",
"WebSocketNode",
"WrenNode",
+246
View File
@@ -0,0 +1,246 @@
"""Service: a server kept running on a worker while its flow is live.
A Python node is a call that returns, in a process the worker kills afterwards,
so a model it loads is loaded again on every message. A service is the other
shape: one long call that runs a command, such as a speech server, and keeps
it up. Other nodes reach it over HTTP at the url it publishes.
"""
from __future__ import annotations
import asyncio
import logging
import shlex
import uuid
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any
import httpx
from pydantic import BaseModel, ConfigDict, Field
from fluksio.core.config import settings
from fluksio.flow.messages import MessageSpec
from fluksio.flow.nodes.base import Node
if TYPE_CHECKING:
from fastapi import FastAPI
from fluksio.flow.remote import RemoteWorker, RemoteWorkerHub
logger = logging.getLogger(__name__)
#: Seconds between restarts, one step per start in a row that never got
#: healthy. A service that served and then died starts again after the first.
BACKOFF = settings.FLOW_RESTART_BACKOFF
#: Starts in a row that never answered healthy before the node reads down.
DOWN_AFTER = 3
#: How often a starting server's health is asked, and how long one ask may take.
HEALTH_POLL_S = 1.0
HEALTH_TIMEOUT_S = 2.0
#: How often to look again when no worker carries the label.
WORKER_POLL_S = 2.0
#: How long a stopping flow waits for the worker to say the server is gone, so
#: a flow started again finds the port free. Under the supervisor's grace.
STOP_WAIT_S = 4.0
class ServiceNode(Node):
"""Keep a server running on a worker and publish the url it listens on.
The command runs on a worker matched by label, in one call that takes none
of the worker's ``--parallel`` slots, and starts again whenever it ends.
Every output port carries the url once the server answers its health
check, and an empty string the moment the call ends.
"""
class Params(BaseModel):
model_config = ConfigDict(extra="allow")
command: str = Field(
description=(
"The server to run, such as `parrot serve --host 0.0.0.0 "
"--port {port}`. {port} and {weights} are filled in."
)
)
port: int = Field(
default=8100, ge=1, le=65535, description="The port the server listens on."
)
worker: str = Field(
description="The label of the worker to run it on, such as gpu."
)
health: str = Field(
default="/health",
description="The path that answers 200 once the server is ready.",
)
env: dict[str, str] = Field(
default_factory=dict,
description="Environment variables for the server.",
)
weights: str = Field(
default="",
pattern=r"^(sha256:[0-9a-f]{64})?$",
description=(
"An artifact digest, fetched to the worker. Its local path "
"is {weights} in the command."
),
)
__slots__ = ("cfg", "argv", "hub", "_task", "_url")
def __init__(
self,
requires: MessageSpec | Iterable[MessageSpec] = (),
provides: MessageSpec | Iterable[MessageSpec] = (),
params: dict[str, Any] | None = None,
name: str | None = None,
):
self.cfg = self.Params.model_validate(params or {})
self.argv = shlex.split(self.cfg.command.replace("{port}", str(self.cfg.port)))
if not self.argv:
raise ValueError("a service needs a command")
if "{weights}" in self.cfg.command and not self.cfg.weights:
raise ValueError("the command uses {weights}, but no weights are set")
# Set from the app on start; a test sets it directly.
self.hub: RemoteWorkerHub | None = None
self._task: asyncio.Task[None] | None = None
self._url = ""
super().__init__(
f=self._never_called,
requires=requires,
provides=provides,
params=params,
name=name or "service",
)
@staticmethod
def _never_called(params: dict[str, Any], **kwargs: Any) -> dict[str, Any] | None:
"""Its flow starts it, not a message, and it publishes on its own."""
return None
# -- lifecycle ------------------------------------------------------------
async def start(self, app: FastAPI | None = None) -> None:
if app is not None:
self.hub = getattr(app.state, "worker_hub", None)
# Spawned on every start, as the websocket node's: the supervisor
# ignores a duplicate.
if self._task is not None and not self._task.done():
return
self._task = self._run_supervised("service", self._keep_running)
async def stop(self, app: FastAPI | None = None) -> None:
if self._task is not None:
await self._cancel_task(self._task)
self._task = None
async def _keep_running(self) -> None:
"""Run the server, and run it again whenever it ends."""
failures = 0
while True:
worker = self.hub.pick(self.cfg.worker) if self.hub is not None else None
if worker is None:
self.report_health(
"degraded", f"no worker labelled '{self.cfg.worker}' is attached"
)
await asyncio.sleep(WORKER_POLL_S)
continue
self.report_health("degraded", f"starting on '{worker.name}'")
healthy, reason = await self._serve_once(worker)
# Only starts that never got healthy lead to down: one that served
# for a while is restarted as often as it dies.
failures = 0 if healthy else failures + 1
delay = BACKOFF[min(failures, len(BACKOFF) - 1)]
logger.warning("Service '%s' ended: %s", self.id, reason)
self.report_health(
"down" if failures >= DOWN_AFTER else "degraded",
f"{reason}; restarting in {delay:g}s",
)
await asyncio.sleep(delay)
async def _serve_once(self, worker: RemoteWorker) -> tuple[bool, str]:
"""One call on this worker, until it ends: whether it got healthy, and why."""
call_id = f"{self.id}:{uuid.uuid4().hex[:8]}"
host = f"[{worker.host}]" if ":" in worker.host else worker.host
url = f"http://{host}:{self.cfg.port}"
payload = {
"op": "serve",
"call_id": call_id,
"argv": self.argv,
"env": self.cfg.env,
"weights": self.cfg.weights,
}
# On a thread, as every call is, and holding no slot.
call = asyncio.ensure_future(
asyncio.to_thread(worker.request, payload, 0, slot=False)
)
# Retrieved however it ends, so one that outlives a stop is not
# reported at exit.
call.add_done_callback(lambda done: done.cancelled() or done.exception())
healthy = False
try:
healthy = await self._await_health(call, worker.name, url)
if healthy:
self._url = url
await asyncio.to_thread(self._publish, url)
self.report_health("ok")
await asyncio.wait({call})
error = call.result().get("error") or {}
return healthy, str(error.get("message") or "the service stopped")
except Exception as exc:
# The worker went away or went silent, or the url was refused.
return healthy, f"{type(exc).__name__}: {exc}"
finally:
if not call.done():
# The flow stopped, or this side failed. Ending the call kills
# the server's process group on the worker, and its answer says
# that is done, so the next start finds the port free.
await asyncio.to_thread(worker.cancel, call_id)
await asyncio.wait({call}, timeout=STOP_WAIT_S)
await asyncio.to_thread(self._withdraw)
async def _await_health(
self, call: asyncio.Future[Any], worker: str, url: str
) -> bool:
"""Ask the server's health until it answers 200, or the call ends first."""
async with httpx.AsyncClient(timeout=HEALTH_TIMEOUT_S) as client:
while not call.done():
problem = await self._probe(client, url + self.cfg.health)
if problem is None:
return True
self.report_health(
"degraded",
f"starting on '{worker}': {self.cfg.health} {problem}",
)
await asyncio.wait({call}, timeout=HEALTH_POLL_S)
return False
@staticmethod
async def _probe(client: httpx.AsyncClient, url: str) -> str | None:
"""None once the url answers 200, else what it did instead."""
try:
response = await client.get(url)
except httpx.HTTPError as exc:
return f"is not answering ({type(exc).__name__})"
if response.status_code == 200:
return None
return f"answers {response.status_code}"
# -- the url ----------------------------------------------------------------
def _publish(self, url: str) -> None:
self.emit({spec.port: url for spec in self.output_ports})
def _withdraw(self) -> None:
"""Empty the url the moment the server is gone.
Unless its value is gone already: a deleted flow's values are cleared
before its nodes stop, and writing the url would bring one back.
"""
if not self._url:
return
self._url = ""
pipeline = self._pipeline
if pipeline is not None and any(
pipeline.state.get(spec.name) for spec in self.output_ports
):
self._publish("")
+18 -4
View File
@@ -45,8 +45,9 @@ SEND_TIMEOUT_S = 30.0
#: whose node has no timeout of its own.
SILENCE_S = 90.0
#: Protocol version this engine speaks. A worker announcing anything else is
#: refused rather than half-understood.
PROTOCOL = 2
#: refused rather than half-understood. 3: the ``serve`` call, and calls that
#: end with their process group.
PROTOCOL = 3
class NoWorker(RemoteError):
@@ -102,9 +103,13 @@ class RemoteWorker:
max_parallel: int = 1,
info: dict[str, Any] | None = None,
inventory: WorkerInventory | None = None,
host: str = "",
) -> None:
self.name = name
self.labels = set(labels)
#: Where the engine reaches what this worker serves: the address its
#: socket came from, unless the worker says otherwise.
self.host = host
self.info = info or {}
self.inventory = inventory or WorkerInventory()
# This machine's books, held here so that a worker reconnecting — which
@@ -163,14 +168,21 @@ class RemoteWorker:
payload: dict[str, Any],
timeout: float,
on_event: Callable[[dict[str, Any]], None] | None = None,
slot: bool = True,
) -> dict[str, Any]:
"""Send a call and wait for its answer, on a thread rather than the loop.
``slot`` off is for what must not queue behind running calls, nor hold
one: a compile, and a service, which would hold one for good.
"""
if self._gone:
raise RemoteError(f"worker '{self.name}' is no longer attached")
call_id = str(payload["call_id"])
inbox: queue.Queue[dict[str, Any] | None] = queue.Queue()
# Blocking here is the backpressure, exactly as taking a slot is in the
# local pool.
self._slots.acquire()
if slot:
self._slots.acquire()
with self._lock:
self._pending[call_id] = inbox
try:
@@ -226,7 +238,8 @@ class RemoteWorker:
finally:
with self._lock:
self._pending.pop(call_id, None)
self._slots.release()
if slot:
self._slots.release()
@property
def compiled(self) -> set[str]:
@@ -440,6 +453,7 @@ class RemoteWorkerHub:
"source": source,
},
timeout=timeout,
slot=False,
)
except RemoteError as exc:
return f"{type(exc).__name__}: {exc}"