Service node: a server kept running on a worker (openpanel V2)
- `service` node type: command, port, worker label, health path, env and
weights (an artifact digest the worker fetches as {weights}). Its flow
starts it; it restarts with backoff, reads degraded while starting or
without a worker and down after three starts that never got healthy, and
publishes http://<worker>:<port> once /health answers 200, "" when it ends.
- Worker protocol 3: a `serve` call, every call in its own process group,
killed when the engine ends the call, the socket drops or the agent gets
SIGTERM. `--service-host` overrides the address the engine sees.
- Services and compiles take no `--parallel` slot, and a flow's detail is
built off the event loop, so a save no longer stalls the engine.
This commit is contained in:
@@ -220,6 +220,11 @@ def _endpoints(controller: FlowController, definition: FlowDef) -> list[Endpoint
|
||||
|
||||
|
||||
def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail:
|
||||
"""The flow and how its nodes are doing. Blocking: call it off the loop.
|
||||
|
||||
A draft is built to report on it, which asks a worker to compile its
|
||||
nodes, and that waits on the socket the loop serves.
|
||||
"""
|
||||
name = definition.name
|
||||
running = {
|
||||
"enabled": controller.is_enabled(name),
|
||||
@@ -534,7 +539,7 @@ async def save_flow(
|
||||
status_code=409,
|
||||
detail={"message": str(exc), "current_version": exc.current},
|
||||
)
|
||||
return _detail(controller, stored)
|
||||
return await run_in_threadpool(_detail, controller, stored)
|
||||
|
||||
|
||||
@router.post("/{name}/publish", response_model=FlowDetail)
|
||||
@@ -561,7 +566,7 @@ async def publish_flow(
|
||||
)
|
||||
_audit("published", name, user)
|
||||
await controller.reload_flow(name)
|
||||
return _detail(controller, published)
|
||||
return await run_in_threadpool(_detail, controller, published)
|
||||
|
||||
|
||||
@router.post("/{name}/discard-draft", response_model=FlowDetail)
|
||||
@@ -581,7 +586,7 @@ async def discard_draft(name: str, controller: FlowControllerDep) -> Any:
|
||||
),
|
||||
)
|
||||
published = await run_in_threadpool(controller.store.discard_draft, name)
|
||||
return _detail(controller, published)
|
||||
return await run_in_threadpool(_detail, controller, published)
|
||||
|
||||
|
||||
@router.delete("/{name}", response_model=Message)
|
||||
@@ -644,7 +649,7 @@ async def rename_flow(
|
||||
# run, so this is cleanup rather than a migration.
|
||||
await run_in_threadpool(controller.forget_flow, name)
|
||||
await controller.reload()
|
||||
return _detail(controller, renamed)
|
||||
return await run_in_threadpool(_detail, controller, renamed)
|
||||
|
||||
|
||||
@router.post("/{name}/duplicate", response_model=FlowDetail)
|
||||
@@ -662,7 +667,7 @@ async def duplicate_flow(
|
||||
copy = await run_in_threadpool(controller.store.duplicate_flow, name)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
|
||||
return _detail(controller, copy)
|
||||
return await run_in_threadpool(_detail, controller, copy)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -810,7 +815,7 @@ async def share_node(
|
||||
)
|
||||
except LibExists as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
return _detail(controller, _read_flow(controller, name))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/unshare", response_model=FlowDetail)
|
||||
@@ -825,7 +830,7 @@ async def unshare_node(
|
||||
await run_in_threadpool(controller.store.unshare_node, name, node_id)
|
||||
except LibNotFound as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
return _detail(controller, _read_flow(controller, name))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -841,7 +846,7 @@ async def start_flow(
|
||||
_read_flow(controller, name)
|
||||
await controller.set_enabled(name, True)
|
||||
_audit("started", name, user)
|
||||
return _detail(controller, _read_flow(controller, name))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
|
||||
|
||||
@router.post("/{name}/stop", response_model=FlowDetail)
|
||||
@@ -850,7 +855,7 @@ async def stop_flow(name: str, controller: FlowControllerDep, user: CurrentUser)
|
||||
_read_flow(controller, name)
|
||||
await controller.set_enabled(name, False)
|
||||
_audit("stopped", name, user)
|
||||
return _detail(controller, _read_flow(controller, name))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
|
||||
|
||||
@router.post("/{name}/pause", response_model=Message)
|
||||
|
||||
@@ -337,6 +337,10 @@ async def _attached_session(websocket: WebSocket, claims: dict[str, Any]) -> Non
|
||||
"version": hello.get("version"),
|
||||
},
|
||||
inventory=WorkerInventory.from_hello(hello.get("inventory")),
|
||||
host=str(
|
||||
hello.get("service_host")
|
||||
or (websocket.client.host if websocket.client else "")
|
||||
),
|
||||
)
|
||||
hub.attach(worker)
|
||||
_behind.pop(name, None)
|
||||
|
||||
@@ -56,6 +56,7 @@ from fluksio.flow.nodes import (
|
||||
Node,
|
||||
NtfyNode,
|
||||
RbeNode,
|
||||
ServiceNode,
|
||||
SwitchNode,
|
||||
WebSocketNode,
|
||||
WrenNode,
|
||||
@@ -425,6 +426,13 @@ NODE_TYPES: dict[str, NodeType] = {
|
||||
runs_code=True,
|
||||
params_schema=_schema_of(ExecNode),
|
||||
),
|
||||
"service": NodeType(
|
||||
title="Service",
|
||||
description="Keep a server running on a worker, and publish its url.",
|
||||
cls=ServiceNode,
|
||||
runs_code=True,
|
||||
params_schema=_schema_of(ServiceNode),
|
||||
),
|
||||
"file": NodeType(
|
||||
title="File",
|
||||
description="Read a file into the flow, or write one out of it.",
|
||||
|
||||
@@ -17,6 +17,7 @@ from fluksio.flow.nodes.logic import ChangeNode, JoinNode, RbeNode, SwitchNode
|
||||
from fluksio.flow.nodes.mlp import MLPNode
|
||||
from fluksio.flow.nodes.mqtt import MqttNode
|
||||
from fluksio.flow.nodes.ntfy import NtfyNode
|
||||
from fluksio.flow.nodes.service import ServiceNode
|
||||
from fluksio.flow.nodes.websocket import WebSocketNode
|
||||
from fluksio.flow.nodes.wren import WrenNode
|
||||
|
||||
@@ -37,6 +38,7 @@ __all__ = [
|
||||
"NtfyNode",
|
||||
"RESERVED_SETTINGS",
|
||||
"RbeNode",
|
||||
"ServiceNode",
|
||||
"SwitchNode",
|
||||
"WebSocketNode",
|
||||
"WrenNode",
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Service: a server kept running on a worker while its flow is live.
|
||||
|
||||
A Python node is a call that returns, in a process the worker kills afterwards,
|
||||
so a model it loads is loaded again on every message. A service is the other
|
||||
shape: one long call that runs a command, such as a speech server, and keeps
|
||||
it up. Other nodes reach it over HTTP at the url it publishes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import shlex
|
||||
import uuid
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.flow.messages import MessageSpec
|
||||
from fluksio.flow.nodes.base import Node
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
|
||||
from fluksio.flow.remote import RemoteWorker, RemoteWorkerHub
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Seconds between restarts, one step per start in a row that never got
|
||||
#: healthy. A service that served and then died starts again after the first.
|
||||
BACKOFF = settings.FLOW_RESTART_BACKOFF
|
||||
#: Starts in a row that never answered healthy before the node reads down.
|
||||
DOWN_AFTER = 3
|
||||
#: How often a starting server's health is asked, and how long one ask may take.
|
||||
HEALTH_POLL_S = 1.0
|
||||
HEALTH_TIMEOUT_S = 2.0
|
||||
#: How often to look again when no worker carries the label.
|
||||
WORKER_POLL_S = 2.0
|
||||
#: How long a stopping flow waits for the worker to say the server is gone, so
|
||||
#: a flow started again finds the port free. Under the supervisor's grace.
|
||||
STOP_WAIT_S = 4.0
|
||||
|
||||
|
||||
class ServiceNode(Node):
|
||||
"""Keep a server running on a worker and publish the url it listens on.
|
||||
|
||||
The command runs on a worker matched by label, in one call that takes none
|
||||
of the worker's ``--parallel`` slots, and starts again whenever it ends.
|
||||
Every output port carries the url once the server answers its health
|
||||
check, and an empty string the moment the call ends.
|
||||
"""
|
||||
|
||||
class Params(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
command: str = Field(
|
||||
description=(
|
||||
"The server to run, such as `parrot serve --host 0.0.0.0 "
|
||||
"--port {port}`. {port} and {weights} are filled in."
|
||||
)
|
||||
)
|
||||
port: int = Field(
|
||||
default=8100, ge=1, le=65535, description="The port the server listens on."
|
||||
)
|
||||
worker: str = Field(
|
||||
description="The label of the worker to run it on, such as gpu."
|
||||
)
|
||||
health: str = Field(
|
||||
default="/health",
|
||||
description="The path that answers 200 once the server is ready.",
|
||||
)
|
||||
env: dict[str, str] = Field(
|
||||
default_factory=dict,
|
||||
description="Environment variables for the server.",
|
||||
)
|
||||
weights: str = Field(
|
||||
default="",
|
||||
pattern=r"^(sha256:[0-9a-f]{64})?$",
|
||||
description=(
|
||||
"An artifact digest, fetched to the worker. Its local path "
|
||||
"is {weights} in the command."
|
||||
),
|
||||
)
|
||||
|
||||
__slots__ = ("cfg", "argv", "hub", "_task", "_url")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
requires: MessageSpec | Iterable[MessageSpec] = (),
|
||||
provides: MessageSpec | Iterable[MessageSpec] = (),
|
||||
params: dict[str, Any] | None = None,
|
||||
name: str | None = None,
|
||||
):
|
||||
self.cfg = self.Params.model_validate(params or {})
|
||||
self.argv = shlex.split(self.cfg.command.replace("{port}", str(self.cfg.port)))
|
||||
if not self.argv:
|
||||
raise ValueError("a service needs a command")
|
||||
if "{weights}" in self.cfg.command and not self.cfg.weights:
|
||||
raise ValueError("the command uses {weights}, but no weights are set")
|
||||
# Set from the app on start; a test sets it directly.
|
||||
self.hub: RemoteWorkerHub | None = None
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._url = ""
|
||||
super().__init__(
|
||||
f=self._never_called,
|
||||
requires=requires,
|
||||
provides=provides,
|
||||
params=params,
|
||||
name=name or "service",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _never_called(params: dict[str, Any], **kwargs: Any) -> dict[str, Any] | None:
|
||||
"""Its flow starts it, not a message, and it publishes on its own."""
|
||||
return None
|
||||
|
||||
# -- lifecycle ------------------------------------------------------------
|
||||
|
||||
async def start(self, app: FastAPI | None = None) -> None:
|
||||
if app is not None:
|
||||
self.hub = getattr(app.state, "worker_hub", None)
|
||||
# Spawned on every start, as the websocket node's: the supervisor
|
||||
# ignores a duplicate.
|
||||
if self._task is not None and not self._task.done():
|
||||
return
|
||||
self._task = self._run_supervised("service", self._keep_running)
|
||||
|
||||
async def stop(self, app: FastAPI | None = None) -> None:
|
||||
if self._task is not None:
|
||||
await self._cancel_task(self._task)
|
||||
self._task = None
|
||||
|
||||
async def _keep_running(self) -> None:
|
||||
"""Run the server, and run it again whenever it ends."""
|
||||
failures = 0
|
||||
while True:
|
||||
worker = self.hub.pick(self.cfg.worker) if self.hub is not None else None
|
||||
if worker is None:
|
||||
self.report_health(
|
||||
"degraded", f"no worker labelled '{self.cfg.worker}' is attached"
|
||||
)
|
||||
await asyncio.sleep(WORKER_POLL_S)
|
||||
continue
|
||||
self.report_health("degraded", f"starting on '{worker.name}'")
|
||||
healthy, reason = await self._serve_once(worker)
|
||||
# Only starts that never got healthy lead to down: one that served
|
||||
# for a while is restarted as often as it dies.
|
||||
failures = 0 if healthy else failures + 1
|
||||
delay = BACKOFF[min(failures, len(BACKOFF) - 1)]
|
||||
logger.warning("Service '%s' ended: %s", self.id, reason)
|
||||
self.report_health(
|
||||
"down" if failures >= DOWN_AFTER else "degraded",
|
||||
f"{reason}; restarting in {delay:g}s",
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
async def _serve_once(self, worker: RemoteWorker) -> tuple[bool, str]:
|
||||
"""One call on this worker, until it ends: whether it got healthy, and why."""
|
||||
call_id = f"{self.id}:{uuid.uuid4().hex[:8]}"
|
||||
host = f"[{worker.host}]" if ":" in worker.host else worker.host
|
||||
url = f"http://{host}:{self.cfg.port}"
|
||||
payload = {
|
||||
"op": "serve",
|
||||
"call_id": call_id,
|
||||
"argv": self.argv,
|
||||
"env": self.cfg.env,
|
||||
"weights": self.cfg.weights,
|
||||
}
|
||||
# On a thread, as every call is, and holding no slot.
|
||||
call = asyncio.ensure_future(
|
||||
asyncio.to_thread(worker.request, payload, 0, slot=False)
|
||||
)
|
||||
# Retrieved however it ends, so one that outlives a stop is not
|
||||
# reported at exit.
|
||||
call.add_done_callback(lambda done: done.cancelled() or done.exception())
|
||||
healthy = False
|
||||
try:
|
||||
healthy = await self._await_health(call, worker.name, url)
|
||||
if healthy:
|
||||
self._url = url
|
||||
await asyncio.to_thread(self._publish, url)
|
||||
self.report_health("ok")
|
||||
await asyncio.wait({call})
|
||||
error = call.result().get("error") or {}
|
||||
return healthy, str(error.get("message") or "the service stopped")
|
||||
except Exception as exc:
|
||||
# The worker went away or went silent, or the url was refused.
|
||||
return healthy, f"{type(exc).__name__}: {exc}"
|
||||
finally:
|
||||
if not call.done():
|
||||
# The flow stopped, or this side failed. Ending the call kills
|
||||
# the server's process group on the worker, and its answer says
|
||||
# that is done, so the next start finds the port free.
|
||||
await asyncio.to_thread(worker.cancel, call_id)
|
||||
await asyncio.wait({call}, timeout=STOP_WAIT_S)
|
||||
await asyncio.to_thread(self._withdraw)
|
||||
|
||||
async def _await_health(
|
||||
self, call: asyncio.Future[Any], worker: str, url: str
|
||||
) -> bool:
|
||||
"""Ask the server's health until it answers 200, or the call ends first."""
|
||||
async with httpx.AsyncClient(timeout=HEALTH_TIMEOUT_S) as client:
|
||||
while not call.done():
|
||||
problem = await self._probe(client, url + self.cfg.health)
|
||||
if problem is None:
|
||||
return True
|
||||
self.report_health(
|
||||
"degraded",
|
||||
f"starting on '{worker}': {self.cfg.health} {problem}",
|
||||
)
|
||||
await asyncio.wait({call}, timeout=HEALTH_POLL_S)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
async def _probe(client: httpx.AsyncClient, url: str) -> str | None:
|
||||
"""None once the url answers 200, else what it did instead."""
|
||||
try:
|
||||
response = await client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
return f"is not answering ({type(exc).__name__})"
|
||||
if response.status_code == 200:
|
||||
return None
|
||||
return f"answers {response.status_code}"
|
||||
|
||||
# -- the url ----------------------------------------------------------------
|
||||
|
||||
def _publish(self, url: str) -> None:
|
||||
self.emit({spec.port: url for spec in self.output_ports})
|
||||
|
||||
def _withdraw(self) -> None:
|
||||
"""Empty the url the moment the server is gone.
|
||||
|
||||
Unless its value is gone already: a deleted flow's values are cleared
|
||||
before its nodes stop, and writing the url would bring one back.
|
||||
"""
|
||||
if not self._url:
|
||||
return
|
||||
self._url = ""
|
||||
pipeline = self._pipeline
|
||||
if pipeline is not None and any(
|
||||
pipeline.state.get(spec.name) for spec in self.output_ports
|
||||
):
|
||||
self._publish("")
|
||||
@@ -45,8 +45,9 @@ SEND_TIMEOUT_S = 30.0
|
||||
#: whose node has no timeout of its own.
|
||||
SILENCE_S = 90.0
|
||||
#: Protocol version this engine speaks. A worker announcing anything else is
|
||||
#: refused rather than half-understood.
|
||||
PROTOCOL = 2
|
||||
#: refused rather than half-understood. 3: the ``serve`` call, and calls that
|
||||
#: end with their process group.
|
||||
PROTOCOL = 3
|
||||
|
||||
|
||||
class NoWorker(RemoteError):
|
||||
@@ -102,9 +103,13 @@ class RemoteWorker:
|
||||
max_parallel: int = 1,
|
||||
info: dict[str, Any] | None = None,
|
||||
inventory: WorkerInventory | None = None,
|
||||
host: str = "",
|
||||
) -> None:
|
||||
self.name = name
|
||||
self.labels = set(labels)
|
||||
#: Where the engine reaches what this worker serves: the address its
|
||||
#: socket came from, unless the worker says otherwise.
|
||||
self.host = host
|
||||
self.info = info or {}
|
||||
self.inventory = inventory or WorkerInventory()
|
||||
# This machine's books, held here so that a worker reconnecting — which
|
||||
@@ -163,14 +168,21 @@ class RemoteWorker:
|
||||
payload: dict[str, Any],
|
||||
timeout: float,
|
||||
on_event: Callable[[dict[str, Any]], None] | None = None,
|
||||
slot: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Send a call and wait for its answer, on a thread rather than the loop.
|
||||
|
||||
``slot`` off is for what must not queue behind running calls, nor hold
|
||||
one: a compile, and a service, which would hold one for good.
|
||||
"""
|
||||
if self._gone:
|
||||
raise RemoteError(f"worker '{self.name}' is no longer attached")
|
||||
call_id = str(payload["call_id"])
|
||||
inbox: queue.Queue[dict[str, Any] | None] = queue.Queue()
|
||||
# Blocking here is the backpressure, exactly as taking a slot is in the
|
||||
# local pool.
|
||||
self._slots.acquire()
|
||||
if slot:
|
||||
self._slots.acquire()
|
||||
with self._lock:
|
||||
self._pending[call_id] = inbox
|
||||
try:
|
||||
@@ -226,7 +238,8 @@ class RemoteWorker:
|
||||
finally:
|
||||
with self._lock:
|
||||
self._pending.pop(call_id, None)
|
||||
self._slots.release()
|
||||
if slot:
|
||||
self._slots.release()
|
||||
|
||||
@property
|
||||
def compiled(self) -> set[str]:
|
||||
@@ -440,6 +453,7 @@ class RemoteWorkerHub:
|
||||
"source": source,
|
||||
},
|
||||
timeout=timeout,
|
||||
slot=False,
|
||||
)
|
||||
except RemoteError as exc:
|
||||
return f"{type(exc).__name__}: {exc}"
|
||||
|
||||
Reference in New Issue
Block a user