Service node: a server kept running on a worker (openpanel V2)
- `service` node type: command, port, worker label, health path, env and
weights (an artifact digest the worker fetches as {weights}). Its flow
starts it; it restarts with backoff, reads degraded while starting or
without a worker and down after three starts that never got healthy, and
publishes http://<worker>:<port> once /health answers 200, "" when it ends.
- Worker protocol 3: a `serve` call, every call in its own process group,
killed when the engine ends the call, the socket drops or the agent gets
SIGTERM. `--service-host` overrides the address the engine sees.
- Services and compiles take no `--parallel` slot, and a flow's detail is
built off the event loop, so a save no longer stalls the engine.
This commit is contained in:
@@ -45,8 +45,9 @@ SEND_TIMEOUT_S = 30.0
|
||||
#: whose node has no timeout of its own.
|
||||
SILENCE_S = 90.0
|
||||
#: Protocol version this engine speaks. A worker announcing anything else is
|
||||
#: refused rather than half-understood.
|
||||
PROTOCOL = 2
|
||||
#: refused rather than half-understood. 3: the ``serve`` call, and calls that
|
||||
#: end with their process group.
|
||||
PROTOCOL = 3
|
||||
|
||||
|
||||
class NoWorker(RemoteError):
|
||||
@@ -102,9 +103,13 @@ class RemoteWorker:
|
||||
max_parallel: int = 1,
|
||||
info: dict[str, Any] | None = None,
|
||||
inventory: WorkerInventory | None = None,
|
||||
host: str = "",
|
||||
) -> None:
|
||||
self.name = name
|
||||
self.labels = set(labels)
|
||||
#: Where the engine reaches what this worker serves: the address its
|
||||
#: socket came from, unless the worker says otherwise.
|
||||
self.host = host
|
||||
self.info = info or {}
|
||||
self.inventory = inventory or WorkerInventory()
|
||||
# This machine's books, held here so that a worker reconnecting — which
|
||||
@@ -163,14 +168,21 @@ class RemoteWorker:
|
||||
payload: dict[str, Any],
|
||||
timeout: float,
|
||||
on_event: Callable[[dict[str, Any]], None] | None = None,
|
||||
slot: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Send a call and wait for its answer, on a thread rather than the loop.
|
||||
|
||||
``slot`` off is for what must not queue behind running calls, nor hold
|
||||
one: a compile, and a service, which would hold one for good.
|
||||
"""
|
||||
if self._gone:
|
||||
raise RemoteError(f"worker '{self.name}' is no longer attached")
|
||||
call_id = str(payload["call_id"])
|
||||
inbox: queue.Queue[dict[str, Any] | None] = queue.Queue()
|
||||
# Blocking here is the backpressure, exactly as taking a slot is in the
|
||||
# local pool.
|
||||
self._slots.acquire()
|
||||
if slot:
|
||||
self._slots.acquire()
|
||||
with self._lock:
|
||||
self._pending[call_id] = inbox
|
||||
try:
|
||||
@@ -226,7 +238,8 @@ class RemoteWorker:
|
||||
finally:
|
||||
with self._lock:
|
||||
self._pending.pop(call_id, None)
|
||||
self._slots.release()
|
||||
if slot:
|
||||
self._slots.release()
|
||||
|
||||
@property
|
||||
def compiled(self) -> set[str]:
|
||||
@@ -440,6 +453,7 @@ class RemoteWorkerHub:
|
||||
"source": source,
|
||||
},
|
||||
timeout=timeout,
|
||||
slot=False,
|
||||
)
|
||||
except RemoteError as exc:
|
||||
return f"{type(exc).__name__}: {exc}"
|
||||
|
||||
Reference in New Issue
Block a user