Service node: a server kept running on a worker (openpanel V2)
- `service` node type: command, port, worker label, health path, env and
weights (an artifact digest the worker fetches as {weights}). Its flow
starts it; it restarts with backoff, reads degraded while starting or
without a worker and down after three starts that never got healthy, and
publishes http://<worker>:<port> once /health answers 200, "" when it ends.
- Worker protocol 3: a `serve` call, every call in its own process group,
killed when the engine ends the call, the socket drops or the agent gets
SIGTERM. `--service-host` overrides the address the engine sees.
- Services and compiles take no `--parallel` slot, and a flow's detail is
built off the event loop, so a save no longer stalls the engine.
This commit is contained in:
@@ -243,6 +243,43 @@ def test_cancelling_a_run_reaches_only_that_run(loop):
|
||||
assert [frame["call_id"] for frame in cancels] == ["run-a:flow.node"]
|
||||
|
||||
|
||||
def test_a_service_and_a_compile_leave_the_only_slot_to_calls(loop):
|
||||
"""A service holding the slot for good would leave the worker blocked."""
|
||||
hub = RemoteWorkerHub()
|
||||
socket = FakeSocket()
|
||||
worker = RemoteWorker(
|
||||
name="gpu1", labels=["gpu"], send=socket.send_json, loop=loop, max_parallel=1
|
||||
)
|
||||
hub.attach(worker)
|
||||
call_in_thread(
|
||||
lambda: worker.request({"op": "serve", "call_id": "svc"}, 0, slot=False)
|
||||
)
|
||||
assert socket.arrived.wait(5)
|
||||
socket.arrived.clear()
|
||||
result: dict = {}
|
||||
thread = call_in_thread(
|
||||
lambda: result.update(
|
||||
value=hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=5)
|
||||
)
|
||||
)
|
||||
assert socket.arrived.wait(5)
|
||||
# Neither waits for the slot the call above holds.
|
||||
compiled = call_in_thread(
|
||||
lambda: result.update(problem=hub.compile("gpu", "flow", "other", "src"))
|
||||
)
|
||||
deadline = time.monotonic() + 5
|
||||
while len(socket.sent) < 3 and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
for frame in socket.sent[1:]:
|
||||
worker.deliver({"call_id": frame["call_id"], "ok": True, "result": None})
|
||||
thread.join(timeout=5)
|
||||
compiled.join(timeout=5)
|
||||
|
||||
assert [frame["op"] for frame in socket.sent] == ["serve", "run", "compile"]
|
||||
assert result == {"value": None, "problem": None}
|
||||
assert worker.in_flight == 1
|
||||
|
||||
|
||||
def test_a_cancel_after_the_loop_closed_leaves_no_coroutine_behind():
|
||||
closed = asyncio.new_event_loop()
|
||||
closed.close()
|
||||
|
||||
Reference in New Issue
Block a user