Service node: a server kept running on a worker (openpanel V2)

- `service` node type: command, port, worker label, health path, env and
  weights (an artifact digest the worker fetches as {weights}). Its flow
  starts it; it restarts with backoff, reads degraded while starting or
  without a worker and down after three starts that never got healthy, and
  publishes http://<worker>:<port> once /health answers 200, "" when it ends.
- Worker protocol 3: a `serve` call, every call in its own process group,
  killed when the engine ends the call, the socket drops or the agent gets
  SIGTERM. `--service-host` overrides the address the engine sees.
- Services and compiles take no `--parallel` slot, and a flow's detail is
  built off the event loop, so a save no longer stalls the engine.

backend/tests/flow/test_node_types.py

@@ -85,6 +85,11 @@ FIXTURES: dict[str, dict] = {
         "requires": [MessageSpec(name="go", dtype=DType.BOOL)],
         "provides": [MessageSpec(name="stdout", dtype=DType.STR)],
     },
+    "service": {
+        "params": {"command": "parrot serve --port {port}", "worker": "gpu"},
+        "requires": [],
+        "provides": [MessageSpec(name="url", dtype=DType.STR)],
+    },
     "file": {
         "params": {"path": "readings.log", "mode": "append"},
         "requires": [MessageSpec(name="line", dtype=DType.STR)],

backend/tests/flow/test_remote.py

@@ -243,6 +243,43 @@ def test_cancelling_a_run_reaches_only_that_run(loop):
     assert [frame["call_id"] for frame in cancels] == ["run-a:flow.node"]
 
 
+def test_a_service_and_a_compile_leave_the_only_slot_to_calls(loop):
+    """A service holding the slot for good would leave the worker blocked."""
+    hub = RemoteWorkerHub()
+    socket = FakeSocket()
+    worker = RemoteWorker(
+        name="gpu1", labels=["gpu"], send=socket.send_json, loop=loop, max_parallel=1
+    )
+    hub.attach(worker)
+    call_in_thread(
+        lambda: worker.request({"op": "serve", "call_id": "svc"}, 0, slot=False)
+    )
+    assert socket.arrived.wait(5)
+    socket.arrived.clear()
+    result: dict = {}
+    thread = call_in_thread(
+        lambda: result.update(
+            value=hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=5)
+        )
+    )
+    assert socket.arrived.wait(5)
+    # Neither waits for the slot the call above holds.
+    compiled = call_in_thread(
+        lambda: result.update(problem=hub.compile("gpu", "flow", "other", "src"))
+    )
+    deadline = time.monotonic() + 5
+    while len(socket.sent) < 3 and time.monotonic() < deadline:
+        time.sleep(0.01)
+    for frame in socket.sent[1:]:
+        worker.deliver({"call_id": frame["call_id"], "ok": True, "result": None})
+    thread.join(timeout=5)
+    compiled.join(timeout=5)
+
+    assert [frame["op"] for frame in socket.sent] == ["serve", "run", "compile"]
+    assert result == {"value": None, "problem": None}
+    assert worker.in_flight == 1
+
+
 def test_a_cancel_after_the_loop_closed_leaves_no_coroutine_behind():
     closed = asyncio.new_event_loop()
     closed.close()

backend/tests/flow/test_service_node.py

@@ -0,0 +1,134 @@
+"""The service node: a server on a worker, restarted when it ends, its url following."""
+
+import asyncio
+from collections.abc import Callable
+
+from fluksio.flow.messages import DType, MessageSpec
+from fluksio.flow.nodes import ServiceNode, service
+from fluksio.flow.pipeline import Pipeline
+from fluksio.flow.remote import RemoteWorker, RemoteWorkerHub
+
+
+class Agent:
+    """The worker's side of the socket: records calls, and ends one on cancel."""
+
+    def __init__(self) -> None:
+        self.sent: list[dict] = []
+        self.worker: RemoteWorker | None = None
+
+    async def send_json(self, payload: dict) -> None:
+        self.sent.append(payload)
+        if payload["op"] == "cancel":
+            self.end(payload["call_id"], "the service was stopped")
+
+    def end(self, call_id: str, reason: str) -> None:
+        assert self.worker is not None
+        self.worker.deliver(
+            {"call_id": call_id, "ok": False, "error": {"message": reason}}
+        )
+
+    def serves(self) -> list[dict]:
+        return [frame for frame in self.sent if frame["op"] == "serve"]
+
+
+async def until(check: Callable[[], bool]) -> None:
+    for _ in range(500):
+        if check():
+            return
+        await asyncio.sleep(0.01)
+    raise AssertionError("never happened")
+
+
+def test_a_service_comes_back_and_its_url_follows(monkeypatch):
+    monkeypatch.setattr(service, "BACKOFF", (0.01,))
+
+    async def healthy(client, url):
+        return None
+
+    monkeypatch.setattr(ServiceNode, "_probe", staticmethod(healthy))
+
+    async def scenario() -> None:
+        agent = Agent()
+        agent.worker = RemoteWorker(
+            name="amcomp",
+            labels=["gpu"],
+            send=agent.send_json,
+            loop=asyncio.get_running_loop(),
+            host="10.0.0.5",
+        )
+        hub = RemoteWorkerHub()
+        node = ServiceNode(
+            provides=[MessageSpec(name="url", dtype=DType.STR)],
+            params={"command": "serve --port {port}", "worker": "gpu", "port": 8110},
+        )
+        node.assign_flow("voice", "parrot")
+        pipeline = Pipeline(nodes=[node])
+        health: list[str] = []
+        node._on_health = lambda _node, status, _detail: health.append(status)
+        node.hub = hub
+
+        def url() -> object:
+            return pipeline.state.get("voice.url")
+
+        await node.start()
+        await asyncio.sleep(0.05)
+        # No worker carries the label yet.
+        assert health[-1] == "degraded"
+        assert not agent.sent
+
+        hub.attach(agent.worker)
+        await until(lambda: bool(agent.serves()))
+        await until(lambda: url() == "http://10.0.0.5:8110")
+        first = agent.serves()[0]
+        assert first["argv"] == ["serve", "--port", "8110"]
+        assert health[-1] == "ok"
+
+        # The server dies: the url goes at once, and a second call starts.
+        agent.end(first["call_id"], "the service exited with code 1")
+        await until(lambda: url() == "")
+        assert "degraded" in health[health.index("ok") :]
+        await until(lambda: len(agent.serves()) == 2)
+        await until(lambda: url() == "http://10.0.0.5:8110")
+
+        # Stopping the flow ends the call on the worker, and empties the url.
+        await node.stop()
+        second = agent.serves()[1]["call_id"]
+        assert {"op": "cancel", "call_id": second} in agent.sent
+        assert url() == ""
+
+    asyncio.run(scenario())
+
+
+def test_a_service_that_never_gets_healthy_reads_down(monkeypatch):
+    monkeypatch.setattr(service, "BACKOFF", (0.01,))
+
+    async def refused(client, url):
+        return "is not answering"
+
+    monkeypatch.setattr(ServiceNode, "_probe", staticmethod(refused))
+
+    async def scenario() -> None:
+        agent = Agent()
+        agent.worker = RemoteWorker(
+            name="amcomp",
+            labels=["gpu"],
+            send=agent.send_json,
+            loop=asyncio.get_running_loop(),
+        )
+        hub = RemoteWorkerHub()
+        hub.attach(agent.worker)
+        node = ServiceNode(params={"command": "serve", "worker": "gpu"})
+        node.assign_flow("voice", "parrot")
+        health: list[tuple[str, str]] = []
+        node._on_health = lambda _node, status, detail: health.append((status, detail))
+        node.hub = hub
+
+        await node.start()
+        for attempt in range(1, service.DOWN_AFTER + 1):
+            await until(lambda n=attempt: len(agent.serves()) == n)
+            agent.end(agent.serves()[-1]["call_id"], "Address already in use")
+        await until(lambda: health[-1][0] == "down")
+        assert "Address already in use" in health[-1][1]
+        await node.stop()
+
+    asyncio.run(scenario())

backend/tests/flow/test_worker_agent.py

@@ -1,6 +1,10 @@
 """The remote worker's own guards: where its token goes, and what node code gets."""
 
 import argparse
+import asyncio
+import json
+import time
+from pathlib import Path
 
 import pytest
 from fluksio_worker import agent
@@ -75,3 +79,59 @@ def test_only_a_whole_digest_is_opened(digest: str) -> None:
     """The digest is joined into a path, so anything else could read any file."""
     with pytest.raises(ValueError, match="not an artifact reference"):
         _Reporter("fluksio").load_artifact({"digest": digest})
+
+
+def _gone(pid: int) -> bool:
+    """No such process, or one only waiting to be reaped."""
+    try:
+        return Path(f"/proc/{pid}/stat").read_text().split()[2] == "Z"
+    except FileNotFoundError:
+        return True
+
+
+def test_stopping_a_service_takes_its_process_group_with_it(tmp_path: Path) -> None:
+    """A child of the server left running would keep its port."""
+    args = argparse.Namespace(
+        url="wss://engine.example.com/api/v1/workers/attach",
+        artifact_url="",
+        token="t",
+    )
+    worker = agent.Agent(args)
+    child = tmp_path / "child"
+    sent: list[dict] = []
+
+    class Socket:
+        async def send(self, text: str) -> None:
+            sent.append(json.loads(text))
+
+    async def scenario() -> None:
+        script = f"sleep 300 & echo $! > {child}; wait"
+        request = {"op": "serve", "call_id": "svc", "argv": ["sh", "-c", script]}
+        task = asyncio.create_task(worker._serve(Socket(), request))
+        for _ in range(500):
+            if child.exists() and child.read_text().strip():
+                break
+            await asyncio.sleep(0.01)
+        worker._cancel("svc")
+        await task
+
+    asyncio.run(scenario())
+
+    pid = int(child.read_text())
+    deadline = time.monotonic() + 5
+    while not _gone(pid) and time.monotonic() < deadline:
+        time.sleep(0.05)
+    assert _gone(pid)
+    assert not worker.serving
+    assert sent == [
+        {
+            "call_id": "svc",
+            "ok": False,
+            "error": {
+                "type": "ServiceStopped",
+                "message": "the service was stopped",
+                "short": "the service was stopped",
+                "traceback": "",
+            },
+        }
+    ]