diff --git a/backend/fluksio/api/routes/flows.py b/backend/fluksio/api/routes/flows.py index 63cc189..02d85aa 100644 --- a/backend/fluksio/api/routes/flows.py +++ b/backend/fluksio/api/routes/flows.py @@ -1026,7 +1026,10 @@ async def flow_events(websocket: WebSocket, token: str = "") -> None: out.append(event) if out: await _send(websocket, out) - except WebSocketDisconnect: + except (WebSocketDisconnect, RuntimeError): + # A peer that goes away mid-send takes the RuntimeError route + # ("websocket.send after websocket.close") rather than the clean + # disconnect. Either way the socket is gone and the loop is over. pass finally: receiver.cancel() diff --git a/backend/fluksio/flow/logs.py b/backend/fluksio/flow/logs.py index 5afa50a..9cb60bf 100644 --- a/backend/fluksio/flow/logs.py +++ b/backend/fluksio/flow/logs.py @@ -34,10 +34,20 @@ class _Tee(io.TextIOBase): sink = _sink.get() if sink is not None and text: sink(text) - return self._real.write(text) + try: + return self._real.write(text) + except OSError: + # A dead stdout — `fluksio serve` runs the engine as a child of the + # dashboard, which holds the far end of that pipe — must not fail + # the node whose output was being teed. The capture above has it, + # and there is nowhere left to report the loss to anyway. + return len(text) def flush(self) -> None: - self._real.flush() + try: + self._real.flush() + except OSError: + pass def isatty(self) -> bool: return self._real.isatty() diff --git a/backend/tests/flow/test_logs.py b/backend/tests/flow/test_logs.py index 1611919..93ad3ac 100644 --- a/backend/tests/flow/test_logs.py +++ b/backend/tests/flow/test_logs.py @@ -122,6 +122,25 @@ def test_printing_outside_a_node_still_reaches_the_real_stream(capsys): assert "server talking" in capsys.readouterr().out +def test_a_dead_real_stream_does_not_fail_the_node_being_teed(): + """`fluksio serve` runs the engine as a child of the dashboard.""" + + class Broken: + def write(self, text: str) -> int: + raise BrokenPipeError(32, "Broken pipe") + + def flush(self) -> None: + raise BrokenPipeError(32, "Broken pipe") + + collected: list[str] = [] + tee = logs._Tee(Broken()) + with logs.capture(collected.append): + assert tee.write("still teed") == len("still teed") + tee.flush() + + assert collected == ["still teed"] + + def test_installing_twice_does_not_stack_tees(): logs.install() once = sys.stdout diff --git a/worker/fluksio_worker/agent.py b/worker/fluksio_worker/agent.py index f2e7058..dc7ae4b 100644 --- a/worker/fluksio_worker/agent.py +++ b/worker/fluksio_worker/agent.py @@ -257,7 +257,11 @@ class Agent: heartbeat = asyncio.create_task(beat()) try: - await loop.run_in_executor(None, worker.send, request) + # A subprocess that died before it could be written to is reported + # by the read below, which says so and ends the call — rather than + # raising here and leaving the engine waiting out its silence. + with contextlib.suppress(OSError): + await loop.run_in_executor(None, worker.send, request) while True: line = await loop.run_in_executor(None, worker.read_line) if not line: