Backend: real alert test results, state cleanup on delete/rename, node trigger errors, queue and collector fixes

`AlertManager.send` swallowed every delivery failure, so the alerts screen's
Test button answered 200 whatever happened — the one thing it exists for. It
takes `raise_on_error` now, which only the test route passes; the per-channel
loop keeps the swallow, because one dead channel must not stop the others
hearing about the same fault. A refused delivery answers 502 with whatever the
sender said.

Renaming a flow left its values under the old name for good: the delete path
already swept them, the rename path never did. It calls the same `forget_flow`,
which covers the messages and the `__ts__`/`__version__`/`__history__`
bookkeeping keyed by message name. Cleanup, not migration — they repopulate
under the new name on the next run.

Triggering a node by hand ran `Node.__call__` with nothing catching it, so a
node that raised produced a 500 and a stack trace in the server log, and
nothing at all on the canvas. `Pipeline.publish_error` is the reporting half of
`_execute_node` lifted out; both paths go through it, so a manual failure now
reads the same on the canvas and in the metrics as a queued one. The route
answers 400 with the node's error.

`MemoryWorkQueue.stats()` counts claimed-but-unacknowledged work rather than
reporting zero, so the health tile means something without Redis. The metrics
collector's held tracebacks are capped at `DETAIL_CAP` and swept on the same
`RUN_STALE_S` cutoff the open runs use, instead of one untruncated traceback
per node kept for the life of the process — a traceback still survives the
flush between the log and the failure it belongs to.

`GET /observability/events` takes `since`/`until`, the window `/runs` already
took, so a failures list can cover the span the charts beside it are drawn from.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XC2jX6Hdj7pxGGKzBTrbqB

backend/tests/api/routes/test_alerts.py

@@ -0,0 +1,66 @@
+"""The Test button: it exists to tell a working channel from a broken one."""
+
+from typing import Any
+
+import pytest
+from fastapi.testclient import TestClient
+
+from app.core.config import settings
+from app.flow.alerts import AlertsConfig, Channel
+
+PREFIX = f"{settings.API_V1_STR}/alerts"
+
+
+def _one_channel(client: TestClient) -> Any:
+    alerts = client.app.state.flow_controller.alerts  # type: ignore[attr-defined]
+    alerts.config = AlertsConfig(
+        channels=[Channel(name="phone", kind="ntfy", config={"topic": "t"})]
+    )
+    return alerts
+
+
+def test_a_channel_that_cannot_deliver_is_not_a_success(
+    client: TestClient,
+    superuser_token_headers: dict[str, str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    alerts = _one_channel(client)
+
+    async def refuse(_config: dict[str, Any], _alert: Any) -> None:
+        raise RuntimeError("nobody home")
+
+    monkeypatch.setattr(alerts, "_send_ntfy", refuse)
+
+    response = client.post(f"{PREFIX}/test/phone", headers=superuser_token_headers)
+
+    assert response.status_code == 502
+    assert "nobody home" in response.json()["detail"]
+
+
+def test_a_channel_that_delivers_says_so(
+    client: TestClient,
+    superuser_token_headers: dict[str, str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    alerts = _one_channel(client)
+    sent = []
+
+    async def accept(_config: dict[str, Any], alert: Any) -> None:
+        sent.append(alert)
+
+    monkeypatch.setattr(alerts, "_send_ntfy", accept)
+
+    response = client.post(f"{PREFIX}/test/phone", headers=superuser_token_headers)
+
+    assert response.status_code == 200
+    assert len(sent) == 1
+
+
+def test_an_unknown_channel_is_a_404(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    _one_channel(client)
+
+    response = client.post(f"{PREFIX}/test/nowhere", headers=superuser_token_headers)
+
+    assert response.status_code == 404

backend/tests/api/routes/test_flows.py

@@ -357,3 +357,69 @@ def test_rename_rejects_an_invalid_name(
         json={"new_name": "Not A Flow Name"},
     )
     assert response.status_code == 400
+
+
+def test_a_node_that_raises_answers_with_its_error(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    """A manual run fails the way a scheduled one does, not as a 500."""
+    saved = client.put(
+        f"{PREFIX}/failing", headers=superuser_token_headers, json=a_flow("failing")
+    ).json()
+    client.put(
+        f"{PREFIX}/failing/nodes/sensor/source",
+        headers=superuser_token_headers,
+        json={"code": BROKEN_NODE},
+    )
+    client.post(
+        f"{PREFIX}/failing/publish",
+        headers=superuser_token_headers,
+        json={"version": saved["definition"]["version"]},
+    )
+
+    response = client.post(
+        f"{PREFIX}/failing/nodes/sensor/trigger",
+        headers=superuser_token_headers,
+        json={"values": {}},
+    )
+
+    assert response.status_code == 400
+    assert "boom" in response.json()["detail"]
+
+    client.delete(f"{PREFIX}/failing", headers=superuser_token_headers)
+
+
+def test_renaming_a_flow_leaves_nothing_under_the_old_name(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    saved = client.put(
+        f"{PREFIX}/movable", headers=superuser_token_headers, json=a_flow("movable")
+    ).json()
+    client.put(
+        f"{PREFIX}/movable/nodes/sensor/source",
+        headers=superuser_token_headers,
+        json={"code": WORKING_NODE},
+    )
+    client.post(
+        f"{PREFIX}/movable/publish",
+        headers=superuser_token_headers,
+        json={"version": saved["definition"]["version"]},
+    )
+    client.post(
+        f"{PREFIX}/movable/run", headers=superuser_token_headers, json={"inputs": {}}
+    )
+
+    state = client.app.state.flow_controller.state
+    assert [key for key in state.keys() if "movable." in key]
+
+    client.post(
+        f"{PREFIX}/movable/rename",
+        headers=superuser_token_headers,
+        json={"new_name": "moved"},
+    )
+
+    # The values repopulate under the new name on the next run; what the old
+    # name left behind would sit there for good.
+    assert [key for key in state.keys() if "movable." in key] == []
+
+    client.delete(f"{PREFIX}/moved", headers=superuser_token_headers)

backend/tests/api/routes/test_observability.py

@@ -134,3 +134,35 @@ def test_runs_narrow_to_one_minute(
     # The upper bound is exclusive, so the run starting the next minute is not
     # in this one.
     assert [run["id"] for run in runs] == ["minute-in"]
+
+
+def test_events_narrow_to_one_minute(
+    client: TestClient, superuser_token_headers: dict[str, str], db: Session
+) -> None:
+    """The failures list reaches as far back as the charts beside it."""
+    minute = datetime.now(timezone.utc).replace(second=0, microsecond=0) - timedelta(
+        hours=4
+    )
+    db.add(EngineEvent(ts=minute, type="node_error", flow=FLOW, detail="minute-in"))
+    db.add(
+        EngineEvent(
+            ts=minute + timedelta(minutes=1),
+            type="node_error",
+            flow=FLOW,
+            detail="minute-after",
+        )
+    )
+    db.commit()
+
+    events = client.get(
+        f"{PREFIX}/events",
+        headers=superuser_token_headers,
+        params={
+            "flow": FLOW,
+            "since": minute.isoformat(),
+            "until": (minute + timedelta(minutes=1)).isoformat(),
+        },
+    ).json()
+
+    # The upper bound is exclusive, so the failure a minute later is not in it.
+    assert [event["detail"] for event in events] == ["minute-in"]