diff --git a/backend/fluksio/api/routes/alerts.py b/backend/fluksio/api/routes/alerts.py index 97410fd..4ea3ca5 100644 --- a/backend/fluksio/api/routes/alerts.py +++ b/backend/fluksio/api/routes/alerts.py @@ -5,9 +5,11 @@ from typing import Any from fastapi import APIRouter, Depends, HTTPException from fastapi.concurrency import run_in_threadpool +from pydantic import BaseModel from fluksio.api.deps import FlowControllerDep, get_current_user from fluksio.core.config import settings +from fluksio.flow import webpush from fluksio.flow.alerts import Alert, AlertsConfig from fluksio.models import Message @@ -63,6 +65,36 @@ async def save_alerts_config(body: AlertsConfig, controller: FlowControllerDep) return body +class WebPushKey(BaseModel): + key: str + + +class Unsubscribe(BaseModel): + endpoint: str + + +@router.get("/webpush/key", response_model=WebPushKey) +async def read_webpush_key() -> Any: + """The key a browser subscribes against. Made on the first ask.""" + return WebPushKey(key=await run_in_threadpool(webpush.public_key)) + + +@router.post("/webpush/subscriptions", response_model=Message) +async def add_webpush_subscription(body: webpush.Subscription) -> Any: + """Remember this browser, so a `webpush` channel can reach it.""" + if not body.keys.get("p256dh") or not body.keys.get("auth"): + raise HTTPException(status_code=422, detail="The subscription has no keys") + await run_in_threadpool(webpush.add_subscription, body) + return Message(message="This browser will be notified") + + +@router.post("/webpush/unsubscribe", response_model=Message) +async def remove_webpush_subscription(body: Unsubscribe) -> Any: + """Forget this browser.""" + await run_in_threadpool(webpush.remove_subscription, body.endpoint) + return Message(message="This browser will no longer be notified") + + @router.post("/test/{channel_name}", response_model=Message) async def test_channel(channel_name: str, controller: FlowControllerDep) -> Any: """Send one alert, to prove the channel works before relying on it.""" diff --git a/backend/fluksio/core/config.py b/backend/fluksio/core/config.py index 22b7c83..024c426 100644 --- a/backend/fluksio/core/config.py +++ b/backend/fluksio/core/config.py @@ -22,6 +22,7 @@ DERIVED_PATHS = { "FLOWS_DIR": "flows", "SECRETS_FILE": "secrets.enc", "ALERTS_FILE": "alerts.json", + "WEBPUSH_FILE": "webpush.json", "PANELS_FILE": "panels.json", "OAUTH_PRIVATE_KEY_FILE": "oauth-key.pem", "CLOUD_CONFIG_FILE": "cloud.json", @@ -71,6 +72,10 @@ class Settings(BaseSettings): # Which failures reach which channel. Beside the flows, not in them: # alerting is the deployment's concern, not any one flow's. ALERTS_FILE: Path = Path("flow-data/alerts.json") + # This installation's web push keypair and the browsers subscribed to it. + # Beside the alerts it serves; deleting it makes every device subscribe + # again. + WEBPUSH_FILE: Path = Path("flow-data/webpush.json") # Where machines can be started from when a node needs one and nothing that # could take it is attached. Operator-authored, like the alerts beside it, # and absent on an installation that has nowhere to start one. diff --git a/backend/fluksio/flow/alerts.py b/backend/fluksio/flow/alerts.py index 6213215..8d31cb1 100644 --- a/backend/fluksio/flow/alerts.py +++ b/backend/fluksio/flow/alerts.py @@ -36,6 +36,7 @@ ALERTING_EVENTS = { "engine_degraded", "cascade_dropped", "queue_unavailable", + "run_finished", } # The same fault repeating is the same alert. @@ -52,10 +53,11 @@ class Channel(BaseModel): """Somewhere to send an alert.""" name: str - kind: Literal["ntfy", "smtp", "webhook", "dashboard"] + kind: Literal["ntfy", "smtp", "webhook", "dashboard", "webpush"] enabled: bool = True # ntfy: server + topic + optional token. smtp: to. webhook: url. # dashboard: message — the record a notification widget reads. + # webpush: nothing — it goes to whichever browsers subscribed here. config: dict[str, Any] = Field(default_factory=dict) @@ -144,6 +146,15 @@ def describe(event: dict[str, Any]) -> Alert | None: ) if kind == "queue_unavailable": return Alert(title="The work queue is unreachable", body=error or "") + if kind == "run_finished": + # Every run ends; only the ones that ended badly are worth a phone. + if event.get("status") != "error": + return None + return Alert( + title=f"A run of '{flow}' failed", + body=f"Run {event.get('run')} finished with errors.", + flow=flow, + ) return None @@ -304,6 +315,8 @@ class AlertManager: await self._send_webhook(config, alert) elif channel.kind == "dashboard": await self._send_dashboard(config, alert) + elif channel.kind == "webpush": + await self._send_webpush(alert) else: await self._send_email(config, alert) except Exception as exc: @@ -344,6 +357,18 @@ class AlertManager: # Blocking: it reads the store to find the declared message. await asyncio.to_thread(self.publish, message, alert.model_dump()) + async def _send_webpush(self, alert: Alert) -> None: + """Wake every browser that subscribed to this installation. + + No settings of its own: a browser subscribes by pressing a button on + the alerts screen, and this channel goes to whichever ones did. Saying + so when none have is the point of the test button. + """ + from fluksio.flow import webpush + + if await webpush.send_to_all(alert.model_dump()) == 0: + raise ValueError("No browsers are subscribed") + async def _send_webhook(self, config: dict[str, Any], alert: Alert) -> None: url = config.get("url") if not url: diff --git a/backend/fluksio/flow/webpush.py b/backend/fluksio/flow/webpush.py new file mode 100644 index 0000000..5c9913c --- /dev/null +++ b/backend/fluksio/flow/webpush.py @@ -0,0 +1,241 @@ +"""Web Push: an alert reaching a phone that has this installation installed. + +The browser hands us a subscription — an endpoint URL at its own push service, +plus two keys — and from then on the engine can wake that device without it +holding a connection open. Two pieces of crypto are involved and both are +specified: the payload is encrypted to the subscription's keys (RFC 8291, +aes128gcm) so the push service carries something it cannot read, and the +request is signed with this installation's own keypair (VAPID, RFC 8292) so the +service knows who is sending. + +Only ``http-ece`` is new here; the signing is `pyjwt` and the request is +`httpx`, both of which the engine already carries. ``pywebpush`` does all of +this in one call, but brings `requests` *and* `aiohttp` with it — two more HTTP +stacks on a machine that may well be a Raspberry Pi. + +The keypair is this installation's identity to the push services and lives with +the subscriptions in one file. Losing it means every browser has to subscribe +again; it is regenerated on the spot if the file goes missing. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import logging +import threading +import time +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +import http_ece +import httpx +import jwt +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec +from pydantic import BaseModel, Field + +from fluksio.core.config import settings + +logger = logging.getLogger(__name__) + +#: How long the push service keeps a notification for a device that is offline. +TTL_S = 86400 +#: A VAPID token is good for twelve hours; well under the 24 the RFC allows. +TOKEN_LIFETIME_S = 12 * 60 * 60 + +# ponytail: one lock over the whole file. Subscriptions change when somebody +# presses a button, and pruning happens once per send — per-record locking +# would be machinery for a file with single-digit rows in it. +_lock = threading.Lock() + + +class Subscription(BaseModel): + """What a browser handed us. Opaque apart from the endpoint's host.""" + + endpoint: str + #: The subscription's public key and its authentication secret, both + #: base64url as `PushSubscription.toJSON()` writes them. + keys: dict[str, str] = Field(default_factory=dict) + + +class _Store(BaseModel): + #: This installation's VAPID private key: the raw P-256 scalar, base64url. + private_key: str = "" + subscriptions: list[Subscription] = Field(default_factory=list) + + +def _b64url(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + +def _unb64url(text: str) -> bytes: + # A browser writes these unpadded; base64 refuses them that way. + return base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)) + + +def _path() -> Path: + return settings.WEBPUSH_FILE + + +def _read() -> _Store: + """Blocking. Call under `_lock` when a write follows.""" + path = _path() + if not path.exists(): + return _Store() + try: + return _Store.model_validate_json(path.read_text()) + except Exception: + # A file that no longer parses must not stop the engine alerting + # through its other channels. + logger.error("The web push store is unreadable; starting a new one") + return _Store() + + +def _write(store: _Store) -> None: + path = _path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(store.model_dump_json(indent=2)) + + +def _private_key(store: _Store) -> ec.EllipticCurvePrivateKey: + return ec.derive_private_key( + int.from_bytes(_unb64url(store.private_key), "big"), ec.SECP256R1() + ) + + +def _public_bytes(key: ec.EllipticCurvePrivateKey) -> bytes: + return key.public_key().public_bytes( + serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint + ) + + +def public_key() -> str: + """This installation's VAPID public key, generating the pair on first ask. + + Base64url of the uncompressed point, which is the shape + `pushManager.subscribe` wants for `applicationServerKey`. Blocking. + """ + with _lock: + store = _read() + if not store.private_key: + key = ec.generate_private_key(ec.SECP256R1()) + store.private_key = _b64url( + key.private_numbers().private_value.to_bytes(32, "big") + ) + _write(store) + logger.info("Generated this installation's web push keypair") + return _b64url(_public_bytes(_private_key(store))) + + +def add_subscription(subscription: Subscription) -> None: + """Remember a browser, replacing what we knew about that endpoint. Blocking.""" + with _lock: + store = _read() + store.subscriptions = [ + s for s in store.subscriptions if s.endpoint != subscription.endpoint + ] + store.subscriptions.append(subscription) + _write(store) + + +def remove_subscription(endpoint: str) -> None: + """Blocking.""" + with _lock: + store = _read() + store.subscriptions = [s for s in store.subscriptions if s.endpoint != endpoint] + _write(store) + + +def subscriptions() -> list[Subscription]: + """Blocking.""" + with _lock: + return _read().subscriptions + + +def _vapid_headers(key: ec.EllipticCurvePrivateKey, endpoint: str) -> dict[str, str]: + """Prove to the push service which installation is sending (RFC 8292).""" + origin = urlparse(endpoint) + token = jwt.encode( + { + "aud": f"{origin.scheme}://{origin.netloc}", + "exp": int(time.time()) + TOKEN_LIFETIME_S, + # Who to shout at about this traffic. A push service will not read + # it unless something has gone wrong. + "sub": f"mailto:{settings.EMAILS_FROM_EMAIL or 'admin@fluksio.invalid'}", + }, + key, + algorithm="ES256", + ) + return {"Authorization": f"vapid t={token},k={_b64url(_public_bytes(key))}"} + + +def _encrypt(payload: bytes, subscription: Subscription) -> bytes: + """Encrypt to this subscription's keys, so only that browser can read it.""" + # Ephemeral, per message: the whole point of the exchange is that this key + # and the subscription's agree on a secret nobody else can derive. + ours = ec.generate_private_key(ec.SECP256R1()) + encrypted: bytes = http_ece.encrypt( + payload, + private_key=ours, + dh=_unb64url(subscription.keys["p256dh"]), + auth_secret=_unb64url(subscription.keys["auth"]), + version="aes128gcm", + ) + return encrypted + + +async def send_to_all(payload: dict[str, Any]) -> int: + """Push one payload to every subscribed browser. Returns how many took it. + + A subscription the push service has retired (404/410) is dropped here — + that is the only signal we get that a browser is never coming back, and + without acting on it the store grows a dead entry per reinstalled phone. + """ + # The keypair exists by now — nothing can have subscribed without asking + # for it — but generating it here costs nothing and keeps this callable. + await asyncio.to_thread(public_key) + with _lock: + store = _read() + known = store.subscriptions + if not known: + return 0 + key = _private_key(store) + + body = json.dumps(payload).encode() + delivered = 0 + gone: list[str] = [] + async with httpx.AsyncClient(timeout=10) as client: + for subscription in known: + try: + encrypted = await asyncio.to_thread(_encrypt, body, subscription) + response = await client.post( + subscription.endpoint, + content=encrypted, + headers={ + "Content-Encoding": "aes128gcm", + "Content-Type": "application/octet-stream", + "TTL": str(TTL_S), + **_vapid_headers(key, subscription.endpoint), + }, + ) + except Exception as exc: + logger.error("Could not push to a subscription: %s", exc) + continue + if response.status_code in (404, 410): + gone.append(subscription.endpoint) + elif response.is_success: + delivered += 1 + else: + logger.error( + "A push service refused the notification: %s %s", + response.status_code, + response.text[:200], + ) + + for endpoint in gone: + await asyncio.to_thread(remove_subscription, endpoint) + logger.info("Dropped a subscription the push service has retired") + return delivered diff --git a/backend/pyproject.toml b/backend/pyproject.toml index a25dfbf..ba3ce16 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -72,6 +72,11 @@ dependencies = [ "rich>=13", # `fluksio serve` opens a dashboard with it at a terminal. "textual>=1.0", + # The encrypted-payload half of web push (RFC 8291). It brings only + # cryptography, which is already here. `pywebpush` would do the signing and + # the request too, at the cost of `requests` *and* `aiohttp` — two HTTP + # stacks beside httpx, on a machine that may be a Raspberry Pi. + "http-ece>=1.2", ] [project.optional-dependencies] @@ -143,6 +148,11 @@ implicit_reexport = true module = ["pyarrow", "pyarrow.*"] ignore_missing_imports = true +# The web push payload encoder carries no annotations. +[[tool.mypy.overrides]] +module = ["http_ece"] +ignore_missing_imports = true + [tool.ruff] target-version = "py312" diff --git a/backend/tests/flow/test_alerts.py b/backend/tests/flow/test_alerts.py index a310b33..4f07326 100644 --- a/backend/tests/flow/test_alerts.py +++ b/backend/tests/flow/test_alerts.py @@ -188,6 +188,13 @@ def test_alerting_can_be_switched_off(): "The work queue is unreachable", ), ({"type": "engine_degraded", "reason": "lag"}, "The engine is struggling"), + # Every run ends. Only a failed one is worth a phone buzzing. + ( + {"type": "run_finished", "flow": "nightly", "run": 7, "status": "error"}, + "A run of 'nightly' failed", + ), + ({"type": "run_finished", "flow": "nightly", "status": "ok"}, None), + ({"type": "run_finished", "flow": "nightly", "status": "cancelled"}, None), ({"type": "node_health", "health": "ok"}, None), # The engine publishes `health`, not `status`: reading the wrong key # meant a device dropping never alerted anyone. diff --git a/backend/tests/flow/test_webpush.py b/backend/tests/flow/test_webpush.py new file mode 100644 index 0000000..a83d46c --- /dev/null +++ b/backend/tests/flow/test_webpush.py @@ -0,0 +1,142 @@ +"""Web push: the crypto is the part that cannot be checked by reading it.""" + +import asyncio +import os + +import http_ece +import httpx +import jwt +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec + +from fluksio.core.config import settings +from fluksio.flow import webpush + + +@pytest.fixture(autouse=True) +def store(tmp_path, monkeypatch): + """A store per test, so nothing writes into the real installation.""" + monkeypatch.setattr(settings, "WEBPUSH_FILE", tmp_path / "webpush.json") + return tmp_path / "webpush.json" + + +def browser( + name: str = "one", +) -> tuple[ec.EllipticCurvePrivateKey, bytes, webpush.Subscription]: + """What a browser hands over when it subscribes.""" + key = ec.generate_private_key(ec.SECP256R1()) + auth = os.urandom(16) + point = key.public_key().public_bytes( + serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint + ) + return ( + key, + auth, + webpush.Subscription( + endpoint=f"https://push.example.com/{name}", + keys={"p256dh": webpush._b64url(point), "auth": webpush._b64url(auth)}, + ), + ) + + +def test_only_the_subscribed_browser_can_read_the_payload(): + """RFC 8291, and the reason the push service is not trusted with content.""" + key, auth, subscription = browser() + payload = b'{"title":"A run failed"}' + + sealed = webpush._encrypt(payload, subscription) + + assert sealed != payload + assert ( + http_ece.decrypt(sealed, private_key=key, auth_secret=auth, version="aes128gcm") + == payload + ) + + +def test_the_request_is_signed_for_the_push_service_that_gets_it(): + """RFC 8292: the audience is the endpoint's origin, never its path.""" + key = ec.generate_private_key(ec.SECP256R1()) + + headers = webpush._vapid_headers(key, "https://push.example.com/quite/a/long/path") + + scheme, rest = headers["Authorization"].split(" ", 1) + parts = dict(part.split("=", 1) for part in rest.split(",")) + assert scheme == "vapid" + claims = jwt.decode( + parts["t"], + key.public_key(), + algorithms=["ES256"], + audience="https://push.example.com", + ) + assert claims["sub"].startswith("mailto:") + # The service looks this key up against the one the browser subscribed with. + assert webpush._unb64url(parts["k"]) == webpush._public_bytes(key) + + +def test_the_keypair_survives_the_next_ask(): + first = webpush.public_key() + assert webpush.public_key() == first + + +def push_to(handler) -> int: + """Send one alert to two browsers, the push services answering `handler`.""" + webpush.add_subscription(browser("gone")[2]) + webpush.add_subscription(browser("here")[2]) + + transport = httpx.MockTransport(handler) + original = httpx.AsyncClient + + def client(**kwargs): + kwargs["transport"] = transport + return original(**kwargs) + + webpush.httpx.AsyncClient = client # type: ignore[misc] + try: + return asyncio.run(webpush.send_to_all({"title": "t", "body": "b"})) + finally: + webpush.httpx.AsyncClient = original # type: ignore[misc] + + +def test_every_subscribed_browser_is_pushed_to(): + seen: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(str(request.url)) + assert request.headers["content-encoding"] == "aes128gcm" + assert request.headers["authorization"].startswith("vapid t=") + return httpx.Response(201) + + assert push_to(handler) == 2 + assert len(seen) == 2 + + +def test_a_retired_subscription_is_dropped(): + """410 is the only word we get that a browser is never coming back.""" + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(410 if request.url.path.endswith("gone") else 201) + + delivered = push_to(handler) + + assert delivered == 1 + assert [s.endpoint for s in webpush.subscriptions()] == [ + "https://push.example.com/here" + ] + + +def test_a_channel_with_nobody_subscribed_says_so(): + """What the test button on the alerts screen reports.""" + from fluksio.flow.alerts import Alert, AlertManager, Channel + from fluksio.flow.events import EventBus + + alerts = AlertManager(EventBus()) + + with pytest.raises(ValueError, match="No browsers"): + asyncio.run( + alerts.send( + Channel(name="phones", kind="webpush"), + Alert(title="t", body="b"), + raise_on_error=True, + ) + ) diff --git a/docker/compose.yml b/docker/compose.yml index 75f5e03..328d073 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -129,6 +129,10 @@ services: # be lost on the next rebuild — un-pairing every screen. - ALERTS_FILE=/data/alerts.json - PANELS_FILE=/data/panels.json + # The push keypair and the browsers subscribed to it. Off the volume, a + # rebuild would silently stop every phone being notified: the key they + # subscribed against would be gone. + - WEBPUSH_FILE=/data/webpush.json # Schedules are written in local time: "off at 02:00" means the house's # two in the morning, not the container's. Unset, an image is UTC, and a # cron would be right twice a year. diff --git a/frontend/index.html b/frontend/index.html index 5e1f483..ad124a6 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -5,6 +5,11 @@ + + +