From 0b4eb704521528bb1344a8ff332f895f2845e17f Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:24:32 +0200 Subject: [PATCH 001/103] LAN proxy: serve /hooks next to /api, so webhooks and served sockets reach the backend instead of the SPA Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docker/compose.lan.yml | 3 ++- docker/nginx-api-proxy.conf | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docker/compose.lan.yml b/docker/compose.lan.yml index 6dcea53..5b2eba6 100644 --- a/docker/compose.lan.yml +++ b/docker/compose.lan.yml @@ -3,7 +3,8 @@ # For reaching the app from a machine that cannot be given a DNS record or a # hosts entry. Traefik routes on the Host header and a bare IP matches no # router, so the frontend is published on a host port instead and proxies the -# API on that same port: http://:${APP_PORT} is the whole app. +# API and the hooks on that same port: http://:${APP_PORT} is the +# whole app. services: frontend: diff --git a/docker/nginx-api-proxy.conf b/docker/nginx-api-proxy.conf index 1c36445..b09b499 100644 --- a/docker/nginx-api-proxy.conf +++ b/docker/nginx-api-proxy.conf @@ -4,7 +4,10 @@ # client asks for /api/v1/... on whatever origin served it, and never needs a # resolvable api.${DOMAIN} or a CORS allowance. -location /api { +# /hooks is where the flows' webhooks and served sockets live, next to the API +# on the backend. One regex takes both, and a regex outranks `location /`, +# which would otherwise answer /hooks with the SPA. +location ~ ^/(api|hooks) { # Docker's embedded DNS, re-resolved: nginx otherwise pins the backend's # address at startup and 502s for good once that container is recreated. resolver 127.0.0.11 valid=10s ipv6=off; @@ -18,8 +21,9 @@ location /api { # header naively would take it for the caller. proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; - # The flow websocket (/api/v1/flows/ws) upgrades through here; passing the - # client's own Connection header covers both it and ordinary requests. + # The flow websocket (/api/v1/flows/ws) and a served socket upgrade through + # here; passing the client's own Connection header covers both and + # ordinary requests. proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $http_connection; # A live socket outlives the 60s default by design. From 02f43bf7c4a76471d52bdc94f69a4d217051dc78 Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:27:53 +0200 Subject: [PATCH 002/103] test-frontend: refuse to run when the api container answers on another domain than the frontend Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e8b9a77..df55045 100644 --- a/Makefile +++ b/Makefile @@ -175,7 +175,9 @@ PW_VERSION = $(shell sed -n 's/.*"@playwright\/test": "[^0-9]*\([0-9.]*\)".*/\1/ # This suite creates and deletes flows, dashboards and users, so the hostname it # is pointed at is not taken from any file: it is read off the running stack # (the frontend container's own Traefik rule) and mapped onto the local Traefik -# by address, so DNS never decides. tests/guard.ts is the second line. +# by address, so DNS never decides. tests/guard.ts is the second line. The +# suite calls api. even on a LAN stack, so an api container recreated +# under another DOMAIN would answer every call with Traefik's 404. test-frontend: ## Run frontend tests (Playwright e2e) against the local stack @ip=$$(docker network inspect proxy \ --format '{{range .Containers}}{{if eq .Name "fluksio-app-proxy-1"}}{{.IPv4Address}}{{end}}{{end}}' \ @@ -184,6 +186,9 @@ test-frontend: ## Run frontend tests (Playwright e2e) against the local stack domain=$$(docker inspect fluksio-app 2>/dev/null \ | grep -o 'Host(`app\.[^`]*`)' | head -1 | sed 's/Host(`app\.//;s/`)//'); \ [ -n "$$domain" ] || { echo " ✗ no running app stack to test — 'make dev' first"; exit 1; }; \ + api=$$(docker inspect fluksio-api 2>/dev/null \ + | grep -o 'Host(`api\.[^`]*`)' | head -1 | sed 's/Host(`api\.//;s/`)//'); \ + [ "$$api" = "$$domain" ] || { echo " ✗ api container is on api.$${api:-?}, the app on app.$$domain — recreate the stack with 'make dev'"; exit 1; }; \ docker run --rm --network proxy --ipc=host \ --user $$(id -u):$$(id -g) -e HOME=/tmp \ --add-host app.$$domain:$$ip --add-host api.$$domain:$$ip \ From 8eec58d1d64e8e4861dcfa959f110505dd0c668d Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:29:48 +0200 Subject: [PATCH 003/103] Runs export: a truncated one counts the selection into X-Total-Count; the SDK and `fluksio export` warn how many runs were left out Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/runs.py | 34 +++++++++++++++------------ backend/fluksio/sdk/cli.py | 7 +++++- backend/fluksio/sdk/client.py | 13 ++++++++++ backend/tests/api/routes/test_runs.py | 23 ++++++++++++++++++ docs/code/api.md | 4 +++- docs/code/cli.md | 2 ++ 6 files changed, 66 insertions(+), 17 deletions(-) diff --git a/backend/fluksio/api/routes/runs.py b/backend/fluksio/api/routes/runs.py index e1c1993..56711e1 100644 --- a/backend/fluksio/api/routes/runs.py +++ b/backend/fluksio/api/routes/runs.py @@ -512,15 +512,22 @@ def _selected( ids: str, since: datetime | None, until: datetime | None, -) -> list[Run]: - """The runs an export covers, newest first — the filters the list takes. +) -> tuple[list[Run], int]: + """The runs an export covers, newest first, and how many the filters match. Capped: the filters bound a *sensible* request and nothing bounded an unfiltered one, so asking for everything read every row of the table into - memory before a byte was sent. A truncated export says so in a header. + memory before a byte was sent. A truncated export says so in a header, and + only then is the selection counted, since otherwise the rows are the count. """ statement = _selection(flow, status, group, ids, since, until) - return list(session.exec(statement.limit(EXPORT_CAP + 1))) + runs = list(session.exec(statement.limit(EXPORT_CAP + 1))) + if len(runs) <= EXPORT_CAP: + return runs, len(runs) + total = session.exec( + select(func.count()).select_from(statement.order_by(None).subquery()) + ).one() + return runs[:EXPORT_CAP], total def _cell(value: Any) -> Any: @@ -613,7 +620,7 @@ def _stream( name: str, columns: list[str], chunks: Iterator[list[dict[str, Any]]], - truncated: bool = False, + total: int = 0, ) -> StreamingResponse: """The rows out, a chunk at a time rather than a list. @@ -629,11 +636,12 @@ def _stream( body = _csv(columns, chunks) media = "text/csv; charset=utf-8" headers = {"Content-Disposition": f'attachment; filename="{name}.{fmt}"'} - if truncated: + if total > EXPORT_CAP: # The body is otherwise indistinguishable from a complete one, and an # export quietly missing its oldest runs is the wrong thing to hand - # somebody who is about to plot it. + # somebody who is about to plot it. The total says how many it missed. headers["X-Truncated"] = str(EXPORT_CAP) + headers["X-Total-Count"] = str(total) return StreamingResponse(body, media_type=media, headers=headers) @@ -656,9 +664,7 @@ def export_metrics( the metrics it lists; ``stride`` thins each curve — per series, so asking for every tenth point of two metrics gives every tenth point of both. """ - runs = _selected(session, flow, status, group, ids, since, until) - truncated = len(runs) > EXPORT_CAP - runs = runs[:EXPORT_CAP] + runs, total = _selected(session, flow, status, group, ids, since, until) wanted = [part for part in name.split(",") if part] step = max(1, stride) @@ -682,7 +688,7 @@ def export_metrics( ) yield rows - return _stream(format, "metrics", list(METRIC_COLUMNS), chunks(), truncated) + return _stream(format, "metrics", list(METRIC_COLUMNS), chunks(), total) @router.get("/export/runs") @@ -710,9 +716,7 @@ def export_runs( three fields rather than two blobs, and the defaults reach the same depth. """ - runs = _selected(session, flow, status, group, ids, since, until) - truncated = len(runs) > EXPORT_CAP - runs = runs[:EXPORT_CAP] + runs, total = _selected(session, flow, status, group, ids, since, until) inputs = [part for part in params.split(",") if part] or sorted( {path for run in runs for path, _ in _leaves(run.params)} ) @@ -730,7 +734,7 @@ def export_runs( row.update((f"metric.{k}", _cell(_dig(run.result, k))) for k in scores) yield [row] - return _stream(format, "runs", columns, chunks(), truncated) + return _stream(format, "runs", columns, chunks(), total) @router.get("/metrics/names", response_model=list[str]) diff --git a/backend/fluksio/sdk/cli.py b/backend/fluksio/sdk/cli.py index 7393d56..4c651a1 100644 --- a/backend/fluksio/sdk/cli.py +++ b/backend/fluksio/sdk/cli.py @@ -17,6 +17,7 @@ import pkgutil import shutil import sys import time +import warnings from collections.abc import Callable, Iterable, Iterator from contextlib import contextmanager, nullcontext from datetime import UTC, datetime @@ -1326,7 +1327,11 @@ def _export( try: if getattr(args, "list_names", False): return _list_names(client, args) - rows = fetch(client) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + rows = fetch(client) + for warning in caught: + print(f"fluksio: warning: {warning.message}", file=sys.stderr) except ApiError as exc: if exc.status != 404: raise diff --git a/backend/fluksio/sdk/client.py b/backend/fluksio/sdk/client.py index 0ef0ded..fa71835 100644 --- a/backend/fluksio/sdk/client.py +++ b/backend/fluksio/sdk/client.py @@ -12,6 +12,7 @@ import os import subprocess import time import uuid +import warnings from collections.abc import Iterable from pathlib import Path from typing import Any @@ -511,6 +512,8 @@ class Client: jsonl on the wire rather than csv: it keeps the types the engine has, and it is the format the CLI converts to parquet from. Read whole — the streaming is the engine's side, and a notebook wants a list. + A selection past the engine's cap comes back without its oldest runs, + and that is a warning, since the rows alone look complete. """ if flow: query["flow"] = flow @@ -523,6 +526,16 @@ class Client: response = self._request("GET", path, idempotent=True, params=query) if response.status_code >= 400: raise ApiError(response.status_code, _detail(response)) + cap = response.headers.get("X-Truncated") + if cap: + # An engine older than the count sends the cap alone. + total = response.headers.get("X-Total-Count") + left = f"{int(total) - int(cap)} older runs" if total else "older runs" + warnings.warn( + f"the export stops at {cap} runs, so {left} of this selection " + "were left out; narrow it by flow, status, since or until", + stacklevel=3, + ) return [json.loads(line) for line in response.text.splitlines() if line] def cancel(self, run_id: str) -> Any: diff --git a/backend/tests/api/routes/test_runs.py b/backend/tests/api/routes/test_runs.py index 35bd238..e34f331 100644 --- a/backend/tests/api/routes/test_runs.py +++ b/backend/tests/api/routes/test_runs.py @@ -1477,3 +1477,26 @@ def test_runs_page_by_when_they_were_created(client, superuser_token_headers, pa "page-2", "page-1", ] + + +@pytest.mark.usefixtures("paged") +def test_a_truncated_export_says_how_many_runs_it_left_out( + client, superuser_token_headers, monkeypatch +): + """The cap is in one header, the selection's size in another, and the SDK + turns the two into a warning naming what is missing.""" + from fluksio.sdk.client import Client + + monkeypatch.setattr("fluksio.api.routes.runs.EXPORT_CAP", 2) + answer = client.get( + f"{settings.API_V1_STR}/runs/export/runs", + params={"flow": "paged-study", "format": "jsonl"}, + headers=superuser_token_headers, + ) + assert answer.headers["X-Truncated"] == "2" + assert answer.headers["X-Total-Count"] == "3" + assert [row["id"] for row in _lines(answer)] == ["page-2", "page-1"] + + sdk = Client(http=client, token=superuser_token_headers["Authorization"][7:]) + with pytest.warns(UserWarning, match="1 older runs"): + assert len(sdk.export_runs(flow="paged-study")) == 2 diff --git a/docs/code/api.md b/docs/code/api.md index e608e89..065205c 100644 --- a/docs/code/api.md +++ b/docs/code/api.md @@ -143,7 +143,9 @@ row's `created_at` as `before` for stable pagination. Exports also accept `ids` and `until`, with `format=csv` or `jsonl`. The metrics export is a long table; the runs export is one row per run. Use `params` and `metrics` to choose columns, including dotted paths into -record outputs. +record outputs. An export covers at most the newest 10 000 runs. When a +selection matches more, the response carries `X-Truncated` with that cap and +`X-Total-Count` with the number of runs the selection matched. Import takes `{"runs": [...]}`, records completed results without executing code, and deduplicates by `external_id`. See diff --git a/docs/code/cli.md b/docs/code/cli.md index 301d7cb..7810998 100644 --- a/docs/code/cli.md +++ b/docs/code/cli.md @@ -297,6 +297,8 @@ select columns, including dotted paths such as `final_metrics.train_loss`. Both support `--flow`, repeated `--run ID`, `--group`, `--status`, `--since`, `--until` and `--local`. The run ID joins results to metrics. +An export stops at the newest 10 000 runs, and when a selection matches more, +the CLI and the Python client warn how many were left out. Output defaults to CSV on stdout. `-o FILE` writes a file; `--format jsonl` writes JSON Lines. Parquet requires `pip install 'fluksio[parquet]'` and From 98f4669b82ea1024aa72d248feb7960382b6f4f3 Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:30:46 +0200 Subject: [PATCH 004/103] Artifact sweep keeps what a flow input's initial references, published or draft Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/runs.py | 29 +++++++++++++++++++++-------- backend/fluksio/main.py | 1 + backend/tests/test_artifacts.py | 24 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/backend/fluksio/flow/runs.py b/backend/fluksio/flow/runs.py index 1082af5..a465891 100644 --- a/backend/fluksio/flow/runs.py +++ b/backend/fluksio/flow/runs.py @@ -56,7 +56,7 @@ from fluksio.flow.pipeline import CacheHit, NodeOutcome, Pipeline from fluksio.flow.queue import WorkItem, WorkQueue from fluksio.flow.schemas import FlowDef, NodeDef from fluksio.flow.state import MemoryState, StateBackend -from fluksio.flow.store import FlowStore +from fluksio.flow.store import FlowNotFound, FlowStore from fluksio.models import Run, RunArtifact, RunMetric, RunNode from fluksio.sdk.imports import digest_of as files_digest @@ -375,16 +375,20 @@ def _references_in(value: Any, found: set[str], depth: int = 0) -> None: def sweep_artifacts( - store: ArtifactStore, state: StateBackend, grace_s: float = 0.0 + store: ArtifactStore, + state: StateBackend, + grace_s: float = 0.0, + flows: FlowStore | None = None, ) -> int: """Remove artifact bytes nothing refers to any more. Returns how many went. - Two things refer to an artifact: a run that recorded it, and a message - currently holding it. The second is what makes a stream of media - affordable — a camera publishing a frame a second replaces the reference - each time, so yesterday's frames are unreferenced by definition and the - store does not grow without bound. What a node *returns* is recorded - against its run and kept; what it emits along the way is not. + Three things refer to an artifact: a run that recorded it, a message + currently holding it, and a flow input's ``initial``, published or draft, + which a flow that has never started holds nowhere else. The second is what + makes a stream of media affordable — a camera publishing a frame a second + replaces the reference each time, so yesterday's frames are unreferenced by + definition and the store does not grow without bound. What a node *returns* + is recorded against its run and kept, what it emits along the way is not. Skipped entirely while anything is running: a node that stores a checkpoint an hour before it returns has neither a row nor a message naming it yet, @@ -399,6 +403,15 @@ def sweep_artifacts( return 0 keep = set(session.exec(select(col(RunArtifact.digest)).distinct()).all()) + for name in flows.list_flows() if flows else (): + for draft in (False, True): + try: + flow = flows.read_flow(name, draft=draft) + except FlowNotFound: + continue + for declared in flow.inputs: + _references_in(declared.initial, keep) + keys = state.keys() for start in range(0, len(keys), _SWEEP_BATCH): for value in state.get_multi(keys[start : start + _SWEEP_BATCH]).values(): diff --git a/backend/fluksio/main.py b/backend/fluksio/main.py index 638f1d9..dd534d5 100644 --- a/backend/fluksio/main.py +++ b/backend/fluksio/main.py @@ -145,6 +145,7 @@ async def _sweep_artifacts(store: ArtifactStore, controller: FlowController) -> store, controller.state, settings.ARTIFACT_GC_GRACE_S, + controller.store, ) except Exception: logger.exception("Artifact sweep failed") diff --git a/backend/tests/test_artifacts.py b/backend/tests/test_artifacts.py index ecaa3c4..cbe145c 100644 --- a/backend/tests/test_artifacts.py +++ b/backend/tests/test_artifacts.py @@ -9,8 +9,11 @@ from sqlmodel import Session, col, select from fluksio.core.config import settings from fluksio.core.db import engine as db_engine from fluksio.flow.artifacts import ArtifactStore, VolatileStore +from fluksio.flow.messages import MessageSpec from fluksio.flow.runs import new_run_id, sweep_artifacts +from fluksio.flow.schemas import FlowDef, FlowInput from fluksio.flow.state import MemoryState +from fluksio.flow.store import FlowStore from fluksio.models import Run, RunArtifact @@ -112,6 +115,27 @@ def test_a_streamed_chunk_falls_out_once_the_message_moves_on(store): assert store.path(second["digest"]) is not None +def test_a_sweep_keeps_what_a_flow_input_starts_with(store, tmp_path): + """A flow that has never run holds its `initial` in no message yet.""" + published = store.put([b"published initial"]) + drafted = store.put([b"draft initial"]) + orphan = store.put([b"referred to by nothing"]) + + flows = FlowStore(tmp_path / "flows") + clip = MessageSpec(name="clip", dtype="artifact") + flows.write_flow( + FlowDef(name="f", inputs=[FlowInput(spec=clip, initial=published)]) + ) + flows.write_draft( + FlowDef(name="g", inputs=[FlowInput(spec=clip, initial=drafted)]), 0 + ) + + assert sweep_artifacts(store, MemoryState(), grace_s=0, flows=flows) == 1 + assert store.path(orphan["digest"]) is None + for kept in (published, drafted): + assert store.path(kept["digest"]) is not None + + def test_a_ring_drops_the_oldest_once_it_is_full(tmp_path): """What makes a camera affordable: the room is fixed, not the history.""" ring = VolatileStore(tmp_path / "ring", limit_bytes=3000) From 6e4e96e7899c9351d121435d076e7749313735f5 Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:31:47 +0200 Subject: [PATCH 005/103] Remote exception names are cut to 128 identifier characters before a class is built for them Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/workers.py | 4 ++++ backend/tests/flow/test_remote.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/fluksio/flow/workers.py b/backend/fluksio/flow/workers.py index 7cb4b78..e678ae0 100644 --- a/backend/fluksio/flow/workers.py +++ b/backend/fluksio/flow/workers.py @@ -17,6 +17,7 @@ import itertools import logging import os import queue +import re import select import signal import subprocess @@ -102,6 +103,8 @@ class NodeCancelled(RemoteError): #: node code chose, so an unbounded cache is a node raising a class named after #: its loop counter. MAX_REMOTE_TYPES = 256 +#: How long one of those names may be. A remote worker's comes off the wire. +MAX_REMOTE_NAME = 128 _remote_types: dict[str, type[RemoteError]] = {} @@ -113,6 +116,7 @@ def _remote_class(name: str) -> type[RemoteError]: the author still reads ``ValueError: bad input`` rather than the name of the transport that carried it. """ + name = re.sub(r"\W", "_", name[:MAX_REMOTE_NAME]) or "RemoteError" cls = _remote_types.get(name) if cls is not None: return cls diff --git a/backend/tests/flow/test_remote.py b/backend/tests/flow/test_remote.py index 8edeb4b..16b6177 100644 --- a/backend/tests/flow/test_remote.py +++ b/backend/tests/flow/test_remote.py @@ -21,7 +21,12 @@ from fluksio.flow.remote import ( RemoteWorkerHub, WorkerInventory, ) -from fluksio.flow.workers import NodeTimeout, RemoteError +from fluksio.flow.workers import ( + MAX_REMOTE_NAME, + NodeTimeout, + RemoteError, + _remote_class, +) @pytest.fixture @@ -149,6 +154,13 @@ def test_a_failure_keeps_its_class_across_the_socket(loop): assert str(caught[0]) == "bad input" +def test_a_class_name_off_the_wire_is_bounded_and_identifier_safe(): + cls = _remote_class("Bad Name\n" + "x" * 10_000) + assert cls.__name__.isidentifier() + assert len(cls.__name__) == MAX_REMOTE_NAME + assert issubclass(cls, RemoteError) + + def test_a_worker_that_goes_away_fails_the_call_rather_than_hanging(loop): hub = RemoteWorkerHub() worker, socket = attach(hub, loop) From f2b7a23b4f989319527e79cb305bcc4a22ff86ff Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:31:54 +0200 Subject: [PATCH 006/103] Socket snapshot carries the nodes waiting for a machine, so a reconnecting browser restores their queued dots Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/flows.py | 4 ++++ backend/tests/flow/test_placement.py | 20 +++++++++++++++++++ frontend/src/components/Flow/useFlowSocket.ts | 10 ++++++++++ 3 files changed, 34 insertions(+) diff --git a/backend/fluksio/api/routes/flows.py b/backend/fluksio/api/routes/flows.py index d4f2a6f..2150ba8 100644 --- a/backend/fluksio/api/routes/flows.py +++ b/backend/fluksio/api/routes/flows.py @@ -907,6 +907,8 @@ def snapshot_payload( two have to agree, so they build the message here rather than each their own. ``emits`` is what the bus counted while nobody was listening — a client that reconnects between two pages would otherwise start from zero. + ``queued`` is the nodes waiting for a machine, for the same reason: the bus + announces a wait once, and a client that missed it would draw the node idle. ``only`` bounds it to a set of message names, which is what a wall panel gets: the values its own dashboards draw, and none of the rest — no node @@ -925,6 +927,7 @@ def snapshot_payload( "paused": [], "logs": [], "emits": {}, + "queued": [], } return { "type": "snapshot", @@ -934,6 +937,7 @@ def snapshot_payload( "paused": controller.paused_flows(), "logs": list(event_bus.recent_logs), "emits": dict(event_bus.emits), + "queued": controller.placer.snapshot()["waiting"] if controller.placer else [], } diff --git a/backend/tests/flow/test_placement.py b/backend/tests/flow/test_placement.py index 73dabf5..27705ed 100644 --- a/backend/tests/flow/test_placement.py +++ b/backend/tests/flow/test_placement.py @@ -193,6 +193,26 @@ def test_a_node_waits_for_a_busy_machine_and_says_so(loop): assert placer.snapshot()["waiting"] == [] +def test_a_reconnecting_client_is_told_what_is_waiting(tmp_path): + """The wait is announced once, so the socket's snapshot carries it again. + + A panel is sent none of it, the same as node status. + """ + from fluksio.api.routes.flows import snapshot_payload + from fluksio.flow.controller import FlowController + from fluksio.flow.store import FlowStore + + placer = Placer(local=ResourceAccountant(cpus=2)) + placer._announce("study.b", "r1", time.monotonic(), "waiting for 2 cpu(s)") + controller = FlowController(FlowStore(tmp_path / "flows"), placer=placer) + + queued = snapshot_payload(controller)["queued"] + assert [(e["node"], e["reason"]) for e in queued] == [ + ("study.b", "waiting for 2 cpu(s)") + ] + assert snapshot_payload(controller, set())["queued"] == [] + + def test_a_worker_attaching_wakes_what_was_waiting_for_one(loop): """A node waiting inside one machine's books could not hear this.""" hub = RemoteWorkerHub() diff --git a/frontend/src/components/Flow/useFlowSocket.ts b/frontend/src/components/Flow/useFlowSocket.ts index 43b1a0f..7f5c364 100644 --- a/frontend/src/components/Flow/useFlowSocket.ts +++ b/frontend/src/components/Flow/useFlowSocket.ts @@ -43,6 +43,8 @@ type FlowEvent = logs?: LogLine[] /** Emission counts per qualified node. Absent on older instances. */ emits?: Record + /** Nodes waiting for a machine. Absent on older instances. */ + queued?: { node: string; reason: string; seconds: number }[] } | { type: "message_value" @@ -275,6 +277,14 @@ function open(url: string) { // Missing against an instance older than this bundle; the graph // then starts from zero the way it always did. liveStore.setEmits(message.emits ?? {}) + // `node_queued` is announced once per wait, so a socket that opens + // mid-wait learns of it here or draws the node idle. + for (const entry of message.queued ?? []) { + liveStore.setStatus(entry.node, { + status: "queued", + detail: entry.reason, + }) + } break case "message_value": liveStore.setValue(message.name, { From 9145b7d1c03b623818a7f482615dd899a4c8ef7c Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:32:32 +0200 Subject: [PATCH 007/103] ValueSource kind includes panel; Panels dialog no longer suggests the screen port openpanel does not serve Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/pipeline.py | 2 +- frontend/src/components/Dashboard/PanelsDialog.tsx | 9 +-------- frontend/src/components/Flow/liveStore.ts | 2 +- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/backend/fluksio/flow/pipeline.py b/backend/fluksio/flow/pipeline.py index 9faf9cc..94e27c9 100644 --- a/backend/fluksio/flow/pipeline.py +++ b/backend/fluksio/flow/pipeline.py @@ -85,7 +85,7 @@ class ValueSource(BaseModel): flow, an agent — be shown at all, since none of those is a node here. """ - #: node, dashboard, flow, agent, chat or api. + #: node, dashboard, flow, agent, chat, api or panel. kind: str = "node" #: Node id, dashboard name, or whatever identifies the caller. id: str = "" diff --git a/frontend/src/components/Dashboard/PanelsDialog.tsx b/frontend/src/components/Dashboard/PanelsDialog.tsx index f6f8805..255d3f0 100644 --- a/frontend/src/components/Dashboard/PanelsDialog.tsx +++ b/frontend/src/components/Dashboard/PanelsDialog.tsx @@ -49,14 +49,7 @@ const slugify = (value: string) => /** Port names the panel firmware knows; `gpio.` takes a pin number. */ // ponytail: free-text ports, no device capability report; add PUT /panels/{id}/device when a second device kind exists -const PORT_SUGGESTIONS = [ - "gpio.", - "backlight", - "screen", - "touch", - "mic", - "speaker", -] +const PORT_SUGGESTIONS = ["gpio.", "backlight", "touch", "mic", "speaker"] const DIRECTIONS = [ ["in", "In"], diff --git a/frontend/src/components/Flow/liveStore.ts b/frontend/src/components/Flow/liveStore.ts index 5cf5bfb..31bf5ce 100644 --- a/frontend/src/components/Flow/liveStore.ts +++ b/frontend/src/components/Flow/liveStore.ts @@ -13,7 +13,7 @@ import { request } from "@/client/core/request" /** Who caused a value. The canvas needs it to pulse the right edge. */ export type ValueSource = { - kind: "node" | "dashboard" | "flow" | "agent" | "chat" | "api" + kind: "node" | "dashboard" | "flow" | "agent" | "chat" | "api" | "panel" id: string label: string detail?: string From f4512aeebc00926954af37789b8963d9e5285094 Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:33:25 +0200 Subject: [PATCH 008/103] Memory-queue ids carry a per-instance boot token, so a restart without Redis no longer overwrites earlier run rows Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/queue.py | 6 +++++- backend/tests/flow/test_queue.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/fluksio/flow/queue.py b/backend/fluksio/flow/queue.py index 7c0ef72..6837616 100644 --- a/backend/fluksio/flow/queue.py +++ b/backend/fluksio/flow/queue.py @@ -18,6 +18,7 @@ from __future__ import annotations import heapq import logging +import secrets import threading import time from abc import ABC, abstractmethod @@ -272,12 +273,15 @@ class MemoryWorkQueue(WorkQueue): self._in_flight = 0 self._counter = 0 self._seq = 0 + # The ids become `FlowRun.id`, a primary key. Counting from zero alone, + # a restart would overwrite the previous boot's rows. + self._boot = secrets.token_hex(4) self._lock = threading.Lock() self._wake = threading.Condition(self._lock) def _next_id(self) -> str: self._seq += 1 - return f"mem-{self._seq}" + return f"mem-{self._boot}-{self._seq}" def add(self, item: WorkItem) -> None: with self._wake: diff --git a/backend/tests/flow/test_queue.py b/backend/tests/flow/test_queue.py index 71cb25e..6922a01 100644 --- a/backend/tests/flow/test_queue.py +++ b/backend/tests/flow/test_queue.py @@ -25,6 +25,18 @@ def test_items_come_back_in_the_order_they_went_in(): assert all(item.entry_id for item in claimed) +def test_two_queues_never_hand_out_the_same_id(): + """A restart without Redis must not reuse the last boot's run ids.""" + first, second = MemoryWorkQueue(), MemoryWorkQueue() + for queue in (first, second): + queue.add(WorkItem(kind="cascade", node="f.n", flow="f")) + + ids = [queue.claim(1, 10)[0].entry_id for queue in (first, second)] + + assert ids[0] != ids[1] + assert all(len(entry_id) <= 64 for entry_id in ids) + + def test_claiming_an_empty_queue_waits_and_gives_up(): queue = MemoryWorkQueue() started = time.monotonic() From f8d14be8e6523818599c666a1217af1a1d208708 Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:34:07 +0200 Subject: [PATCH 009/103] Dev backend reloads on worker edits too: fastapi dev with --reload-dir over backend and worker, worker in develop.watch Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- development.md | 7 ++++--- docker/compose.dev.yml | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/development.md b/development.md index 01348e7..3bd3abc 100644 --- a/development.md +++ b/development.md @@ -66,9 +66,10 @@ Standalone (`cd app && make dev`) publishes the full set: ## Hot reload, and what it misses -`compose.dev.yml` bind-mounts `backend/fluksio` into the api container and runs uvicorn with -`--reload`, so a backend edit is live. The `develop.watch` block next to it syncs the same -directory, but only under `docker compose watch`, which no target runs — the detached +`compose.dev.yml` bind-mounts `backend/fluksio` and `worker/fluksio_worker` into the api +container and runs uvicorn with `--reload` over both, so a backend edit is live and a worker +edit restarts the engine along with its local workers. The `develop.watch` block next to it +syncs the same directories, but only under `docker compose watch`, which no target runs — the detached `up -d` flow never triggers it. The mount is what does the work, so an api container created before the mount existed keeps serving the source baked into its image and has to be recreated once. diff --git a/docker/compose.dev.yml b/docker/compose.dev.yml index e9fa1e7..0388de1 100644 --- a/docker/compose.dev.yml +++ b/docker/compose.dev.yml @@ -92,10 +92,18 @@ services: build: context: .. dockerfile: backend/Dockerfile + # `fastapi dev`, because `run` takes no `--reload-dir`. Watching the worker + # too makes a worker edit restart the engine, which respawns the local + # worker pool with the new code. command: - fastapi - - run - - --reload + - dev + - --host + - 0.0.0.0 + - --reload-dir + - /app/backend + - --reload-dir + - /app/worker - "fluksio/main.py" develop: watch: @@ -107,6 +115,9 @@ services: - .venv - path: ../backend/pyproject.toml action: rebuild + - path: ../worker/fluksio_worker + action: sync + target: /app/worker/fluksio_worker volumes: # What makes --reload above mean anything in the detached flow (`make dev`, # `make dev-local`): it starts the stack with `up -d`, which never runs the From 1735a3651e5aab22f8100f6a3dfe14e227ef333f Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:34:47 +0200 Subject: [PATCH 010/103] Shared node changes rebuild the flows whose published copy uses them, not their drafts Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/controller.py | 3 ++- backend/fluksio/flow/store.py | 10 +++++++--- backend/tests/flow/test_library.py | 26 ++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/backend/fluksio/flow/controller.py b/backend/fluksio/flow/controller.py index 3e8cdfc..ee434aa 100644 --- a/backend/fluksio/flow/controller.py +++ b/backend/fluksio/flow/controller.py @@ -837,7 +837,8 @@ class FlowController: async def reload_lib_users(self, ref: str) -> list[str]: """Rebuild the flows running a shared node whose source has changed.""" - flows = sorted({usage.split(".", 1)[0] for usage in self.store.usages(ref)}) + usages = self.store.usages(ref, draft=False) + flows = sorted({usage.split(".", 1)[0] for usage in usages}) for flow in flows: await self.reload_flow(flow) return flows diff --git a/backend/fluksio/flow/store.py b/backend/fluksio/flow/store.py index d414c63..e100ede 100644 --- a/backend/fluksio/flow/store.py +++ b/backend/fluksio/flow/store.py @@ -355,12 +355,16 @@ class FlowStore: self._set_source_ref(flow, node_id, None) self._commit(f"Unshare node '{flow}.{node_id}'") - def usages(self, lib_name: str) -> list[str]: - """Every ``flow.node`` pointing at this shared source.""" + def usages(self, lib_name: str, draft: bool = True) -> list[str]: + """Every ``flow.node`` pointing at this shared source. + + Read from the drafts the editor shows, or with ``draft`` off from the + published flows the engine runs. + """ used = [] for name in self.list_flows(): try: - definition = self.read_flow(name, draft=True) + definition = self.read_flow(name, draft=draft) except FlowNotFound: continue used += [ diff --git a/backend/tests/flow/test_library.py b/backend/tests/flow/test_library.py index 03eb208..6a0619d 100644 --- a/backend/tests/flow/test_library.py +++ b/backend/tests/flow/test_library.py @@ -1,9 +1,11 @@ """Nodes shared across flows: one source, many instances.""" +import asyncio from pathlib import Path import pytest +from fluksio.flow.controller import FlowController from fluksio.flow.messages import MessageSpec from fluksio.flow.schemas import FlowDef, NodeDef from fluksio.flow.store import FlowStore, LibExists, LibNotFound @@ -67,6 +69,30 @@ def test_a_second_flow_can_use_the_same_source(store: FlowStore): assert store.usages("read_temp") == ["cooling.sensor", "heating.sensor"] +def test_a_shared_source_change_rebuilds_only_what_is_published(store: FlowStore): + store.write_flow(a_flow()) + store.write_node_source("heating", "sensor", SOURCE) + store.share_node("heating", "sensor", "read_temp") + # Published with the shared node, which the draft has since dropped. + store.publish_flow("heating", store.read_flow("heating", draft=True).version) + store.unshare_node("heating", "sensor") + # Shared in a draft that was never published. + cooling = a_flow("cooling") + cooling.nodes[0].source_ref = "read_temp" + store.write_draft(cooling, 0) + + controller = FlowController(store) + rebuilt: list[str] = [] + + async def reload_flow(name: str) -> None: + rebuilt.append(name) + + controller.reload_flow = reload_flow # type: ignore[method-assign] + asyncio.run(controller.reload_lib_users("read_temp")) + + assert rebuilt == ["heating"] + + def test_a_shared_name_is_not_taken_twice(store: FlowStore): store.write_flow(a_flow()) store.write_node_source("heating", "sensor", SOURCE) From 1c1e2dc1a2c8401911f8a1bb419c23699c76e85f Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:40:03 +0200 Subject: [PATCH 011/103] make clean goes on past a root-owned backend/htmlcov, with a hint to sudo rm it Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index df55045..45830f2 100644 --- a/Makefile +++ b/Makefile @@ -253,5 +253,6 @@ docs-serve: ## Serve the documentation site locally with live reload clean: ## Remove build artifacts and caches rm -rf frontend/dist frontend/blob-report frontend/test-results - rm -rf backend/.pytest_cache backend/htmlcov site + rm -rf backend/.pytest_cache backend/htmlcov site \ + || echo " ! not all removed (htmlcov/ root-owned? sudo rm -rf backend/htmlcov)" >&2 find backend -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true From 43fb446297f8cab6938e84cc2bc7979cc1bd5c5b Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:41:28 +0200 Subject: [PATCH 012/103] Stat unit sized against the hero numerals: dui-hero on the readout's outer span, so the unit's 0.4em is of the hero rather than the 14px body text Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- .../src/components/Dashboard/ui/fluksio/Data.tsx | 15 ++++++++------- .../src/components/Dashboard/ui/glass/Data.tsx | 15 ++++++++------- .../src/components/Dashboard/ui/material/Data.tsx | 15 ++++++++------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/frontend/src/components/Dashboard/ui/fluksio/Data.tsx b/frontend/src/components/Dashboard/ui/fluksio/Data.tsx index 5076548..acde297 100644 --- a/frontend/src/components/Dashboard/ui/fluksio/Data.tsx +++ b/frontend/src/components/Dashboard/ui/fluksio/Data.tsx @@ -31,13 +31,14 @@ export function Readout({ }: ReadoutProps) { const { label, numeric } = useAnimatedNumber(value, precision) return ( - - + + {numeric ? ( {label} ) : ( diff --git a/frontend/src/components/Dashboard/ui/glass/Data.tsx b/frontend/src/components/Dashboard/ui/glass/Data.tsx index 859f522..5f499e3 100644 --- a/frontend/src/components/Dashboard/ui/glass/Data.tsx +++ b/frontend/src/components/Dashboard/ui/glass/Data.tsx @@ -30,13 +30,14 @@ export function Readout({ }: ReadoutProps) { const { label, numeric } = useAnimatedNumber(value, precision) return ( - - + + {numeric ? ( {label} ) : ( diff --git a/frontend/src/components/Dashboard/ui/material/Data.tsx b/frontend/src/components/Dashboard/ui/material/Data.tsx index 86b0cd2..98a013a 100644 --- a/frontend/src/components/Dashboard/ui/material/Data.tsx +++ b/frontend/src/components/Dashboard/ui/material/Data.tsx @@ -29,13 +29,14 @@ export function Readout({ }: ReadoutProps) { const { label, numeric } = useAnimatedNumber(value, precision) return ( - - + + {numeric ? ( {label} ) : ( From b0c07a35e9b2cc873bff33841e89c09dfea052bc Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:41:29 +0200 Subject: [PATCH 013/103] Locked slider dims its rail: :has() on the slider, since the rail comes before the input and no sibling selector reaches it Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/src/components/Dashboard/ui/core/core.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/Dashboard/ui/core/core.css b/frontend/src/components/Dashboard/ui/core/core.css index f382f09..7303dbb 100644 --- a/frontend/src/components/Dashboard/ui/core/core.css +++ b/frontend/src/components/Dashboard/ui/core/core.css @@ -576,8 +576,9 @@ outline-offset: 2px; } +/* The rail comes before the input, so no sibling selector reaches it. */ .dui-slider-input:disabled ~ .dui-slider-thumb, -.dui-slider-input:disabled ~ .dui-slider-rail { +.dui-slider:has(.dui-slider-input:disabled) .dui-slider-rail { opacity: 0.45; } From f2305ef61937ba51885dadac005cdedcb872c943 Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:41:30 +0200 Subject: [PATCH 014/103] Phone header: assistant button in the corner, gated on an available provider as the sidebar entry is; header triggers on the right at 44px Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/src/routes/_layout.tsx | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/frontend/src/routes/_layout.tsx b/frontend/src/routes/_layout.tsx index f208679..fa27657 100644 --- a/frontend/src/routes/_layout.tsx +++ b/frontend/src/routes/_layout.tsx @@ -1,16 +1,23 @@ +import { useQuery } from "@tanstack/react-query" import { createFileRoute, Outlet, redirect, useRouterState, } from "@tanstack/react-router" +import { Bot } from "lucide-react" -import { ControlTrigger } from "@/components/Chat/ChatPanel" +import { + aiStatusQueryOptions, + ControlTrigger, +} from "@/components/Chat/ChatPanel" +import { chatStore } from "@/components/Chat/chatStore" import { ConnectionNotice } from "@/components/Common/ConnectionNotice" import ErrorComponent from "@/components/Common/ErrorComponent" import { Footer } from "@/components/Common/Footer" import { useFlowSocket } from "@/components/Flow/useFlowSocket" import AppSidebar from "@/components/Sidebar/AppSidebar" +import { Button } from "@/components/ui/button" import { SidebarInset, SidebarProvider, @@ -38,6 +45,8 @@ function Layout() { // here rather than only inside the editor. useFlowSocket() const home = useRouterState({ select: (s) => s.location.pathname === "/" }) + // Gated as the sidebar's entry is: with no provider there is nothing to open. + const assistant = useQuery(aiStatusQueryOptions()) return ( @@ -48,10 +57,25 @@ function Layout() { {/* The sidebar carries its own collapse control; a phone has no sidebar on screen to carry it. Home's chat sits opposite, where - the brain view has it on a wider screen. */} + the brain view has it on a wider screen, and the assistant in the + corner, one tap away rather than behind the sidebar sheet. */}
- {home ? : null} +
+ {home ? : null} + {assistant.data?.available ? ( + + ) : null} +
From 8e65ce41cb2302daedd89167402f8656996953a2 Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:41:31 +0200 Subject: [PATCH 015/103] Test hygiene: forget the refused study, quiet _Tee on a closed stream, wait for connector recovery Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/logs.py | 8 +++++--- backend/tests/flow/test_connector.py | 26 ++++++++++++++++++++++---- backend/tests/test_cli.py | 13 ++++++++++--- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/backend/fluksio/flow/logs.py b/backend/fluksio/flow/logs.py index 9cb60bf..e8f1957 100644 --- a/backend/fluksio/flow/logs.py +++ b/backend/fluksio/flow/logs.py @@ -36,17 +36,19 @@ class _Tee(io.TextIOBase): sink(text) try: return self._real.write(text) - except OSError: + except (OSError, ValueError): # A dead stdout — `fluksio serve` runs the engine as a child of the # dashboard, which holds the far end of that pipe — must not fail # the node whose output was being teed. The capture above has it, - # and there is nowhere left to report the loss to anyway. + # and there is nowhere left to report the loss to anyway. A closed + # one (ValueError) is the same at interpreter exit, when whatever + # was wrapped — pytest's capture, say — has been closed first. return len(text) def flush(self) -> None: try: self._real.flush() - except OSError: + except (OSError, ValueError): pass def isatty(self) -> bool: diff --git a/backend/tests/flow/test_connector.py b/backend/tests/flow/test_connector.py index 4abde58..258ceb6 100644 --- a/backend/tests/flow/test_connector.py +++ b/backend/tests/flow/test_connector.py @@ -1,6 +1,7 @@ """The connector contract: polling, deduplication, health and discovery.""" import asyncio +from collections.abc import Callable from types import SimpleNamespace from typing import Any @@ -48,12 +49,28 @@ def a_sensor(readings: list[Any], **params: Any) -> Sensor: return node -def run_briefly(node: ConnectorNode, seconds: float = 0.12, app: Any = None) -> None: - """Start the poll loop, let it tick a few times, stop it.""" +def run_briefly( + node: ConnectorNode, + seconds: float = 0.12, + app: Any = None, + until: Callable[[], bool] | None = None, +) -> None: + """Start the poll loop, let it tick a few times, stop it. + + With ``until``, stop once that holds instead, giving up after 5 s: a fixed + pause is too short when the machine is busy. The caller's asserts then say + what never happened. + """ async def cycle() -> None: await node.start(app) - await asyncio.sleep(seconds) + if until is None: + await asyncio.sleep(seconds) + else: + for _ in range(500): + if until(): + break + await asyncio.sleep(0.01) await node.stop() asyncio.run(cycle()) @@ -97,7 +114,8 @@ def test_a_failing_poll_reports_down_and_keeps_going(): node._on_health = lambda _node, status, detail: health.append((status, detail)) Pipeline(nodes=[node]) - run_briefly(node) + # The first poll fails, so an "ok" after it is the recovery. + run_briefly(node, until=lambda: len(health) > 1 and health[-1][0] == "ok") assert ("down", "RuntimeError: device unplugged") in health # It recovered rather than giving up. diff --git a/backend/tests/test_cli.py b/backend/tests/test_cli.py index 61cb816..355d218 100644 --- a/backend/tests/test_cli.py +++ b/backend/tests/test_cli.py @@ -636,6 +636,8 @@ def test_two_files_of_one_name_are_refused(tmp_path) -> None: where it sits; this is the spelling that still gets there — two files named on the command line, each rooted at its own directory. """ + import sys + import pytest from fluksio.sdk import SyncError @@ -647,9 +649,14 @@ def test_two_files_of_one_name_are_refused(tmp_path) -> None: first = tmp_path / "s1" / "study.py" second = tmp_path / "s2" / "study.py" - _import(*_module_of(first), first) - with pytest.raises(SyncError, match="both import as 'study'"): - _import(*_module_of(second), second) + roots = {str(tmp_path / "s1"), str(tmp_path / "s2")} + try: + _import(*_module_of(first), first) + with pytest.raises(SyncError, match="both import as 'study'"): + _import(*_module_of(second), second) + finally: + sys.modules.pop("study", None) + sys.path[:] = [entry for entry in sys.path if entry not in roots] def test_how_long_ago_reads_like_a_duration() -> None: From 9a5d7a0d0cb0c5af2bdcd7bcbe2f0186ffc5137c Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:49:07 +0200 Subject: [PATCH 016/103] Soak engine scenario: a sleeping fixture node keeps cascades in flight for the kill, and the report says how many were unacknowledged Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/scripts/soak.py | 46 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/backend/scripts/soak.py b/backend/scripts/soak.py index a22a29d..813e2c3 100644 --- a/backend/scripts/soak.py +++ b/backend/scripts/soak.py @@ -149,8 +149,9 @@ PREFIX = "soak_" FLOW_LOAD = f"{PREFIX}load" FLOW_CHAIN = f"{PREFIX}chain" FLOW_MQTT = f"{PREFIX}mqtt" +FLOW_SLOW = f"{PREFIX}slow" #: Deletion order: whatever reads another flow's messages goes first. -FLOWS = (FLOW_MQTT, FLOW_CHAIN, FLOW_LOAD) +FLOWS = (FLOW_SLOW, FLOW_MQTT, FLOW_CHAIN, FLOW_LOAD) COUNT_SOURCE = '''"""Count what made it through, reading the total it also writes.""" @@ -159,6 +160,16 @@ def process(tick, total=0, params=None): return {"total": int(total) + 1} ''' +SLOW_SOURCE = '''"""Hold the cascade open, so an engine kill finds work in flight.""" + +import time + + +def process(total, pause): + time.sleep(float(pause)) + return {"done": total} +''' + def spec(name: str, dtype: str = "int", trigger: bool = True) -> dict[str, Any]: return {"name": name, "dtype": dtype, "trigger": trigger} @@ -214,6 +225,30 @@ def chain_flow() -> dict[str, Any]: } +def slow_flow(pause: float) -> dict[str, Any]: + """A sleep at the end of the load's cascade, for the engine scenario only. + + It reads the total, so it runs after the count and never holds a node + worker the count is waiting for. + """ + return { + "name": FLOW_SLOW, + "title": "Soak slow", + "version": 0, + "nodes": [ + { + "id": "pause", + "type": "python", + "title": "Pause", + "position": {"x": 40, "y": 40}, + "params": {"pause": pause}, + "requires": [spec(f"{FLOW_LOAD}.total")], + "provides": [spec("done")], + } + ], + } + + def mqtt_flow() -> dict[str, Any]: """A subscriber on the dev stack's own broker, for the broker scenario.""" return { @@ -691,6 +726,12 @@ def scenario_engine(client: httpx.Client, args: argparse.Namespace, url: str) -> report = Report("engine") dead_before = dead_letters(client) + # The count alone finishes in a few milliseconds, so a kill used to catch + # next to nothing in flight. This pause keeps about three quarters of the + # engine's cascade slots busy at this rate, and the engine still keeps up. + pause = 0.75 * settings.FLOW_MAX_CASCADES / args.rps + create_flow(client, slow_flow(pause), {"pause": SLOW_SOURCE}) + driver = Driver(url, args.rps) log.info("engine: %d rps, killing %s halfway through", args.rps, API) driver.start() @@ -700,6 +741,8 @@ def scenario_engine(client: httpx.Client, args: argparse.Namespace, url: str) -> # Read after the total, so work already accepted but not yet counted makes # the claim below easier rather than impossible. accepted_at_kill = driver.accepted + # Claimed and not acknowledged, so what the reaper has to hand back. + unacked = queue_stats(client).get("pending", "?") docker(API, "kill", API) time.sleep(2.0) @@ -707,6 +750,7 @@ def scenario_engine(client: httpx.Client, args: argparse.Namespace, url: str) -> # it back — and it must never be left down. docker(API, "start", API) recovered = wait_for(lambda: health_code(client) == 200, HEALTH_RECOVER_S, 2.0) + report.stat(f"unacknowledged at the kill {unacked}, stack-wide") report.stat(f"engine answered healthy again {seconds(recovered)} after restart") report.check( f"health back to 200 within {HEALTH_RECOVER_S:.0f}s of the restart", From 810290e5239a11e423d6a8dbae84f36aae8de20c Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:51:24 +0200 Subject: [PATCH 017/103] soak: pass flow names as SQL parameters so observability cleanup parses Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/scripts/soak.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/scripts/soak.py b/backend/scripts/soak.py index 813e2c3..dc1f83d 100644 --- a/backend/scripts/soak.py +++ b/backend/scripts/soak.py @@ -849,13 +849,13 @@ def clear_redis() -> None: def clear_observability() -> None: """The collector wrote rollups, runs and events for the soak flows.""" - names = ", ".join(repr(flow) for flow in FLOWS) + marks = ", ".join("?" for _ in FLOWS) script = "\n".join( [ "import sqlite3", "db = sqlite3.connect('/data/fluksio.db')", *( - f"db.execute('DELETE FROM {table} WHERE flow IN ({names})')" + f"db.execute('DELETE FROM {table} WHERE flow IN ({marks})', {FLOWS!r})" for table in ("metric_minute", "flow_run", "engine_event") ), "db.commit()", From 9f7b552885038611b680937eeca7768ea8faaf6f Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 14 Oct 2026 23:59:24 +0200 Subject: [PATCH 018/103] Colour tile tests: the disc on the shared widgets panel for every look's screenshot, and a key's release sending rgb and hex Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/tests/color-drag.spec.ts | 79 +++++++++++++++++++++++++++---- frontend/tests/widgets.spec.ts | 12 ++++- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/frontend/tests/color-drag.spec.ts b/frontend/tests/color-drag.spec.ts index 6edf754..29cc784 100644 --- a/frontend/tests/color-drag.spec.ts +++ b/frontend/tests/color-drag.spec.ts @@ -12,7 +12,8 @@ import { api, apiPage, deleteAll } from "./utils/api" * * The arithmetic behind the wheel is checked in `color.check.ts`, which needs * no browser. What needs one is the gesture: how many messages a drag sends, - * and whether the one it does send is where the pointer was let go. + * and whether the one it does send is where the pointer was let go. And the + * wiring: a tile set to rgb or hex reads and sends that form, not the wheel's. */ const flowName = `test_color_${Date.now().toString(36)}` const dashboard = `${flowName}_d` @@ -36,7 +37,11 @@ test.beforeAll(async ({ browser }) => { { id: "lamp", type: "python", - provides: [{ name: "tint", dtype: "list", item: "float" }], + provides: [ + { name: "tint", dtype: "list", item: "float" }, + { name: "rgb", dtype: "list", item: "int" }, + { name: "hex", dtype: "str" }, + ], }, ], }, @@ -48,6 +53,16 @@ test.beforeAll(async ({ browser }) => { method: "POST", data: { version: flow.definition.version }, }) + // The same amber in either form, as a flow speaking it would set it. + for (const [name, value] of [ + ["rgb", [204, 150, 57]], + ["hex", "#cc9639"], + ] as const) { + await api(page, `/messages/${flowName}.${name}`, { + method: "POST", + data: { value }, + }) + } // A leftover from a run that failed before its teardown would answer the // create with a version conflict. @@ -66,6 +81,20 @@ test.beforeAll(async ({ browser }) => { layout: { lg: { x: 0, y: 0, w: 6, h: 4 } }, config: { target, dtype: "list", format: "hsv" }, }, + { + id: "rgb", + type: "color", + title: "RGB", + layout: { lg: { x: 6, y: 0, w: 3, h: 4 } }, + config: { target: `${flowName}.rgb`, dtype: "list", format: "rgb" }, + }, + { + id: "hex", + type: "color", + title: "Hex", + layout: { lg: { x: 9, y: 0, w: 3, h: 4 } }, + config: { target: `${flowName}.hex`, dtype: "str", format: "hex" }, + }, ], }, }) @@ -87,10 +116,13 @@ test.afterAll(async ({ browser }) => { await deleteAll(browser, [`/dashboards/${dashboard}`, `/flows/${flowName}`]) }) -/** Open the panel, settled, and collect every publish the widget makes. */ -async function openBoard(page: Page) { +/** Open the panel, settled, and collect every publish made to `name`. */ +async function openBoard( + page: Page, + name = target, +) { await page.goto(`/view/${dashboard}`) - await page.getByTestId("color-wheel").waitFor({ timeout: 20000 }) + await page.getByTestId("color-wheel").first().waitFor({ timeout: 20000 }) // A tile fades and scales in, so where the disc is drawn is only settled // once that has finished — and the gestures below are aimed in pixels. await page.waitForFunction( @@ -104,11 +136,11 @@ async function openBoard(page: Page) { // Every publish this panel makes, as the browser sends it. The setup above // talks to the API out of band, so nothing but the widget is counted here. - const sent: [number, number, number][] = [] + const sent: Value[] = [] page.on("request", (request) => { if ( request.method() === "POST" && - request.url().includes(`/messages/${target}`) + request.url().includes(`/messages/${name}`) ) sent.push(request.postDataJSON()?.value) }) @@ -117,7 +149,7 @@ async function openBoard(page: Page) { test("a dragged disc publishes once, where it was let go", async ({ page }) => { const sent = await openBoard(page) - const disc = page.getByTestId("color-wheel") + const disc = page.getByRole("slider", { name: "Tint hue and saturation" }) const box = (await disc.boundingBox())! const middle = { x: box.x + box.width / 2, y: box.y + box.height / 2 } @@ -211,3 +243,34 @@ test("a dragged brightness column publishes where it was let go", async ({ .toBe(1) expect(sent[0][2], "the release sent another brightness").toBe(letGo) }) + +for (const { title, sends } of [ + { title: "RGB", sends: [204, 162, 57] }, + { title: "Hex", sends: "#cca239" }, +]) { + test(`a key's release publishes the colour as ${title}`, async ({ page }) => { + const sent = await openBoard( + page, + `${flowName}.${title.toLowerCase()}`, + ) + const disc = page.getByRole("slider", { + name: `${title} hue and saturation`, + }) + // The amber the setup published, read back from the tile's own form. + await expect(disc).toHaveAttribute( + "aria-valuetext", + "38 degrees, 72% saturated", + ) + + // A key moves the hue five degrees on the way down and commits on the + // way up, so one press is one message. + await disc.press("ArrowRight") + await expect + .poll(() => sent.length, { + timeout: 10000, + message: "letting go of the key published nothing", + }) + .toBe(1) + expect(sent[0], "the key sent another colour").toEqual(sends) + }) +} diff --git a/frontend/tests/widgets.spec.ts b/frontend/tests/widgets.spec.ts index c89f10f..0f23dde 100644 --- a/frontend/tests/widgets.spec.ts +++ b/frontend/tests/widgets.spec.ts @@ -24,7 +24,7 @@ const lockedName = `${flowName}_locked` const w = (name: string) => `${flowName}.${name}` /** Every tile the dashboard carries, including the deliberately unbound one. */ -const TILES = 11 +const TILES = 12 test.use({ storageState: "playwright/.auth/user.json" }) @@ -59,6 +59,7 @@ test.beforeAll(async ({ browser }) => { { name: "lamp", dtype: "bool" }, { name: "setpoint", dtype: "float" }, { name: "idle", dtype: "float" }, + { name: "tint", dtype: "list", item: "float" }, ], }, ], @@ -78,6 +79,8 @@ test.beforeAll(async ({ browser }) => { await publish(page, w("condition"), "sun") await publish(page, w("setpoint"), 21.5) await publish(page, w("idle"), 0) + // Amber, so the disc's handle and shade have something to show. + await publish(page, w("tint"), [38, 72, 80]) await publish(page, w("days"), [ { label: "Mon", icon: "sun", value: "21°" }, { label: "Tue", icon: "cloudy", value: "18°" }, @@ -214,6 +217,13 @@ test.beforeAll(async ({ browser }) => { layout: { lg: { x: 6, y: 4, w: 6, h: 3 } }, config: { series: [{ message: w("level"), dtype: "float" }] }, }, + { + id: "tint", + type: "color", + title: "Tint", + layout: { lg: { x: 6, y: 7, w: 3, h: 3 } }, + config: { target: w("tint"), dtype: "list", format: "hsv" }, + }, ] const draft = await ( await api(page, `/dashboards/${dashboardName}`, { From 0a9ea24b9355075f6e2a21f472895a38d90c7871 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 00:04:03 +0200 Subject: [PATCH 019/103] Stored state lists its keys, each with a two-press delete beside Clear Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/reference/connector-contract.md | 4 +- frontend/src/components/Flow/NodePanel.tsx | 54 +++++++++++++++++++++- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/docs/reference/connector-contract.md b/docs/reference/connector-contract.md index 95aff46..3e75e41 100644 --- a/docs/reference/connector-contract.md +++ b/docs/reference/connector-contract.md @@ -164,8 +164,8 @@ await asyncio.to_thread(self.store.delete, "token:device") 64 KiB and each package allows 256 keys; exceeding a limit raises `ValueError`. - Calls block on the database: use `asyncio.to_thread` on the event loop. -Superusers can list keys and clear them under **Stored state** in the node -panel. Values are never returned. The equivalent API paths are: +Superusers can list keys, delete one, or clear them all under **Stored state** +in the node panel. Values are never returned. The equivalent API paths are: | Method | Path | |---|---| diff --git a/frontend/src/components/Flow/NodePanel.tsx b/frontend/src/components/Flow/NodePanel.tsx index 237cf30..b5cdffc 100644 --- a/frontend/src/components/Flow/NodePanel.tsx +++ b/frontend/src/components/Flow/NodePanel.tsx @@ -4,6 +4,7 @@ import { CornerDownRight, Maximize2, Minimize2, + Trash2, X, } from "lucide-react" import { @@ -1236,12 +1237,15 @@ const SCAFFOLD_SHAPE = (() => { /** * What a connector keeps across restarts, such as a certificate it pinned on - * first contact. Clearing it is how a replaced device gets trusted again. + * first contact. Deleting a key is how one replaced device gets trusted again; + * clearing them all starts every device over. */ function StoredStateSection({ nodeType }: { nodeType: string }) { const queryClient = useQueryClient() const { showSuccessToast, showErrorToast } = useCustomToast() const [confirming, setConfirming] = useState(false) + // The key whose delete was pressed once; a second press deletes it. + const [armed, setArmed] = useState(null) const queryKey = ["connector-state", nodeType] const { data } = useQuery({ queryKey, @@ -1256,6 +1260,15 @@ function StoredStateSection({ nodeType }: { nodeType: string }) { }, onError: () => showErrorToast("The stored state could not be cleared."), }) + const forget = useMutation({ + mutationFn: (key: string) => + ConnectorsService.deleteConnectorState({ nodeType, key }), + onSuccess: (_, key) => { + showSuccessToast(`Deleted '${key}'`) + queryClient.invalidateQueries({ queryKey }) + }, + onError: () => showErrorToast("The key could not be deleted."), + }) if (!data) return null return ( @@ -1308,6 +1321,45 @@ function StoredStateSection({ nodeType }: { nodeType: string }) { )}
+ {data.data.length > 0 ? ( +
    + {data.data.map(({ key }) => { + // The icon says nothing of its own, so the label carries the + // confirm step. + const label = + armed === key ? `Confirm: delete ${key}` : `Delete ${key}` + return ( +
  • + + {key} + + +
  • + ) + })} +
+ ) : null} ) } From 98416dc516e6cb1346755c7e352f0f595c3f02fa Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 00:07:58 +0200 Subject: [PATCH 020/103] =?UTF-8?q?Dashboard=20editor=20undo:=20=E2=8C=98Z?= =?UTF-8?q?=20/=20=E2=87=A7=E2=8C=98Z=20over=20a=2050-entry=20history=20of?= =?UTF-8?q?=20commits,=20folding=20keystrokes=20and=20the=20grid's=20own?= =?UTF-8?q?=20correction=20as=20the=20flow=20editor=20does?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/interface/dashboards.md | 1 + .../components/Dashboard/DashboardEditor.tsx | 65 ++++++++++++++++++- frontend/tests/editor.spec.ts | 24 +++++++ 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/docs/interface/dashboards.md b/docs/interface/dashboards.md index 0997e2b..811dd30 100644 --- a/docs/interface/dashboards.md +++ b/docs/interface/dashboards.md @@ -14,6 +14,7 @@ to a paired screen, such as a wall tablet. Edits stay in a draft; paired screens show the published version. On phones, tiles stack in one column and dragging is disabled, but settings remain editable. +In edit mode, ⌘Z / Ctrl-Z undoes an edit and ⌘⇧Z / Ctrl-Shift-Z redoes it. Controls can publish real values while you edit, so choose their targets with care. Your new Switch may already be flipping something! diff --git a/frontend/src/components/Dashboard/DashboardEditor.tsx b/frontend/src/components/Dashboard/DashboardEditor.tsx index 78bb103..ed4edae 100644 --- a/frontend/src/components/Dashboard/DashboardEditor.tsx +++ b/frontend/src/components/Dashboard/DashboardEditor.tsx @@ -288,6 +288,41 @@ const scaledStrategy = (scale: number) => ({ calcStyle: setTransform as (pos: Position) => CSSProperties, }) +/** Dashboards to step back and forward through, newest last. */ +type History = { + past: Dashboard[] + future: Dashboard[] + /** When the last entry was recorded, so a burst of typing stays one edit. */ + at: number +} + +const HISTORY_LIMIT = 50 +const TYPING_WINDOW = 500 + +/** + * Remember the dashboard as it was before a change. + * + * Fields commit on every keystroke, and the grid writes back its own + * correction a moment after a widget comes or goes, so consecutive edits that + * leave the same widgets in place fold into the entry already on the stack. + * Adding, removing or reordering a widget is a finished action and starts its + * own. + */ +function record(history: History, previous: Dashboard, next: Dashboard) { + const before = widgetsOf(previous) + const after = widgetsOf(next) + const sameWidgets = + before.length === after.length && + before.every((widget, index) => widget.id === after[index].id) + if (!sameWidgets || Date.now() - history.at >= TYPING_WINDOW) { + history.past.push(previous) + if (history.past.length > HISTORY_LIMIT) history.past.shift() + } + history.at = Date.now() + // A new change is a new branch: what was undone is not coming back. + history.future = [] +} + const same = (a: Layout, b: Layout) => a.length === b.length && a.every((item, index) => { @@ -349,6 +384,7 @@ export function DashboardEditor({ // The saved version is what the next save is based on; without following it // the second save of a session is always a conflict. const version = useRef(dashboard.version) + const history = useRef({ past: [], future: [], at: 0 }) useEffect(() => { version.current = dashboard.version @@ -369,7 +405,8 @@ export function DashboardEditor({ return request } - const commit = (next: Dashboard) => { + /** Put a dashboard on the canvas and on its way to the server. */ + const apply = (next: Dashboard) => { setDraft(next) if (timer.current) clearTimeout(timer.current) timer.current = setTimeout(() => { @@ -378,6 +415,25 @@ export function DashboardEditor({ }, AUTOSAVE_MS) } + /** Apply a change and make it undoable. */ + const commit = (next: Dashboard) => { + record(history.current, draft, next) + apply(next) + } + + /** + * Step through the history. Undo and redo save like any other edit, so the + * autosave debounce collapses a run of them into one write. + */ + const step = (back: boolean) => { + const { past, future } = history.current + const remembered = (back ? past : future).pop() + if (!remembered) return + ;(back ? future : past).push(draft) + history.current.at = 0 + apply(remembered) + } + /** Send what is queued and wait for the server to have it. */ const flush = async () => { if (timer.current) { @@ -439,10 +495,13 @@ export function DashboardEditor({ setSettingsOpen(true) } - // As on the flow canvas, and reachable while typing for the same reasons. - // Outside edit mode both keys stay the browser's. + // As on the flow canvas: undo and redo step aside while typing, since a + // field keeps its own, and ⌘P and ⌘S stay reachable for the same reasons. + // Outside edit mode every key stays the browser's. useShortcuts( { + "mod+z": () => (edit ? step(true) : false), + "mod+shift+z": () => (edit ? step(false) : false), "mod+p": () => (edit ? setPaletteOpen((open) => !open) : false), // An edit still waiting for its autosave counts: publishing flushes it. "mod+s": () => { diff --git a/frontend/tests/editor.spec.ts b/frontend/tests/editor.spec.ts index 9cc1a86..51470cf 100644 --- a/frontend/tests/editor.spec.ts +++ b/frontend/tests/editor.spec.ts @@ -11,6 +11,8 @@ import { api, apiPage, deleteAll } from "./utils/api" * * The third is what a widget drawn without its title is dragged by, since the * header it used to be dragged by is the thing that is gone. + * + * The last is the undo a mis-click needs: a removed widget comes back. */ const dashboardName = `test_editor_${Date.now().toString(36)}` @@ -201,3 +203,25 @@ test("a palette is pasted as a link and read as roles", async ({ page }) => { "", ) }) + +test("a removed widget comes back on undo and goes again on redo", async ({ + page, +}) => { + await openEditor(page) + const frames = page.getByTestId("widget-frame") + const top = frames.filter({ hasText: "Top" }) + + await top.click() + await page.getByTestId("delete-widget").click() + await expect(top).toHaveCount(0) + + // Removing closes the panel, so the keys reach the canvas rather than a + // field, which keeps its own undo. + await page.keyboard.press("ControlOrMeta+z") + await expect(top).toBeVisible() + await expect(frames).toHaveCount(2) + + await page.keyboard.press("ControlOrMeta+Shift+z") + await expect(top).toHaveCount(0) + await expect(frames).toHaveCount(1) +}) From 278a480a5ccba6a43a9d16215451d45a33e39493 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 00:23:01 +0200 Subject: [PATCH 021/103] Endpoint labels sweep instead of pulse: left to right on a label publishing into the flow, right to left on one reading out of it, one pulse long, off under reduced motion; snapshot values are marked prior so catching up sweeps nothing Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/src/components/Flow/EndpointNode.tsx | 52 +++++++++++++-- frontend/src/components/Flow/flow.css | 63 +++++++++++++++++++ frontend/src/components/Flow/liveStore.ts | 7 ++- 3 files changed, 117 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/Flow/EndpointNode.tsx b/frontend/src/components/Flow/EndpointNode.tsx index 849d866..44d8bdb 100644 --- a/frontend/src/components/Flow/EndpointNode.tsx +++ b/frontend/src/components/Flow/EndpointNode.tsx @@ -1,11 +1,12 @@ import { Handle, type NodeProps, Position } from "@xyflow/react" import { LayoutDashboard, LogIn, LogOut, Workflow } from "lucide-react" -import { memo } from "react" +import { memo, useEffect, useMemo, useRef, useState } from "react" import { useIsMobile } from "@/hooks/useMobile" import { cn } from "@/lib/utils" import type { EndpointNodeData } from "./endpoints" import { handleOffset, nodeHeight } from "./layout" +import { useLiveValues } from "./liveStore" const KIND_ICONS = { dashboard: LayoutDashboard, @@ -22,10 +23,21 @@ const KIND_ICONS = { * the same reason dashboards are their own documents. It is here to account * for a value, not to compete with the nodes for attention. */ -function EndpointNodeComponent({ data, selected }: NodeProps) { +function EndpointNodeComponent({ id, data, selected }: NodeProps) { const { label, kind, detail, provides, requires } = data as EndpointNodeData const Icon = KIND_ICONS[kind as keyof typeof KIND_ICONS] ?? Workflow - const messages = [...provides, ...requires] + const messages = useMemo( + () => [...provides, ...requires], + [provides, requires], + ) + const values = useLiveValues(messages) + // Which way the last value went, and a count whose change remounts the sweep + // — remounting is what restarts it. See `.endpoint-sweep` in flow.css. + const [sweep, setSweep] = useState<{ dir: "in" | "out"; n: number }>() + // When each message last carried a value, as far as this label has seen. + // The first look only takes notes: mounting is not a value passing, and + // neither is a snapshot catching the page up. + const seen = useRef>(null) // Follows the graph's own direction; see DESIGN-GUIDELINES.md → Responsive. const vertical = useIsMobile() @@ -34,12 +46,35 @@ function EndpointNodeComponent({ data, selected }: NodeProps) { // node's ports are, so the layout can reserve exactly this much. const ports = Math.max(provides.length, requires.length) + useEffect(() => { + const first = seen.current === null + seen.current ??= new Map() + let dir: "in" | "out" | undefined + for (const [index, name] of messages.entries()) { + const live = values[index] + // An empty store is a reconnect on its way, not a value. + if (!live?.ts || seen.current.get(name) === live.ts) continue + seen.current.set(name, live.ts) + if (first || live.prior) continue + // Published by this label, by the rule its own edges pulse on (LiveEdge). + if (provides.includes(name) && (!live.source || live.source.id === id)) { + dir = "in" + } else if (requires.includes(name)) { + dir ??= "out" + } + } + if (!dir) return + const next = dir + setSweep((last) => ({ dir: next, n: (last?.n ?? 0) + 1 })) + }, [values, messages, provides, requires, id]) + return (
+ ) : null} {label} diff --git a/frontend/src/components/Flow/flow.css b/frontend/src/components/Flow/flow.css index e151c4c..aa26557 100644 --- a/frontend/src/components/Flow/flow.css +++ b/frontend/src/components/Flow/flow.css @@ -223,6 +223,69 @@ } } +/* + * An endpoint label has no stroke to pulse and no rim to lap, and lighting all + * of it at once would not say which way the value went. So a value passing + * sweeps across it: left to right on a label publishing into the flow, right + * to left on one reading out of it, then fades back to rest. One pulse long, + * like everything else on the canvas that answers a message. + * + * Where the light has got to is registered so it can travel, outside the guard + * below for the same reason as the node's angle: it is a type, not motion. + */ +@property --endpoint-sweep { + syntax: ""; + inherits: false; + initial-value: 0%; +} + +@media (prefers-reduced-motion: no-preference) { + /* + * Behind the text rather than over it, so the label stays as legible as it + * was; the label isolates itself, so behind does not mean behind the canvas. + * A brighter leading edge is what reads as travel; the wash it leaves is + * what reads as lit. + */ + .endpoint-sweep { + --endpoint-sweep-to: to right; + position: absolute; + inset: 0; + z-index: -1; + pointer-events: none; + background: linear-gradient( + var(--endpoint-sweep-to), + color-mix(in srgb, var(--primary) 14%, transparent) + calc(var(--endpoint-sweep) - 40%), + color-mix(in srgb, var(--primary) 32%, transparent) + calc(var(--endpoint-sweep) - 8%), + transparent var(--endpoint-sweep) + ); + animation: endpoint-sweep var(--duration-pulse) linear forwards; + } + + .endpoint-sweep.endpoint-sweep-out { + --endpoint-sweep-to: to left; + } + + /* + * Constant rate while it travels, as the node's arc does. It stays mounted + * until the next value remounts it, so it has to go out on its own. + */ + @keyframes endpoint-sweep { + from { + --endpoint-sweep: 0%; + } + 60% { + --endpoint-sweep: 140%; + opacity: 1; + } + to { + --endpoint-sweep: 140%; + opacity: 0; + } + } +} + /* Handles are neutral: the one brand-secondary affordance here is Run. */ .react-flow__handle { width: 12px; diff --git a/frontend/src/components/Flow/liveStore.ts b/frontend/src/components/Flow/liveStore.ts index 31bf5ce..0e3fc10 100644 --- a/frontend/src/components/Flow/liveStore.ts +++ b/frontend/src/components/Flow/liveStore.ts @@ -22,6 +22,11 @@ export type LiveValue = { value: unknown ts: number | null source?: ValueSource + /** + * Caught up from a snapshot rather than seen arriving. It happened, but not + * while anyone was watching — the same distinction as `priorEmits`. + */ + prior?: boolean } export type LiveStatus = { status: "active" | "error" | "queued" | "running" | "success" @@ -127,7 +132,7 @@ export const liveStore = { }, setValues(entries: Record) { for (const [name, value] of Object.entries(entries)) { - values.set(name, value) + values.set(name, { ...value, prior: true }) notify(`value:${name}`) } }, From 0e69f50a7a82d5f7a68372728b4484bca83b21b5 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 00:23:14 +0200 Subject: [PATCH 022/103] Docs: endpoint labels sweep when a value passes Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/interface/flow-editor.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/interface/flow-editor.md b/docs/interface/flow-editor.md index 5ab659e..eb9168b 100644 --- a/docs/interface/flow-editor.md +++ b/docs/interface/flow-editor.md @@ -10,7 +10,9 @@ The graph runs left to right on desktop and top to bottom on phones. - **Nodes** show their execution status. - **Edges** pulse when values pass through. Click one to inspect its payload. - **Endpoints** show flow inputs, dashboard widgets and nodes in other flows - that exchange messages with this flow. + that exchange messages with this flow. A value passing lights one up in a + sweep: left to right when it sends into the flow, right to left when it reads + out of it. Values update over a websocket. A connection banner tells you when the editor cannot reach the engine. From 82466cad9a772d1c2647c3b1828eabf6494e3286 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 08:36:38 +0200 Subject: [PATCH 023/103] Verify shoots the flow canvas and media tile from the Demo, not the media example The canvas shot opens the Demo's music flow; the media shot reads the Demo's Media screen, so neither needs the `camera` dashboard. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/scripts/capture-screenshots.mjs | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/frontend/scripts/capture-screenshots.mjs b/frontend/scripts/capture-screenshots.mjs index 831baba..615bfa5 100644 --- a/frontend/scripts/capture-screenshots.mjs +++ b/frontend/scripts/capture-screenshots.mjs @@ -223,10 +223,11 @@ async function capturePanels(page, dir) { } /** - * A media tile drawing what a camera published, where there is one. + * A media tile drawing what a camera published: the Demo's Media screen, + * whose house is drawn by a node as heat. * - * Skipped unless the media example is seeded (root `make seed-example-media`), - * since it is the one shot that needs a source of frames. The bytes arrive as + * Skipped unless the Demo is seeded (root `make seed-hosted-demo`), since it + * is the one shot that needs a source of frames. The bytes arrive as * a blob — pushed down the socket, or fetched with the session's credential, * which no `img` could carry on its own — so a `blob:` source is the proof the * whole path ran rather than that a picture is merely present. @@ -237,25 +238,23 @@ async function capturePanels(page, dir) { * where idleness only ever said the page stopped asking. */ async function captureMedia(page, dir) { - const answer = await page.goto(`${APP_URL}/view/camera`, { + const answer = await page.goto(`${APP_URL}/view/media`, { waitUntil: "domcontentloaded", }) if (!answer?.ok()) return - const picture = page.locator("img[alt='Test camera']") + const picture = page.locator("img[alt='The house, as heat']") try { await picture.waitFor({ timeout: 15000 }) await page.waitForFunction( () => document - .querySelector("img[alt='Test camera']") + .querySelector("img[alt='The house, as heat']") ?.src?.startsWith("blob:") ?? false, { timeout: 15000 }, ) } catch { - console.warn( - " media tile drew nothing — is `make seed-example-media` run?", - ) + console.warn(" media tile drew nothing — is the Demo seeded?") return } await page.waitForTimeout(500) @@ -296,7 +295,13 @@ async function captureWorkers(page, dir) { async function captureFlows(page, dir) { await page.goto(`${APP_URL}/flows`, { waitUntil: "networkidle" }) - if (await page.getByTestId("flow-card").count()) { + // The Demo's music flow, where it is seeded: two nodes, live values on the + // edges and the tiles at both ends, all still legible once fitted. The house + // has fifteen nodes, which fitted to the screen read as dots. + const demo = page.locator("[data-testid=flow-card][href$='/flows/music']") + if (await demo.count()) { + await demo.click() + } else if (await page.getByTestId("flow-card").count()) { await page.getByTestId("flow-card").first().click() } else { await page.getByTestId("new-flow").click() From 7d28b4445167d0c0d5ec51ff56ff63128cf1680c Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 08:37:32 +0200 Subject: [PATCH 024/103] development.md: verify takes its domain off the running stack Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- development.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/development.md b/development.md index 3bd3abc..6e2f00a 100644 --- a/development.md +++ b/development.md @@ -153,8 +153,8 @@ creates and deletes flows, dashboards and users. `PLAYWRIGHT_ALLOW_PUBLIC=1` ove `app/frontend/screenshots/{light,dark}/`. It is an alias for `make verify-docker`: the run always happens in the Playwright container, which pins the browser by reading `frontend/package.json`. Running Playwright on the host is unsupported — the workspace pins a -browser revision whose install is incomplete. The domain comes from `.env`, and -`make verify DOMAIN=example.com` still wins. +browser revision whose install is incomplete. The domain comes off the running stack (the +frontend container's Traefik rule), so the shots are of whatever `make dev` last started. ## Lint, hooks and the generated client From afcbb63a56ef658eacbbc66ff9fc052218ce3c6e Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:32:35 +0200 Subject: [PATCH 025/103] Delay node: hold the latest value inside the interval and send it at the window's end The interval now rides the output ports, so the pipeline's own hold and flush keep what arrives inside the window instead of the node dropping it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/nodes/delay.py | 25 ++++++++++--------- backend/tests/flow/test_discretization.py | 29 ++++++++++++++++++++++- docs/reference/node-types.md | 9 +++---- 3 files changed, 45 insertions(+), 18 deletions(-) diff --git a/backend/fluksio/flow/nodes/delay.py b/backend/fluksio/flow/nodes/delay.py index a9d448e..d8d23f7 100644 --- a/backend/fluksio/flow/nodes/delay.py +++ b/backend/fluksio/flow/nodes/delay.py @@ -23,7 +23,8 @@ class DelayNode(Node): This node can: - Add a fixed delay before forwarding messages - - Rate-limit messages to a minimum interval between forwards + - Rate-limit messages to a minimum interval between forwards, sending the + latest one held inside the window when the window ends - Schedule emissions using crontab syntax (like a cron job) The node passes through all input values to outputs with matching names. @@ -34,7 +35,7 @@ class DelayNode(Node): - Without ``requires``: Emits current timestamp at each cron tick - With ``requires``: Emits last received input at each cron tick - **Execution order:** rate check → delay → forward + **Execution order:** delay → rate check → forward :param params: Parameters dict containing: - ``delay`` (int): Fixed delay in seconds before forwarding (default: 0) @@ -89,7 +90,6 @@ class DelayNode(Node): __slots__ = ( "delay", "interval", - "ts", "mapping", "cron_expr", "last_input", @@ -113,11 +113,18 @@ class DelayNode(Node): self.interval = cfg.interval self.mapping = cfg.mapping self.cron_expr = cfg.cron - self.ts = 0.0 self.last_input: dict[str, Any] = {} self._cron_task: asyncio.Task[None] | None = None self._stop_cron: asyncio.Event | None = None + if cfg.interval > 0: + # The engine's own rate limit, on every output: it keeps the latest + # value inside the window and publishes it when the window ends. + kwargs["provides"] = [ + spec.model_copy(update={"interval": max(spec.interval, cfg.interval)}) + for spec in self._normalize_ports(kwargs.get("provides", ())) + ] + super().__init__(f=self._f, params=params, **kwargs) def _port_pairs(self) -> list[tuple[str, str]]: @@ -130,7 +137,7 @@ class DelayNode(Node): ] def _f(self, params: dict[str, Any], **kwargs: Any) -> dict[str, Any] | None: - """Forward messages with optional delay, rate-limiting, and alarm.""" + """Forward messages, after a delay if one is set.""" logger.debug("[%s] Received %s", self.name, kwargs) @@ -138,14 +145,6 @@ class DelayNode(Node): if kwargs: self.last_input = dict(kwargs) - # Apply rate limiting first - if self.interval > 0: - ts = time.time() - if ts <= self.ts + self.interval: - logger.debug("[%s] Stashing %s", self.name, kwargs) - return None - self.ts = ts - if not kwargs: return None diff --git a/backend/tests/flow/test_discretization.py b/backend/tests/flow/test_discretization.py index cae205c..6fe3f87 100644 --- a/backend/tests/flow/test_discretization.py +++ b/backend/tests/flow/test_discretization.py @@ -4,7 +4,7 @@ import time from fluksio.flow.executor import ExecutionService from fluksio.flow.messages import DType, MessageSpec -from fluksio.flow.nodes import Node +from fluksio.flow.nodes import DelayNode, Node from fluksio.flow.pipeline import Pipeline from fluksio.flow.queue import MemoryWorkQueue @@ -213,6 +213,33 @@ def test_a_node_whose_inputs_did_not_change_is_not_run(): assert seen == [1.0, 2.0] +def test_a_delay_interval_sends_the_latest_value_when_the_window_ends(): + """It used to drop a value inside the window rather than hold it.""" + seen: list[float] = [] + source = make_node("source", lambda params: None, provides=[spec("raw")]) + delay = DelayNode( + params={"interval": WINDOW}, + requires=[spec("raw")], + provides=[spec("out")], + name="delay", + ) + delay.assign_flow("demo", "delay") + watcher = make_node( + "watcher", lambda out, params: seen.append(out), requires=[spec("out")] + ) + _pipeline, queue, service = _running([source, delay, watcher]) + + for value in (1.0, 2.0, 3.0): + source.inject({"raw": value}) + for item in queue.claim(10, 10): + service._run_item(item) + assert seen == [1.0] + + time.sleep(WINDOW * 2) + assert _run_due(queue, service) == 1 + assert seen == [1.0, 3.0] + + def test_a_join_behind_a_skipped_branch_still_runs(): """Skipping a node frees its consumers, which may already have been passed.""" seen: list[tuple[float, float]] = [] diff --git a/docs/reference/node-types.md b/docs/reference/node-types.md index 7366d04..6f2e861 100644 --- a/docs/reference/node-types.md +++ b/docs/reference/node-types.md @@ -171,13 +171,14 @@ Use `cron` for scheduled emissions, or trigger the node by hand for testing. | Setting | Default | Notes | |---|---|---| | `delay` | `0` | seconds to hold each message, fractional | -| `interval` | `0` | minimum seconds between forwards | +| `interval` | `0` | minimum seconds between forwards, per output | | `mapping` | `{}` | input port → output port; paired in order when empty | | `cron` | — | five-field expression | -Order of operations: rate check → delay → forward. With a `cron` and no inputs -it emits the current time on each tick; with inputs it emits the last value it -received. +Order of operations: delay → rate check → forward. A message arriving inside +the `interval` window is held, and when the window ends the latest one held is +sent. With a `cron` and no inputs it emits the current time on each tick; with +inputs it emits the last value it received. Timers wait until their deadline; busy workers can delay execution. From c7e38a8a449a547f0afaadff234e6dbf7a8948b1 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:34:40 +0200 Subject: [PATCH 026/103] Rename the trigger node type to debounce, migrating stored flows The store rewrites `"type": "trigger"` to `debounce` in every published and draft flow when it opens, bumps the version and commits once. No alias stays behind, so `trigger` is now an unknown type. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/controller.py | 10 +++--- backend/fluksio/flow/nodes/__init__.py | 4 +-- .../flow/nodes/{trigger.py => debounce.py} | 16 ++++----- backend/fluksio/flow/store.py | 34 +++++++++++++++++++ backend/tests/flow/test_house_vocabulary.py | 26 +++++++------- backend/tests/flow/test_node_types.py | 2 +- backend/tests/flow/test_store.py | 28 +++++++++++++++ docs/reference/node-types.md | 11 +++--- frontend/src/components/Flow/FlowNode.tsx | 2 +- 9 files changed, 98 insertions(+), 35 deletions(-) rename backend/fluksio/flow/nodes/{trigger.py => debounce.py} (90%) diff --git a/backend/fluksio/flow/controller.py b/backend/fluksio/flow/controller.py index ee434aa..ac14ecb 100644 --- a/backend/fluksio/flow/controller.py +++ b/backend/fluksio/flow/controller.py @@ -40,6 +40,7 @@ from fluksio.flow.messages import MessageSpec, flow_of, qualify from fluksio.flow.nodes import ( RESERVED_SETTINGS, ChangeNode, + DebounceNode, DelayNode, ExecNode, FileNode, @@ -53,7 +54,6 @@ from fluksio.flow.nodes import ( NtfyNode, RbeNode, SwitchNode, - TriggerNode, WebSocketNode, WrenNode, ) @@ -408,11 +408,11 @@ NODE_TYPES: dict[str, NodeType] = { cls=JoinNode, params_schema=_schema_of(JoinNode), ), - "trigger": NodeType( - title="Trigger", + "debounce": NodeType( + title="Debounce", description="Send one value now and another once things go quiet.", - cls=TriggerNode, - params_schema=_schema_of(TriggerNode), + cls=DebounceNode, + params_schema=_schema_of(DebounceNode), ), "exec": NodeType( title="Command", diff --git a/backend/fluksio/flow/nodes/__init__.py b/backend/fluksio/flow/nodes/__init__.py index 4f91e03..1973a54 100644 --- a/backend/fluksio/flow/nodes/__init__.py +++ b/backend/fluksio/flow/nodes/__init__.py @@ -5,6 +5,7 @@ Split by the outside world each one talks to. Importing from """ from fluksio.flow.nodes.base import RESERVED_SETTINGS, Node +from fluksio.flow.nodes.debounce import DebounceNode from fluksio.flow.nodes.delay import DelayNode from fluksio.flow.nodes.exec import ExecNode from fluksio.flow.nodes.file import FileNode @@ -15,12 +16,12 @@ from fluksio.flow.nodes.logic import ChangeNode, JoinNode, RbeNode, SwitchNode from fluksio.flow.nodes.mlp import MLPNode from fluksio.flow.nodes.mqtt import MqttNode from fluksio.flow.nodes.ntfy import NtfyNode -from fluksio.flow.nodes.trigger import TriggerNode from fluksio.flow.nodes.websocket import WebSocketNode from fluksio.flow.nodes.wren import WrenNode __all__ = [ "ChangeNode", + "DebounceNode", "DelayNode", "ExecNode", "FileNode", @@ -35,7 +36,6 @@ __all__ = [ "RESERVED_SETTINGS", "RbeNode", "SwitchNode", - "TriggerNode", "WebSocketNode", "WrenNode", ] diff --git a/backend/fluksio/flow/nodes/trigger.py b/backend/fluksio/flow/nodes/debounce.py similarity index 90% rename from backend/fluksio/flow/nodes/trigger.py rename to backend/fluksio/flow/nodes/debounce.py index d85d21e..3c61e66 100644 --- a/backend/fluksio/flow/nodes/trigger.py +++ b/backend/fluksio/flow/nodes/debounce.py @@ -1,8 +1,8 @@ -"""Trigger: send one thing now, another if nothing follows. +"""Debounce: send one thing now, another if nothing follows. -Node-RED's *trigger*. The shape it exists for is "the door opened — turn the -light on, and off again in two minutes unless it opens again", which is -awkward to express any other way. +Node-RED calls this node *trigger*. The shape it exists for is "the door +opened — turn the light on, and off again in two minutes unless it opens +again", which is awkward to express any other way. """ from __future__ import annotations @@ -19,7 +19,7 @@ from fluksio.flow.nodes.base import Node logger = logging.getLogger(__name__) -class TriggerNode(Node): +class DebounceNode(Node): """Emit ``first`` on arrival, then ``then`` once the wait runs out. A value arriving during the wait extends it, so the second emission only @@ -67,17 +67,17 @@ class TriggerNode(Node): ): self.cfg = self.Params.model_validate(params or {}) super().__init__( - f=self._trigger, + f=self._debounce, requires=requires, provides=provides, params=params, - name=name or "trigger", + name=name or "debounce", ) def _outputs(self, value: Any) -> dict[str, Any]: return {spec.port: value for spec in self.output_ports} - def _trigger(self, params: dict[str, Any], **kwargs: Any) -> dict[str, Any] | None: + def _debounce(self, params: dict[str, Any], **kwargs: Any) -> dict[str, Any] | None: if not kwargs or not self.output_ports: return None diff --git a/backend/fluksio/flow/store.py b/backend/fluksio/flow/store.py index e100ede..6d2a48f 100644 --- a/backend/fluksio/flow/store.py +++ b/backend/fluksio/flow/store.py @@ -166,6 +166,40 @@ class FlowStore: if not (self.root / ".git").exists(): self._git("init", "-q") self._commit("Initialise flow store", allow_empty=True) + self._migrate() + + def _migrate(self) -> None: + """Rewrite node types stored under a name they no longer have. + + The node type ``trigger`` became ``debounce`` on 2026-10-15, with no + alias left for the old name. Each published and draft document is read + as plain JSON and only the ``type`` of a matching node changes, so a + field this version does not model survives, and the old document stays + in git. The version goes up by one, so an editor still holding the old + document gets a conflict on its next save instead of writing the old + name back. A store with nothing to rename is left alone. + """ + changed = [] + paths = [*self.root.glob("*/flow.json"), *self.root.glob("*/flow.draft.json")] + for path in sorted(paths): + try: + data = json.loads(path.read_text()) + nodes = [n for n in data["nodes"] if n.get("type") == "trigger"] + except Exception: + # Unreadable to the engine too, which reports it when it loads. + continue + if not nodes: + continue + for node in nodes: + node["type"] = "debounce" + data["version"] = data.get("version", 1) + 1 + path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n") + changed.append(f"{path.parent.name}/{path.name}") + if changed: + logger.warning( + "Renamed node type 'trigger' to 'debounce' in %s", ", ".join(changed) + ) + self._commit("Rename node type 'trigger' to 'debounce'") # ------------------------------------------------------------------------- # git diff --git a/backend/tests/flow/test_house_vocabulary.py b/backend/tests/flow/test_house_vocabulary.py index 10db4b7..8df6fbb 100644 --- a/backend/tests/flow/test_house_vocabulary.py +++ b/backend/tests/flow/test_house_vocabulary.py @@ -8,8 +8,8 @@ come up than to go down. from fluksio.flow.messages import DType, MessageSpec from fluksio.flow.nodes import MqttNode +from fluksio.flow.nodes.debounce import DebounceNode from fluksio.flow.nodes.mqtt import topic_matches -from fluksio.flow.nodes.trigger import TriggerNode # ── MQTT topic filters ──────────────────────────────────────────────────── @@ -72,7 +72,7 @@ def test_without_json_key_the_port_name_is_the_key_as_before(): assert node.json_keys == {} -# ── trigger: passthrough and a per-message wait ─────────────────────────── +# ── debounce: passthrough and a per-message wait ────────────────────────── class _Pipeline: @@ -87,8 +87,8 @@ class _Pipeline: return True -def _trigger(**params) -> tuple[TriggerNode, _Pipeline]: - node = TriggerNode( +def _debounce(**params) -> tuple[DebounceNode, _Pipeline]: + node = DebounceNode( requires=[ MessageSpec(name="run", port="run", dtype=DType.STR), MessageSpec(name="run_for", port="run_for", dtype=DType.FLOAT), @@ -103,30 +103,30 @@ def _trigger(**params) -> tuple[TriggerNode, _Pipeline]: def test_the_incoming_value_passes_through_and_the_wait_comes_from_a_port(): - node, pipeline = _trigger() + node, pipeline = _debounce() - assert node._trigger({}, run="DOWN", run_for=28.0) == {"cmd": "DOWN"} + assert node._debounce({}, run="DOWN", run_for=28.0) == {"cmd": "DOWN"} (outputs, seconds, _guard) = pipeline.deferred[-1] assert outputs == {"shutters.cmd": "STOP"} and seconds == 28.0 - node._trigger({}, run="UP", run_for=26.0) + node._debounce({}, run="UP", run_for=26.0) assert pipeline.deferred[-1][1] == 26.0 def test_a_second_command_invalidates_the_stop_the_first_one_scheduled(): - node, pipeline = _trigger() - node._trigger({}, run="DOWN", run_for=28.0) + node, pipeline = _debounce() + node._debounce({}, run="DOWN", run_for=28.0) first = pipeline.deferred[-1][2] - node._trigger({}, run="UP", run_for=26.0) + node._debounce({}, run="UP", run_for=26.0) assert pipeline.deferred[-1][2] != first def test_a_wait_of_zero_sends_nothing_afterwards(): """STOP is commanded once. Nothing follows it, forever.""" - node, pipeline = _trigger() - node._trigger({}, run="DOWN", run_for=28.0) + node, pipeline = _debounce() + node._debounce({}, run="DOWN", run_for=28.0) scheduled = len(pipeline.deferred) - assert node._trigger({}, run="STOP", run_for=0.0) == {"cmd": "STOP"} + assert node._debounce({}, run="STOP", run_for=0.0) == {"cmd": "STOP"} assert len(pipeline.deferred) == scheduled assert node.recall("armed", 0) == 0 diff --git a/backend/tests/flow/test_node_types.py b/backend/tests/flow/test_node_types.py index 00b9d7b..face945 100644 --- a/backend/tests/flow/test_node_types.py +++ b/backend/tests/flow/test_node_types.py @@ -75,7 +75,7 @@ FIXTURES: dict[str, dict] = { ], "provides": [MessageSpec(name="reading", dtype=DType.JSON)], }, - "trigger": { + "debounce": { "params": {"first": True, "then": False, "wait": 120}, "requires": [MessageSpec(name="motion", dtype=DType.BOOL)], "provides": [MessageSpec(name="light", dtype=DType.BOOL)], diff --git a/backend/tests/flow/test_store.py b/backend/tests/flow/test_store.py index 0734b03..13abd34 100644 --- a/backend/tests/flow/test_store.py +++ b/backend/tests/flow/test_store.py @@ -131,3 +131,31 @@ def test_renaming_onto_an_existing_name_is_refused(store: FlowStore): store.rename_flow("heating", "warmth") assert store.list_flows() == ["heating", "warmth"] + + +def test_opening_the_store_renames_trigger_nodes_to_debounce(store: FlowStore): + """The old name has no alias, so what is stored under it is rewritten once.""" + store.write_flow( + FlowDef( + name="hall", + nodes=[NodeDef(id="hold", type="trigger"), NodeDef(id="lamp")], + ) + ) + store.write_draft( + FlowDef(name="hall", nodes=[NodeDef(id="hold", type="trigger")]), + base_version=1, + ) + commits = commit_count(store) + + reopened = FlowStore(store.root) + + published = reopened.read_flow("hall") + draft = reopened.read_flow("hall", draft=True) + assert [node.type for node in published.nodes] == ["debounce", "python"] + assert [node.type for node in draft.nodes] == ["debounce"] + # Bumped, so an editor holding the old draft cannot save `trigger` back. + assert (published.version, draft.version) == (2, 3) + assert commit_count(store) == commits + 1 + + FlowStore(store.root) + assert commit_count(store) == commits + 1 diff --git a/docs/reference/node-types.md b/docs/reference/node-types.md index 6f2e861..9300fed 100644 --- a/docs/reference/node-types.md +++ b/docs/reference/node-types.md @@ -187,11 +187,12 @@ Timers wait until their deadline; busy workers can delay execution. Batch flows reject rate-limited non-streaming ports. Runs have no live timer to release a held value. -### Trigger +### Debounce -**`trigger`** — send one value now and another once things go quiet. Despite -the name, a debounce and hold rather than a scheduler: everything it sends -starts from a value arriving. For a cron tick, see [`inject`](#inject). +**`debounce`** — send one value now and another once things go quiet. A +debounce and hold rather than a scheduler: everything it sends starts from a +value arriving. For a cron tick, see [`inject`](#inject). Node-RED calls this +node *trigger*. | Setting | Default | Notes | |---|---|---| @@ -363,7 +364,7 @@ Each record in `actions` contains `message`, `value`, `applied`, `delay_s`, applied. Keep `apply=false` while checking how the model interprets commands. A call with a delay ("in 10m") is held by the engine's work queue, like the -Trigger node's second value: it survives a restart where the queue is in +Debounce node's second value: it survives a restart where the queue is in Redis, and not otherwise. The node adds default devices to the model's view for missing expected devices. diff --git a/frontend/src/components/Flow/FlowNode.tsx b/frontend/src/components/Flow/FlowNode.tsx index d40dbaf..c002285 100644 --- a/frontend/src/components/Flow/FlowNode.tsx +++ b/frontend/src/components/Flow/FlowNode.tsx @@ -55,7 +55,7 @@ const NODE_ICONS = { change: Shuffle, rbe: Filter, join: Merge, - trigger: Timer, + debounce: Timer, exec: Terminal, file: FileText, ntfy: Bell, From 13567f899092c507324ffb462d3c30cfc0c0b042 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:37:30 +0200 Subject: [PATCH 027/103] Give a rate-limit flush its own run, linked to the run that held the value A flush ran its cascade under no id, so its executions and failures belonged to no run. It now records a run with source "flush", and the holding run's id travels on the queue item into a new `flow_run.parent` column, also on the `/observability/runs` rows. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- .../versions/930ffa094aad_flow_run_parent.py | 37 ++++++++++++ backend/fluksio/api/routes/observability.py | 2 + backend/fluksio/flow/executor.py | 15 +++-- backend/fluksio/flow/metrics.py | 1 + backend/fluksio/flow/pipeline.py | 60 +++++++++++++------ backend/fluksio/flow/queue.py | 5 ++ backend/fluksio/models.py | 3 + backend/tests/test_metrics.py | 54 +++++++++++++++++ frontend/src/client/schemas.gen.ts | 5 ++ frontend/src/client/types.gen.ts | 1 + 10 files changed, 157 insertions(+), 26 deletions(-) create mode 100644 backend/fluksio/alembic/versions/930ffa094aad_flow_run_parent.py diff --git a/backend/fluksio/alembic/versions/930ffa094aad_flow_run_parent.py b/backend/fluksio/alembic/versions/930ffa094aad_flow_run_parent.py new file mode 100644 index 0000000..0018c81 --- /dev/null +++ b/backend/fluksio/alembic/versions/930ffa094aad_flow_run_parent.py @@ -0,0 +1,37 @@ +"""flow_run.parent + +A rate-limit flush is a run of its own now, and this names the run whose value +it released. Rows written before this carry an empty string, the same as every +run that continues no other. + +Revision ID: 930ffa094aad +Revises: b7d2e9c4a1f3 +Create Date: 2026-10-15 + +""" + +import sqlalchemy as sa +import sqlmodel.sql.sqltypes +from alembic import op + +# revision identifiers, used by Alembic. +revision = "930ffa094aad" +down_revision = "b7d2e9c4a1f3" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "flow_run", + sa.Column( + "parent", + sqlmodel.sql.sqltypes.AutoString(length=64), + nullable=False, + server_default="", + ), + ) + + +def downgrade(): + op.drop_column("flow_run", "parent") diff --git a/backend/fluksio/api/routes/observability.py b/backend/fluksio/api/routes/observability.py index 154a989..f72733c 100644 --- a/backend/fluksio/api/routes/observability.py +++ b/backend/fluksio/api/routes/observability.py @@ -85,6 +85,8 @@ class RunRow(BaseModel): id: str flow: str source: str + #: The run this one continues: for a flush, the run whose value it released. + parent: str = "" status: str started_at: datetime finished_at: datetime | None = None diff --git a/backend/fluksio/flow/executor.py b/backend/fluksio/flow/executor.py index 0778dfb..c183a9c 100644 --- a/backend/fluksio/flow/executor.py +++ b/backend/fluksio/flow/executor.py @@ -440,13 +440,6 @@ class ExecutionService: self.queue.park(node.flow, item) return True - if item.kind == "flush": - # A rate-limit window ended; nothing to replay, only to let out. - # ponytail: no run record for a flush — it is the tail of the run - # that scheduled it, not a run of its own. - pipeline.flush(node) - return True - if item.guard_key and str(node.recall(item.guard_key, "")) != item.guard_value: # The node moved on while this waited — a restarted timer, say. logger.debug("Guard no longer holds for '%s', dropped", item.node) @@ -461,6 +454,7 @@ class ExecutionService: "flow": item.flow, "node": item.node, "cause": item.cause, + "parent": item.parent, "deliveries": item.deliveries, "ts": now, } @@ -483,6 +477,11 @@ class ExecutionService: replay = item.deliveries > 1 emission = item.kind == "emission" try: + if item.kind == "flush": + # A rate-limit window ended; nothing to replay, only to let out. + # A run of its own, so what it sets off is recorded somewhere. + pipeline.flush(node, item.entry_id) + return True # An emission's values went into state when the node produced them; # this item carries them so its readers get the chunk that caused # the wave rather than whichever is newest by the time they run. @@ -491,7 +490,7 @@ class ExecutionService: published = ( set(item.outputs) if emission - else pipeline.apply_outputs(node, item.outputs or None) + else pipeline.apply_outputs(node, item.outputs or None, item.entry_id) ) pipeline.run_downstream( node, diff --git a/backend/fluksio/flow/metrics.py b/backend/fluksio/flow/metrics.py index 7aaa441..6eacba9 100644 --- a/backend/fluksio/flow/metrics.py +++ b/backend/fluksio/flow/metrics.py @@ -291,6 +291,7 @@ class MetricsCollector: "id": run_id, "flow": str(event.get("flow") or "")[:NAME_MAX], "source": str(event.get("cause") or ""), + "parent": str(event.get("parent") or "")[:64], "started_ts": ts, "started_at": datetime.fromtimestamp(ts, UTC), "finished_at": None, diff --git a/backend/fluksio/flow/pipeline.py b/backend/fluksio/flow/pipeline.py index 94e27c9..8076def 100644 --- a/backend/fluksio/flow/pipeline.py +++ b/backend/fluksio/flow/pipeline.py @@ -670,7 +670,7 @@ class Pipeline: return passed, held, due_at, stamps.get(flush_key) def _throttled( - self, node: Node, result: dict[str, Any] + self, node: Node, result: dict[str, Any], entry_id: str = "" ) -> tuple[dict[str, Any], list[str]]: """Hold back the outputs whose port is not due to publish yet. @@ -679,6 +679,9 @@ class Pipeline: than leaving the consumer on the one before it. Nothing declaring an interval means nothing to look up. + ``entry_id`` is the run doing the holding, which the flush names as + its parent. + Returns what passed and the held keys the publish makes stale — the caller drops them in the same round trip it writes the values with, rather than one DEL per port here. @@ -699,28 +702,35 @@ class Pipeline: self._state.update( {self._held_key(name): value for name, value in held.items()} ) - self._schedule_flush(node, due_at, now, pending) + self._schedule_flush(node, due_at, now, pending, entry_id) return passed, stale def _schedule_flush( - self, node: Node, at: float, now: float, pending: float | None + self, + node: Node, + at: float, + now: float, + pending: float | None, + entry_id: str = "", ) -> None: """Come back to this node when its rate-limit window ends. One timer in flight per node: a pending one that is early enough is left alone, and one that turns out to be too late simply finds nothing - to do when it fires. + to do when it fires. So the parent a flush names is the run that booked + it, which held the first value of the window, not necessarily the last. """ if self._queue is None or (pending and now < pending <= at): return self._state[self._flush_key(node.id)] = at - self.defer(node, {}, at - now, kind="flush") + self.defer(node, {}, at - now, kind="flush", parent=entry_id) - def flush(self, node: Node) -> None: + def flush(self, node: Node, entry_id: str = "") -> None: """Let through what this node's rate limits held back. Runs when a window ends: the last value a limited output kept back is published, and a node whose inputs were all held back gets its run. + ``entry_id`` is the flush's own run. """ limited_out = [name for name, s in node.provides.items() if s.interval > 0] stored = ( @@ -737,14 +747,17 @@ class Pipeline: # Publishing runs the limit again, which is what clears the hold — # or puts the timer back, if this fired a hair early. In that case # nothing came out and there is nothing downstream to wake. - published = self.apply_outputs(node, held) + published = self.apply_outputs(node, held, entry_id) if published: - self.run_downstream(node, changed=published) + self.run_downstream(node, entry_id=entry_id, changed=published) if any(spec.interval > 0 for spec in node.requires.values()): # A wake-up that was held back, not a value that just changed — # so this one deliberately walks everything reachable. self._execute_parallel( - {node, *self._get_downstream(node)}, self._state, check_ready=True + {node, *self._get_downstream(node)}, + self._state, + check_ready=True, + entry_id=entry_id, ) def _increment_message_versions(self, outputs: dict[str, Any]) -> None: @@ -791,7 +804,7 @@ class Pipeline: return self._state.compare_and_swap_multi(expected, updates) - def _input_is_due(self, node: Node) -> bool: + def _input_is_due(self, node: Node, entry_id: str = "") -> bool: """Has any rate-limited input waited out its interval? A node runs when *any* of its inputs is due, so a slow port next to a @@ -827,6 +840,7 @@ class Pipeline: ), now, stamps.get(flush_key), + entry_id, ) return False @@ -834,7 +848,7 @@ class Pipeline: return True def _is_node_ready( - self, node: Node, state: StateBackend + self, node: Node, state: StateBackend, entry_id: str = "" ) -> tuple[bool, dict[str, Any]]: """Whether the node may run, and the inputs the answer was read from. @@ -851,7 +865,7 @@ class Pipeline: if any(msg not in values for msg in waited_on): return False, values - if not self._input_is_due(node): + if not self._input_is_due(node, entry_id): return False, values if not node.synchronous: @@ -1062,7 +1076,7 @@ class Pipeline: stale: list[str] = [] if result: - result, stale = self._throttled(node, result) + result, stale = self._throttled(node, result, entry_id) if result: self._record_outputs(node, result, state, stale) @@ -1358,7 +1372,7 @@ class Pipeline: # The values are kept whether or not the answer is yes: # a node that may run reads them below instead of # asking state for the same keys again. - ready, ready_inputs = self._is_node_ready(n, state) + ready, ready_inputs = self._is_node_ready(n, state, entry_id) if not ready: # Completed, not merely passed over: a later trigger # may still make it ready, but not running is not @@ -1440,11 +1454,14 @@ class Pipeline: return self._execute_parallel(nodes, self._state, check_ready=False) - def apply_outputs(self, node: Node, outputs: dict[str, Any] | None) -> set[str]: + def apply_outputs( + self, node: Node, outputs: dict[str, Any] | None, entry_id: str = "" + ) -> set[str]: """Record what a node emitted: state, history, versions and events. Shared by the direct path and by the execution service replaying a journaled item, so a value looks the same on the canvas either way. + ``entry_id`` is the run this happens in. Returns the message names that actually reached state — post rate limiting — which is what the cascade behind it has to walk from. @@ -1453,7 +1470,7 @@ class Pipeline: if outputs: # This is where a chatty subscriber gets thinned out, so a port set # to publish every 60s does so whatever the broker sends. - outputs, stale = self._throttled(node, outputs) + outputs, stale = self._throttled(node, outputs, entry_id) if not outputs: return set() @@ -1569,6 +1586,7 @@ class Pipeline: self._state.update( {self._held_key(name): value for name, value in held.items()} ) + # Held before any run began, so its flush names no parent. self._schedule_flush(node, due_at, now, pending) return True except Exception: @@ -1607,7 +1625,9 @@ class Pipeline: ) try: published = ( - set(outputs or {}) if delivered else self.apply_outputs(node, outputs) + set(outputs or {}) + if delivered + else self.apply_outputs(node, outputs, run_id) ) state = self.run_downstream( node, @@ -1684,6 +1704,7 @@ class Pipeline: seconds: float, guard: tuple[str, Any] | None = None, kind: str = "cascade", + parent: str = "", ) -> bool: """Publish a node's outputs later, without holding a worker thread. @@ -1691,6 +1712,8 @@ class Pipeline: over; if it has moved on, the item is dropped. That is how a wait which gets restarted cancels the one it replaced. + ``parent`` is the run the item continues, recorded beside its own. + Returns False when there is no queue to hold the item, in which case the caller has to wait however it waited before. """ @@ -1703,9 +1726,10 @@ class Pipeline: node=node.id, flow=node.flow, outputs=outputs, - cause="delay", + cause="flush" if kind == "flush" else "delay", guard_key=guard[0] if guard else "", guard_value=str(guard[1]) if guard else "", + parent=parent, ) try: self._queue.add_delayed(item, time.time() + seconds) diff --git a/backend/fluksio/flow/queue.py b/backend/fluksio/flow/queue.py index 6837616..3d46bad 100644 --- a/backend/fluksio/flow/queue.py +++ b/backend/fluksio/flow/queue.py @@ -94,6 +94,9 @@ class WorkItem: #: The run this item is, on a ``run`` item. Everything else about a run is #: in its database row, so the journal only has to name it. run_id: str = "" + #: The run that scheduled this item, on a ``flush``: the one whose value + #: the rate limit held back. + parent: str = "" def to_fields(self) -> dict[str, str]: return { @@ -107,6 +110,7 @@ class WorkItem: "guard_value": self.guard_value, "enqueued_at": str(self.enqueued_at), "run_id": self.run_id, + "parent": self.parent, } @classmethod @@ -126,6 +130,7 @@ class WorkItem: deliveries=deliveries, enqueued_at=float(fields.get("enqueued_at") or time.time()), run_id=fields.get("run_id", ""), + parent=fields.get("parent", ""), ) diff --git a/backend/fluksio/models.py b/backend/fluksio/models.py index 3235b06..16152e6 100644 --- a/backend/fluksio/models.py +++ b/backend/fluksio/models.py @@ -298,6 +298,9 @@ class FlowRun(SQLModel, table=True): flow: str = Field(index=True, max_length=255) #: What caused it: the work item's cause, "manual", or "run" for a batch. source: str = "" + #: The run this one continues: on a rate-limit flush, the run whose value + #: the window held back. Empty for everything else. + parent: str = Field(default="", max_length=64) started_at: datetime = Field( index=True, sa_type=UTCDateTime, diff --git a/backend/tests/test_metrics.py b/backend/tests/test_metrics.py index 688573e..7d8d03a 100644 --- a/backend/tests/test_metrics.py +++ b/backend/tests/test_metrics.py @@ -11,7 +11,12 @@ from datetime import UTC, datetime from sqlmodel import Session, select from fluksio.flow.events import EventBus +from fluksio.flow.executor import ExecutionService +from fluksio.flow.messages import DType, MessageSpec from fluksio.flow.metrics import DETAIL_CAP, RUN_STALE_S, MetricsCollector +from fluksio.flow.nodes import Node +from fluksio.flow.pipeline import Pipeline +from fluksio.flow.queue import MemoryWorkQueue from fluksio.models import EngineEvent, FlowRun, MetricBucket FLOW = "metrics-test" @@ -292,3 +297,52 @@ def test_a_failure_keeps_the_run_it_happened_in(db: Session) -> None: failure = db.exec(select(EngineEvent).where(EngineEvent.run == "run-abc123")).one() assert failure.detail.startswith("ValueError: in a run") + + +def test_a_rate_limit_flush_is_a_run_that_names_its_parent(db: Session) -> None: + """A flush used to run under no id, so what it set off was in no run.""" + collector = MetricsCollector(EventBus()) + + class Bus: + """Hands every event straight to the collector.""" + + def publish(self, event: dict) -> None: + collector.handle(event) + + window = 0.05 + source = Node( + f=lambda params: None, + provides=[MessageSpec(name="raw", dtype=DType.FLOAT)], + name="in", + ) + relay = Node( + f=lambda raw, params: {"level": raw}, + requires=[MessageSpec(name="raw", dtype=DType.FLOAT)], + provides=[MessageSpec(name="level", dtype=DType.FLOAT, interval=window)], + name="relay", + ) + source.assign_flow("metrics-flush", "in") + relay.assign_flow("metrics-flush", "relay") + queue = MemoryWorkQueue() + pipeline = Pipeline(nodes=[source, relay], work_queue=queue, events=Bus()) + service = ExecutionService(queue, events=Bus()) + service.bind(pipeline) + + # The second value lands inside the relay's window and is held. + held_in = "" + for value in (1.0, 2.0): + source.inject({"raw": value}) + for item in queue.claim(10, 10): + service._run_item(item) + held_in = item.entry_id + time.sleep(window * 2) + queue.move_due(time.time()) + for item in queue.claim(10, 10): + service._run_item(item) + asyncio.run(collector.flush()) + + flush = db.exec(select(FlowRun).where(FlowRun.source == "flush")).one() + assert (flush.status, flush.parent) == ("ok", held_in) + parent = db.get(FlowRun, held_in) + assert parent is not None + assert (parent.flow, parent.source) == ("metrics-flush", "external") diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 3349cee..69c579f 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -4777,6 +4777,11 @@ export const fluksio__api__routes__observability__RunRowSchema = { type: 'string', title: 'Source' }, + parent: { + type: 'string', + title: 'Parent', + default: '' + }, status: { type: 'string', title: 'Status' diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 0f8c144..6520c41 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -506,6 +506,7 @@ export type fluksio__api__routes__observability__RunRow = { id: string; flow: string; source: string; + parent?: string; status: string; started_at: string; finished_at?: (string | null); From e98808e9d8103fa4d3d0feef0a5060d8195bd5fb Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:42:01 +0200 Subject: [PATCH 028/103] A supervised loop that crashes reports its node down A push session that failed before it listened left only a task_crashed event. The wrapper in _run_supervised now reports down with the exception, unless the loop already said why it is down. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/nodes/base.py | 23 +++++++++++++++++++-- backend/tests/flow/test_connector.py | 30 ++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/backend/fluksio/flow/nodes/base.py b/backend/fluksio/flow/nodes/base.py index a222143..64e22a1 100644 --- a/backend/fluksio/flow/nodes/base.py +++ b/backend/fluksio/flow/nodes/base.py @@ -90,6 +90,7 @@ class Node: "_pipeline", "synchronous", "_on_health", + "_health", "supervisor", "fingerprint", ) @@ -105,6 +106,9 @@ class Node: self.f = f self._pipeline: Pipeline | None = None self._on_health: Callable[[Node, str, str | None], None] | None = None + # What this node reported since its background loop last started, so + # a crash does not overwrite the reason the loop gave for being down. + self._health: str | None = None # Set by the controller before start(); a node built on its own (tests, # previews) runs its loops unsupervised. self.supervisor: Supervisor | None = None @@ -206,11 +210,25 @@ class Node: Returns the task only when unsupervised, since that is the one case where the caller has to cancel it itself. + + A loop that raises reports the node down with the exception, unless it + already said why it is down. A push session that fails before it + listens would otherwise show up as a crash event and nowhere else. """ + + async def reported() -> None: + self._health = None + try: + await factory() + except Exception as exc: + if self._health != "down": + self.report_health("down", f"{type(exc).__name__}: {exc}") + raise + if self.supervisor is not None: - self.supervisor.spawn(f"{self.id}:{name}", self.flow, factory) + self.supervisor.spawn(f"{self.id}:{name}", self.flow, reported) return None - return asyncio.create_task(factory()) + return asyncio.create_task(reported()) @staticmethod async def _cancel_task(task: asyncio.Task[None]) -> None: @@ -230,6 +248,7 @@ class Node: def report_health(self, status: str, detail: str | None = None) -> None: """Say how this node's connection is doing: ok, degraded or down.""" + self._health = status if self._on_health is not None: self._on_health(self, status, detail) diff --git a/backend/tests/flow/test_connector.py b/backend/tests/flow/test_connector.py index 258ceb6..f530369 100644 --- a/backend/tests/flow/test_connector.py +++ b/backend/tests/flow/test_connector.py @@ -14,6 +14,7 @@ from fluksio.flow.messages import DType, MessageSpec from fluksio.flow.nodes import Node from fluksio.flow.pipeline import Pipeline from fluksio.flow.plugins import load_plugins +from fluksio.flow.supervision import Supervisor class Sensor(ConnectorNode): @@ -156,6 +157,35 @@ def test_an_undeclared_port_keeps_failing_until_the_node_declares_it(): assert len([entry for entry in health if entry[0] == "down"]) > 1 +def test_a_push_session_that_fails_before_it_listens_reports_down(): + """A crashed loop is node health, not only a crash event.""" + + class Push(Sensor): + async def start(self, app: Any = None) -> None: + self._run_supervised("push", self._session) + + async def _session(self) -> None: + raise ConnectionError("bridge unreachable") + + health: list[tuple[str, str | None]] = [] + node = Push(provides=[MessageSpec(name="reading", dtype=DType.FLOAT)]) + node.assign_flow("demo", "push") + node._on_health = lambda _node, status, detail: health.append((status, detail)) + + async def scenario() -> None: + node.supervisor = Supervisor() + await node.start() + for _ in range(500): + if health: + break + await asyncio.sleep(0.01) + await node.supervisor.cancel_all() + + asyncio.run(scenario()) + + assert health == [("down", "ConnectionError: bridge unreachable")] + + class Actuator(ConnectorNode): """A connector that commands something instead of reading it.""" From a1bdae3484a0ee03a25fa62b27fbe32ec1f2ad9e Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:44:22 +0200 Subject: [PATCH 029/103] Connection nodes speak of listener and sender, and of send_queue_size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mqtt and http take websocket's vocabulary: their Mode is LISTENER/SENDER, default names read mqtt_listener_… and http_listener_…, and mqtt's publish_queue_size is send_queue_size. No alias stays behind, so the flow store rewrites the old setting in every published and draft mqtt node when it opens, in the same migration as trigger → debounce. Every mqtt and http setting carries a one-line description for the editor, and the node-types reference uses one wording for direction and says that an http sender publishes a JSON object response on its outputs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/controller.py | 6 +- backend/fluksio/flow/nodes/base.py | 4 +- backend/fluksio/flow/nodes/http.py | 91 ++++++----- backend/fluksio/flow/nodes/mqtt.py | 169 ++++++++++++-------- backend/fluksio/flow/nodes/websocket.py | 8 +- backend/fluksio/flow/store.py | 33 ++-- backend/tests/flow/test_brain_graph.py | 6 +- backend/tests/flow/test_house_vocabulary.py | 8 +- backend/tests/flow/test_senders.py | 28 ++-- backend/tests/flow/test_store.py | 14 ++ docs/getting-started/facility-automation.md | 8 +- docs/reference/node-types.md | 26 +-- 12 files changed, 241 insertions(+), 160 deletions(-) diff --git a/backend/fluksio/flow/controller.py b/backend/fluksio/flow/controller.py index ac14ecb..6675701 100644 --- a/backend/fluksio/flow/controller.py +++ b/backend/fluksio/flow/controller.py @@ -97,7 +97,7 @@ HOOK_PREFIX = "/hooks" def _is_hook(node: Node | None) -> TypeGuard[HttpNode | WebSocketNode]: """Mounted under HOOK_PREFIX: a webhook, or a socket the engine serves.""" - return (isinstance(node, HttpNode) and node.mode is HttpNode.Mode.TRIGGER) or ( + return (isinstance(node, HttpNode) and node.mode is HttpNode.Mode.LISTENER) or ( isinstance(node, WebSocketNode) and node.serves ) @@ -338,13 +338,13 @@ NODE_TYPES: dict[str, NodeType] = { ), "mqtt": NodeType( title="MQTT", - description="Subscribe to topics, or publish what arrives on its inputs.", + description="Listen on topics, or send to them.", cls=MqttNode, params_schema=_schema_of(MqttNode), ), "http": NodeType( title="HTTP", - description="Receive data on a webhook, or send it to a URL.", + description="Listen on a webhook, or send to a URL and pass the reply on.", cls=HttpNode, params_schema=_schema_of(HttpNode), ), diff --git a/backend/fluksio/flow/nodes/base.py b/backend/fluksio/flow/nodes/base.py index 64e22a1..d8fb937 100644 --- a/backend/fluksio/flow/nodes/base.py +++ b/backend/fluksio/flow/nodes/base.py @@ -411,7 +411,7 @@ class Node: Trigger this node externally, executing downstream nodes if dependencies are met. This method is for nodes that receive data from upstream dependencies. - For trigger/subscriber nodes that inject data into the pipeline, use :meth:`inject`. + For listener nodes that inject data into the pipeline, use :meth:`inject`. :param inputs: Input values matching this node's ``requires``. :type inputs: dict | None @@ -429,7 +429,7 @@ class Node: """ Inject data into the pipeline as if this node produced it. - This method is for trigger/subscriber nodes that receive external data + This method is for listener nodes that receive external data (e.g., HTTP requests, MQTT messages) and need to inject it into the pipeline. The data is validated against this node's ``provides`` specification. diff --git a/backend/fluksio/flow/nodes/http.py b/backend/fluksio/flow/nodes/http.py index d9b13c8..9fbf12d 100644 --- a/backend/fluksio/flow/nodes/http.py +++ b/backend/fluksio/flow/nodes/http.py @@ -1,4 +1,4 @@ -"""HTTP nodes: an inbound webhook trigger and an outbound request sender.""" +"""HTTP nodes: an inbound webhook listener and an outbound request sender.""" from __future__ import annotations @@ -81,50 +81,50 @@ def _first_catch_all(app: FastAPI) -> int: class HttpNode(Node): """ - HTTP node that can act as a trigger (receiver) or sender based on configuration. + HTTP node that acts as a listener (webhook) or a sender. This node integrates with FastAPI to either: - - **Trigger mode**: Receive incoming HTTP requests (GET/POST) and inject data + - **Listener**: Receive incoming HTTP requests (GET/POST) and inject data into the pipeline. Used when ``provides`` is specified but ``requires`` is empty. - - **Sender mode**: Make outgoing HTTP requests with pipeline data. Used when - ``requires`` is specified. + - **Sender**: Make outgoing HTTP requests with pipeline data, and publish a + JSON object response on the outputs. Used when ``requires`` is specified. - :param url: The URL endpoint. For trigger mode, this is the route path - (e.g., "/sensors/temperature"). For sender mode, this is the full URL + :param url: The URL endpoint. For a listener, this is the route path + (e.g., "/sensors/temperature"). For a sender, this is the full URL to send requests to. :type url: str :param method: HTTP method - "GET" or "POST". :type method: Literal["GET", "POST"] - :param requires: Messages required by this node (makes it a sender node). + :param requires: Messages required by this node (makes it a sender). :type requires: MessageSpec | list[MessageSpec] | None - :param provides: Messages provided by this node (makes it a trigger node). + :param provides: Messages provided by this node (makes it a listener). :type provides: MessageSpec | list[MessageSpec] | None :param params: Additional parameters for the node. :type params: dict :param name: Optional name for the node. :type name: str | None - :param timeout: Request timeout in seconds (for sender mode). + :param timeout: Request timeout in seconds (sender only). :type timeout: float :param headers: Additional HTTP headers. :type headers: dict[str, str] | None - :param secret: Shared secret callers append to the webhook URL (trigger mode). + :param secret: Shared secret callers append to the webhook URL (listener only). :type secret: str :raises ValueError: If both ``requires`` and ``provides`` are empty, or if the configuration is invalid. :example: - Trigger node (receives POST requests): + Listener (receives POST requests): - >>> trigger = HttpNode( + >>> listener = HttpNode( ... url="/api/sensors/temperature", ... method="POST", ... provides=[MessageSpec(name="temperature", dtype=DType.FLOAT)], ... params={}, ... ) - Sender node (makes POST requests): + Sender (makes POST requests): >>> sender = HttpNode( ... url="https://api.example.com/data", @@ -137,7 +137,7 @@ class HttpNode(Node): class Mode(Enum): """Operating mode of the HTTP node.""" - TRIGGER = "trigger" # Receives HTTP requests + LISTENER = "listener" # Receives HTTP requests SENDER = "sender" # Sends HTTP requests # A repeated request is a repeated request, whatever the endpoint does @@ -159,14 +159,29 @@ class HttpNode(Node): class Params(BaseModel): model_config = ConfigDict(extra="allow") - url: str - method: Literal["GET", "POST"] = "POST" - timeout: float = 30.0 - headers: dict[str, str] = {} + url: str = Field( + description=( + "A path such as /tick, served as a webhook at /hooks//, " + "or the full URL a sender requests." + ) + ) + method: Literal["GET", "POST"] = Field( + default="POST", + description="The method a webhook accepts or a sender uses.", + ) + timeout: float = Field( + default=30.0, description="Seconds a sender waits for the response." + ) + headers: dict[str, str] = Field( + default={}, + description=( + "Request headers, sender only. A value may be a secret reference." + ), + ) query: dict[str, Any] = Field( default_factory=dict, description=( - "Fixed query parameters, sender mode. A value may be a secret " + "Fixed query parameters, sender only. A value may be a secret " "reference, which is how an API key stays out of the flow file." ), ) @@ -228,7 +243,7 @@ class HttpNode(Node): ) # Inputs mean this node sends data out; outputs mean it receives. - self.mode = HttpNode.Mode.SENDER if requires else HttpNode.Mode.TRIGGER + self.mode = HttpNode.Mode.SENDER if requires else HttpNode.Mode.LISTENER self.url = cfg.url self.method = cfg.method.upper() @@ -245,12 +260,12 @@ class HttpNode(Node): name = f"http_{self.mode.value}_{safe_url}" # Initialize parent with appropriate function - # For trigger mode, f is a no-op since data is injected via inject() - # For sender mode, f handles the outgoing HTTP request + # For a listener, f is a no-op since data is injected via inject() + # For a sender, f handles the outgoing HTTP request super().__init__( f=( - self._noop_trigger - if self.mode == HttpNode.Mode.TRIGGER + self._noop_listener + if self.mode == HttpNode.Mode.LISTENER else self._sender_handler ), requires=requires, @@ -260,11 +275,11 @@ class HttpNode(Node): ) @staticmethod - def _noop_trigger(params: dict[str, Any], **kwargs: Any) -> dict[str, Any] | None: + def _noop_listener(params: dict[str, Any], **kwargs: Any) -> dict[str, Any] | None: """ - No-op function for trigger mode nodes. + No-op function for listener nodes. - Trigger mode nodes inject data via :meth:`inject`, not :meth:`__call__`. + Listener nodes inject data via :meth:`inject`, not :meth:`__call__`. This function exists only to satisfy the Node interface. """ return None @@ -273,7 +288,7 @@ class HttpNode(Node): self, params: dict[str, Any], **kwargs: Any ) -> dict[str, Any] | None: """ - Send HTTP request with pipeline data (sender mode). + Send HTTP request with pipeline data (sender). This method is called when upstream dependencies are satisfied. It sends the required data via HTTP request. @@ -332,37 +347,37 @@ class HttpNode(Node): async def start(self, app: FastAPI | None = None) -> None: """A webhook needs a route; a sender reaches out on its own.""" - if self.mode is HttpNode.Mode.TRIGGER and app is not None: + if self.mode is HttpNode.Mode.LISTENER and app is not None: self.register_route(app) async def stop(self, app: FastAPI | None = None) -> None: - if self.mode is HttpNode.Mode.TRIGGER and app is not None: + if self.mode is HttpNode.Mode.LISTENER and app is not None: self.unregister_route(app) def register_route(self, app: FastAPI) -> None: """ Register this node's HTTP endpoint with a FastAPI application. - This method should only be called for trigger mode nodes. It creates + This method should only be called for listener nodes. It creates a route that, when called, triggers the node in the pipeline. :param app: The FastAPI application instance. :type app: FastAPI - :raises RuntimeError: If called on a sender mode node. + :raises RuntimeError: If called on a sender. :example: >>> from fastapi import FastAPI >>> app = FastAPI() - >>> trigger_node = HttpNode( + >>> listener = HttpNode( ... url="/sensors/data", ... method="POST", ... provides=[MessageSpec(name="value", dtype=DType.FLOAT)], ... params={}, ... ) - >>> trigger_node.register_route(app) + >>> listener.register_route(app) """ - if self.mode != HttpNode.Mode.TRIGGER: - raise RuntimeError("Can only register routes for trigger mode nodes") + if self.mode != HttpNode.Mode.LISTENER: + raise RuntimeError("Can only register routes for listener nodes") if self._route_registered: return @@ -408,7 +423,7 @@ class HttpNode(Node): if spec.port in data: typed_data[spec.port] = spec.coerce(data[spec.port]) - # Inject data into the pipeline (trigger mode nodes inject via + # Inject data into the pipeline (listener nodes inject via # provides). Off the loop: journalling is a blocking Redis # round trip, and every webhook was making it on the thread # the whole API answers from. diff --git a/backend/fluksio/flow/nodes/mqtt.py b/backend/fluksio/flow/nodes/mqtt.py index 7a3303f..a6dbc6b 100644 --- a/backend/fluksio/flow/nodes/mqtt.py +++ b/backend/fluksio/flow/nodes/mqtt.py @@ -1,4 +1,4 @@ -"""MQTT nodes: a subscriber that wakes the graph, a publisher that speaks for it.""" +"""MQTT nodes: a listener that wakes the graph, a sender that speaks for it.""" from __future__ import annotations @@ -62,15 +62,14 @@ def topic_matches(filter_: str, topic: str) -> bool: class MqttNode(Node): """ - MQTT node that can act as a subscriber (trigger) or publisher (sender). + MQTT node that acts as a listener or a sender. This node integrates with an MQTT broker to either: - - **Trigger mode (Subscriber)**: Subscribe to MQTT topics and inject received - messages into the pipeline. Used when ``provides`` is specified but - ``requires`` is empty. - - **Sender mode (Publisher)**: Publish pipeline data to MQTT topics. Used when - ``requires`` is specified. + - **Listener**: Subscribe to MQTT topics and inject received messages into + the pipeline. Used when ``provides`` is specified but ``requires`` is empty. + - **Sender**: Publish pipeline data to MQTT topics. Used when ``requires`` + is specified. The ``topic`` parameter in ``params`` controls the mapping between pipeline message names and MQTT topics: @@ -78,11 +77,11 @@ class MqttNode(Node): - **dict**: Explicit mapping from message name to MQTT topic, e.g. ``{"temperature": "sensors/room1/temp", "humidity": "sensors/room1/hum"}``. - **str** (legacy): A single topic string. All messages are mapped to this - one topic (subscriber receives from it, publisher sends to it). + one topic (a listener subscribes to it, a sender publishes to it). - :param requires: Messages required by this node (makes it a publisher node). + :param requires: Messages required by this node (makes it a sender). :type requires: MessageSpec | list[MessageSpec] | None - :param provides: Messages provided by this node (makes it a subscriber node). + :param provides: Messages provided by this node (makes it a listener). :type provides: MessageSpec | list[MessageSpec] | None :param params: Parameters dict containing: - ``topic`` (str | dict): MQTT topic(s). A dict maps message names to @@ -97,7 +96,7 @@ class MqttNode(Node): - ``keepalive`` (int): Keepalive interval in seconds (default: 60) - ``timeout`` (float): Deadline for a broker operation in seconds (default: 10.0) - - ``publish_queue_size`` (int): Publisher backlog depth (default: 256) + - ``send_queue_size`` (int): Sender backlog depth (default: 256) :type params: dict :param name: Optional name for the node. :type name: str | None @@ -105,9 +104,9 @@ class MqttNode(Node): :raises ValueError: If both ``requires`` and ``provides`` are empty. :example: - Subscriber with per-message topics: + Listener with per-message topics: - >>> subscriber = MqttNode( + >>> listener = MqttNode( ... provides=[ ... MessageSpec(name="inverter_input", dtype=DType.FLOAT), ... MessageSpec(name="inverter_output", dtype=DType.FLOAT), @@ -121,9 +120,9 @@ class MqttNode(Node): ... }, ... ) - Publisher with per-message topics: + Sender with per-message topics: - >>> publisher = MqttNode( + >>> sender = MqttNode( ... requires=[ ... MessageSpec(name="target_temp", dtype=DType.FLOAT), ... MessageSpec(name="fan_speed", dtype=DType.INT), @@ -138,9 +137,9 @@ class MqttNode(Node): ... }, ... ) - Legacy single-topic subscriber: + Legacy single-topic listener: - >>> subscriber = MqttNode( + >>> listener = MqttNode( ... provides=[MessageSpec(name="temperature", dtype=DType.FLOAT)], ... params={"topic": "sensors/temperature", "broker_host": "localhost"}, ... ) @@ -149,8 +148,8 @@ class MqttNode(Node): class Mode(Enum): """Operating mode of the MQTT node.""" - SUBSCRIBER = "subscriber" # Receives MQTT messages (trigger) - PUBLISHER = "publisher" # Sends MQTT messages (sender) + LISTENER = "listener" # Subscribes and injects what arrives + SENDER = "sender" # Publishes its inputs # Publishing again is a second command to whatever is listening. idempotent = False @@ -167,14 +166,14 @@ class MqttNode(Node): "retain", "keepalive", "timeout", - "publish_queue_size", + "send_queue_size", "json_keys", "_topic_to_ports", "_wildcards", "_subscription_task", "_mqtt_client", "_stop_event", - "_publish_queue", + "_send_queue", "_publisher_task", "_loop", ) @@ -183,15 +182,41 @@ class MqttNode(Node): model_config = ConfigDict(extra="allow") # One topic for every port, or a per-port mapping. - topic: str | dict[str, str] = "*" - broker_host: str = "localhost" - broker_port: int = 1883 - username: str | None = None - password: str | None = Field(default=None, json_schema_extra={"x-secret": True}) - client_id: str | None = None - qos: int = 0 - retain: bool = False - keepalive: int = 60 + topic: str | dict[str, str] = Field( + default="*", + description='One topic for every port, or {"port": "some/topic"} per port.', + ) + broker_host: str = Field( + default="localhost", + description="Hostname or IP of the broker, as the engine reaches it.", + ) + broker_port: int = Field( + default=1883, description="The port the broker listens on." + ) + username: str | None = Field( + default=None, description="Username for the broker, if it asks for one." + ) + password: str | None = Field( + default=None, + description="Password for the broker, kept in the secrets store.", + json_schema_extra={"x-secret": True}, + ) + client_id: str | None = Field( + default=None, + description="Client identifier the broker sees. Empty picks a random one.", + ) + qos: int = Field( + default=0, + description="Quality of service for subscribing and publishing, 0, 1 or 2.", + ) + retain: bool = Field( + default=False, + description="Have the broker keep the last value sent for later subscribers.", + ) + keepalive: int = Field( + default=60, + description="Seconds between pings that keep an idle connection open.", + ) # Bounds every broker operation: subscribe, publish, and the # disconnect acknowledgement on the way out. Without one a client # whose socket died waits for that ack forever, and the task never @@ -202,9 +227,9 @@ class MqttNode(Node): description="Give up on a broker operation after this many seconds.", ) # Deep enough to ride out a broker hiccup, shallow enough that a - # publisher which cannot keep up drops old values instead of growing + # sender which cannot keep up drops old values instead of growing # without bound. A node that bursts wants more than one that trickles. - publish_queue_size: int = Field( + send_queue_size: int = Field( default=256, gt=0, description=( @@ -215,17 +240,23 @@ class MqttNode(Node): # Which key to lift out of a JSON object payload. A device that wraps # its reading — Victron's ``{"value": 5}`` — is otherwise a Python node # per port. One key for every port, or a per-port mapping. - json_key: str | dict[str, str] = "" + json_key: str | dict[str, str] = Field( + default="", + description=( + "Key to lift out of a JSON object payload, for every port or as " + '{"port": "key"}.' + ), + ) @classmethod def target_key(cls, params: dict[str, Any]) -> str | None: """The broker and topic, which is one physical thing. - A publisher and a subscriber on the same topic get the same key on + A sender and a listener on the same topic get the same key on purpose: they are two ends of one wire, and drawing them as one neuron is the only way the path through the broker shows up at all. """ - # ponytail: publisher and subscriber merge into one neuron; key on mode + # ponytail: sender and listener merge into one neuron; key on mode # as well if the two directions ever need telling apart. fields = cls.Params.model_fields topic = params.get("topic", fields["topic"].default) @@ -253,9 +284,9 @@ class MqttNode(Node): ) # Inputs mean this node publishes; outputs mean it subscribes. - self.mode = MqttNode.Mode.PUBLISHER if requires else MqttNode.Mode.SUBSCRIBER + self.mode = MqttNode.Mode.SENDER if requires else MqttNode.Mode.LISTENER - ports = provides if self.mode == MqttNode.Mode.SUBSCRIBER else requires + ports = provides if self.mode == MqttNode.Mode.LISTENER else requires if isinstance(cfg.topic, dict): self.topics: dict[str, str] = dict(cfg.topic) else: @@ -285,13 +316,13 @@ class MqttNode(Node): self.retain = cfg.retain self.keepalive = cfg.keepalive self.timeout = cfg.timeout - self.publish_queue_size = cfg.publish_queue_size + self.send_queue_size = cfg.send_queue_size # Runtime state self._subscription_task: asyncio.Task[None] | None = None self._mqtt_client = None self._stop_event: asyncio.Event | None = None - self._publish_queue: asyncio.Queue[dict[str, Any]] | None = None + self._send_queue: asyncio.Queue[dict[str, Any]] | None = None self._publisher_task: asyncio.Task[None] | None = None self._loop: asyncio.AbstractEventLoop | None = None @@ -311,13 +342,13 @@ class MqttNode(Node): name = f"mqtt_{self.mode.value}_{safe_topic}" # Initialize parent with appropriate function - # For subscriber mode, f is a no-op since data is injected via inject() - # For publisher mode, f handles the outgoing MQTT publish + # For a listener, f is a no-op since data is injected via inject() + # For a sender, f handles the outgoing MQTT publish super().__init__( f=( - self._noop_subscriber - if self.mode == MqttNode.Mode.SUBSCRIBER - else self._publisher_handler + self._noop_listener + if self.mode == MqttNode.Mode.LISTENER + else self._sender_handler ), requires=requires, provides=provides, @@ -326,13 +357,11 @@ class MqttNode(Node): ) @staticmethod - def _noop_subscriber( - params: dict[str, Any], **kwargs: Any - ) -> dict[str, Any] | None: + def _noop_listener(params: dict[str, Any], **kwargs: Any) -> dict[str, Any] | None: """ - No-op function for subscriber mode nodes. + No-op function for listener nodes. - Subscriber mode nodes inject data via :meth:`inject`, not :meth:`__call__`. + Listener nodes inject data via :meth:`inject`, not :meth:`__call__`. This function exists only to satisfy the Node interface and should not be called directly. @@ -345,11 +374,11 @@ class MqttNode(Node): """ return None - def _publisher_handler( + def _sender_handler( self, params: dict[str, Any], **kwargs: Any ) -> dict[str, Any] | None: """ - Publish pipeline data to MQTT topic (publisher mode). + Publish pipeline data to MQTT topic (sender). This method is called when upstream dependencies are satisfied. Handing the payload to the node's publisher task is all that happens @@ -363,7 +392,7 @@ class MqttNode(Node): :returns: None (publishing is fire-and-forget). :rtype: dict | None """ - loop, queue = self._loop, self._publish_queue + loop, queue = self._loop, self._send_queue if loop is not None and queue is not None: loop.call_soon_threadsafe(self._enqueue, queue, dict(kwargs)) return None @@ -386,21 +415,21 @@ class MqttNode(Node): if queue.full(): try: queue.get_nowait() - logger.warning("Publish queue full for node '%s', dropped", self.name) - self.report_health("degraded", "publish queue full") + logger.warning("Send queue full for node '%s', dropped", self.name) + self.report_health("degraded", "send queue full") except asyncio.QueueEmpty: pass queue.put_nowait(data) async def _publisher_loop(self) -> None: - """Hold one connection and drain the publish queue over it. + """Hold one connection and drain the send queue over it. A dropped connection raises, and the supervisor decides when to - reconnect — the same arrangement the subscriber uses. + reconnect — the same arrangement the listener uses. """ aiomqtt = _aiomqtt() - queue = self._publish_queue + queue = self._send_queue if queue is None: return @@ -461,7 +490,7 @@ class MqttNode(Node): continue # A string goes on the wire as it stands. Devices on a shared - # broker expect bare values, and the subscriber below already + # broker expect bare values, and the listener below already # falls back to the raw text when it is not JSON, so a # fluksio-to-fluksio round trip is unaffected. payload = value if isinstance(value, str) else json.dumps(value) @@ -479,8 +508,8 @@ class MqttNode(Node): ) async def start(self, app: FastAPI | None = None) -> None: - """A subscriber listens; a publisher opens the connection it will reuse.""" - if self.mode is MqttNode.Mode.SUBSCRIBER: + """A listener subscribes; a sender opens the connection it will reuse.""" + if self.mode is MqttNode.Mode.LISTENER: await self.start_subscription() else: await self.start_publisher() @@ -491,41 +520,41 @@ class MqttNode(Node): async def start_publisher(self) -> None: """Run the task that owns this node's connection to the broker.""" - if self._publish_queue is not None: + if self._send_queue is not None: return - self._publish_queue = asyncio.Queue(maxsize=self.publish_queue_size) + self._send_queue = asyncio.Queue(maxsize=self.send_queue_size) self._loop = asyncio.get_running_loop() self._publisher_task = self._run_supervised("mqtt-out", self._publisher_loop) async def stop_publisher(self) -> None: """Drop the queue and let the connection go.""" - if self._publish_queue is None: + if self._send_queue is None: return if self._publisher_task is not None: await self._cancel_task(self._publisher_task) self._publisher_task = None - self._publish_queue = None + self._send_queue = None self._loop = None async def start_subscription(self) -> None: """ - Start the MQTT subscription for trigger mode nodes. + Start the MQTT subscription for listener nodes. This method starts a background task that listens for messages on the subscribed topic and triggers the pipeline when messages arrive. - :raises RuntimeError: If called on a publisher mode node. + :raises RuntimeError: If called on a sender. :example: - >>> subscriber = MqttNode( + >>> listener = MqttNode( ... topic="sensors/#", ... provides=[MessageSpec(name="value", dtype=DType.FLOAT)], ... params={"broker_host": "localhost"}, ... ) - >>> await subscriber.start_subscription() + >>> await listener.start_subscription() """ - if self.mode != MqttNode.Mode.SUBSCRIBER: - raise RuntimeError("Can only start subscription for subscriber mode nodes") + if self.mode != MqttNode.Mode.LISTENER: + raise RuntimeError("Can only start subscription for listener nodes") if self._subscription_task is not None: return # Already running diff --git a/backend/fluksio/flow/nodes/websocket.py b/backend/fluksio/flow/nodes/websocket.py index 15732f6..6593758 100644 --- a/backend/fluksio/flow/nodes/websocket.py +++ b/backend/fluksio/flow/nodes/websocket.py @@ -121,7 +121,13 @@ class WebSocketNode(Node): ), ) # Same as the mqtt node: a device that wraps its reading in an object. - json_key: str | dict[str, str] = "" + json_key: str | dict[str, str] = Field( + default="", + description=( + "Key to lift out of a JSON object frame, for every port or as " + '{"port": "key"}.' + ), + ) @classmethod def target_key(cls, params: dict[str, Any]) -> str | None: diff --git a/backend/fluksio/flow/store.py b/backend/fluksio/flow/store.py index 6d2a48f..aee1d9b 100644 --- a/backend/fluksio/flow/store.py +++ b/backend/fluksio/flow/store.py @@ -169,15 +169,16 @@ class FlowStore: self._migrate() def _migrate(self) -> None: - """Rewrite node types stored under a name they no longer have. + """Rewrite node types and settings stored under a name they no longer have. - The node type ``trigger`` became ``debounce`` on 2026-10-15, with no - alias left for the old name. Each published and draft document is read - as plain JSON and only the ``type`` of a matching node changes, so a - field this version does not model survives, and the old document stays - in git. The version goes up by one, so an editor still holding the old - document gets a conflict on its next save instead of writing the old - name back. A store with nothing to rename is left alone. + On 2026-10-15 the node type ``trigger`` became ``debounce`` and the mqtt + setting ``publish_queue_size`` became ``send_queue_size``, with no alias + left for the old names. Each published and draft document is read as + plain JSON and only the matching node changes, so a field this version + does not model survives, and the old document stays in git. The version + goes up by one, so an editor still holding the old document gets a + conflict on its next save instead of writing the old name back. A store + with nothing to rename is left alone. """ changed = [] paths = [*self.root.glob("*/flow.json"), *self.root.glob("*/flow.draft.json")] @@ -185,21 +186,31 @@ class FlowStore: try: data = json.loads(path.read_text()) nodes = [n for n in data["nodes"] if n.get("type") == "trigger"] + queues = [ + n["params"] + for n in data["nodes"] + if n.get("type") == "mqtt" + and "publish_queue_size" in (n.get("params") or {}) + ] except Exception: # Unreadable to the engine too, which reports it when it loads. continue - if not nodes: + if not nodes and not queues: continue for node in nodes: node["type"] = "debounce" + for params in queues: + params["send_queue_size"] = params.pop("publish_queue_size") data["version"] = data.get("version", 1) + 1 path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n") changed.append(f"{path.parent.name}/{path.name}") if changed: logger.warning( - "Renamed node type 'trigger' to 'debounce' in %s", ", ".join(changed) + "Renamed node type 'trigger' to 'debounce' and mqtt " + "'publish_queue_size' to 'send_queue_size' in %s", + ", ".join(changed), ) - self._commit("Rename node type 'trigger' to 'debounce'") + self._commit("Rename 'trigger' nodes and mqtt 'publish_queue_size'") # ------------------------------------------------------------------------- # git diff --git a/backend/tests/flow/test_brain_graph.py b/backend/tests/flow/test_brain_graph.py index 72876d7..ec20a45 100644 --- a/backend/tests/flow/test_brain_graph.py +++ b/backend/tests/flow/test_brain_graph.py @@ -19,7 +19,7 @@ from fluksio.flow.store import FlowStore BROKER = {"broker_host": "mosquitto", "broker_port": 1883, "topic": "sensors/temp"} -def subscriber(node_id: str) -> NodeDef: +def listener(node_id: str) -> NodeDef: return NodeDef( id=node_id, type="mqtt", @@ -35,7 +35,7 @@ def controller(tmp_path: Path) -> FlowController: FlowDef( name="house", nodes=[ - subscriber("sensor"), + listener("sensor"), NodeDef( id="scale", type="change", @@ -45,7 +45,7 @@ def controller(tmp_path: Path) -> FlowController: ], ) ) - store.write_flow(FlowDef(name="shed", nodes=[subscriber("sensor")])) + store.write_flow(FlowDef(name="shed", nodes=[listener("sensor")])) return FlowController(store) diff --git a/backend/tests/flow/test_house_vocabulary.py b/backend/tests/flow/test_house_vocabulary.py index 8df6fbb..aff89a6 100644 --- a/backend/tests/flow/test_house_vocabulary.py +++ b/backend/tests/flow/test_house_vocabulary.py @@ -29,7 +29,7 @@ def test_a_filter_does_not_cover_a_neighbouring_topic(): assert not topic_matches("sensors/living/temp", "sensors/living") -def _subscriber(topic, **params) -> MqttNode: +def _listener(topic, **params) -> MqttNode: node = MqttNode( provides=[MessageSpec(name="reading", port="reading", dtype=DType.FLOAT)], params={"topic": {"reading": topic}, **params}, @@ -40,7 +40,7 @@ def _subscriber(topic, **params) -> MqttNode: def test_a_wildcard_subscription_routes_to_its_port(): """It used to subscribe and then drop every message it was sent.""" - node = _subscriber("shellypm/status/#") + node = _listener("shellypm/status/#") assert node._ports_for("shellypm/status/pm1:0") == ["reading"] assert node._ports_for("shellypmc/status/pm1:0") == [] @@ -63,12 +63,12 @@ def test_an_exact_topic_still_wins_over_a_filter(): def test_json_key_lifts_the_value_a_device_wraps(): """Victron publishes {"value": 47} on every one of its topics.""" - node = _subscriber("N/x/battery/Soc", json_key="value") + node = _listener("N/x/battery/Soc", json_key="value") assert node.json_keys == {"reading": "value"} def test_without_json_key_the_port_name_is_the_key_as_before(): - node = _subscriber("sensors/temp") + node = _listener("sensors/temp") assert node.json_keys == {} diff --git a/backend/tests/flow/test_senders.py b/backend/tests/flow/test_senders.py index 655066a..1cbfbb8 100644 --- a/backend/tests/flow/test_senders.py +++ b/backend/tests/flow/test_senders.py @@ -20,7 +20,7 @@ def test_http_senders_share_one_pooled_client(): close_shared_client() -def _publisher() -> MqttNode: +def _sender() -> MqttNode: node = MqttNode( requires=[MessageSpec(name="setpoint", port="setpoint", dtype=DType.FLOAT)], params={"topic": {"setpoint": "heating/setpoint"}}, @@ -29,27 +29,27 @@ def _publisher() -> MqttNode: return node -def test_a_started_publisher_queues_instead_of_connecting(): +def test_a_started_sender_queues_instead_of_connecting(): """The handler runs on a worker thread; it must not block on the broker.""" - node = _publisher() + node = _sender() async def scenario() -> None: - node._publish_queue = asyncio.Queue(maxsize=4) + node._send_queue = asyncio.Queue(maxsize=4) node._loop = asyncio.get_running_loop() - await asyncio.to_thread(node._publisher_handler, {}, setpoint=21.0) + await asyncio.to_thread(node._sender_handler, {}, setpoint=21.0) # call_soon_threadsafe lands on the next loop pass. await asyncio.sleep(0) - assert node._publish_queue.qsize() == 1 - assert node._publish_queue.get_nowait() == {"setpoint": 21.0} + assert node._send_queue.qsize() == 1 + assert node._send_queue.get_nowait() == {"setpoint": 21.0} asyncio.run(scenario()) -def test_a_full_publish_queue_drops_the_oldest(): +def test_a_full_send_queue_drops_the_oldest(): """A broker that cannot keep up must not grow the queue without bound.""" - node = _publisher() + node = _sender() health: list[tuple[str, str | None]] = [] node._on_health = lambda _n, status, detail: health.append((status, detail)) @@ -61,7 +61,7 @@ def test_a_full_publish_queue_drops_the_oldest(): assert queue.qsize() == 2 assert queue.get_nowait() == {"setpoint": 2.0} assert queue.get_nowait() == {"setpoint": 3.0} - assert health == [("degraded", "publish queue full")] + assert health == [("degraded", "send queue full")] asyncio.run(scenario()) @@ -121,7 +121,7 @@ def test_the_configured_timeout_reaches_the_broker_client(monkeypatch): assert seen["timeout"] == 2.5 -def test_the_configured_backlog_reaches_the_publish_queue(monkeypatch): +def test_the_configured_backlog_reaches_the_send_queue(monkeypatch): """The depth is read when the queue is built, so it has to be per node.""" import aiomqtt @@ -139,14 +139,14 @@ def test_the_configured_backlog_reaches_the_publish_queue(monkeypatch): node = MqttNode( requires=[MessageSpec(name="setpoint", port="setpoint", dtype=DType.FLOAT)], - params={"topic": {"setpoint": "heating/setpoint"}, "publish_queue_size": 8}, + params={"topic": {"setpoint": "heating/setpoint"}, "send_queue_size": 8}, ) node.assign_flow("heating", "out") async def scenario() -> int: await node.start_publisher() - assert node._publish_queue is not None - size = node._publish_queue.maxsize + assert node._send_queue is not None + size = node._send_queue.maxsize await node.stop_publisher() return size diff --git a/backend/tests/flow/test_store.py b/backend/tests/flow/test_store.py index 13abd34..a5e37d3 100644 --- a/backend/tests/flow/test_store.py +++ b/backend/tests/flow/test_store.py @@ -159,3 +159,17 @@ def test_opening_the_store_renames_trigger_nodes_to_debounce(store: FlowStore): FlowStore(store.root) assert commit_count(store) == commits + 1 + + +def test_opening_the_store_renames_the_mqtt_publish_queue_size(store: FlowStore): + """The mqtt setting is now `send_queue_size`, with no alias for the old name.""" + relay = NodeDef(id="relay", type="mqtt", params={"publish_queue_size": 8}) + store.write_flow(FlowDef(name="hall", nodes=[relay])) + commits = commit_count(store) + + reopened = FlowStore(store.root) + + published = reopened.read_flow("hall") + assert published.nodes[0].params == {"send_queue_size": 8} + assert published.version == 2 + assert commit_count(store) == commits + 1 diff --git a/docs/getting-started/facility-automation.md b/docs/getting-started/facility-automation.md index e0b850b..ac5a02f 100644 --- a/docs/getting-started/facility-automation.md +++ b/docs/getting-started/facility-automation.md @@ -86,10 +86,10 @@ Publish the change. The Function now responds to sensor readings instead of the test value. If your sensor sends a JSON object such as `{"temperature": 18.0}`, set `json_key` to `temperature`. -An MQTT node with input ports publishes to its configured topic; one with -output ports subscribes. Add a publishing node only after checking the payload -format your device expects. A relay may need `ON` and `OFF` rather than a -boolean. +An MQTT node with inputs is a sender and publishes to its configured topic; +one with outputs only is a listener and subscribes. Add a sender only after +checking the payload format your device expects. A relay may need `ON` and +`OFF` rather than a boolean. !!! tip "Keep the first test on screen" diff --git a/docs/reference/node-types.md b/docs/reference/node-types.md index 9300fed..0157174 100644 --- a/docs/reference/node-types.md +++ b/docs/reference/node-types.md @@ -25,9 +25,11 @@ arrive as keyword arguments. ### MQTT -**`mqtt`** — subscribe to topics, or publish what arrives on its inputs. +**`mqtt`** — listen on topics, or send to them. -A node with *outputs only* subscribes; a node with *inputs* publishes. +The ports pick the direction: outputs only make a **listener**, inputs a +**sender**. A listener subscribes to its topics, a sender publishes its inputs +to them. | Setting | Default | Notes | |---|---|---| @@ -40,7 +42,7 @@ A node with *outputs only* subscribes; a node with *inputs* publishes. | `retain` | `false` | on published messages | | `keepalive` | `60` | seconds | | `timeout` | `10.0` | seconds before a broker operation is given up on | -| `publish_queue_size` | `256` | payloads that may wait for the broker; past this the oldest is dropped and the node reports degraded | +| `send_queue_size` | `256` | payloads that may wait for the broker; past this the oldest is dropped and the node reports degraded | | `json_key` | — | key to lift out of an object payload; one for every port, or `{"port": "key"}` | A topic may be a filter: `+` matches one level, `#` the rest. Everything a @@ -55,18 +57,22 @@ unwrapped only if it contains the port's name as a key. ### HTTP -**`http`** — receive data on a webhook, or send it to a URL. +**`http`** — listen on a webhook, or send to a URL and pass the reply on. -Outputs only makes it a **webhook**: the engine mounts a route at -`/hooks/{flow}/{url}/{secret}` while the flow runs. Inputs make it a **sender**. +The ports pick the direction: outputs only make a **listener**, inputs a +**sender**. A listener is a **webhook**: the engine mounts a route at +`/hooks/{flow}/{url}/{secret}` while the flow runs. A sender requests `url` +and publishes a JSON object response on its outputs, one key per output port. +A key no port declares fails the call, and a response that is not an object +publishes nothing. | Setting | Default | Notes | |---|---|---| -| `url` | — | the route path in webhook mode, the full URL in sender mode | +| `url` | — | the route path for a webhook, the full URL for a sender | | `method` | `POST` | `GET` or `POST` | -| `timeout` | `30` | seconds, sender mode | -| `headers` | `{}` | | -| `query` | `{}` | fixed query parameters; a value may be a secret reference | +| `timeout` | `30` | seconds, sender only | +| `headers` | `{}` | request headers, sender only; a value may be a secret reference | +| `query` | `{}` | fixed query parameters, sender only; a value may be a secret reference | | `send_inputs` | `true` | off when the inputs only trigger the request | | `secret` | — | shared secret appended to the webhook URL; takes a secret reference | From 0746c94e0ae290acf7d65b47ccef415f20c99b7a Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:45:24 +0200 Subject: [PATCH 030/103] Warn when a node that runs no code names a device A device on a node without source was accepted and silently ignored. The build now reports it as ignored_device, an advisory issue. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/controller.py | 20 ++++++++++++++++++++ backend/fluksio/flow/pipeline.py | 3 ++- backend/tests/flow/test_drafts.py | 23 +++++++++++++++++++++++ docs/concepts/flows.md | 3 ++- frontend/src/client/schemas.gen.ts | 2 +- frontend/src/client/types.gen.ts | 4 ++-- 6 files changed, 50 insertions(+), 5 deletions(-) diff --git a/backend/fluksio/flow/controller.py b/backend/fluksio/flow/controller.py index 6675701..1e1eb12 100644 --- a/backend/fluksio/flow/controller.py +++ b/backend/fluksio/flow/controller.py @@ -174,6 +174,8 @@ class LoadedNode: #: Published, and with no body of its own — so what runs is the new-node #: template, which returns nothing at all. missing_source: bool = False + #: The device this node names although its type never runs on a worker. + ignored_device: str = "" @dataclass @@ -1041,6 +1043,8 @@ class FlowController: node_type = NODE_TYPES.get(node_def.type) if node_type is None: raise ValueError(f"Unknown node type '{node_def.type}'") + if node_def.device and not node_type.has_source: + entry.ignored_device = node_def.device params = resolve_params(node_def.params) @@ -1923,6 +1927,22 @@ def _collect_issues( for entry in loaded.values() if entry.missing_source ] + # Only code travels to a worker, so a device on any other node is a field + # nothing reads. Saying so beats a setting that looks like it took. + issues += [ + ValidationIssue( + code="ignored_device", + message=( + f"Node '{entry.id.rpartition('.')[2]}' names device " + f"'{entry.ignored_device}', but only a Function node runs on a " + "worker. The setting is ignored." + ), + flow=entry.flow, + node=entry.id, + ) + for entry in loaded.values() + if entry.ignored_device + ] return issues diff --git a/backend/fluksio/flow/pipeline.py b/backend/fluksio/flow/pipeline.py index 8076def..76375a6 100644 --- a/backend/fluksio/flow/pipeline.py +++ b/backend/fluksio/flow/pipeline.py @@ -46,7 +46,7 @@ MANUAL_RUN_PREFIX = "manual-" # neither the brain graph nor the health summary treats them as a fault. # Lives here rather than beside the controller so `ValidationIssue` can carry # the distinction itself, and every reader gets it for free. -ADVISORY_ISSUES = frozenset({"unauthenticated_hook"}) +ADVISORY_ISSUES = frozenset({"unauthenticated_hook", "ignored_device"}) class ValidationIssue(BaseModel): @@ -61,6 +61,7 @@ class ValidationIssue(BaseModel): "self_loop_needs_initial", "missing_source", "node_unhealthy", + "ignored_device", ] message: str flow: str = "" diff --git a/backend/tests/flow/test_drafts.py b/backend/tests/flow/test_drafts.py index b3d085f..5f54da5 100644 --- a/backend/tests/flow/test_drafts.py +++ b/backend/tests/flow/test_drafts.py @@ -125,6 +125,29 @@ def test_a_published_node_with_no_body_is_reported(store: FlowStore): assert [i for i in controller.issues if i.code == "missing_source"] == [] +def test_a_device_on_a_node_without_code_is_reported(store: FlowStore): + """Only code travels to a worker, so the device would be ignored silently.""" + store.write_flow( + FlowDef( + name="heating", + nodes=[ + NodeDef( + id="relay", + type="change", + device="gpu", + provides=[MessageSpec(name="temp")], + ) + ], + ) + ) + controller = FlowController(store) + + asyncio.run(controller.reload()) + (issue,) = [i for i in controller.issues if i.code == "ignored_device"] + assert issue.node == "heating.relay" + assert issue.advisory + + def test_a_node_being_written_in_the_editor_is_not_reported(store: FlowStore): """A draft legitimately has no published body yet — that is what a draft is.""" store.write_draft(a_flow(), 0) diff --git a/docs/concepts/flows.md b/docs/concepts/flows.md index 70353cd..50e6b62 100644 --- a/docs/concepts/flows.md +++ b/docs/concepts/flows.md @@ -132,9 +132,10 @@ The canvas reports issues on the affected nodes: | `missing_source` | a Function node has no saved code | | `node_error` | code or configuration could not load | | `unauthenticated_hook` | a webhook or served websocket has no shared secret; advisory | +| `ignored_device` | a node other than a Function node names a device, which nothing reads; advisory | | `node_unhealthy` | a loaded node reports a runtime problem, such as a lost connection | -Build issues prevent the affected flow from running. The last two are runtime +Build issues prevent the affected flow from running. The last three are runtime or advisory issues: other work continues, and node health clears when the connection recovers. diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 69c579f..46f8c6a 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -4325,7 +4325,7 @@ export const ValidationIssueSchema = { properties: { code: { type: 'string', - enum: ['cycle', 'unconnected_input', 'missing_initial_value', 'node_error', 'unauthenticated_hook', 'self_loop_needs_initial', 'missing_source', 'node_unhealthy'], + enum: ['cycle', 'unconnected_input', 'missing_initial_value', 'node_error', 'unauthenticated_hook', 'self_loop_needs_initial', 'missing_source', 'node_unhealthy', 'ignored_device'], title: 'Code' }, message: { diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 6520c41..38b9681 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -1406,7 +1406,7 @@ export type ValidationError = { * Something wrong with a flow — a fault, or merely advisory. */ export type ValidationIssue = { - code: 'cycle' | 'unconnected_input' | 'missing_initial_value' | 'node_error' | 'unauthenticated_hook' | 'self_loop_needs_initial' | 'missing_source' | 'node_unhealthy'; + code: 'cycle' | 'unconnected_input' | 'missing_initial_value' | 'node_error' | 'unauthenticated_hook' | 'self_loop_needs_initial' | 'missing_source' | 'node_unhealthy' | 'ignored_device'; message: string; flow?: string; nodes?: Array<(string)>; @@ -1419,7 +1419,7 @@ export type ValidationIssue = { readonly advisory: boolean; }; -export type code = 'cycle' | 'unconnected_input' | 'missing_initial_value' | 'node_error' | 'unauthenticated_hook' | 'self_loop_needs_initial' | 'missing_source' | 'node_unhealthy'; +export type code = 'cycle' | 'unconnected_input' | 'missing_initial_value' | 'node_error' | 'unauthenticated_hook' | 'self_loop_needs_initial' | 'missing_source' | 'node_unhealthy' | 'ignored_device'; export type ValidationResult = { issues?: Array; From be85c6c92ac1f25857f8b21b5108804a26f6d5e5 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:48:47 +0200 Subject: [PATCH 031/103] Retire a shared node in use with ?detach=true DELETE /flows/library/{name} still answers 409 while nodes use the source. With detach=true each of them keeps a copy of the code, in the published flow and the draft, the source is deleted and the running users rebuild. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/flows.py | 19 ++++++++++++++-- backend/fluksio/flow/store.py | 31 ++++++++++++++++++++++++++ backend/tests/api/routes/test_flows.py | 31 ++++++++++++++++++++++++++ backend/tests/flow/test_library.py | 24 ++++++++++++++++++++ docs/code/api.md | 3 +++ frontend/src/client/sdk.gen.ts | 7 ++++++ frontend/src/client/types.gen.ts | 1 + 7 files changed, 114 insertions(+), 2 deletions(-) diff --git a/backend/fluksio/api/routes/flows.py b/backend/fluksio/api/routes/flows.py index 2150ba8..69094f6 100644 --- a/backend/fluksio/api/routes/flows.py +++ b/backend/fluksio/api/routes/flows.py @@ -336,8 +336,23 @@ def read_library(controller: FlowControllerDep) -> Any: @router.delete("/library/{lib_name}", response_model=Message) -async def delete_shared_node(lib_name: str, controller: FlowControllerDep) -> Any: - """Remove a shared source, as long as no flow still runs it.""" +async def delete_shared_node( + lib_name: str, controller: FlowControllerDep, detach: bool = False +) -> Any: + """Remove a shared source, as long as no flow still runs it. + + With ``detach``, every node using it keeps a copy of the code as its own + first, so the source can be retired while flows still use it. + """ + if detach: + running = await run_in_threadpool(controller.store.usages, lib_name, False) + try: + await run_in_threadpool(controller.store.detach_lib, lib_name) + except LibNotFound: + raise HTTPException(status_code=404, detail=f"No shared node '{lib_name}'") + for flow in sorted({usage.split(".", 1)[0] for usage in running}): + await controller.reload_flow(flow) + return Message(message=f"Deleted shared node '{lib_name}'") used_by = controller.store.usages(lib_name) if used_by: raise HTTPException( diff --git a/backend/fluksio/flow/store.py b/backend/fluksio/flow/store.py index aee1d9b..9edef65 100644 --- a/backend/fluksio/flow/store.py +++ b/backend/fluksio/flow/store.py @@ -400,6 +400,37 @@ class FlowStore: self._set_source_ref(flow, node_id, None) self._commit(f"Unshare node '{flow}.{node_id}'") + def detach_lib(self, name: str) -> None: + """Give every node running this shared source a copy, then delete it. + + Published flows and drafts alike, so a running flow keeps the same code + and nothing waits on a publish. Each document changed goes up a version, + so an editor holding the old one gets a conflict on its next save + instead of pointing a node back at a source that is gone. + """ + with self._write_lock: + code = self.read_lib_source(name) + for flow in self.list_flows(): + for path, node_file in ( + (self._flow_file(flow), self._node_file), + (self._draft_file(flow), self._draft_node_file), + ): + if not path.exists(): + continue + definition = FlowDef.model_validate_json(path.read_text()) + users = [n for n in definition.nodes if n.source_ref == name] + if not users: + continue + for node in users: + target = node_file(flow, node.id) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(code) + node.source_ref = None + definition.version += 1 + path.write_text(definition.model_dump_json(indent=2) + "\n") + self._lib_file(name).unlink() + self._commit(f"Detach and delete shared node '{name}'") + def usages(self, lib_name: str, draft: bool = True) -> list[str]: """Every ``flow.node`` pointing at this shared source. diff --git a/backend/tests/api/routes/test_flows.py b/backend/tests/api/routes/test_flows.py index 1e8f708..80fbfe1 100644 --- a/backend/tests/api/routes/test_flows.py +++ b/backend/tests/api/routes/test_flows.py @@ -550,3 +550,34 @@ async def test_the_live_socket_ends_when_its_client_goes( await asyncio.sleep(0.05) parked = [t for t in asyncio.all_tasks() if "Queue.get" in repr(t.get_coro())] assert parked == [] + + +def test_a_shared_node_in_use_is_retired_only_with_detach( + client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + client.put( + f"{PREFIX}/shared", headers=superuser_token_headers, json=a_flow("shared") + ) + client.put( + f"{PREFIX}/shared/nodes/sensor/source", + headers=superuser_token_headers, + json={"code": WORKING_NODE}, + ) + client.post( + f"{PREFIX}/shared/nodes/sensor/share", + headers=superuser_token_headers, + json={"lib_name": "reading"}, + ) + + url = f"{PREFIX}/library/reading" + assert client.delete(url, headers=superuser_token_headers).status_code == 409 + response = client.delete(f"{url}?detach=true", headers=superuser_token_headers) + assert response.status_code == 200 + + assert client.get(f"{PREFIX}/library", headers=superuser_token_headers).json() == [] + source = client.get( + f"{PREFIX}/shared/nodes/sensor/source", headers=superuser_token_headers + ) + assert source.json()["code"] == WORKING_NODE + + client.delete(f"{PREFIX}/shared", headers=superuser_token_headers) diff --git a/backend/tests/flow/test_library.py b/backend/tests/flow/test_library.py index 6a0619d..501fff1 100644 --- a/backend/tests/flow/test_library.py +++ b/backend/tests/flow/test_library.py @@ -120,3 +120,27 @@ def test_unsharing_takes_a_private_copy_back(store: FlowStore): def test_a_missing_shared_source_is_reported(store: FlowStore): with pytest.raises(LibNotFound): store.read_lib_source("nothing_here") + + +def test_retiring_a_shared_source_leaves_each_user_a_copy(store: FlowStore): + store.write_flow(a_flow()) + store.write_node_source("heating", "sensor", SOURCE) + store.share_node("heating", "sensor", "read_temp") + store.publish_flow("heating", store.read_flow("heating", draft=True).version) + version = store.read_flow("heating").version + # Shared in a draft that was never published. + cooling = a_flow("cooling") + cooling.nodes[0].source_ref = "read_temp" + store.write_draft(cooling, 0) + + store.detach_lib("read_temp") + + assert store.list_lib() == [] + heating = store.read_flow("heating") + assert heating.nodes[0].source_ref is None + assert heating.version == version + 1 + assert store.read_node_source("heating", "sensor") == SOURCE + # What runs is the same code, so there is nothing to publish. + assert not store.has_draft("heating") + assert store.read_flow("cooling", draft=True).nodes[0].source_ref is None + assert store.read_node_source("cooling", "sensor", draft=True) == SOURCE diff --git a/docs/code/api.md b/docs/code/api.md index 065205c..5c790b0 100644 --- a/docs/code/api.md +++ b/docs/code/api.md @@ -83,6 +83,9 @@ Ordinary source edits stay in the draft until publishing. Shared library source changes affect its users immediately. `POST /flows/{name}/nodes/{id}/share` promotes a source to the library; `/unshare` makes it private again. `GET /flows/library` lists shared sources. +`DELETE /flows/library/{name}` retires one, and answers 409 while nodes still +use it. With `?detach=true` each of those nodes keeps a copy of the code, in +the published flow and the draft, and the shared source is deleted. ### Running and controlling diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index 8cab379..9fcd840 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -819,8 +819,12 @@ export class FlowsService { /** * Delete Shared Node * Remove a shared source, as long as no flow still runs it. + * + * With ``detach``, every node using it keeps a copy of the code as its own + * first, so the source can be retired while flows still use it. * @param data The data for the request. * @param data.libName + * @param data.detach * @returns Message Successful Response * @throws ApiError */ @@ -831,6 +835,9 @@ export class FlowsService { path: { lib_name: data.libName }, + query: { + detach: data.detach + }, errors: { 422: 'Validation Error' } diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 38b9681..778f84a 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -1716,6 +1716,7 @@ export type FlowsReadGraphResponse = (BrainGraph); export type FlowsReadLibraryResponse = (Array); export type FlowsDeleteSharedNodeData = { + detach?: boolean; libName: string; }; From 2dc9295c260ad9644f1bac1a5f7c1515491a9a50 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:50:04 +0200 Subject: [PATCH 032/103] Audit shared-library source writes A shared source goes live in every flow using it the moment it is saved, so the write is recorded in the audit trail like a publish. Private source saves stay drafts and are not audited. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/flows.py | 5 +++- backend/tests/api/routes/test_flows.py | 34 ++++++++++++++++++++++++++ docs/code/api.md | 3 ++- frontend/src/client/sdk.gen.ts | 3 ++- 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/backend/fluksio/api/routes/flows.py b/backend/fluksio/api/routes/flows.py index 69094f6..bc352af 100644 --- a/backend/fluksio/api/routes/flows.py +++ b/backend/fluksio/api/routes/flows.py @@ -625,6 +625,7 @@ async def save_node_source( node_id: str, source: NodeSource, controller: FlowControllerDep, + user: CurrentUser, ) -> Any: """Save a node's source as an unpublished edit and report whether it loads. @@ -634,7 +635,8 @@ async def save_node_source( A shared node writes to the library, so the fix reaches every flow using it — and that one is live immediately rather than waiting for a publish, - because the copy is not any single flow's to hold back. + because the copy is not any single flow's to hold back. Being live, it is + audited like a publish. """ definition = _read_flow(controller, name) ref = _source_ref(definition, node_id) @@ -643,6 +645,7 @@ async def save_node_source( controller.store.write_lib_source, ref, source.code ) if changed: + _audit(f"edited shared node '{ref}'", name, user) await controller.reload_lib_users(ref) else: await run_in_threadpool( diff --git a/backend/tests/api/routes/test_flows.py b/backend/tests/api/routes/test_flows.py index 80fbfe1..d28ada2 100644 --- a/backend/tests/api/routes/test_flows.py +++ b/backend/tests/api/routes/test_flows.py @@ -581,3 +581,37 @@ def test_a_shared_node_in_use_is_retired_only_with_detach( assert source.json()["code"] == WORKING_NODE client.delete(f"{PREFIX}/shared", headers=superuser_token_headers) + + +def test_a_shared_source_write_is_audited( + client: TestClient, + superuser_token_headers: dict[str, str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """It goes live at once, so it is recorded like a publish.""" + client.put( + f"{PREFIX}/audited", headers=superuser_token_headers, json=a_flow("audited") + ) + client.post( + f"{PREFIX}/audited/nodes/sensor/share", + headers=superuser_token_headers, + json={"lib_name": "audited_reading"}, + ) + + published: list[dict] = [] + monkeypatch.setattr(event_bus, "publish", published.append) + client.put( + f"{PREFIX}/audited/nodes/sensor/source", + headers=superuser_token_headers, + json={"code": WORKING_NODE}, + ) + + (audit,) = [e for e in published if e["type"] == "audit"] + assert audit["action"] == "edited shared node 'audited_reading'" + assert audit["flow"] == "audited" + + client.delete( + f"{PREFIX}/library/audited_reading?detach=true", + headers=superuser_token_headers, + ) + client.delete(f"{PREFIX}/audited", headers=superuser_token_headers) diff --git a/docs/code/api.md b/docs/code/api.md index 5c790b0..3d6df91 100644 --- a/docs/code/api.md +++ b/docs/code/api.md @@ -80,7 +80,8 @@ Function and Command nodes, changing Command settings, or changing `resources.env`. Other users can edit wiring, titles, and ordinary settings. Ordinary source edits stay in the draft until publishing. Shared library -source changes affect its users immediately. +source changes affect its users immediately, and the audit trail records them +like a publish. `POST /flows/{name}/nodes/{id}/share` promotes a source to the library; `/unshare` makes it private again. `GET /flows/library` lists shared sources. `DELETE /flows/library/{name}` retires one, and answers 409 while nodes still diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index 9fcd840..fbad4f3 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -1017,7 +1017,8 @@ export class FlowsService { * * A shared node writes to the library, so the fix reaches every flow using * it — and that one is live immediately rather than waiting for a publish, - * because the copy is not any single flow's to hold back. + * because the copy is not any single flow's to hold back. Being live, it is + * audited like a publish. * @param data The data for the request. * @param data.name * @param data.nodeId From 025123a219c6392ee8ef5e1894f800f9c3b8e105 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:50:18 +0200 Subject: [PATCH 033/103] Engine timings move to FLOW_* settings NODE_STOP_TIMEOUT, REBUILD_WAIT, FLOW_REBUILD_WAIT, BACKOFF and CANCEL_GRACE keep their defaults and are read from Settings. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/core/config.py | 20 ++++++++++++++++++++ backend/fluksio/flow/controller.py | 6 +++--- backend/fluksio/flow/supervision.py | 5 +++-- backend/tests/flow/test_supervision.py | 8 ++++++++ docs/reference/configuration.md | 5 +++++ 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/backend/fluksio/core/config.py b/backend/fluksio/core/config.py index eac2267..f88d8af 100644 --- a/backend/fluksio/core/config.py +++ b/backend/fluksio/core/config.py @@ -8,6 +8,7 @@ from pydantic import ( AnyUrl, BeforeValidator, EmailStr, + Field, HttpUrl, PositiveInt, computed_field, @@ -163,6 +164,25 @@ class Settings(BaseSettings): # grew it then gives the memory back, and the next call pays its imports # again. 0, the default, keeps every worker warm. FLOW_WORKER_MAX_RSS_MB: int = 0 + # How long a node's `stop` may take before a rebuild moves on without it. + # A broker that has gone away can leave a node waiting on an + # acknowledgement that never comes. + FLOW_NODE_STOP_TIMEOUT: float = 5.0 + # How long a request waits for a rebuild that is already running before it + # is answered as busy: one for the whole pipeline, one for a single flow's. + # Past either, the engine is wedged rather than busy. + FLOW_PIPELINE_REBUILD_WAIT: float = 15.0 + FLOW_REBUILD_WAIT: float = 5.0 + # Seconds between restarts of a node's crashed loop, one step per attempt + # with the last repeating. A JSON list in the environment. + FLOW_RESTART_BACKOFF: Annotated[tuple[float, ...], Field(min_length=1)] = ( + 1.0, + 5.0, + 30.0, + 60.0, + ) + # How long a cancelled node loop gets to stop before it is abandoned. + FLOW_CANCEL_GRACE: float = 5.0 # How long the engine's own metrics, events and run records are kept. #: The largest body `PUT /artifacts` will take, in bytes. A checkpoint or a #: video segment is a legitimate artifact, so this is generous rather than diff --git a/backend/fluksio/flow/controller.py b/backend/fluksio/flow/controller.py index 1e1eb12..9665cd9 100644 --- a/backend/fluksio/flow/controller.py +++ b/backend/fluksio/flow/controller.py @@ -106,7 +106,7 @@ def _is_hook(node: Node | None) -> TypeGuard[HttpNode | WebSocketNode]: # A node's `stop` talks to whatever it connected to, and a broker that has gone # away can leave it waiting for an acknowledgement that never arrives — which # used to hold the rebuild, and everything queued behind it, forever. -NODE_STOP_TIMEOUT = 5.0 +NODE_STOP_TIMEOUT = settings.FLOW_NODE_STOP_TIMEOUT # How long a rebuild asked for by a request waits for one already running. # Generous on purpose: a rebuild of a populated instance reconnects every @@ -114,14 +114,14 @@ NODE_STOP_TIMEOUT = 5.0 # healthy one of those should not be turned away. Past that the controller is # wedged rather than busy, and an error the caller can act on beats a request # that never ends. -REBUILD_WAIT = 15.0 +REBUILD_WAIT = settings.FLOW_PIPELINE_REBUILD_WAIT # How long a caller waits for a *flow* rebuild that is already running. A # per-flow rebuild reconnects one flow's nodes rather than the instance's, # so this is a queueing budget — several of them back to back, which is what # seeding does — not the room a single one needs. Fifteen seconds was sized for # the whole-pipeline rebuild and would let a wedge sit unreported. -FLOW_REBUILD_WAIT = 5.0 +FLOW_REBUILD_WAIT = settings.FLOW_REBUILD_WAIT # How long a node has to have been down before it is shown as down. A device # that misses one poll and comes back is a flake, not a fault, and without this diff --git a/backend/fluksio/flow/supervision.py b/backend/fluksio/flow/supervision.py index ff2d624..b526271 100644 --- a/backend/fluksio/flow/supervision.py +++ b/backend/fluksio/flow/supervision.py @@ -26,11 +26,12 @@ from collections import deque from collections.abc import Callable, Coroutine from typing import Any +from fluksio.core.config import settings from fluksio.flow.events import EventBus logger = logging.getLogger(__name__) -BACKOFF = (1.0, 5.0, 30.0, 60.0) +BACKOFF = settings.FLOW_RESTART_BACKOFF # A task that burns through this many restarts in the window is not going to # recover by being restarted again *now*. # @@ -49,7 +50,7 @@ QUARANTINE_BACKOFF = (300.0, 900.0, 3600.0) # this is not going to stop on its own — a client closing a socket the broker # no longer answers on is the case seen in the wild — and the rebuild asking # for it must not wait on that forever. -CANCEL_GRACE = 5.0 +CANCEL_GRACE = settings.FLOW_CANCEL_GRACE TaskFactory = Callable[[], Coroutine[Any, Any, None]] diff --git a/backend/tests/flow/test_supervision.py b/backend/tests/flow/test_supervision.py index a5a6746..94d214f 100644 --- a/backend/tests/flow/test_supervision.py +++ b/backend/tests/flow/test_supervision.py @@ -234,3 +234,11 @@ def test_cancelling_one_flow_lifts_only_its_quarantine(): await supervisor.cancel_all() asyncio.run(scenario()) + + +def test_the_restart_steps_are_read_from_the_environment(monkeypatch): + """A list in an environment variable, so it arrives as JSON.""" + from fluksio.core.config import Settings + + monkeypatch.setenv("FLOW_RESTART_BACKOFF", "[2, 10]") + assert Settings().FLOW_RESTART_BACKOFF == (2.0, 10.0) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 3e37778..5932fa3 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -129,6 +129,11 @@ Set `TZ` in `.env` for schedules that should follow local time, such as | `FLOW_GPUS` | `0` | local GPU count; `0` detects NVIDIA device nodes | | `FLOW_RAM` | `0` | memory budget in MB; `0` detects host/container memory and reserves 1 GiB | | `FLOW_WORKER_MAX_RSS_MB` | `0` | retire a local worker above this memory use after a call; `0` disables | +| `FLOW_NODE_STOP_TIMEOUT` | `5` | seconds a node gets to stop during a rebuild before the rebuild moves on | +| `FLOW_PIPELINE_REBUILD_WAIT` | `15` | seconds a request waits for a running whole-pipeline rebuild before it gets a 503 | +| `FLOW_REBUILD_WAIT` | `5` | the same for a single flow's rebuild, such as a deploy | +| `FLOW_RESTART_BACKOFF` | `[1,5,30,60]` | seconds between restarts of a crashed node loop; the last step repeats | +| `FLOW_CANCEL_GRACE` | `5` | seconds a cancelled node loop gets to stop before it is abandoned | | `OBS_RETENTION_DAYS` | `30` | retention for metrics, events and run records | The three concurrency limits must be at least `1`. They also have CLI flags: From efe80cb3c2e31760a9b90632c230da7e4cd8fd98 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:51:38 +0200 Subject: [PATCH 034/103] Warn at startup when worker or cascade limits are far past the cores More than four times os.cpu_count() logs one warning; the value is kept. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/resources.py | 25 +++++++++++++++++++++++++ backend/fluksio/main.py | 7 +++++++ backend/tests/flow/test_resources.py | 11 +++++++++++ docs/reference/configuration.md | 4 +++- 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/backend/fluksio/flow/resources.py b/backend/fluksio/flow/resources.py index be51011..3951007 100644 --- a/backend/fluksio/flow/resources.py +++ b/backend/fluksio/flow/resources.py @@ -65,12 +65,37 @@ ENGINE_RAM_RESERVE_MB = 1024 #: Where a container finds its own memory limit (cgroup v2). CGROUP_MEMORY_MAX = Path("/sys/fs/cgroup/memory.max") +#: How many times the core count a concurrency limit may reach before startup +#: warns. Above one on purpose: a node waiting on the network holds no core. +OVERSUBSCRIBE_FACTOR = 4 + def machine_cpus() -> int: """Cores a node may be given here, when nobody said.""" return max(1, (os.cpu_count() or 1) - ENGINE_RESERVE) +def warn_oversubscribed(limits: dict[str, int]) -> None: + """Warn, never clamp, when a concurrency limit is far past the cores. + + A high number can be right for nodes that wait on devices, but one far + past the machine is more often a zero too many. + """ + cores = os.cpu_count() or 1 + high = [ + f"{name}={value}" + for name, value in limits.items() + if value > OVERSUBSCRIBE_FACTOR * cores + ] + if high: + logger.warning( + "Concurrency above %dx this machine's %d cores: %s", + OVERSUBSCRIBE_FACTOR, + cores, + ", ".join(high), + ) + + def machine_gpus() -> int: """Cards a node may be given here, when nobody said. diff --git a/backend/fluksio/main.py b/backend/fluksio/main.py index dd534d5..39833df 100644 --- a/backend/fluksio/main.py +++ b/backend/fluksio/main.py @@ -49,6 +49,7 @@ from fluksio.flow.resources import ( fair_share_env, machine_gpus, machine_ram_mb, + warn_oversubscribed, ) from fluksio.flow.runs import RUN_STATE_TTL, RunService, sweep_artifacts from fluksio.flow.secrets import init_secrets @@ -372,6 +373,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: accountant.ram_mb, settings.FLOW_WORKER_MAX_RSS_MB, ) + warn_oversubscribed( + { + "FLOW_MAX_WORKERS": settings.FLOW_MAX_WORKERS, + "FLOW_MAX_CASCADES": settings.FLOW_MAX_CASCADES, + } + ) watchdog = LoopWatchdog(event_bus) app.state.watchdog = watchdog diff --git a/backend/tests/flow/test_resources.py b/backend/tests/flow/test_resources.py index f4ee54e..d03ee5d 100644 --- a/backend/tests/flow/test_resources.py +++ b/backend/tests/flow/test_resources.py @@ -20,6 +20,7 @@ from fluksio.flow.resources import ( fair_share_env, machine_gpus, machine_ram_mb, + warn_oversubscribed, ) from fluksio.flow.schemas import NodeDef, Resources @@ -45,6 +46,16 @@ def test_the_cards_are_counted_from_the_devices_the_driver_makes(monkeypatch): assert machine_gpus() == 0 +def test_a_limit_far_past_the_cores_is_warned_about(monkeypatch, caplog): + """Said once, never clamped: nodes waiting on devices can want a high one.""" + monkeypatch.setattr(os, "cpu_count", lambda: 2) + warn_oversubscribed({"FLOW_MAX_WORKERS": 8, "FLOW_MAX_CASCADES": 9}) + + assert [r.getMessage() for r in caplog.records] == [ + "Concurrency above 4x this machine's 2 cores: FLOW_MAX_CASCADES=9" + ] + + def test_memory_is_the_containers_limit_where_that_is_lower(monkeypatch, tmp_path): """The host's total is memory a container would be killed for using.""" from fluksio.flow import resources diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 5932fa3..b0502ff 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -137,7 +137,9 @@ Set `TZ` in `.env` for schedules that should follow local time, such as | `OBS_RETENTION_DAYS` | `30` | retention for metrics, events and run records | The three concurrency limits must be at least `1`. They also have CLI flags: -`--max-workers`, `--max-cascades` and `--max-runs`. Resource flags are `--gpus`, +`--max-workers`, `--max-cascades` and `--max-runs`. When `FLOW_MAX_WORKERS` or +`FLOW_MAX_CASCADES` is more than four times the machine's core count, the +engine logs a warning at startup and uses the value anyway. Resource flags are `--gpus`, `--ram` and `--worker-max-rss`. Nodes with declared resources are scheduled against the available budget. From 0d382edcbf0fda9f2a73698784db6ef7aa1137fd Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:53:19 +0200 Subject: [PATCH 035/103] Superuser route to flush the metrics collector now POST /observability/flush handles what the collector has queued and writes it, so the soak harness no longer sleeps FLUSH_INTERVAL_S before clearing its own rows. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/observability.py | 24 ++++++++++++-- backend/fluksio/flow/metrics.py | 17 ++++++++++ backend/fluksio/main.py | 4 ++- backend/scripts/soak.py | 19 +++++++---- .../tests/api/routes/test_observability.py | 32 ++++++++++++++++++- docs/code/api.md | 1 + frontend/src/client/sdk.gen.ts | 18 ++++++++++- frontend/src/client/types.gen.ts | 2 ++ 8 files changed, 106 insertions(+), 11 deletions(-) diff --git a/backend/fluksio/api/routes/observability.py b/backend/fluksio/api/routes/observability.py index f72733c..ad315f4 100644 --- a/backend/fluksio/api/routes/observability.py +++ b/backend/fluksio/api/routes/observability.py @@ -19,12 +19,17 @@ from sqlalchemy import select as sa_select from sqlmodel import col, select from fluksio import __version__ -from fluksio.api.deps import FlowControllerDep, SessionDep, get_current_user +from fluksio.api.deps import ( + FlowControllerDep, + SessionDep, + get_current_active_superuser, + get_current_user, +) from fluksio.api.routes.runs import elapsed_ms from fluksio.core.config import settings from fluksio.flow.controller import NodeStatus from fluksio.flow.pipeline import ADVISORY_ISSUES -from fluksio.models import EngineEvent, FlowRun, MetricBucket +from fluksio.models import EngineEvent, FlowRun, Message, MetricBucket router = APIRouter( prefix="/observability", @@ -508,3 +513,18 @@ async def read_dead_letters( if controller.execution is None: return [] return await run_in_threadpool(controller.execution.queue.dead_letters, limit) + + +@router.post( + "/flush", + response_model=Message, + dependencies=[Depends(get_current_active_superuser)], +) +async def flush_metrics(request: Request) -> Any: + """Write what the metrics collector holds now, not at its next interval. + + For tooling that reads or clears these tables right after causing events, + such as the soak harness clearing its own rows. + """ + await request.app.state.metrics_collector.flush_now() + return Message(message="Metrics flushed") diff --git a/backend/fluksio/flow/metrics.py b/backend/fluksio/flow/metrics.py index 6eacba9..b2f7d3f 100644 --- a/backend/fluksio/flow/metrics.py +++ b/backend/fluksio/flow/metrics.py @@ -103,6 +103,7 @@ class MetricsCollector: # held with the time it arrived so an unpaired one does not stay put. self._tracebacks: dict[tuple[str, str], tuple[float, str]] = {} self._last_prune = 0.0 + self._queue: asyncio.Queue[dict[str, Any]] | None = None # ------------------------------------------------------------------------- # The loop @@ -112,6 +113,7 @@ class MetricsCollector: """Consume the bus until cancelled, flushing on a fixed interval.""" last = time.monotonic() async with self._events.subscribe() as queue: + self._queue = queue while True: # A busy bus never idles, so the flush is on a deadline rather # than on the timeout alone. @@ -129,6 +131,21 @@ class MetricsCollector: await self.flush() last = time.monotonic() + async def flush_now(self) -> None: + """Write what has been heard so far, without waiting for the interval. + + Events the bus has handed over and the loop has not reached yet are + taken first, so whoever caused an event and then asks for a flush finds + it written. + """ + while self._queue is not None and not self._queue.empty(): + event = self._queue.get_nowait() + try: + self.handle(event) + except Exception: + logger.exception("Could not record %s", event.get("type")) + await self.flush() + # ------------------------------------------------------------------------- # Accumulating # ------------------------------------------------------------------------- diff --git a/backend/fluksio/main.py b/backend/fluksio/main.py index 39833df..bb0836b 100644 --- a/backend/fluksio/main.py +++ b/backend/fluksio/main.py @@ -384,7 +384,9 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: _background(watchdog.run(), "loop-watchdog") _background(alerts.run(), "alert-manager") - _background(MetricsCollector(event_bus).run(), "metrics-collector") + # On the app so a superuser can ask it to flush now. + app.state.metrics_collector = MetricsCollector(event_bus) + _background(app.state.metrics_collector.run(), "metrics-collector") _background(_sweep_artifacts(artifacts, controller), "artifact-gc") _background(_trim_volatile(volatile), "artifact-ring") await controller.start() diff --git a/backend/scripts/soak.py b/backend/scripts/soak.py index dc1f83d..8ba85ac 100644 --- a/backend/scripts/soak.py +++ b/backend/scripts/soak.py @@ -36,7 +36,6 @@ from typing import Any import httpx from fluksio.core.config import settings -from fluksio.flow.metrics import FLUSH_INTERVAL_S log = logging.getLogger("soak") @@ -374,6 +373,14 @@ def create_flow( log.info("created flow '%s'", name) +def flush_metrics(client: httpx.Client) -> None: + """Have the metrics collector write what it holds now.""" + try: + client.post("/api/v1/observability/flush").raise_for_status() + except httpx.HTTPError as exc: + log.error("could not flush the metrics collector: %s", exc) + + def delete_flows(client: httpx.Client) -> None: for name in FLOWS: try: @@ -887,11 +894,11 @@ def cleanup(client: httpx.Client | None) -> None: delete_flows(client) clear_redis() - # Deleting a flow publishes an audit event of its own, and the metrics - # collector buffers what it hears for up to FLUSH_INTERVAL_S before writing - # it. Clearing the tables any sooner races that flush and leaves the rows - # this very cleanup caused. - time.sleep(FLUSH_INTERVAL_S + 5.0) + # Deleting a flow publishes an audit event of its own, which the metrics + # collector holds until its next flush. Clearing the tables before it is + # written would leave the rows this very cleanup caused. + if client is not None: + flush_metrics(client) clear_observability() diff --git a/backend/tests/api/routes/test_observability.py b/backend/tests/api/routes/test_observability.py index 6e8f963..77dafba 100644 --- a/backend/tests/api/routes/test_observability.py +++ b/backend/tests/api/routes/test_observability.py @@ -1,9 +1,11 @@ +import time from datetime import UTC, datetime, timedelta from fastapi.testclient import TestClient -from sqlmodel import Session +from sqlmodel import Session, select from fluksio.core.config import settings +from fluksio.flow.events import event_bus from fluksio.models import EngineEvent, FlowRun, MetricBucket PREFIX = f"{settings.API_V1_STR}/observability" @@ -482,3 +484,31 @@ def test_a_node_failing_on_every_call_makes_the_summary_degraded( assert body["nodes"]["failing"] == 1 (problem,) = [p for p in body["problems"] if "failing" in p] assert "house.calc" in problem and "house.old" not in problem + + +def test_a_superuser_can_flush_the_collector_now( + client: TestClient, + superuser_token_headers: dict[str, str], + normal_user_token_headers: dict[str, str], + db: Session, +) -> None: + """Whoever reads or clears the tables need not wait out the interval.""" + event_bus.publish( + { + "type": "audit", + "action": "flushed", + "flow": FLOW, + "user": "a@example.com", + "ts": time.time(), + } + ) + url = f"{PREFIX}/flush" + assert client.post(url, headers=normal_user_token_headers).status_code == 403 + assert client.post(url, headers=superuser_token_headers).status_code == 200 + + written = db.exec( + select(EngineEvent).where( + EngineEvent.flow == FLOW, EngineEvent.detail == "flushed" + ) + ).all() + assert len(written) == 1 diff --git a/docs/code/api.md b/docs/code/api.md index 3d6df91..a418e09 100644 --- a/docs/code/api.md +++ b/docs/code/api.md @@ -208,6 +208,7 @@ Connector state keys can also be listed and cleared by a superuser; see | `GET` | `/observability/summary` | health and engine version; always returns `200` | | `GET` | `/observability/timeseries`, `/observability/flows` | execution metrics or flow rollups | | `GET` | `/observability/runs`, `/observability/events`, `/observability/dead-letter` | cascades, events, or abandoned work | +| `POST` | `/observability/flush` | write buffered metrics and events now; superuser only | | `GET` | `/utils/health/` | deep health check; fails on loop or state-backend problems | | `GET` | `/workers`, `/workers/resources` | attached workers and resource availability | | `GET`, `POST` | `/workers/tokens` | list or mint worker credentials; superuser only | diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index fbad4f3..66ce387 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { AiReadAiConfigResponse, AiUpdateAiConfigData, AiUpdateAiConfigResponse, AiDeleteAiConfigResponse, AiListAiModelsData, AiListAiModelsResponse, AiReadAiStatusResponse, AiReadControlNodesResponse, AiReadChatResponse, AiChatData, AiChatResponse, AiClearChatResponse, AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsReadWebpushKeyResponse, AlertsAddWebpushSubscriptionData, AlertsAddWebpushSubscriptionResponse, AlertsRemoveWebpushSubscriptionData, AlertsRemoveWebpushSubscriptionResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudDisconnectResponse, ConnectorsReadConnectorStateData, ConnectorsReadConnectorStateResponse, ConnectorsClearConnectorStateData, ConnectorsClearConnectorStateResponse, ConnectorsDeleteConnectorStateData, ConnectorsDeleteConnectorStateResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, FlowsIssueSocketTicketResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsImportRunsData, RunsImportRunsResponse, RunsReadRunsData, RunsReadRunsResponse, RunsDeleteRunsData, RunsDeleteRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadMetricNamesData, RunsReadMetricNamesResponse, RunsReadRunData, RunsReadRunResponse, RunsDeleteRunData, RunsDeleteRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsRetryRunData, RunsRetryRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SearchReadSearchIndexResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsReadConfigResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersReadTokensResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersRevokeTokenData, WorkersRevokeTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; +import type { AiReadAiConfigResponse, AiUpdateAiConfigData, AiUpdateAiConfigResponse, AiDeleteAiConfigResponse, AiListAiModelsData, AiListAiModelsResponse, AiReadAiStatusResponse, AiReadControlNodesResponse, AiReadChatResponse, AiChatData, AiChatResponse, AiClearChatResponse, AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsReadWebpushKeyResponse, AlertsAddWebpushSubscriptionData, AlertsAddWebpushSubscriptionResponse, AlertsRemoveWebpushSubscriptionData, AlertsRemoveWebpushSubscriptionResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudDisconnectResponse, ConnectorsReadConnectorStateData, ConnectorsReadConnectorStateResponse, ConnectorsClearConnectorStateData, ConnectorsClearConnectorStateResponse, ConnectorsDeleteConnectorStateData, ConnectorsDeleteConnectorStateResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, FlowsIssueSocketTicketResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, ObservabilityFlushMetricsResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsImportRunsData, RunsImportRunsResponse, RunsReadRunsData, RunsReadRunsResponse, RunsDeleteRunsData, RunsDeleteRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadMetricNamesData, RunsReadMetricNamesResponse, RunsReadRunData, RunsReadRunResponse, RunsDeleteRunData, RunsDeleteRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsRetryRunData, RunsRetryRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SearchReadSearchIndexResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsReadConfigResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersReadTokensResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersRevokeTokenData, WorkersRevokeTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; export class AiService { /** @@ -1907,6 +1907,22 @@ export class ObservabilityService { } }); } + + /** + * Flush Metrics + * Write what the metrics collector holds now, not at its next interval. + * + * For tooling that reads or clears these tables right after causing events, + * such as the soak harness clearing its own rows. + * @returns Message Successful Response + * @throws ApiError + */ + public static flushMetrics(): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/api/v1/observability/flush' + }); + } } export class PanelsService { diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 778f84a..7258c8b 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -2001,6 +2001,8 @@ export type ObservabilityReadDeadLettersData = { export type ObservabilityReadDeadLettersResponse = (Array); +export type ObservabilityFlushMetricsResponse = (Message); + export type PanelsReadPanelsResponse = (PanelsPublic); export type PanelsSavePanelsData = { From 78792a95bb9966b13d76610b09bf902134fadff0 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:53:38 +0200 Subject: [PATCH 036/103] GPUs follow the engine's own CUDA_VISIBLE_DEVICES Its entries are the cards present, and a local child's allocation is named through them, so under =2,3 card 0 is device 2. Remote workers keep mapping against their own mask. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/controller.py | 2 +- backend/fluksio/flow/resources.py | 34 +++++++++++++++++++++++++--- backend/tests/flow/test_resources.py | 28 ++++++++++++++++++++++- docs/code/cli.md | 5 ++-- docs/reference/configuration.md | 10 +++++--- 5 files changed, 69 insertions(+), 10 deletions(-) diff --git a/backend/fluksio/flow/controller.py b/backend/fluksio/flow/controller.py index 9665cd9..91d046e 100644 --- a/backend/fluksio/flow/controller.py +++ b/backend/fluksio/flow/controller.py @@ -979,7 +979,7 @@ class FlowController: with placer.claim( # type: ignore[union-attr] wanted, device=device, policy=policy, node=node_id, run=run_id ) as (target, allocation): - env = derive_env(wanted, allocation) + env = derive_env(wanted, allocation, local=target.worker is None) if target.worker is None: return pool.for_env(env).run( # type: ignore[union-attr] owner, diff --git a/backend/fluksio/flow/resources.py b/backend/fluksio/flow/resources.py index 3951007..0e05c22 100644 --- a/backend/fluksio/flow/resources.py +++ b/backend/fluksio/flow/resources.py @@ -96,15 +96,32 @@ def warn_oversubscribed(limits: dict[str, int]) -> None: ) +def visible_gpus() -> list[str] | None: + """The cards the engine's own ``CUDA_VISIBLE_DEVICES`` lists, if it is set. + + Indices, UUIDs or MIG instances, kept as written: a child is handed the + same names, so whatever the driver took here it takes there. + """ + mask = os.environ.get("CUDA_VISIBLE_DEVICES") + if mask is None: + return None + return [part.strip() for part in mask.split(",") if part.strip()] + + def machine_gpus() -> int: """Cards a node may be given here, when nobody said. - The device nodes the NVIDIA driver creates, counted — no vendor tool, so + The engine's own ``CUDA_VISIBLE_DEVICES`` where it has one, since a card + the process can see but was not given is not its to hand out. Otherwise + the device nodes the NVIDIA driver creates, counted — no vendor tool, so the one dependency does not become two. It answers for the case that actually bit people (a box with cards, an engine told nothing, every GPU node clamped to zero and running at once) and for nothing else: another vendor, or a card this process cannot see, is still `--gpus N`. """ + visible = visible_gpus() + if visible is not None: + return len(visible) return len(glob.glob("/dev/nvidia[0-9]*")) @@ -229,7 +246,9 @@ class ResourceAccountant: # ----------------------------------------------------------------------------- -def derive_env(wanted: Resources, allocation: Allocation) -> dict[str, str]: +def derive_env( + wanted: Resources, allocation: Allocation, local: bool = True +) -> dict[str, str]: """The environment a worker running this node is started with. Precedence is engine environment, then what the allocation implies, then @@ -240,10 +259,19 @@ def derive_env(wanted: Resources, allocation: Allocation) -> dict[str, str]: contents depend on the version installed, so writing it here would silently replace whatever the author had put there. It travels in ``resources.env``, where it is visible. + + ``local`` is whether the allocation is this machine's. A child's + ``CUDA_VISIBLE_DEVICES`` replaces the engine's rather than narrowing it, so + here a card is named the way the engine's own list names it: under ``=2,3`` + card 0 is device 2. A remote worker does the same against its own list. """ env = {var: str(allocation.cpus) for var in THREAD_VARS} if wanted.gpus: - env["CUDA_VISIBLE_DEVICES"] = ",".join(str(index) for index in allocation.gpus) + names = (visible_gpus() if local else None) or [] + env["CUDA_VISIBLE_DEVICES"] = ",".join( + names[index] if index < len(names) else str(index) + for index in allocation.gpus + ) env.update(wanted.env) return env diff --git a/backend/tests/flow/test_resources.py b/backend/tests/flow/test_resources.py index d03ee5d..ffcc2fa 100644 --- a/backend/tests/flow/test_resources.py +++ b/backend/tests/flow/test_resources.py @@ -33,6 +33,7 @@ def test_the_cards_are_counted_from_the_devices_the_driver_makes(monkeypatch): """ from fluksio.flow import resources + monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False) monkeypatch.setattr( resources.glob, "glob", @@ -150,12 +151,37 @@ def test_the_share_becomes_the_thread_limit(): assert "CUDA_VISIBLE_DEVICES" not in env -def test_a_gpu_node_is_told_which_card_is_its(): +def test_a_gpu_node_is_told_which_card_is_its(monkeypatch): + monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False) env = derive_env(Resources(cpus=1, gpus=2), Allocation(cpus=1, gpus=(1, 3))) assert env["CUDA_VISIBLE_DEVICES"] == "1,3" +@pytest.mark.parametrize( + ("mask", "cards", "named"), + [(None, 2, "0"), ("2,3", 2, "2"), ("GPU-a1b2, MIG-c3d4", 2, "GPU-a1b2")], +) +def test_the_engines_own_mask_is_the_cards_it_has(monkeypatch, mask, cards, named): + """Under ``CUDA_VISIBLE_DEVICES=2,3`` card 0 is device 2, not device 0. + + A child's mask replaces the engine's, so an unmapped 0 would be a card + the engine was never given. A remote worker maps against its own mask. + """ + from fluksio.flow import resources + + monkeypatch.setattr(resources.glob, "glob", lambda pattern: ["/dev/nvidia0"] * 2) + if mask is None: + monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False) + else: + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", mask) + wanted, allocation = Resources(cpus=1, gpus=1), Allocation(cpus=1, gpus=(0,)) + + assert machine_gpus() == cards + assert derive_env(wanted, allocation)["CUDA_VISIBLE_DEVICES"] == named + assert derive_env(wanted, allocation, local=False)["CUDA_VISIBLE_DEVICES"] == "0" + + def test_what_the_node_asked_for_wins(): """A declaration outranks what the allocation would imply — it is deliberate.""" wanted = Resources( diff --git a/docs/code/cli.md b/docs/code/cli.md index 7810998..34a8cd9 100644 --- a/docs/code/cli.md +++ b/docs/code/cli.md @@ -65,8 +65,9 @@ that instance's running engine; plain mode refuses to start a duplicate. | `--worker-max-rss MB` | `0` (disabled) | Retire a worker above this resident memory after its call | Resource defaults can also come from the corresponding `FLOW_*` settings; -see [Configuration](../reference/configuration.md). GPU detection uses -NVIDIA device nodes. Set `--gpus` when your hardware is not detected. +see [Configuration](../reference/configuration.md). GPU detection counts the +entries of `CUDA_VISIBLE_DEVICES` when it is set, otherwise NVIDIA device +nodes. Set `--gpus` when your hardware is not detected. Use local disk for the data directory on a cluster: SQLite's write-ahead log does not support NFS. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index b0502ff..f135896 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -126,7 +126,7 @@ Set `TZ` in `.env` for schedules that should follow local time, such as | `FLOW_MAX_RUNS` | `4` | concurrent batch runs | | `FLOW_NODE_TIMEOUT` | `0` | maximum node silence in seconds, unless the node overrides it; `0` disables | | `FLOW_CPUS` | `0` | CPU budget; `0` detects cores and reserves two for the engine | -| `FLOW_GPUS` | `0` | local GPU count; `0` detects NVIDIA device nodes | +| `FLOW_GPUS` | `0` | local GPU count; `0` counts the engine's `CUDA_VISIBLE_DEVICES`, or NVIDIA device nodes without it | | `FLOW_RAM` | `0` | memory budget in MB; `0` detects host/container memory and reserves 1 GiB | | `FLOW_WORKER_MAX_RSS_MB` | `0` | retire a local worker above this memory use after a call; `0` disables | | `FLOW_NODE_STOP_TIMEOUT` | `5` | seconds a node gets to stop during a rebuild before the rebuild moves on | @@ -139,14 +139,18 @@ Set `TZ` in `.env` for schedules that should follow local time, such as The three concurrency limits must be at least `1`. They also have CLI flags: `--max-workers`, `--max-cascades` and `--max-runs`. When `FLOW_MAX_WORKERS` or `FLOW_MAX_CASCADES` is more than four times the machine's core count, the -engine logs a warning at startup and uses the value anyway. Resource flags are `--gpus`, -`--ram` and `--worker-max-rss`. +engine logs a warning at startup and uses the value anyway. Resource flags are +`--gpus`, `--ram` and `--worker-max-rss`. Nodes with declared resources are scheduled against the available budget. Undeclared nodes use the shared pool and receive thread caps. Attached workers report their own inventory; the engine's resource settings describe only its own machine. See [Remote workers](../code/workers.md) and the **Workers** screen. +When the engine runs with `CUDA_VISIBLE_DEVICES`, a node's card is named the +way that list names it, so under `2,3` the first card is device `2`. UUID and +MIG entries are passed on as written. + ## Artifacts | Variable | Default | Purpose | From f57b6f695b0a29da538c25ffe3eaef5235dc2791 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:55:02 +0200 Subject: [PATCH 037/103] wren runs its model on one BLAS thread threadpoolctl scopes the limit to wren.act, so other numpy calls in the engine keep their threads. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/nodes/wren.py | 7 ++++++- backend/pyproject.toml | 9 +++++++++ backend/tests/flow/test_wren.py | 22 ++++++++++++++++++++++ uv.lock | 11 +++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/backend/fluksio/flow/nodes/wren.py b/backend/fluksio/flow/nodes/wren.py index 1f9e2c7..2048508 100644 --- a/backend/fluksio/flow/nodes/wren.py +++ b/backend/fluksio/flow/nodes/wren.py @@ -14,6 +14,7 @@ from collections.abc import Iterable from typing import Any from pydantic import BaseModel, ConfigDict, Field +from threadpoolctl import threadpool_limits from fluksio.flow.messages import DType, MessageSpec from fluksio.flow.nodes.base import Node @@ -306,7 +307,11 @@ class WrenNode(Node): wren = _wren() specs = {spec.port: spec for spec in self.output_ports} house = self._house(wren, specs, presence, outdoor, dark) - calls, confidence = wren.act(prompt, house) + # One BLAS thread answers in about half the time for a model this + # small. Scoped to the call, so numpy elsewhere in the process keeps + # its threads. + with threadpool_limits(1): + calls, confidence = wren.act(prompt, house) confidence = round(float(confidence), 3) write = self.cfg.apply and confidence >= self.cfg.min_confidence writes: dict[str, Any] = {} diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 18d7911..8e26983 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -58,6 +58,10 @@ dependencies = [ # The `mlp` node, which is registered at boot. Kept out of the extras # because a data-science environment has it already. "numpy>=2.2.6", + # The `wren` node runs its model on one BLAS thread, which answers in about + # half the time. Scoped to the call: OPENBLAS_NUM_THREADS would bind every + # other numpy call in the engine too, and only counts before numpy loads. + "threadpoolctl>=3.5", # The `inject` node's cron. Imported where it is used, behind a message # that cannot fire while this line is here. "croniter>=1.3.0", @@ -164,6 +168,11 @@ ignore_missing_imports = true module = ["wren"] ignore_missing_imports = true +# What scopes the `wren` node's BLAS threads to its call; it ships no types. +[[tool.mypy.overrides]] +module = ["threadpoolctl"] +ignore_missing_imports = true + [tool.ruff] target-version = "py312" diff --git a/backend/tests/flow/test_wren.py b/backend/tests/flow/test_wren.py index 2f3fff3..44c048b 100644 --- a/backend/tests/flow/test_wren.py +++ b/backend/tests/flow/test_wren.py @@ -143,3 +143,25 @@ def test_nothing_is_written_unless_applied_and_sure(fake_wren, params, confidenc } ] } + + +def test_the_model_runs_on_one_blas_thread(fake_wren, monkeypatch): + """About twice as fast for a model this small; numpy is loaded by `mlp`.""" + from threadpoolctl import threadpool_info + + threads: list[int] = [] + act = fake_wren.act + + def counted(command: str, house: Any) -> tuple[list[Any], float]: + threads.extend( + pool["num_threads"] + for pool in threadpool_info() + if pool["user_api"] == "blas" + ) + return act(command, house) + + monkeypatch.setattr(fake_wren, "act", counted) + _node(("kitchen_lamp", DType.BOOL)).execute({"house.command": "lights on"}) + + assert threads + assert set(threads) == {1} diff --git a/uv.lock b/uv.lock index 10e0f43..be2104a 100644 --- a/uv.lock +++ b/uv.lock @@ -946,6 +946,7 @@ dependencies = [ { name = "rich" }, { name = "sqlmodel" }, { name = "textual" }, + { name = "threadpoolctl" }, { name = "uv" }, { name = "websockets" }, ] @@ -998,6 +999,7 @@ requires-dist = [ { name = "sentry-sdk", extras = ["fastapi"], marker = "extra == 'server'", specifier = ">=2.20.0" }, { name = "sqlmodel", specifier = ">=0.0.21,<1.0.0" }, { name = "textual", specifier = ">=1.0" }, + { name = "threadpoolctl", specifier = ">=3.5" }, { name = "uv", specifier = ">=0.5" }, { name = "websockets", specifier = ">=14" }, ] @@ -2840,6 +2842,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/be/35261223d9416a0751cdff1c7b4a6f881387218a12d439fe22fefebc8c04/textual-8.2.8-py3-none-any.whl", hash = "sha256:267375fd402dc8d981457212efa71f0e3365fd17bba144ba9bb3ed7563cb374a", size = 731418, upload-time = "2026-06-30T06:51:26.364Z" }, ] +[[package]] +name = "threadpoolctl" +version = "3.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/dc/6c58154c1c65f758ea979e7139cb76993a9cfc662d14e9be3c4a667cfb77/threadpoolctl-3.7.0.tar.gz", hash = "sha256:61348cfb77d53b9242e0017029244b559b810c142ced65b4e21eeca1843959a7", size = 31961, upload-time = "2026-09-15T15:46:20.263Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/3f/f88a53f60a472b46f4023f56d204dd7de33d34c5d2acbfa0d70a674e639e/threadpoolctl-3.7.0-py3-none-any.whl", hash = "sha256:cd8b60b5641b45c67bbf73c64c843235fc2d8a480c87389f52f5dbee893b86be", size = 26362, upload-time = "2026-09-15T15:46:19.168Z" }, +] + [[package]] name = "typer" version = "0.21.1" From b4ae333b6a49fe82cdc2bfb5badcc5a21d3de981 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:55:41 +0200 Subject: [PATCH 038/103] sweep_artifacts: narrow the optional flow store for mypy Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/runs.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/fluksio/flow/runs.py b/backend/fluksio/flow/runs.py index a465891..0ac7d13 100644 --- a/backend/fluksio/flow/runs.py +++ b/backend/fluksio/flow/runs.py @@ -403,14 +403,15 @@ def sweep_artifacts( return 0 keep = set(session.exec(select(col(RunArtifact.digest)).distinct()).all()) - for name in flows.list_flows() if flows else (): - for draft in (False, True): - try: - flow = flows.read_flow(name, draft=draft) - except FlowNotFound: - continue - for declared in flow.inputs: - _references_in(declared.initial, keep) + if flows is not None: + for name in flows.list_flows(): + for draft in (False, True): + try: + flow = flows.read_flow(name, draft=draft) + except FlowNotFound: + continue + for declared in flow.inputs: + _references_in(declared.initial, keep) keys = state.keys() for start in range(0, len(keys), _SWEEP_BATCH): From 5a5f4c5a800d72bff742086db52c7f3eae98b26b Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 09:57:51 +0200 Subject: [PATCH 039/103] http sender drops reply keys no output port declares A sender without outputs no longer fails on a {"status": "ok"} reply. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/nodes/http.py | 11 ++++++++--- backend/tests/flow/test_senders.py | 26 +++++++++++++++++++++++++- docs/reference/node-types.md | 2 +- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/backend/fluksio/flow/nodes/http.py b/backend/fluksio/flow/nodes/http.py index 9fbf12d..9ceba3d 100644 --- a/backend/fluksio/flow/nodes/http.py +++ b/backend/fluksio/flow/nodes/http.py @@ -301,7 +301,7 @@ class HttpNode(Node): :type params: dict :param kwargs: Pipeline data to send (from required messages). :type kwargs: Any - :returns: Response data if the endpoint returns JSON, None otherwise. + :returns: The reply's values for the declared output ports, or None. :rtype: dict | None """ client = shared_client() @@ -330,8 +330,13 @@ class HttpNode(Node): except Exception: return None # Outputs are keyed by port, so only an object can be one. A bare - # scalar or list is a valid reply, just not something to publish. - return body if isinstance(body, dict) else None + # scalar or list is a valid reply, just not something to publish, + # and neither is a key no port declares: the reply is the + # endpoint's, and a `status` beside the values is common. + if not isinstance(body, dict): + return None + ports = {spec.port for spec in self.output_ports} + return {key: value for key, value in body.items() if key in ports} or None except httpx.HTTPStatusError as e: logger.error( diff --git a/backend/tests/flow/test_senders.py b/backend/tests/flow/test_senders.py index 1cbfbb8..e59cd6c 100644 --- a/backend/tests/flow/test_senders.py +++ b/backend/tests/flow/test_senders.py @@ -2,8 +2,10 @@ import asyncio +import httpx + from fluksio.flow.messages import DType, MessageSpec -from fluksio.flow.nodes import MqttNode +from fluksio.flow.nodes import HttpNode, MqttNode from fluksio.flow.nodes.http import close_shared_client, shared_client @@ -20,6 +22,28 @@ def test_http_senders_share_one_pooled_client(): close_shared_client() +def test_an_http_reply_keeps_only_the_declared_ports(monkeypatch): + """A `status` beside the value is the endpoint's business, not an error.""" + reply = {"temp": 21.5, "status": "ok"} + client = httpx.Client( + transport=httpx.MockTransport(lambda _: httpx.Response(200, json=reply)) + ) + monkeypatch.setattr("fluksio.flow.nodes.http.shared_client", lambda: client) + + def sender(*provides: MessageSpec) -> HttpNode: + node = HttpNode( + requires=[MessageSpec(name="setpoint", dtype=DType.FLOAT)], + provides=provides, + params={"url": "http://thermostat.test/set"}, + ) + node.assign_flow("heating", "thermostat") + return node + + temp = MessageSpec(name="temp", dtype=DType.FLOAT) + assert sender(temp).execute({"heating.setpoint": 21.0}) == {"heating.temp": 21.5} + assert sender().execute({"heating.setpoint": 21.0}) is None + + def _sender() -> MqttNode: node = MqttNode( requires=[MessageSpec(name="setpoint", port="setpoint", dtype=DType.FLOAT)], diff --git a/docs/reference/node-types.md b/docs/reference/node-types.md index 0157174..ea57a43 100644 --- a/docs/reference/node-types.md +++ b/docs/reference/node-types.md @@ -63,7 +63,7 @@ The ports pick the direction: outputs only make a **listener**, inputs a **sender**. A listener is a **webhook**: the engine mounts a route at `/hooks/{flow}/{url}/{secret}` while the flow runs. A sender requests `url` and publishes a JSON object response on its outputs, one key per output port. -A key no port declares fails the call, and a response that is not an object +Keys no port declares are dropped, and a response that is not an object publishes nothing. | Setting | Default | Notes | From 0da9a65eacf8a95de0ae654aa5fff24a5165a6d7 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 10:05:21 +0200 Subject: [PATCH 040/103] Library delete 409s on published usages as well as draft ones Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/flows.py | 5 ++++- backend/tests/api/routes/test_flows.py | 30 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/backend/fluksio/api/routes/flows.py b/backend/fluksio/api/routes/flows.py index bc352af..756233b 100644 --- a/backend/fluksio/api/routes/flows.py +++ b/backend/fluksio/api/routes/flows.py @@ -353,7 +353,10 @@ async def delete_shared_node( for flow in sorted({usage.split(".", 1)[0] for usage in running}): await controller.reload_flow(flow) return Message(message=f"Deleted shared node '{lib_name}'") - used_by = controller.store.usages(lib_name) + # The published flows too: a draft that unshared a node still runs it. + used_by = sorted( + {*controller.store.usages(lib_name), *controller.store.usages(lib_name, False)} + ) if used_by: raise HTTPException( status_code=409, diff --git a/backend/tests/api/routes/test_flows.py b/backend/tests/api/routes/test_flows.py index d28ada2..b2cb0d7 100644 --- a/backend/tests/api/routes/test_flows.py +++ b/backend/tests/api/routes/test_flows.py @@ -583,6 +583,36 @@ def test_a_shared_node_in_use_is_retired_only_with_detach( client.delete(f"{PREFIX}/shared", headers=superuser_token_headers) +def test_a_shared_node_only_the_published_flow_uses_is_kept( + client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + """An unshare in the draft leaves the running flow on the source.""" + client.put( + f"{PREFIX}/unshared", headers=superuser_token_headers, json=a_flow("unshared") + ) + shared = client.post( + f"{PREFIX}/unshared/nodes/sensor/share", + headers=superuser_token_headers, + json={"lib_name": "published_reading"}, + ).json() + client.post( + f"{PREFIX}/unshared/publish", + headers=superuser_token_headers, + json={"version": shared["definition"]["version"]}, + ) + client.post( + f"{PREFIX}/unshared/nodes/sensor/unshare", headers=superuser_token_headers + ) + + url = f"{PREFIX}/library/published_reading" + response = client.delete(url, headers=superuser_token_headers) + assert response.status_code == 409 + assert "unshared.sensor" in response.json()["detail"] + + client.delete(f"{url}?detach=true", headers=superuser_token_headers) + client.delete(f"{PREFIX}/unshared", headers=superuser_token_headers) + + def test_a_shared_source_write_is_audited( client: TestClient, superuser_token_headers: dict[str, str], From ccc7081b089307ea0d8a69e2c40d1d5b4c270027 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 10:07:50 +0200 Subject: [PATCH 041/103] PUT /panels/ carries a version and refuses a stale one with 409 The Panels dialog re-reads the list on 409. A panels.json without a version reads as 0. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/panels.py | 25 ++++++++++-- backend/fluksio/flow/panels.py | 3 ++ backend/tests/api/routes/test_panels.py | 39 ++++++++++++++++++- docs/code/api.md | 2 +- frontend/src/client/schemas.gen.ts | 10 +++++ frontend/src/client/sdk.gen.ts | 5 +++ frontend/src/client/types.gen.ts | 2 + .../src/components/Dashboard/PanelsDialog.tsx | 14 +++++-- frontend/src/components/Dashboard/queries.ts | 13 ++++++- frontend/tests/panel.spec.ts | 5 ++- 10 files changed, 108 insertions(+), 10 deletions(-) diff --git a/backend/fluksio/api/routes/panels.py b/backend/fluksio/api/routes/panels.py index 56ef9c7..8cb6b78 100644 --- a/backend/fluksio/api/routes/panels.py +++ b/backend/fluksio/api/routes/panels.py @@ -269,6 +269,8 @@ class PanelsPublic(BaseModel): """ panels: list[PanelDef] = Field(default_factory=list) + #: What a save of this list has to send back. + version: int = 0 #: Whatever this instance was told it is reachable at. The same setting #: the password-reset links are built from, so an instance that has it #: wrong has it wrong in both places. @@ -277,7 +279,9 @@ class PanelsPublic(BaseModel): def _public(config: PanelsConfig) -> PanelsPublic: return PanelsPublic( - panels=config.panels, frontend_host=settings.FRONTEND_HOST.rstrip("/") + panels=config.panels, + version=config.version, + frontend_host=settings.FRONTEND_HOST.rstrip("/"), ) @@ -298,6 +302,11 @@ async def save_panels(body: PanelsConfig) -> Any: A panel that disappears here takes its credential with it, so this is also how a device is unpaired along with its assignment. Re-pairing one screen and keeping the assignment is ``/{panel_id}/unpair`` below. + + The body carries the ``version`` it was based on, and a save someone else + has moved past is refused with 409, as a dashboard's is. Two clients each + writing back the whole list would otherwise drop each other's panels, and + a dropped panel revokes its screen at once. """ seen = set() for panel in body.panels: @@ -314,9 +323,19 @@ async def save_panels(body: PanelsConfig) -> Any: # saving an arrangement — and reading it in a separate step from # writing it is exactly how an unpair in between was undone. with edit_lock: - stored = {p.id: p.nonce for p in read_config().panels} + stored = read_config() + if body.version != stored.version: + raise HTTPException( + status_code=409, + detail={ + "message": "Someone else saved the panels first", + "current_version": stored.version, + }, + ) + nonces = {p.id: p.nonce for p in stored.panels} for panel in body.panels: - panel.nonce = stored.get(panel.id, 0) + panel.nonce = nonces.get(panel.id, 0) + body.version = stored.version + 1 write_config(body) await run_in_threadpool(_save) diff --git a/backend/fluksio/flow/panels.py b/backend/fluksio/flow/panels.py index f8f62a9..91bc4f4 100644 --- a/backend/fluksio/flow/panels.py +++ b/backend/fluksio/flow/panels.py @@ -96,6 +96,9 @@ class PanelsConfig(BaseModel): """Every panel this instance knows about.""" panels: list[PanelDef] = Field(default_factory=list) + #: Bumped on every save; a save based on an older one is refused, as a + #: dashboard's is. A file written before there was one reads as 0. + version: int = 0 #: Held across a read-modify-write of the panels file. diff --git a/backend/tests/api/routes/test_panels.py b/backend/tests/api/routes/test_panels.py index a2622ed..bde5a16 100644 --- a/backend/tests/api/routes/test_panels.py +++ b/backend/tests/api/routes/test_panels.py @@ -11,7 +11,11 @@ DASHBOARDS = f"{settings.API_V1_STR}/dashboards" def _panels(client: TestClient, headers: dict[str, str], config: dict) -> None: - response = client.put(f"{PREFIX}/", headers=headers, json=config) + """Replace the panels, based on whatever is stored now.""" + version = client.get(f"{PREFIX}/", headers=headers).json()["version"] + response = client.put( + f"{PREFIX}/", headers=headers, json={**config, "version": version} + ) assert response.status_code == 200, response.text @@ -130,6 +134,39 @@ def test_duplicate_panel_is_refused( assert response.status_code == 422 +def test_a_stale_save_is_refused( + client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + """Two clients writing back the whole list must not drop each other's panels.""" + read = client.get(f"{PREFIX}/", headers=superuser_token_headers).json() + base = read["version"] + + saved = client.put( + f"{PREFIX}/", + headers=superuser_token_headers, + json={"panels": [*read["panels"], {"id": "mine"}], "version": base}, + ) + assert saved.status_code == 200, saved.text + assert saved.json()["version"] == base + 1 + + stale = client.put( + f"{PREFIX}/", + headers=superuser_token_headers, + json={"panels": [*read["panels"], {"id": "theirs"}], "version": base}, + ) + assert stale.status_code == 409 + assert stale.json()["detail"]["current_version"] == base + 1 + + ids = [ + p["id"] + for p in client.get(f"{PREFIX}/", headers=superuser_token_headers).json()[ + "panels" + ] + ] + assert "mine" in ids + assert "theirs" not in ids + + def test_polling_needs_the_secret(client: TestClient) -> None: started = client.post(f"{PREFIX}/pair").json() assert len(started["code"]) == 6 diff --git a/docs/code/api.md b/docs/code/api.md index a418e09..16a8a50 100644 --- a/docs/code/api.md +++ b/docs/code/api.md @@ -177,7 +177,7 @@ stored once. Node code uses `save_artifact` and `load_artifact` instead. | `POST` | `/dashboards/from-flow/{flow}` | create a batch flow's results dashboard | | `PUT` | `/dashboards/{name}` | save a draft | | `POST` | `/dashboards/{name}/publish`, `/dashboards/{name}/discard`, `/dashboards/{name}/rename` | publish, discard, or rename | -| `GET`, `PUT` | `/panels/` | read or update panel assignments | +| `GET`, `PUT` | `/panels/` | read or replace panel assignments; send the current `version` | | `POST` | `/panels/pair` | start device pairing | | `GET` | `/panels/pair/{code}` | poll pairing with its `secret` | | `POST` | `/panels/{id}/pair` | approve the pairing | diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 46f8c6a..9276ad3 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -2933,6 +2933,11 @@ export const PanelsConfigSchema = { }, type: 'array', title: 'Panels' + }, + version: { + type: 'integer', + title: 'Version', + default: 0 } }, type: 'object', @@ -2949,6 +2954,11 @@ export const PanelsPublicSchema = { type: 'array', title: 'Panels' }, + version: { + type: 'integer', + title: 'Version', + default: 0 + }, frontend_host: { type: 'string', title: 'Frontend Host', diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index 66ce387..48b1be5 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -1946,6 +1946,11 @@ export class PanelsService { * A panel that disappears here takes its credential with it, so this is also * how a device is unpaired along with its assignment. Re-pairing one screen * and keeping the assignment is ``/{panel_id}/unpair`` below. + * + * The body carries the ``version`` it was based on, and a save someone else + * has moved past is refused with 409, as a dashboard's is. Two clients each + * writing back the whole list would otherwise drop each other's panels, and + * a dropped panel revokes its screen at once. * @param data The data for the request. * @param data.requestBody * @returns PanelsPublic Successful Response diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 7258c8b..5e02060 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -994,6 +994,7 @@ export type direction = 'in' | 'out'; */ export type PanelsConfig = { panels?: Array; + version?: number; }; /** @@ -1009,6 +1010,7 @@ export type PanelsConfig = { */ export type PanelsPublic = { panels?: Array; + version?: number; frontend_host?: string; }; diff --git a/frontend/src/components/Dashboard/PanelsDialog.tsx b/frontend/src/components/Dashboard/PanelsDialog.tsx index 255d3f0..6eba6e1 100644 --- a/frontend/src/components/Dashboard/PanelsDialog.tsx +++ b/frontend/src/components/Dashboard/PanelsDialog.tsx @@ -107,9 +107,17 @@ export function PanelsDialog() { const host = config?.frontend_host ?? "" const write = (next: PanelsConfig) => - save.mutate(next, { - onError: (error) => handleError.call(showErrorToast, error as ApiError), - }) + save.mutate( + { ...next, version: config?.version }, + { + onError: (error) => + (error as ApiError).status === 409 + ? showErrorToast( + "Someone else changed the panels first. Their version is shown now, so make the change again.", + ) + : handleError.call(showErrorToast, error as ApiError), + }, + ) const replace = (id: string, panel: PanelDef) => write({ panels: panels.map((p) => (p.id === id ? panel : p)) }) diff --git a/frontend/src/components/Dashboard/queries.ts b/frontend/src/components/Dashboard/queries.ts index fffca1d..4cc77bb 100644 --- a/frontend/src/components/Dashboard/queries.ts +++ b/frontend/src/components/Dashboard/queries.ts @@ -1,6 +1,7 @@ import { useMutation, useQueryClient } from "@tanstack/react-query" import { + ApiError, type DashboardDef_Input, DashboardsService, MessagesService, @@ -55,7 +56,13 @@ export const panelQueryOptions = (id: string) => ({ queryFn: () => PanelsService.readPanel({ panelId: id }), }) -/** Replace the panels. Superuser-only on the server. */ +/** + * Replace the panels, carrying the version they were read at. Superuser-only + * on the server. + * + * A 409 means someone else saved first, so the list is read again and the + * next change is based on theirs. + */ export function useSavePanels() { const queryClient = useQueryClient() return useMutation({ @@ -65,6 +72,10 @@ export function useSavePanels() { queryClient.setQueryData(panelKeys.all, saved) queryClient.invalidateQueries({ queryKey: panelKeys.all }) }, + onError: (error) => { + if (error instanceof ApiError && error.status === 409) + queryClient.invalidateQueries({ queryKey: panelKeys.all }) + }, }) } diff --git a/frontend/tests/panel.spec.ts b/frontend/tests/panel.spec.ts index 24f5da3..8632086 100644 --- a/frontend/tests/panel.spec.ts +++ b/frontend/tests/panel.spec.ts @@ -107,6 +107,7 @@ test.beforeAll(async ({ browser }) => { ...existing, { id: flowName, title: "Hall", dashboards: [first, second] }, ], + version: config.version, }, }) await page.close() @@ -116,7 +117,8 @@ test.afterAll(async ({ browser }) => { // Restoring the list is also what removes this spec's own panel. if (panels) { const page = await apiPage(browser) - await api(page, "/panels/", { method: "PUT", data: { panels } }) + const { version } = await (await api(page, "/panels/")).json() + await api(page, "/panels/", { method: "PUT", data: { panels, version } }) await page.close() } await deleteAll(browser, [ @@ -240,6 +242,7 @@ async function setTouch(page: Page, touch: boolean) { panels: (config.panels ?? []).map((panel: PanelDef) => panel.id === flowName ? { ...panel, touch } : panel, ), + version: config.version, }, }) } From f5be932df35acb35fc5c83ede85954bf4ff80c1d Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 10:08:46 +0200 Subject: [PATCH 042/103] Flow inputs carry an origin hint, run or panel The Inputs editor and the canvas label show it, and the run forms leave panel inputs out. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/schemas.py | 9 +++ docs/concepts/flows.md | 7 +- docs/concepts/values.md | 6 +- docs/interface/flow-editor.md | 2 +- frontend/src/client/schemas.gen.ts | 14 ++++ frontend/src/client/types.gen.ts | 13 ++++ frontend/src/components/Flow/FlowBoundary.tsx | 78 ++++++++++++------- frontend/src/components/Flow/RunDialog.tsx | 11 ++- frontend/src/components/Flow/deriveEdges.ts | 7 +- frontend/src/components/Flow/endpoints.ts | 3 +- 10 files changed, 111 insertions(+), 39 deletions(-) diff --git a/backend/fluksio/flow/schemas.py b/backend/fluksio/flow/schemas.py index 73c6ab3..00d5f1d 100644 --- a/backend/fluksio/flow/schemas.py +++ b/backend/fluksio/flow/schemas.py @@ -267,6 +267,15 @@ class FlowInput(BaseModel): spec: MessageSpec initial: Any | None = None + origin: Literal["run", "panel"] = Field( + default="run", + description=( + "What writes it. A run input is a parameter a run passes in, a " + "panel input is written by a dashboard or panel control. A hint " + "only: run forms leave panel inputs out, the engine treats both " + "the same." + ), + ) class FlowDef(BaseModel): diff --git a/docs/concepts/flows.md b/docs/concepts/flows.md index 50e6b62..34b00e9 100644 --- a/docs/concepts/flows.md +++ b/docs/concepts/flows.md @@ -96,13 +96,18 @@ Declare messages supplied by dashboard controls, the API or run parameters as the flow's **Inputs**. Give live inputs a starting value: ```json -{"inputs": [{"spec": {"name": "setpoint", "dtype": "float"}, "initial": 21.0}]} +{"inputs": [{"spec": {"name": "setpoint", "dtype": "float"}, "initial": 21.0, "origin": "panel"}]} ``` This seeds `house.setpoint` in flow `house` if it has no value yet. Later publications replace it. Batch inputs are run parameters, and batch outputs select the run's result. See [Where a node's values come from](values.md). +`origin` says what writes an input: `run` (the default) for a parameter a run +passes in, `panel` for a dashboard or panel control. The canvas labels each +input with it, and the run form leaves panel inputs out, so a run keeps their +starting value. The engine treats both the same. + ## Drafts and publishing - **Saving** stores a draft; the engine keeps running the published version. diff --git a/docs/concepts/values.md b/docs/concepts/values.md index 1628f71..33ff8a8 100644 --- a/docs/concepts/values.md +++ b/docs/concepts/values.md @@ -47,8 +47,10 @@ The initial value seeds an unset live message. Later publications take over; changing the starting value does not overwrite a value already in state. Use **Publish this value now** in the input row to send it immediately. -The canvas draws flow inputs as labelled endpoints. Without a producer or -declared input, a consuming node reports `unconnected_input`. +The canvas draws flow inputs as labelled endpoints, marked `run` or `panel` +by what writes them. A panel input is one a dashboard or panel control writes, +and the run form does not ask for it. Without a producer or declared input, a +consuming node reports `unconnected_input`. A batch flow's inputs are run parameters. Its **Result** section selects the messages returned when the run finishes. See [Runs](runs.md). diff --git a/docs/interface/flow-editor.md b/docs/interface/flow-editor.md index eb9168b..e13f3b1 100644 --- a/docs/interface/flow-editor.md +++ b/docs/interface/flow-editor.md @@ -80,7 +80,7 @@ Open **Flow settings** from the pencil button in the dock. |---|---| | **Running** | starts or stops the published live flow's subscriptions, schedules and webhooks | | **Mode** | chooses continuous `live` operation or requested `batch` runs | -| **Inputs** | declares external messages and starting values; batch inputs become run parameters | +| **Inputs** | declares external messages, their starting values and whether a run or a panel writes them; run inputs become run parameters | | **Result** | selects batch output messages | | **Contents** | shows the node count and message namespace | diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 9276ad3..5b91d4a 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -1355,6 +1355,13 @@ export const FlowInput_InputSchema = { } ], title: 'Initial' + }, + origin: { + type: 'string', + enum: ['run', 'panel'], + title: 'Origin', + description: 'What writes it. A run input is a parameter a run passes in, a panel input is written by a dashboard or panel control. A hint only: run forms leave panel inputs out, the engine treats both the same.', + default: 'run' } }, type: 'object', @@ -1376,6 +1383,13 @@ export const FlowInput_OutputSchema = { } ], title: 'Initial' + }, + origin: { + type: 'string', + enum: ['run', 'panel'], + title: 'Origin', + description: 'What writes it. A run input is a parameter a run passes in, a panel input is written by a dashboard or panel control. A hint only: run forms leave panel inputs out, the engine treats both the same.', + default: 'run' } }, type: 'object', diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 5e02060..f69e78a 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -391,14 +391,27 @@ export type FlowDetail = { export type FlowInput_Input = { spec: MessageSpec; initial?: (unknown | null); + /** + * What writes it. A run input is a parameter a run passes in, a panel input is written by a dashboard or panel control. A hint only: run forms leave panel inputs out, the engine treats both the same. + */ + origin?: 'run' | 'panel'; }; +/** + * What writes it. A run input is a parameter a run passes in, a panel input is written by a dashboard or panel control. A hint only: run forms leave panel inputs out, the engine treats both the same. + */ +export type origin = 'run' | 'panel'; + /** * A message the flow starts with rather than computes. */ export type FlowInput_Output = { spec: MessageSpec; initial?: (unknown | null); + /** + * What writes it. A run input is a parameter a run passes in, a panel input is written by a dashboard or panel control. A hint only: run forms leave panel inputs out, the engine treats both the same. + */ + origin?: 'run' | 'panel'; }; /** diff --git a/frontend/src/components/Flow/FlowBoundary.tsx b/frontend/src/components/Flow/FlowBoundary.tsx index 5374db9..f647789 100644 --- a/frontend/src/components/Flow/FlowBoundary.tsx +++ b/frontend/src/components/Flow/FlowBoundary.tsx @@ -6,6 +6,7 @@ import { MessagesService } from "@/client" import { Button } from "@/components/ui/button" import { Checkbox } from "@/components/ui/checkbox" import { Input } from "@/components/ui/input" +import { Segmented } from "@/components/ui/segmented" import { Select, SelectContent, @@ -160,6 +161,12 @@ function usePublishInput() { }) } +/** What writes an input: a run passing it in, or a control on a panel. */ +const ORIGINS = [ + ["run", "run"], + ["panel", "panel"], +] as const + function InputRow({ flow, declared, @@ -231,34 +238,44 @@ function InputRow({
- {live && name ? ( -
- now - {current === undefined ? ( - — - ) : ( - - - - )} - - - - - Put this value into the flow now - -
- ) : null} +
+ + onChange({ ...declared, origin: origin as typeof declared.origin }) + } + /> + {live && name ? ( + <> + now + {current === undefined ? ( + — + ) : ( + + + + )} + + + + + Put this value into the flow now + + + ) : null} +
) } @@ -338,8 +355,9 @@ export function BoundarySections({
Inputs - Messages that arrive from outside, such as a dashboard control, a - run or the API, and the value the flow starts from. + Messages that arrive from outside, and the value the flow starts + from. A run passes a run input in, a dashboard or panel control + writes a panel input. +
+ ))} + + )} + + !open && setPending(null)} + > + + + Revoke {pending?.email}? + + They lose access through the portal and need a new invitation to + get it back. + + + + + + + + + + ) +} + +export default PortalMembers diff --git a/frontend/src/routes/_layout/admin.tsx b/frontend/src/routes/_layout/admin.tsx index 69c34b5..fedb3e0 100644 --- a/frontend/src/routes/_layout/admin.tsx +++ b/frontend/src/routes/_layout/admin.tsx @@ -6,6 +6,7 @@ import { type UserPublic, UsersService } from "@/client" import AddUser from "@/components/Admin/AddUser" import { columns, type UserTableData } from "@/components/Admin/columns" import OAuthClients from "@/components/Admin/OAuthClients" +import PortalMembers from "@/components/Admin/PortalMembers" import { DataTable } from "@/components/Common/DataTable" import PendingUsers from "@/components/Pending/PendingUsers" import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" @@ -77,6 +78,7 @@ function Admin() { + From ffa044724bf091f89bc9631f9a662354b7142dba Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 10:31:40 +0200 Subject: [PATCH 050/103] Take the portal's JWKS from the tunnel and verify by kid The welcome frame's keys replace the set pinned at enrolment, but only over wss:// or loopback, the rule enrolment applies to the portal address. Portal tokens are verified against the key their kid names, so the hub can publish a new key beside the old one. A welcome without keys, from an older hub, leaves the pinned set alone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/cloud/config.py | 22 +++++++++++------ backend/fluksio/cloud/connector.py | 24 +++++++++++++++++++ backend/fluksio/cloud/enroll.py | 6 ++--- backend/tests/test_cloud.py | 38 ++++++++++++++++++++++++++++++ docs/interface/portal.md | 5 ++-- 5 files changed, 83 insertions(+), 12 deletions(-) diff --git a/backend/fluksio/cloud/config.py b/backend/fluksio/cloud/config.py index e8abf47..34d0dce 100644 --- a/backend/fluksio/cloud/config.py +++ b/backend/fluksio/cloud/config.py @@ -6,10 +6,12 @@ flows repository. Deleting it is the local, unilateral way to sever the connection — the portal's tokens stop verifying immediately, whatever the portal still believes. -The portal's public keys are stored here rather than fetched: they are pinned -at enrolment, when a person was holding a claim code they had just read off the -portal's own screen. A hub whose DNS or TLS is later hijacked cannot re-key an -instance that already enrolled; it can only fail to verify. +The portal's public keys are stored here rather than fetched per request. They +are pinned at enrolment, when a person was holding a claim code they had just +read off the portal's own screen, and replaced by the set the hub sends each +time the tunnel attaches, so the hub can publish a new key beside the old one. +That makes the tunnel's TLS what a later key rests on: only a hub that answers +at the enrolled address, to this instance's own credential, can hand it one. """ from __future__ import annotations @@ -123,10 +125,16 @@ def decode_portal_token(token: str) -> dict[str, Any]: if config is None: raise InvalidTokenError("this instance is not enrolled with a portal") + # By `kid`, so the hub can publish the next key beside the current one and + # tokens signed with either verify while it rotates. + kid = jwt.get_unverified_header(token).get("kid") try: - key = jwt.PyJWKSet.from_dict(config.jwks).keys[0] - except (IndexError, jwt.PyJWKError) as exc: - raise InvalidTokenError(f"the pinned portal key is unusable: {exc}") from exc + keys = jwt.PyJWKSet.from_dict(config.jwks).keys + except (jwt.PyJWKError, jwt.PyJWKSetError) as exc: + raise InvalidTokenError(f"the pinned portal keys are unusable: {exc}") from exc + key = next((candidate for candidate in keys if candidate.key_id == kid), None) + if key is None: + raise InvalidTokenError("signed with a key the portal has not published") claims: dict[str, Any] = jwt.decode( token, diff --git a/backend/fluksio/cloud/connector.py b/backend/fluksio/cloud/connector.py index 2aa4ceb..10473c9 100644 --- a/backend/fluksio/cloud/connector.py +++ b/backend/fluksio/cloud/connector.py @@ -27,10 +27,12 @@ from datetime import timedelta from typing import Any import httpx +import jwt from fastapi import FastAPI import fluksio from fluksio.cloud import config as cloud_config +from fluksio.cloud.enroll import is_loopback logger = logging.getLogger(__name__) @@ -239,6 +241,7 @@ class CloudConnector: self._owner = welcome.get("owner") self._adopt_owner(config, self._owner) + self._adopt_jwks(welcome.get("jwks")) attached = True self._connected = True @@ -319,6 +322,27 @@ class CloudConnector: # link: everything else this connection does still works. logger.exception("Could not map the enrolling account to the portal owner") + @staticmethod + def _adopt_jwks(jwks: Any) -> None: + """Replace the pinned signing keys with the set the hub sends on attach. + + Taken only from a link that is encrypted or on this machine, the rule + enrolment applies to the portal address. Over cleartext anything on the + path could hand this instance a key of its own. + """ + config = cloud_config.load() + if config is None or not isinstance(jwks, dict) or jwks == config.jwks: + return + if not config.ws_url.startswith("wss://") and not is_loopback(config.ws_url): + return + try: + jwt.PyJWKSet.from_dict(jwks) + except jwt.PyJWTError as exc: + logger.warning("Ignoring unusable portal keys: %s", exc) + return + cloud_config.save(replace(config, jwks=jwks)) + logger.info("Took the portal's current signing keys") + @staticmethod def _forget_member(portal_sub: str, owner: str | None) -> None: """Delete the local user of somebody the portal no longer admits. diff --git a/backend/fluksio/cloud/enroll.py b/backend/fluksio/cloud/enroll.py index 468d1b3..5962b7e 100644 --- a/backend/fluksio/cloud/enroll.py +++ b/backend/fluksio/cloud/enroll.py @@ -38,7 +38,7 @@ class EnrollError(Exception): self.detail = detail -def _is_loopback(url: str) -> bool: +def is_loopback(url: str) -> bool: """Whether the address is this machine. Not a `*.local` name: mDNS answers it with whoever claims it on the LAN, @@ -62,7 +62,7 @@ def redeem_claim( # this address. Over plain http both are readable by anything on the path, # so refuse rather than enrol insecurely — bar a loopback portal, which is # how the stack is developed against itself. - if not base.startswith("https://") and not _is_loopback(base): + if not base.startswith("https://") and not is_loopback(base): raise EnrollError(422, "The portal address must be https:// (or this machine)") try: response = httpx.post( @@ -110,7 +110,7 @@ def enroll( token=data["instance_token"], issuer=data["issuer"], # Pinned here, at the one moment the claim code proves who we are - # talking to. Nothing refreshes this. + # talking to. The tunnel refreshes it on every attach after this. jwks=data["jwks"], local_user_id=str(user.id), enrolled_at=datetime.now(UTC).isoformat(), diff --git a/backend/tests/test_cloud.py b/backend/tests/test_cloud.py index 2e9384a..095479e 100644 --- a/backend/tests/test_cloud.py +++ b/backend/tests/test_cloud.py @@ -772,3 +772,41 @@ def test_admin_lists_and_revokes_portal_members( ) db.delete(arrived) db.commit() + + +def test_the_portal_keys_follow_the_tunnel( + enrolled: User, # noqa: ARG001 (fixture installs the enrolment) + portal_key: rsa.RSAPrivateKey, +) -> None: + """A rotated keypair arrives on attach, and either key verifies by kid.""" + import json + + from cryptography.hazmat.primitives import serialization + + from fluksio.cloud.connector import CloudConnector + + pinned = cloud_config.load() + assert pinned is not None + successor = rsa.generate_private_key(public_exponent=65537, key_size=2048) + jwk = json.loads(jwt.algorithms.RSAAlgorithm.to_jwk(successor.public_key())) + jwk.update({"use": "sig", "alg": "RS256", "kid": "next"}) + rotated = {"keys": [*jwks(portal_key)["keys"], jwk]} + claims = jwt.decode(portal_token(portal_key), options={"verify_signature": False}) + pem = successor.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + signed_by_next = jwt.encode(claims, pem, algorithm="RS256", headers={"kid": "next"}) + + # Over cleartext to another machine, a key is not worth taking. + CloudConnector._adopt_jwks(rotated) + assert cloud_config.load().jwks == pinned.jwks + with pytest.raises(jwt.exceptions.InvalidTokenError): + decode_token(signed_by_next) + + cloud_config.save(replace(pinned, ws_url="wss://hub.example.test/attach")) + CloudConnector._adopt_jwks({"keys": []}) + CloudConnector._adopt_jwks(rotated) + assert decode_token(portal_token(portal_key))["portal_sub"] == "portal-user-1" + assert decode_token(signed_by_next)["portal_sub"] == "portal-user-1" diff --git a/docs/interface/portal.md b/docs/interface/portal.md index b4896c0..67e4c58 100644 --- a/docs/interface/portal.md +++ b/docs/interface/portal.md @@ -76,8 +76,9 @@ status reads **Removed at the portal** and the instance stops dialling. Connect with a new code to restore remote access. Local sign-in and flows continue after disconnecting. Portal requests follow -the instance's normal account permissions, and the instance verifies portal -tokens against the public signing key pinned during enrollment. +the instance's normal account permissions. The instance verifies portal tokens +against the portal's signing keys, which it pins during enrollment and takes +again from the portal each time it connects. ## Custom portals From b4d7ffb9826d9f756ce4e8642e1feaad535df91a Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 10:32:33 +0200 Subject: [PATCH 051/103] Run child tables cascade from run.id Migration ccb21eacf5d5 deletes rows whose run is gone, then adds foreign keys with ON DELETE CASCADE from run_node, run_metric and run_artifact to run.id. Deleting runs is now the one statement. A node or metric written after its run was deleted is dropped with an info line. Migrations run with foreign keys off, since a batch rebuild of run would otherwise cascade into every child row. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- .../ccb21eacf5d5_run_child_foreign_keys.py | 38 ++++++++++++++++++ backend/fluksio/api/routes/runs.py | 11 ++--- backend/fluksio/core/db.py | 40 +++++++++++++------ backend/fluksio/flow/runs.py | 7 ++++ backend/fluksio/models.py | 12 ++++-- backend/tests/api/routes/test_runs.py | 21 +++++++++- 6 files changed, 104 insertions(+), 25 deletions(-) create mode 100644 backend/fluksio/alembic/versions/ccb21eacf5d5_run_child_foreign_keys.py diff --git a/backend/fluksio/alembic/versions/ccb21eacf5d5_run_child_foreign_keys.py b/backend/fluksio/alembic/versions/ccb21eacf5d5_run_child_foreign_keys.py new file mode 100644 index 0000000..c7e15c6 --- /dev/null +++ b/backend/fluksio/alembic/versions/ccb21eacf5d5_run_child_foreign_keys.py @@ -0,0 +1,38 @@ +"""run_node, run_metric, run_artifact → run.id + +A foreign key from each run child table to its run, cascading on delete, so +deleting a run takes its rows with it and nothing can write one for a run that +is not there. + +Revision ID: ccb21eacf5d5 +Revises: 2e7ca7d10044 +Create Date: 2026-10-15 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "ccb21eacf5d5" +down_revision = "2e7ca7d10044" +branch_labels = None +depends_on = None + +TABLES = ("run_node", "run_metric", "run_artifact") + + +def upgrade(): + for table in TABLES: + # Rows a late write left behind after their run was deleted. Nothing + # reads them, and the constraint would refuse them. + op.execute(f"DELETE FROM {table} WHERE run_id NOT IN (SELECT id FROM run)") + with op.batch_alter_table(table) as batch_op: + batch_op.create_foreign_key( + f"fk_{table}_run_id_run", "run", ["run_id"], ["id"], ondelete="CASCADE" + ) + + +def downgrade(): + for table in TABLES: + with op.batch_alter_table(table) as batch_op: + batch_op.drop_constraint(f"fk_{table}_run_id_run", type_="foreignkey") diff --git a/backend/fluksio/api/routes/runs.py b/backend/fluksio/api/routes/runs.py index 64479c9..b843e0f 100644 --- a/backend/fluksio/api/routes/runs.py +++ b/backend/fluksio/api/routes/runs.py @@ -851,10 +851,10 @@ async def retry_run(run_id: str, request: Request, user: CurrentUser) -> Any: def _drop_runs(session: Session, chosen: Any) -> None: """A selection of runs and everything hanging off them. - The run tables carry a plain string ``run_id`` and no foreign key, so - nothing cascades on its own and these four statements are the whole of it. - ``flow_run``, ``metric_minute`` and ``engine_event`` stay — they are the - observability record and are pruned on their own window. + The run tables' foreign keys cascade, so deleting the runs takes their + nodes, metrics and artifact rows with them. ``flow_run``, ``metric_minute`` + and ``engine_event`` stay — they are the observability record and are + pruned on their own window. Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage cache, so a later run loses hits these would have served. And a node @@ -864,9 +864,6 @@ def _drop_runs(session: Session, chosen: Any) -> None: keeps whatever a ``run_artifact`` row or a live message still names, so dropping the rows is enough and the hourly sweep reclaims the blobs. """ - session.execute(delete(RunNode).where(col(RunNode.run_id).in_(chosen))) - session.execute(delete(RunMetric).where(col(RunMetric.run_id).in_(chosen))) - session.execute(delete(RunArtifact).where(col(RunArtifact.run_id).in_(chosen))) session.execute(delete(Run).where(col(Run.id).in_(chosen))) diff --git a/backend/fluksio/core/db.py b/backend/fluksio/core/db.py index 0b2d91a..833f8ba 100644 --- a/backend/fluksio/core/db.py +++ b/backend/fluksio/core/db.py @@ -66,8 +66,9 @@ def _sqlite_pragmas(dbapi_connection: Any, _record: Any) -> None: # Commit without waiting for the platter. A power cut can lose the last # transactions; it cannot corrupt the file. cursor.execute("PRAGMA synchronous=NORMAL") - # Off by default in SQLite, and `ondelete="CASCADE"` on the OAuth tables is - # the whole of revoking a client. + # Off by default in SQLite, and `ondelete="CASCADE"` is the whole of + # revoking an OAuth client and of deleting a run's nodes, metrics and + # artifacts with it. cursor.execute("PRAGMA foreign_keys=ON") # A second writer waits rather than raising "database is locked". cursor.execute("PRAGMA busy_timeout=30000") @@ -109,17 +110,30 @@ def _alembic_config(connection: Any) -> Config: def migrate(engine: Engine) -> None: """Bring the schema up to date, creating it if there is nothing there.""" - with engine.begin() as connection: - config = _alembic_config(connection) - tables = inspect(connection).get_table_names() - if not tables: - # Nothing to upgrade from. Building the schema from the models is - # both faster and exactly what the revisions would have produced. - SQLModel.metadata.create_all(connection) - command.stamp(config, "head") - logger.info("created the database schema") - return - command.upgrade(config, "head") + with engine.connect() as connection: + # Foreign keys off while the schema changes. A batch migration rebuilds + # a table by dropping it, and dropping `run` with them on cascades into + # every run's nodes, metrics and artifacts. SQLite ignores the pragma + # inside a transaction, so it is set before the migration's opens and + # reset after it closes. + connection.exec_driver_sql("PRAGMA foreign_keys=OFF") + connection.commit() + try: + with connection.begin(): + config = _alembic_config(connection) + tables = inspect(connection).get_table_names() + if not tables: + # Nothing to upgrade from. Building the schema from the + # models is both faster and exactly what the revisions + # would have produced. + SQLModel.metadata.create_all(connection) + command.stamp(config, "head") + logger.info("created the database schema") + return + command.upgrade(config, "head") + finally: + connection.exec_driver_sql("PRAGMA foreign_keys=ON") + connection.commit() def init_db(session: Session) -> None: diff --git a/backend/fluksio/flow/runs.py b/backend/fluksio/flow/runs.py index 9ac0ff2..5ee58c3 100644 --- a/backend/fluksio/flow/runs.py +++ b/backend/fluksio/flow/runs.py @@ -585,6 +585,9 @@ class MetricSink: ) ) session.commit() + except IntegrityError: + # Deleted while it was still reporting, as `_record_node` explains. + logger.info("Run %s is gone; not writing its metrics", self.run_id) except Exception: logger.exception( "Could not write %d metric(s) of %s", len(rows), self.run_id @@ -1568,6 +1571,10 @@ class RunService: ) ) session.commit() + except IntegrityError: + # The run was deleted while this node was still going, say once it + # had been swept as abandoned. Its rows have no run to belong to. + logger.info("Run %s is gone; not recording node '%s'", run_id, outcome.node) except Exception: logger.exception( "Could not record node '%s' of run %s", outcome.node, run_id diff --git a/backend/fluksio/models.py b/backend/fluksio/models.py index 6665392..ff6902e 100644 --- a/backend/fluksio/models.py +++ b/backend/fluksio/models.py @@ -432,7 +432,9 @@ class RunNode(SQLModel, table=True): __tablename__ = "run_node" - run_id: str = Field(primary_key=True, max_length=64) + run_id: str = Field( + primary_key=True, max_length=64, foreign_key="run.id", ondelete="CASCADE" + ) node: str = Field(primary_key=True, max_length=255) #: ok, error, skipped, cached or cancelled. status: str = Field(default="ok", max_length=16) @@ -470,7 +472,9 @@ class RunMetric(SQLModel, table=True): __tablename__ = "run_metric" - run_id: str = Field(primary_key=True, max_length=64) + run_id: str = Field( + primary_key=True, max_length=64, foreign_key="run.id", ondelete="CASCADE" + ) name: str = Field(primary_key=True, max_length=128) #: -1 for a value with no step of its own — a final score. step: int = Field(primary_key=True) @@ -484,7 +488,9 @@ class RunArtifact(SQLModel, table=True): __tablename__ = "run_artifact" - run_id: str = Field(primary_key=True, max_length=64) + run_id: str = Field( + primary_key=True, max_length=64, foreign_key="run.id", ondelete="CASCADE" + ) #: The message the bytes left the node on, which is what addresses them. name: str = Field(primary_key=True, max_length=255) #: What the node called the file, when it said. Kept because a reference diff --git a/backend/tests/api/routes/test_runs.py b/backend/tests/api/routes/test_runs.py index 902d87c..dfc8764 100644 --- a/backend/tests/api/routes/test_runs.py +++ b/backend/tests/api/routes/test_runs.py @@ -7,6 +7,7 @@ The pipeline half — what a hit restores and what a key is made of — is in import base64 import csv import json +import logging import threading from datetime import UTC, datetime, timedelta @@ -41,6 +42,10 @@ def test_a_run_cache_finds_what_an_earlier_run_recorded(tmp_path): reference = store.put([b"payload"], name="data.bin") plain, with_artifact, collected = "k-plain", "k-artifact", "k-collected" with Session(db_engine) as session: + for run_id in ("cache-1", "cache-2", "cache-3"): + session.add( + Run(id=run_id, flow="study", status="ok", created_at=datetime.now(UTC)) + ) session.add( RunNode( run_id="cache-1", @@ -891,7 +896,7 @@ def test_overview_is_not_read_as_a_run_id(client, superuser_token_headers): # ----------------------------------------------------------------------------- # Deleting a run # -# The route owns the four statements; what these guard is that it takes the +# The foreign keys cascade; what these guard is that deleting a run takes the # children with it and refuses a run the driver is still writing to. # ----------------------------------------------------------------------------- @@ -928,7 +933,7 @@ def deletable_run(): def test_deleting_a_run_takes_its_children_with_it( client, superuser_token_headers, deletable_run ): - """No foreign key cascades here, so the route has to do it itself.""" + """The foreign keys cascade, which needs them on for every connection.""" answer = client.delete( f"{settings.API_V1_STR}/runs/{deletable_run}", headers=superuser_token_headers ) @@ -943,6 +948,18 @@ def test_deleting_a_run_takes_its_children_with_it( assert left == [], f"{table.__name__} rows outlived the run" +def test_a_node_finishing_after_its_run_was_deleted_is_dropped_quietly(caplog): + """The constraint refuses its row, and that is a no-op rather than an error.""" + service = RunService(controller=_OneFlow(FlowDef(name="gone")), queue=_Collect()) + with caplog.at_level(logging.INFO, logger="fluksio.flow.runs"): + service._record_node("del-gone", NodeOutcome(node="gone.a", ok=True)) + + with Session(db_engine) as session: + assert session.get(RunNode, {"run_id": "del-gone", "node": "gone.a"}) is None + assert "is gone" in caplog.text + assert not [record for record in caplog.records if record.levelno >= logging.ERROR] + + def test_deleting_a_run_that_is_not_there_is_a_404(client, superuser_token_headers): answer = client.delete( f"{settings.API_V1_STR}/runs/nope-1", headers=superuser_token_headers From 3c9aec0982558da98e1390ee589300c57c299c03 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 10:48:59 +0200 Subject: [PATCH 052/103] DELETE /artifacts/{digest}: superuser only, 409 while referenced The keep-set the hourly sweep uses is now referenced_artifacts, shared by the sweep and the route. The volatile ring is not touched. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/artifacts.py | 40 +++++++++++++++++++++++-- backend/fluksio/flow/artifacts.py | 13 ++++++++ backend/fluksio/flow/runs.py | 36 ++++++++++++++-------- backend/tests/test_artifacts.py | 30 +++++++++++++++++++ docs/code/api.md | 5 ++++ frontend/src/client/sdk.gen.ts | 29 +++++++++++++++++- frontend/src/client/types.gen.ts | 6 ++++ 7 files changed, 144 insertions(+), 15 deletions(-) diff --git a/backend/fluksio/api/routes/artifacts.py b/backend/fluksio/api/routes/artifacts.py index 89fb0b6..45573af 100644 --- a/backend/fluksio/api/routes/artifacts.py +++ b/backend/fluksio/api/routes/artifacts.py @@ -10,17 +10,23 @@ import tempfile from pathlib import Path from typing import Any -from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response from fastapi.responses import FileResponse from jwt.exceptions import InvalidTokenError from pydantic import BaseModel from sqlmodel import Session from starlette.concurrency import run_in_threadpool -from fluksio.api.deps import user_from_token, worker_claims +from fluksio.api.deps import ( + FlowControllerDep, + get_current_active_superuser, + user_from_token, + worker_claims, +) from fluksio.core.config import settings from fluksio.core.db import engine from fluksio.flow.artifacts import ArtifactStore +from fluksio.flow.runs import referenced_artifacts #: What may be echoed back as a response content type. The caller holds the #: reference and passes its media type, so this guards a header rather than @@ -153,3 +159,33 @@ def get_artifact( headers["Content-Security-Policy"] = "sandbox" headers["X-Content-Type-Options"] = "nosniff" return FileResponse(path, media_type=media_type, headers=headers) + + +@router.delete( + "/{digest}", + status_code=204, + dependencies=[Depends(get_current_active_superuser)], +) +def delete_artifact( + digest: str, request: Request, controller: FlowControllerDep +) -> Response: + """Remove one artifact's bytes from the store. + + A superuser's call, because the store records no uploader and identical + content from two callers is one file, so nobody else can be said to own + it. Refused while anything still names the digest, the same references the + hourly sweep keeps, so a reference that exists never answers 404. The + volatile ring is left alone: its frames fall out on their own. + """ + store = _store(request) + if digest in referenced_artifacts(controller.state, controller.store): + raise HTTPException( + status_code=409, + detail=( + "The artifact is still referenced by a run, a current message or " + "a flow input's initial value" + ), + ) + if not store.delete(digest): + raise HTTPException(status_code=404, detail="No such artifact") + return Response(status_code=204) diff --git a/backend/fluksio/flow/artifacts.py b/backend/fluksio/flow/artifacts.py index a1d5ad5..6515be3 100644 --- a/backend/fluksio/flow/artifacts.py +++ b/backend/fluksio/flow/artifacts.py @@ -160,6 +160,19 @@ class ArtifactStore: self.put_file(source) return True + def delete(self, digest: str) -> bool: + """Remove one artifact's bytes. Returns whether this store had them. + + The ring is not looked in: its frames fall out on their own. + """ + if not valid_digest(digest): + return False + try: + self._path(digest).unlink() + except FileNotFoundError: + return False + return True + def read(self, digest: str) -> Iterator[bytes]: target = self.path(digest) if target is None: diff --git a/backend/fluksio/flow/runs.py b/backend/fluksio/flow/runs.py index 5ee58c3..43c0b83 100644 --- a/backend/fluksio/flow/runs.py +++ b/backend/fluksio/flow/runs.py @@ -382,13 +382,12 @@ def sweep_artifacts( ) -> int: """Remove artifact bytes nothing refers to any more. Returns how many went. - Three things refer to an artifact: a run that recorded it, a message - currently holding it, and a flow input's ``initial``, published or draft, - which a flow that has never started holds nowhere else. The second is what - makes a stream of media affordable — a camera publishing a frame a second - replaces the reference each time, so yesterday's frames are unreferenced by - definition and the store does not grow without bound. What a node *returns* - is recorded against its run and kept, what it emits along the way is not. + What still refers to one is `referenced_artifacts`. A message holding one is + what makes a stream of media affordable — a camera publishing a frame a + second replaces the reference each time, so yesterday's frames are + unreferenced by definition and the store does not grow without bound. What + a node *returns* is recorded against its run and kept, what it emits along + the way is not. Skipped entirely while anything is running: a node that stores a checkpoint an hour before it returns has neither a row nor a message naming it yet, @@ -401,6 +400,23 @@ def sweep_artifacts( if active is not None: logger.debug("Artifact sweep skipped: run %s is in flight", active) return 0 + + removed = store.collect(referenced_artifacts(state, flows), grace_s) + if removed: + logger.info("Artifact sweep removed %d unreferenced artifacts", removed) + return removed + + +def referenced_artifacts( + state: StateBackend, flows: FlowStore | None = None +) -> set[str]: + """Every artifact digest something still refers to. + + Three things do: a run that recorded it, a message currently holding it, + and a flow input's ``initial``, published or draft, which a flow that has + never started holds nowhere else. + """ + with Session(db_engine) as session: keep = set(session.exec(select(col(RunArtifact.digest)).distinct()).all()) if flows is not None: @@ -417,11 +433,7 @@ def sweep_artifacts( for start in range(0, len(keys), _SWEEP_BATCH): for value in state.get_multi(keys[start : start + _SWEEP_BATCH]).values(): _references_in(value, keep) - - removed = store.collect(keep, grace_s) - if removed: - logger.info("Artifact sweep removed %d unreferenced artifacts", removed) - return removed + return keep def seed_values( diff --git a/backend/tests/test_artifacts.py b/backend/tests/test_artifacts.py index cbe145c..c23a27d 100644 --- a/backend/tests/test_artifacts.py +++ b/backend/tests/test_artifacts.py @@ -230,3 +230,33 @@ def test_a_type_that_can_run_script_is_served_sandboxed( assert headers["content-security-policy"] == "sandbox" assert headers["x-content-type-options"] == "nosniff" assert "content-security-policy" not in served(image, "image/png") + + +def test_a_superuser_deletes_an_artifact_nothing_references( + client: TestClient, + superuser_token_headers: dict[str, str], + normal_user_token_headers: dict[str, str], +) -> None: + url = f"{settings.API_V1_STR}/artifacts" + digest = client.put( + f"{url}?name=x.bin", headers=superuser_token_headers, content=b"delete me" + ).json()["digest"] + + refused = client.delete(f"{url}/{digest}", headers=normal_user_token_headers) + assert refused.status_code == 403 + + run_id = _run_row("ok", digest) + try: + held = client.delete(f"{url}/{digest}", headers=superuser_token_headers) + assert held.status_code == 409 + finally: + _forget(run_id) + + gone = client.delete(f"{url}/{digest}", headers=superuser_token_headers) + assert gone.status_code == 204 + assert ( + client.get(f"{url}/{digest}", headers=superuser_token_headers).status_code + == 404 + ) + again = client.delete(f"{url}/{digest}", headers=superuser_token_headers) + assert again.status_code == 404 diff --git a/docs/code/api.md b/docs/code/api.md index d8f6bd6..311bf3f 100644 --- a/docs/code/api.md +++ b/docs/code/api.md @@ -172,6 +172,11 @@ curl "$FLUKSIO/artifacts/" \ The upload returns a reference with the content digest. Identical bytes are stored once. Node code uses `save_artifact` and `load_artifact` instead. +A superuser can remove stored bytes with `DELETE /artifacts/`. While +a run, a current message or a flow input's initial value still references the +digest, the call returns `409`. The hourly sweep removes unreferenced bytes +on its own, so this is for reclaiming space sooner. + ## Dashboards and panels | Method | Path | Action | diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index 8da7e17..3b0aab8 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { AiReadAiConfigResponse, AiUpdateAiConfigData, AiUpdateAiConfigResponse, AiDeleteAiConfigResponse, AiListAiModelsData, AiListAiModelsResponse, AiReadAiStatusResponse, AiReadControlNodesResponse, AiReadChatResponse, AiChatData, AiChatResponse, AiClearChatResponse, AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsReadWebpushKeyResponse, AlertsAddWebpushSubscriptionData, AlertsAddWebpushSubscriptionResponse, AlertsRemoveWebpushSubscriptionData, AlertsRemoveWebpushSubscriptionResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudReadMembersResponse, CloudRevokeMemberData, CloudRevokeMemberResponse, CloudDisconnectResponse, ConnectorsReadConnectorStateData, ConnectorsReadConnectorStateResponse, ConnectorsClearConnectorStateData, ConnectorsClearConnectorStateResponse, ConnectorsDeleteConnectorStateData, ConnectorsDeleteConnectorStateResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, FlowsIssueSocketTicketResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, ObservabilityFlushMetricsResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsImportRunsData, RunsImportRunsResponse, RunsReadRunsData, RunsReadRunsResponse, RunsDeleteRunsData, RunsDeleteRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadMetricNamesData, RunsReadMetricNamesResponse, RunsReadRunData, RunsReadRunResponse, RunsDeleteRunData, RunsDeleteRunResponse, RunsReadNodeOutputsData, RunsReadNodeOutputsResponse, RunsCancelRunData, RunsCancelRunResponse, RunsRetryRunData, RunsRetryRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SearchReadSearchIndexResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsReadConfigResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersReadTokensResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersRevokeTokenData, WorkersRevokeTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; +import type { AiReadAiConfigResponse, AiUpdateAiConfigData, AiUpdateAiConfigResponse, AiDeleteAiConfigResponse, AiListAiModelsData, AiListAiModelsResponse, AiReadAiStatusResponse, AiReadControlNodesResponse, AiReadChatResponse, AiChatData, AiChatResponse, AiClearChatResponse, AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsReadWebpushKeyResponse, AlertsAddWebpushSubscriptionData, AlertsAddWebpushSubscriptionResponse, AlertsRemoveWebpushSubscriptionData, AlertsRemoveWebpushSubscriptionResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, ArtifactsDeleteArtifactData, ArtifactsDeleteArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudReadMembersResponse, CloudRevokeMemberData, CloudRevokeMemberResponse, CloudDisconnectResponse, ConnectorsReadConnectorStateData, ConnectorsReadConnectorStateResponse, ConnectorsClearConnectorStateData, ConnectorsClearConnectorStateResponse, ConnectorsDeleteConnectorStateData, ConnectorsDeleteConnectorStateResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, FlowsIssueSocketTicketResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, ObservabilityFlushMetricsResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsImportRunsData, RunsImportRunsResponse, RunsReadRunsData, RunsReadRunsResponse, RunsDeleteRunsData, RunsDeleteRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadMetricNamesData, RunsReadMetricNamesResponse, RunsReadRunData, RunsReadRunResponse, RunsDeleteRunData, RunsDeleteRunResponse, RunsReadNodeOutputsData, RunsReadNodeOutputsResponse, RunsCancelRunData, RunsCancelRunResponse, RunsRetryRunData, RunsRetryRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SearchReadSearchIndexResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsReadConfigResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersReadTokensResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersRevokeTokenData, WorkersRevokeTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; export class AiService { /** @@ -320,6 +320,33 @@ export class ArtifactsService { } }); } + + /** + * Delete Artifact + * Remove one artifact's bytes from the store. + * + * A superuser's call, because the store records no uploader and identical + * content from two callers is one file, so nobody else can be said to own + * it. Refused while anything still names the digest, the same references the + * hourly sweep keeps, so a reference that exists never answers 404. The + * volatile ring is left alone: its frames fall out on their own. + * @param data The data for the request. + * @param data.digest + * @returns void Successful Response + * @throws ApiError + */ + public static deleteArtifact(data: ArtifactsDeleteArtifactData): CancelablePromise { + return __request(OpenAPI, { + method: 'DELETE', + url: '/api/v1/artifacts/{digest}', + path: { + digest: data.digest + }, + errors: { + 422: 'Validation Error' + } + }); + } } export class CloudService { diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 4cea20e..0e287dc 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -1620,6 +1620,12 @@ export type ArtifactsGetArtifactData = { export type ArtifactsGetArtifactResponse = (unknown); +export type ArtifactsDeleteArtifactData = { + digest: string; +}; + +export type ArtifactsDeleteArtifactResponse = (void); + export type CloudReadStatusResponse = ({ [key: string]: unknown; }); From 6b0c4a430849d2ecf697e788719960eb385153c2 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 10:52:33 +0200 Subject: [PATCH 053/103] Flow delete offers to delete its runs, history included DELETE /runs?flow= removes every Run row of the flow but left its flow_run rows, so a deleted flow kept appearing in the history. history=true now drops those too, and the flow delete dialogs (editor and bulk) send it when "Also delete its runs" is ticked. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/flows.py | 2 +- backend/fluksio/api/routes/runs.py | 9 +++- backend/tests/api/routes/test_runs.py | 29 +++++++++++- docs/code/api.md | 2 +- docs/concepts/runs.md | 8 ++-- frontend/src/client/sdk.gen.ts | 8 +++- frontend/src/client/types.gen.ts | 1 + .../src/components/Common/OverviewToolbar.tsx | 4 ++ frontend/src/components/Flow/FlowEditor.tsx | 11 ++++- frontend/src/components/Flow/FlowPanel.tsx | 46 +++++++++++++++++-- frontend/src/routes/_layout/flows/index.tsx | 22 +++++++-- 11 files changed, 124 insertions(+), 18 deletions(-) diff --git a/backend/fluksio/api/routes/flows.py b/backend/fluksio/api/routes/flows.py index 756233b..2e8be16 100644 --- a/backend/fluksio/api/routes/flows.py +++ b/backend/fluksio/api/routes/flows.py @@ -556,7 +556,7 @@ async def delete_flow( # Its files are gone; its values and queued work would otherwise linger. # Its runs stay: a run record is a record of what ran, and surviving the # flow it belonged to is the point of keeping one. `DELETE /runs?flow=` - # is what clears them. + # is what clears them, and with `history=true` its cascades too. await run_in_threadpool(controller.forget_flow, name) await controller.reload_flow(name) return Message(message=f"Deleted flow '{name}'") diff --git a/backend/fluksio/api/routes/runs.py b/backend/fluksio/api/routes/runs.py index b843e0f..c895e9b 100644 --- a/backend/fluksio/api/routes/runs.py +++ b/backend/fluksio/api/routes/runs.py @@ -29,7 +29,7 @@ from fluksio.flow.events import event_bus from fluksio.flow.messages import requalify from fluksio.flow.runs import OUTPUT_CAP, RunRejected, RunService, new_run_id from fluksio.flow.store import FlowNotFound -from fluksio.models import Message, Run, RunArtifact, RunMetric, RunNode +from fluksio.models import FlowRun, Message, Run, RunArtifact, RunMetric, RunNode router = APIRouter( prefix="/runs", tags=["runs"], dependencies=[Depends(get_current_user)] @@ -881,6 +881,7 @@ def delete_runs( session: SessionDep, user: CurrentUser, flow: Annotated[str, Query(min_length=1)], + history: bool = False, ) -> Any: """Every run of one flow at once. @@ -888,6 +889,10 @@ def delete_runs( deleting a flow leaves its runs standing on purpose, so this is the only way to clear a history without a call per run. ``flow`` is required — there is no "delete every run of everything" here, by design. + + ``history`` also drops the flow's rows in the run history (``flow_run``), + its cascades included. That is for a flow being deleted, which would + otherwise stay in the history until the retention window passes. """ with writing(session): live = _live_run(session, flow) @@ -906,6 +911,8 @@ def delete_runs( select(func.count()).select_from(Run).where(col(Run.flow) == flow) ).one() _drop_runs(session, chosen) + if history: + session.execute(delete(FlowRun).where(col(FlowRun.flow) == flow)) session.commit() event_bus.publish( { diff --git a/backend/tests/api/routes/test_runs.py b/backend/tests/api/routes/test_runs.py index dfc8764..3f512ff 100644 --- a/backend/tests/api/routes/test_runs.py +++ b/backend/tests/api/routes/test_runs.py @@ -33,7 +33,7 @@ from fluksio.flow.runs import ( ) from fluksio.flow.schemas import FlowDef, FlowInput, NodeDef from fluksio.flow.store import FlowStore -from fluksio.models import Run, RunArtifact, RunMetric, RunNode +from fluksio.models import FlowRun, Run, RunArtifact, RunMetric, RunNode def test_a_run_cache_finds_what_an_earlier_run_recorded(tmp_path): @@ -1041,6 +1041,33 @@ def test_deleting_a_flows_runs_clears_that_flow_only(client, superuser_token_hea session.commit() +def test_history_also_clears_a_flows_cascades(client, superuser_token_headers): + """What deleting a flow together with its runs asks for.""" + made = datetime.now(UTC) + url = f"{settings.API_V1_STR}/runs" + with Session(db_engine) as session: + session.add(Run(id="hist-1", flow="gone", status="ok", created_at=made)) + session.add(FlowRun(id="hist-1", flow="gone", source="run", started_at=made)) + session.add(FlowRun(id="hist-2", flow="gone", started_at=made)) + session.add(FlowRun(id="hist-3", flow="other", started_at=made)) + session.commit() + + plain = client.delete(url, params={"flow": "gone"}, headers=superuser_token_headers) + assert plain.status_code == 200 + with Session(db_engine) as session: + assert session.get(FlowRun, "hist-2") is not None + + answer = client.delete( + url, params={"flow": "gone", "history": True}, headers=superuser_token_headers + ) + assert answer.status_code == 200 + with Session(db_engine) as session: + left = session.exec(select(FlowRun).where(col(FlowRun.flow) == "gone")).all() + assert left == [] + session.delete(session.get(FlowRun, "hist-3")) + session.commit() + + # ----------------------------------------------------------------------------- # A cached node's curve # diff --git a/docs/code/api.md b/docs/code/api.md index 311bf3f..84f42cd 100644 --- a/docs/code/api.md +++ b/docs/code/api.md @@ -131,7 +131,7 @@ values. You can publish only to declared messages, just as dashboard controls do | `GET` | `/runs/{id}/nodes/{node}/outputs` | read what one node returned | | `POST` | `/runs/{id}/cancel`, `/runs/{id}/retry` | cancel, or create a retry | | `DELETE` | `/runs/{id}` | delete a finished run and its records | -| `DELETE` | `/runs?flow={name}` | delete a flow's runs; refused while any are active | +| `DELETE` | `/runs?flow={name}` | delete a flow's runs; refused while any are active. `history=true` also clears its cascades from the history | | `GET` | `/runs/{id}/metrics?name=&stride=` | read metric series | | `GET` | `/runs/metrics/names` | list metric names across a selection | | `GET` | `/runs/series/compare?ids=a,b,c&metric=&x=` | compare curves by step, time, or another metric | diff --git a/docs/concepts/runs.md b/docs/concepts/runs.md index 75125d2..ea77265 100644 --- a/docs/concepts/runs.md +++ b/docs/concepts/runs.md @@ -191,9 +191,11 @@ Select two or more runs to compare curves and the parameters that differ. Plot against step, elapsed seconds or another metric from the same runs. Comparisons have shareable URLs. -Deleting a flow keeps its run history. You can still inspect those runs, but -cannot retry them until the flow exists again. Delete history separately, -after active runs finish or are cancelled. +Deleting a flow keeps its run history, and the artifacts those runs recorded +stay on disk. You can still inspect those runs, but cannot retry them until +the flow exists again. Tick **Also delete its runs** in the delete dialog to +clear them with the flow, or delete history separately after active runs +finish or are cancelled. ### Taking it into a dataframe diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index 3b0aab8..293dd18 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -2343,8 +2343,13 @@ export class RunsService { * deleting a flow leaves its runs standing on purpose, so this is the only * way to clear a history without a call per run. ``flow`` is required — * there is no "delete every run of everything" here, by design. + * + * ``history`` also drops the flow's rows in the run history (``flow_run``), + * its cascades included. That is for a flow being deleted, which would + * otherwise stay in the history until the retention window passes. * @param data The data for the request. * @param data.flow + * @param data.history * @returns Message Successful Response * @throws ApiError */ @@ -2353,7 +2358,8 @@ export class RunsService { method: 'DELETE', url: '/api/v1/runs', query: { - flow: data.flow + flow: data.flow, + history: data.history }, errors: { 422: 'Validation Error' diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 0e287dc..acf17da 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -2125,6 +2125,7 @@ export type RunsReadRunsResponse = (Array); export type RunsDeleteRunsData = { flow: string; + history?: boolean; }; export type RunsDeleteRunsResponse = (Message); diff --git a/frontend/src/components/Common/OverviewToolbar.tsx b/frontend/src/components/Common/OverviewToolbar.tsx index b0c8d05..2512290 100644 --- a/frontend/src/components/Common/OverviewToolbar.tsx +++ b/frontend/src/components/Common/OverviewToolbar.tsx @@ -277,6 +277,7 @@ export function ConfirmDelete({ pending, description, onConfirm, + children, }: { open: boolean onOpenChange: (open: boolean) => void @@ -286,6 +287,8 @@ export function ConfirmDelete({ /** What is actually lost, when the git-backed answer below is not it. */ description?: string onConfirm: () => void + /** An option the delete takes, drawn between the question and the buttons. */ + children?: ReactNode }) { return ( @@ -307,6 +310,7 @@ export function ConfirmDelete({ )} + {children} +
+ {member.linkable && ( + + )} + +
))} @@ -120,6 +147,33 @@ export function PortalMembers() {
+ + !open && setLinking(null)} + > + + + Link {linking?.email}? + + Through the portal they open this instance as the local user with + this email, who keeps their password. Removing them at the portal, + or disconnecting it, then deletes that user. + + + + + + + + ) } From a6736bf484254b28003aa7a3a8dbbdb5c5d33d1b Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 11:09:54 +0200 Subject: [PATCH 061/103] Workers screen shows each agent's version and protocol, and who is behind The hello carries the fluksio-worker version. An agent refused for an older protocol stays listed as Needs upgrade until it attaches again. Docs gain an Upgrading workers section. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/workers.py | 32 +++++++++++++++++-- backend/tests/api/routes/test_workers.py | 21 +++++++++++++ docs/code/workers.md | 19 +++++++---- frontend/src/client/schemas.gen.ts | 15 +++++++++ frontend/src/client/types.gen.ts | 3 ++ frontend/src/routes/_layout/workers.tsx | 40 +++++++++++++++++++----- worker/fluksio_worker/agent.py | 10 ++++++ 7 files changed, 124 insertions(+), 16 deletions(-) diff --git a/backend/fluksio/api/routes/workers.py b/backend/fluksio/api/routes/workers.py index ca6ef74..2388ef9 100644 --- a/backend/fluksio/api/routes/workers.py +++ b/backend/fluksio/api/routes/workers.py @@ -10,6 +10,7 @@ from __future__ import annotations import asyncio import logging +import time import uuid from datetime import UTC, datetime, timedelta from typing import Any @@ -55,6 +56,11 @@ TOKEN_DAYS = 365 #: The sockets attached with each token, by `jti`, so revoking one drops them. _attached: dict[str, set[WebSocket]] = {} +#: Workers refused for an older protocol, by name, until one attaches again. +#: Such an agent exits on the refusal, so this is the one place left that says +#: which host still needs `pip install -U fluksio-worker`. +_behind: dict[str, WorkerInfo] = {} + class WorkerInfo(BaseModel): name: str @@ -68,6 +74,11 @@ class WorkerInfo(BaseModel): cpus: int = 1 gpus: int = 0 ram_mb: int | None = None + protocol: int = 0 + #: The fluksio-worker it runs; empty for an agent too old to say. + version: str = "" + #: Refused for speaking an older protocol than this engine, so not attached. + behind: bool = False class TokenRequest(BaseModel): @@ -109,9 +120,11 @@ def read_workers(request: Request) -> Any: cpus=worker.inventory.cpus, gpus=worker.inventory.gpus, ram_mb=worker.inventory.ram_mb, + protocol=int(worker.info.get("protocol") or 0), + version=str(worker.info.get("version") or ""), ) for worker in _hub(request.app).workers() - ] + ] + list(_behind.values()) class ResourceLevel(BaseModel): @@ -274,7 +287,19 @@ async def _attached_session(websocket: WebSocket, claims: dict[str, Any]) -> Non await websocket.close(code=1002) return - if hello.get("op") != "hello" or int(hello.get("protocol", 0)) != PROTOCOL: + protocol = int(hello.get("protocol", 0)) + if hello.get("op") != "hello" or protocol != PROTOCOL: + if hello.get("op") == "hello" and protocol < PROTOCOL: + name = str(claims.get("sub") or hello.get("name") or "worker") + _behind[name] = WorkerInfo( + name=name, + labels=sorted(str(label) for label in claims.get("labels") or []), + last_seen=time.time(), + python=str(hello.get("python") or ""), + protocol=protocol, + version=str(hello.get("version") or ""), + behind=True, + ) await websocket.send_json( {"op": "refused", "reason": f"this engine speaks protocol {PROTOCOL}"} ) @@ -308,10 +333,13 @@ async def _attached_session(websocket: WebSocket, claims: dict[str, Any]) -> Non info={ "python": hello.get("python"), "venv_digest": hello.get("venv_digest"), + "protocol": protocol, + "version": hello.get("version"), }, inventory=WorkerInventory.from_hello(hello.get("inventory")), ) hub.attach(worker) + _behind.pop(name, None) await websocket.send_json({"op": "welcome", "protocol": PROTOCOL, "name": name}) try: diff --git a/backend/tests/api/routes/test_workers.py b/backend/tests/api/routes/test_workers.py index 2a5b6bb..884e8a8 100644 --- a/backend/tests/api/routes/test_workers.py +++ b/backend/tests/api/routes/test_workers.py @@ -38,6 +38,27 @@ def test_the_token_names_the_worker_and_its_labels( assert [(w["name"], w["labels"]) for w in attached] == [("gpu-box", ["gpu"])] +def test_the_list_names_each_version_and_who_is_behind( + client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + issued = mint(client, superuser_token_headers) + url = f"{PREFIX}/attach?token={issued['token']}" + + def listed() -> list[tuple[object, ...]]: + answer = client.get(PREFIX, headers=superuser_token_headers).json() + return [(w["protocol"], w["version"], w["behind"]) for w in answer] + + with client.websocket_connect(url) as ws: + ws.send_json({**hello(), "protocol": PROTOCOL - 1, "version": "0.1.5"}) + assert ws.receive_json()["op"] == "refused" + assert listed() == [(PROTOCOL - 1, "0.1.5", True)] + + with client.websocket_connect(url) as ws: + ws.send_json({**hello(), "version": "0.2.0"}) + ws.receive_json() + assert listed() == [(PROTOCOL, "0.2.0", False)] + + def test_a_worker_cannot_claim_a_label_its_token_lacks( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: diff --git a/docs/code/workers.md b/docs/code/workers.md index 810c501..b996bda 100644 --- a/docs/code/workers.md +++ b/docs/code/workers.md @@ -127,7 +127,7 @@ curl "$API_URL/workers" -H "Authorization: Bearer $TOKEN" curl "$API_URL/workers/resources" -H "Authorization: Bearer $TOKEN" ``` -## Revoke or upgrade +## Revoke a token List tokens and revoke one by its `jti`: @@ -138,12 +138,19 @@ curl -X POST "$API_URL/workers/tokens/$JTI/revoke" \ ``` Revocation also disconnects attached workers and blocks artifact access. -Replacing the OAuth signing key revokes all worker tokens. +Replacing the OAuth signing key revokes all worker tokens. Older tokens without +signed labels and a `jti` need to be replaced. -Upgrade the engine before its workers, then run `pip install -U fluksio-worker` -on each host. Engine and worker must speak the same protocol; a mismatch is -reported and refused. Older tokens without signed labels and a `jti` need to -be replaced. +## Upgrading workers + +Engine and worker must speak the same protocol. A worker on another protocol is +refused and exits, so a release that bumps the protocol needs +`pip install -U fluksio-worker` on every worker host. Upgrade the engine first, +then the workers. + +The **Workers** screen shows each worker's package version and protocol. A +worker refused for an older protocol stays listed as **Needs upgrade** until it +attaches again, or until the engine restarts. ## Slurm provisioning diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 3b7a8c7..badc07c 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -4677,6 +4677,21 @@ export const WorkerInfoSchema = { } ], title: 'Ram Mb' + }, + protocol: { + type: 'integer', + title: 'Protocol', + default: 0 + }, + version: { + type: 'string', + title: 'Version', + default: '' + }, + behind: { + type: 'boolean', + title: 'Behind', + default: false } }, type: 'object', diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 1734eb0..3f1300e 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -1534,6 +1534,9 @@ export type WorkerInfo = { cpus?: number; gpus?: number; ram_mb?: (number | null); + protocol?: number; + version?: string; + behind?: boolean; }; /** diff --git a/frontend/src/routes/_layout/workers.tsx b/frontend/src/routes/_layout/workers.tsx index 7a0ca8d..7abb438 100644 --- a/frontend/src/routes/_layout/workers.tsx +++ b/frontend/src/routes/_layout/workers.tsx @@ -9,6 +9,7 @@ import { FlavorsService, WorkersService, } from "@/client" +import { Badge } from "@/components/ui/badge" import { Button } from "@/components/ui/button" import { Dialog, @@ -172,15 +173,38 @@ function Attached() { ) : null} - - {worker.cpus} cpu - {worker.gpus ? ` · ${worker.gpus} gpu` : ""} - {worker.ram_mb ? ` · ${Math.round(worker.ram_mb / 1024)} GB` : ""}{" "} - · {worker.in_flight}/{worker.max_parallel} in flight - + {worker.behind ? ( + + Refused for an older protocol. Upgrade it with{" "} + pip install -U fluksio-worker + + ) : ( + + {worker.cpus} cpu + {worker.gpus ? ` · ${worker.gpus} gpu` : ""} + {worker.ram_mb + ? ` · ${Math.round(worker.ram_mb / 1024)} GB` + : ""}{" "} + · {worker.in_flight}/{worker.max_parallel} in flight + + )} - - seen {ago(worker.last_seen ?? 0)} + + {worker.behind ? ( + + Needs upgrade + + ) : null} + + {worker.version ? `v${worker.version} · ` : ""}protocol{" "} + {worker.protocol} + + + {worker.behind ? "refused" : "seen"} {ago(worker.last_seen ?? 0)} + ))} diff --git a/worker/fluksio_worker/agent.py b/worker/fluksio_worker/agent.py index 8c91ca5..3a428f2 100644 --- a/worker/fluksio_worker/agent.py +++ b/worker/fluksio_worker/agent.py @@ -28,6 +28,7 @@ import argparse import asyncio import contextlib import glob +import importlib.metadata import ipaddress import json import logging @@ -167,6 +168,7 @@ class Agent: { "op": "hello", "protocol": PROTOCOL, + "version": _version(), "name": self.args.name, "labels": self.args.labels, "python": self.args.python, @@ -421,6 +423,14 @@ def _remap_cuda(mask: str, requested: str) -> str: return ",".join(out) +def _version() -> str: + """The installed fluksio-worker, or empty when this file was copied over.""" + try: + return importlib.metadata.version("fluksio-worker") + except importlib.metadata.PackageNotFoundError: + return "" + + def _venv_digest(python: str) -> str: """What is installed here, so the engine can say when it has drifted.""" try: From 9586a9fbc3000dc9c10e4198507a100f97f0a95a Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 15:48:22 +0200 Subject: [PATCH 062/103] Bump pytest to 9 for PYSEC-2026-1845, crud user tests delete their superusers pytest 8+ collects files and directories together in name order, so crud/ now runs before test_cloud.py and its leftover superusers made adopting the owner ambiguous. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/pyproject.toml | 2 +- backend/tests/crud/test_user.py | 7 +++++++ uv.lock | 9 +++++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 8e26983..51c3733 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -118,7 +118,7 @@ fluksio-worker = { workspace = true } [dependency-groups] dev = [ - "pytest<8.0.0,>=7.4.3", + "pytest<10.0.0,>=9.0.3", "mypy<2.0.0,>=1.8.0", "ruff<1.0.0,>=0.2.2", "prek>=0.2.24,<1.0.0", diff --git a/backend/tests/crud/test_user.py b/backend/tests/crud/test_user.py index c292ebe..ba30679 100644 --- a/backend/tests/crud/test_user.py +++ b/backend/tests/crud/test_user.py @@ -56,6 +56,9 @@ def test_check_if_user_is_superuser(db: Session) -> None: user_in = UserCreate(email=email, password=password, is_superuser=True) user = crud.create_user(session=db, user_create=user_in) assert user.is_superuser is True + # A stray superuser makes adopting the owner ambiguous (tests/test_cloud.py). + db.delete(user) + db.commit() def test_check_if_user_is_superuser_normal_user(db: Session) -> None: @@ -75,6 +78,8 @@ def test_get_user(db: Session) -> None: assert user_2 assert user.email == user_2.email assert jsonable_encoder(user) == jsonable_encoder(user_2) + db.delete(user) + db.commit() def test_update_user(db: Session) -> None: @@ -91,6 +96,8 @@ def test_update_user(db: Session) -> None: assert user.email == user_2.email verified, _ = verify_password(new_password, user_2.hashed_password) assert verified + db.delete(user) + db.commit() def test_authenticate_user_with_bcrypt_upgrades_to_argon2(db: Session) -> None: diff --git a/uv.lock b/uv.lock index be2104a..d7b447e 100644 --- a/uv.lock +++ b/uv.lock @@ -1010,7 +1010,7 @@ dev = [ { name = "coverage", specifier = ">=7.4.3,<8.0.0" }, { name = "mypy", specifier = ">=1.8.0,<2.0.0" }, { name = "prek", specifier = ">=0.2.24,<1.0.0" }, - { name = "pytest", specifier = ">=7.4.3,<8.0.0" }, + { name = "pytest", specifier = ">=9.0.3,<10.0.0" }, { name = "ruff", specifier = ">=0.2.2,<1.0.0" }, ] @@ -2296,17 +2296,18 @@ crypto = [ [[package]] name = "pytest" -version = "7.4.4" +version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, + { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/80/1f/9d8e98e4133ffb16c90f3b405c43e38d3abb715bb5d7a63a5a684f7e46a3/pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280", size = 1357116, upload-time = "2023-12-31T12:00:18.035Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/51/ff/f6e8b8f39e08547faece4bd80f89d5a8de68a38b2d179cc1c4490ffa3286/pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8", size = 325287, upload-time = "2023-12-31T12:00:13.963Z" }, + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] [[package]] From cea442d125e05ba0275cd9f0acc730533bedfc38 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 15:54:47 +0200 Subject: [PATCH 063/103] Draw a round dashboard in the mosaic as its circle with a dot per face DashboardSummary carries the dashboard's shape, so the list no longer lays a round dashboard's faces out at grid places they never use. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/dashboards.py | 4 +++ backend/tests/flow/test_dashboards.py | 7 +++++ frontend/src/client/schemas.gen.ts | 6 ++++ frontend/src/client/types.gen.ts | 1 + .../src/components/Common/DashboardMosaic.tsx | 28 +++++++++++++++++++ 5 files changed, 46 insertions(+) diff --git a/backend/fluksio/flow/dashboards.py b/backend/fluksio/flow/dashboards.py index 01319a6..a1c34ba 100644 --- a/backend/fluksio/flow/dashboards.py +++ b/backend/fluksio/flow/dashboards.py @@ -598,6 +598,9 @@ class DashboardSummary(BaseModel): version: int = 1 #: The grid `footprint` is placed in — see ``DashboardDef.columns``. columns: int = 12 + #: A round dashboard's widgets are faces rather than tiles, so a list + #: draws them as a circle instead of at the grid places they carry. + shape: Literal["rect", "round"] = "rect" #: Every widget's shape, so a list can draw the dashboard's outline #: without a document read per tile. footprint: list[WidgetFootprint] = Field(default_factory=list) @@ -666,6 +669,7 @@ class DashboardStore: has_draft=defn.has_draft, version=defn.version, columns=defn.columns, + shape=defn.shape, footprint=defn.footprint, updated_at=self.updated_at(name), ) diff --git a/backend/tests/flow/test_dashboards.py b/backend/tests/flow/test_dashboards.py index 858b0da..f528dcc 100644 --- a/backend/tests/flow/test_dashboards.py +++ b/backend/tests/flow/test_dashboards.py @@ -415,6 +415,7 @@ def test_a_summary_carries_the_footprint_of_the_document(store: DashboardStore): (summary,) = store.list() assert summary.columns == saved.columns + assert summary.shape == "rect" assert [(f.type, f.x, f.y, f.w, f.h) for f in summary.footprint] == [ ("chart", 2, 1, 6, 4), ("gauge", 0, 5, 2, 2), @@ -422,6 +423,12 @@ def test_a_summary_carries_the_footprint_of_the_document(store: DashboardStore): ] +def test_a_summary_says_a_dashboard_is_round(store: DashboardStore): + store.write(DashboardDef(name="knob", shape="round")) + + assert store.list()[0].shape == "round" + + def test_a_summary_carries_when_the_working_copy_was_written(store: DashboardStore): published = store.write(default_dashboard("house")) document = store.root / "house" / "dashboard.json" diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index badc07c..e780ce1 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -828,6 +828,12 @@ export const DashboardSummarySchema = { title: 'Columns', default: 12 }, + shape: { + type: 'string', + enum: ['rect', 'round'], + title: 'Shape', + default: 'rect' + }, footprint: { items: { '$ref': '#/components/schemas/WidgetFootprint' diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 3f1300e..7d42afa 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -215,6 +215,7 @@ export type DashboardSummary = { has_draft?: boolean; version?: number; columns?: number; + shape?: 'rect' | 'round'; footprint?: Array; updated_at?: (string | null); }; diff --git a/frontend/src/components/Common/DashboardMosaic.tsx b/frontend/src/components/Common/DashboardMosaic.tsx index a3e5cb7..cd623eb 100644 --- a/frontend/src/components/Common/DashboardMosaic.tsx +++ b/frontend/src/components/Common/DashboardMosaic.tsx @@ -65,6 +65,9 @@ function blocksOf(footprint: WidgetFootprint[]): { * dashboard. The summary carries the shapes, so it costs no request either. */ function Footprint({ dashboard }: { dashboard: DashboardSummary }) { + if (dashboard.shape === "round") { + return + } const { blocks, rows } = blocksOf(dashboard.footprint ?? []) const columns = dashboard.columns || DEFAULT_COLUMNS // Before the editor could place things, every widget was written at 0,0; @@ -112,6 +115,31 @@ function Footprint({ dashboard }: { dashboard: DashboardSummary }) { ) } +/** + * A round dashboard shows one face at a time, so its grid places mean + * nothing: it is drawn as the circle, with a dot per face where a panel puts + * its page dots. + */ +function RoundFootprint({ faces }: { faces: WidgetFootprint[] }) { + return ( +
+
+
+ {faces.map((face, index) => ( + + ))} +
+
+
+ ) +} + /** One dashboard in the mosaic. */ function Tile({ dashboard, From 3df6f5fd8b66aed6b146d79af42f5f1ded56b927 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 15:55:00 +0200 Subject: [PATCH 064/103] Give a Glass Switch or Button face a stronger press in light A face has no card behind its disc, so the tile button's white wash all but vanished into a light ground. The look's root states the press's own wash, tinted with the primary and casting the pane's shadow. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- .../components/Dashboard/ui/glass/glass.css | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/frontend/src/components/Dashboard/ui/glass/glass.css b/frontend/src/components/Dashboard/ui/glass/glass.css index 8c23b93..ce34197 100644 --- a/frontend/src/components/Dashboard/ui/glass/glass.css +++ b/frontend/src/components/Dashboard/ui/glass/glass.css @@ -26,6 +26,9 @@ --gl-radius: 1.25rem; --gl-track: rgb(255 255 255 / 0.16); --gl-blob-opacity: 0.5; + /* A round face's press, which has no card behind it. See below. */ + --gl-press: var(--gl-bg); + --gl-press-shadow: var(--gl-inset); } [data-look="glass"].light { @@ -37,6 +40,8 @@ --gl-track: rgb(0 0 0 / 0.1); /* Diluted by the white it mixes into, so it takes more of it to show. */ --gl-blob-opacity: 0.6; + --gl-press: color-mix(in srgb, var(--primary) 14%, rgb(255 255 255 / 0.85)); + --gl-press-shadow: var(--gl-shadow), var(--gl-inset); } /* --- the ground ---------------------------------------------------------- */ @@ -238,6 +243,19 @@ pointer-events: none; } +/* + * A Switch's or a Button's face on a round dashboard is one large press with + * only the ground behind it. A tile's button has a card for contrast; this has + * none, so in light the white wash all but disappears into the ground. The + * look's root states the press's own wash, so a dashboard keeps its own when + * a round panel draws it beside one in another look or theme. + */ +.dui-face-press +> .gl-button:not([data-variant="filled"], [aria-pressed="true"]) { + background: var(--gl-press); + box-shadow: var(--gl-press-shadow); +} + .gl-switch { position: relative; display: inline-flex; From c5c12376fc96ba130319dc5cbbe60e5bb84bf7bd Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 15:55:02 +0200 Subject: [PATCH 065/103] Opt-in WITH_FFMPEG build arg installs Debian's ffmpeg in the backend image Off by default, since it adds about 400 MB. Documented with the clip recipe and in the configuration reference. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/Dockerfile | 10 ++++++++++ docker/compose.yml | 3 +++ docs/code/nodes.md | 7 ++++--- docs/reference/configuration.md | 6 ++++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 93f978e..290b454 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -26,6 +26,16 @@ ENV PATH="/app/.venv/bin:$PATH" # environment somebody already works in is the case that adopts instead. ENV NODE_VENV=managed +# ffmpeg for node code that transcodes, such as the clip recipe in +# docs/code/nodes.md. Off by default: Debian's package and what it pulls in add +# about 400 MB. Set WITH_FFMPEG=true in .env and rebuild to include it. +ARG WITH_FFMPEG=false +RUN if [ "$WITH_FFMPEG" = true ]; then \ + apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/*; \ + fi + # Install dependencies # Ref: https://docs.astral.sh/uv/guides/integration/docker/#intermediate-layers RUN --mount=type=cache,target=/root/.cache/uv \ diff --git a/docker/compose.yml b/docker/compose.yml index f78ed61..93eabfc 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -163,6 +163,9 @@ services: build: context: .. dockerfile: backend/Dockerfile + args: + # Opt-in: ffmpeg costs the image about 400 MB (backend/Dockerfile). + - WITH_FFMPEG=${WITH_FFMPEG:-false} labels: - autoheal=true - traefik.enable=true diff --git a/docs/code/nodes.md b/docs/code/nodes.md index 498a70e..cc86c81 100644 --- a/docs/code/nodes.md +++ b/docs/code/nodes.md @@ -150,9 +150,10 @@ def process(clip): ``` Use it only for clips headed for such a panel. Browsers play the originals, -and the result plays there too, just smaller. The Docker image does not ship -`ffmpeg` yet, so the node runs only where `ffmpeg` is on the `PATH`, such as a -[remote worker](workers.md). +and the result plays there too, just smaller. The node needs `ffmpeg` on the +`PATH`. The Docker image includes it only when built with +[`WITH_FFMPEG=true`](../reference/configuration.md#image-options) in `.env`. +Otherwise run the node on a [remote worker](workers.md) that has it. ## Printing diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index f135896..1f44773 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -100,6 +100,12 @@ to the backend. An empty value needs that proxy when the frontend and backend are served separately. See [facility automation](../getting-started/facility-automation.md#run-it-long-term) for deployment settings. +## Image options + +`WITH_FFMPEG=true` in `.env` installs `ffmpeg` into the backend image on the +next build, for node code that transcodes video. It is off by default because +it adds about 400 MB. + ## Environment | Variable | Default | Purpose | From e903b1ba1db899f292275408ef2c228f0c5306e5 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 15:56:14 +0200 Subject: [PATCH 066/103] Pin Traefik to 3.6.25 in the dev and production compose files A floating 3.6 tag left a host on an older cached image that refuses --accesslog.fields.queryparameters, so the proxy exited on start. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docker/compose.dev.yml | 4 +++- docker/compose.traefik.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/compose.dev.yml b/docker/compose.dev.yml index 0388de1..a482a9d 100644 --- a/docker/compose.dev.yml +++ b/docker/compose.dev.yml @@ -8,7 +8,9 @@ services: # on http://api.localhost / http://app.localhost via the routers in # compose.yml. Dashboard at http://localhost:8090. proxy: - image: traefik:3.6 + # A patch tag, so a host holding an older cached 3.6 pulls this one. 3.6.25 + # knows --accesslog.fields.queryparameters, which older 3.6 refuses. + image: traefik:3.6.25 volumes: - /var/run/docker.sock:/var/run/docker.sock ports: diff --git a/docker/compose.traefik.yml b/docker/compose.traefik.yml index 0353564..541c80b 100644 --- a/docker/compose.traefik.yml +++ b/docker/compose.traefik.yml @@ -1,6 +1,8 @@ services: traefik: - image: traefik:3.6 + # A patch tag, so a host holding an older cached 3.6 pulls this one. 3.6.25 + # knows --accesslog.fields.queryparameters, which older 3.6 refuses. + image: traefik:3.6.25 ports: # Listen on port 80, default for HTTP, necessary to redirect to HTTPS - 80:80 From e6ce3feabb99df7066b53d238b34e08f1501e380 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 15:56:59 +0200 Subject: [PATCH 067/103] Backend CI tests on Python 3.12 and 3.14, with the extras installed A matrix of the declared floor and the newest release in place of 3.13 alone. The job now syncs --all-extras like `make test-backend`, since the mail, MQTT and Influx tests fail on a fresh venv without the server extra. Both packages list 3.14 among their classifiers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- .gitea/workflows/test-backend.yml | 15 +++++++++++++-- backend/pyproject.toml | 1 + development.md | 2 +- worker/pyproject.toml | 1 + 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/test-backend.yml b/.gitea/workflows/test-backend.yml index 739251c..43d8ef1 100644 --- a/.gitea/workflows/test-backend.yml +++ b/.gitea/workflows/test-backend.yml @@ -27,18 +27,27 @@ env: jobs: test-backend: runs-on: ubuntu-latest + # The floor `requires-python` declares and the newest release, so neither + # end of the supported range goes untested. + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.14"] env: # Per run, so the `down -v` steps reach neither a dev stack nor a run # still going. Unique names cannot be recycled, hence the always-on # teardown below. COMPOSE_PROJECT_NAME: fluksio-app-ci-backend-${{ github.run_id }} + # uv would otherwise take a managed interpreter it already has over the + # one set up below. + UV_PYTHON: ${{ matrix.python-version }} steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: ${{ matrix.python-version }} - name: Set up uv uses: astral-sh/setup-uv@v7 with: @@ -46,7 +55,9 @@ jobs: pyproject.toml backend/pyproject.toml uv.lock - - run: uv sync + # The extras too, as `make test-backend` does: the mail, MQTT and Influx + # tests import what only the `server` extra installs. + - run: uv sync --all-extras # Nothing to start: the suite names its own SQLite file in a temp # directory (backend/tests/__init__.py), the mail paths are patched, and # the flow engine falls back to in-memory state without a Redis host. diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 51c3733..dfbe63f 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Home Automation", "Topic :: Scientific/Engineering", "Topic :: System :: Distributed Computing", diff --git a/development.md b/development.md index 6e2f00a..a43215a 100644 --- a/development.md +++ b/development.md @@ -173,7 +173,7 @@ contract between the two frontends, and that target is the only thing checking i ## CI `.gitea/workflows/` runs four jobs, each writing `.env` from `.env.example` first: -`test-backend.yml` (pytest, plus a check that both wheels build), `playwright.yml` (the e2e suite in +`test-backend.yml` (pytest on Python 3.12 and 3.14, plus a check that both wheels build), `playwright.yml` (the e2e suite in two shards, one compose project each, the browser loading the SPA from the nginx `frontend` service), `pre-commit.yml` (the hooks above, plus strict mypy, which they do not cover) and `test-compose.yml` (a smoke test that the production images actually come up). diff --git a/worker/pyproject.toml b/worker/pyproject.toml index 58850cf..fd344c6 100644 --- a/worker/pyproject.toml +++ b/worker/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Home Automation", "Topic :: Scientific/Engineering", "Topic :: System :: Distributed Computing", From 01afced0b8fe21764ec03fd886272da042b21849 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 15:57:46 +0200 Subject: [PATCH 068/103] Soak's redis scenario refuses to run without --force It stops the Redis the whole stack journals to, so every flow fails briefly, not just the soak fixtures. `all` includes it and needs the flag too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- Makefile | 2 +- backend/scripts/soak.py | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 45830f2..43058cd 100644 --- a/Makefile +++ b/Makefile @@ -204,7 +204,7 @@ test-frontend: ## Run frontend tests (Playwright e2e) against the local stack # Load and chaos against a *running* stack, never part of `make test`: it # restarts this stack's containers. `SOAK_ARGS="--dry-run"` only looks. -soak: ## Run the soak/chaos harness (SOAK_ARGS="--minutes 30 --scenario redis") +soak: ## Run the soak/chaos harness (SOAK_ARGS="--minutes 30 --scenario redis --force") cd backend && uv run python scripts/soak.py $(SOAK_ARGS) bench-startup: ## Time submitting a run (BENCH_ARGS="--kedro ../some/kedro/project") diff --git a/backend/scripts/soak.py b/backend/scripts/soak.py index 8ba85ac..609fca5 100644 --- a/backend/scripts/soak.py +++ b/backend/scripts/soak.py @@ -4,11 +4,14 @@ Standalone on purpose, and never part of the test run. It drives the running stack through the *durable* path — the webhook, which journals every trigger to the work queue — and then takes pieces of the stack away to see what survives. - make soak - make soak SOAK_ARGS="--minutes 30 --rps 25" - make soak SOAK_ARGS="--scenario redis" + make soak SOAK_ARGS="--force" + make soak SOAK_ARGS="--minutes 30 --rps 25 --scenario load" + make soak SOAK_ARGS="--scenario redis --force" make soak SOAK_ARGS="--dry-run" +The redis scenario, which `all` includes, needs `--force`: the Redis it stops +is the one the whole stack journals to, not a copy of its own. + Every docker verb goes through :func:`docker`, which refuses any container that is not part of this project's compose stack. The machine this runs on hosts unrelated services; nothing here may ever reach them. @@ -936,7 +939,18 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: action="store_true", help="Report what the harness can see and change nothing.", ) + parser.add_argument( + "--force", + action="store_true", + help="Allow the redis scenario, which stops the Redis the whole stack shares.", + ) args = parser.parse_args(argv) + if args.scenario in ("all", "redis") and not (args.force or args.dry_run): + parser.error( + f"the redis scenario stops {REDIS}, which the whole stack shares, so " + "every flow briefly fails to journal, not just the soak fixtures. " + "Pass --force on a stack nobody relies on, or pick another --scenario." + ) args.rps = max(1, min(MAX_RPS, args.rps)) args.seconds = max(10.0, args.minutes * 60) return args From c2adc2c526a89ebad69d9b3c8034bcc66dc3a56f Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 15:58:52 +0200 Subject: [PATCH 069/103] make dev and make up warn when .env lacks a key compose would default Only keys .env.example lists, so optional ones such as APP_PORT stay quiet. A missing TZ used to run every cron on UTC without a word. Never fails. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- Makefile | 13 ++++++++++++- docs/reference/configuration.md | 3 ++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 43058cd..be8cae8 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # Convenience targets for development, testing, linting, and deployment. # The workspace root delegates to these (see ../Makefile). -.PHONY: init dev-utils dev dev-local dev-lan rebuild-frontend up down update install dev-backend dev-frontend \ +.PHONY: init env-check dev-utils dev dev-local dev-lan rebuild-frontend up down update install dev-backend dev-frontend \ generate-client sync-example test test-backend test-checks test-frontend soak bench-startup lint lint-backend \ lint-frontend format-frontend umami build docs docs-serve clean help @@ -39,6 +39,17 @@ help: ## Show available targets # `make dev DOMAIN=fluksio.com` still wins. dev dev-utils dev-local dev-lan rebuild-frontend: export DOMAIN ?= localhost +# A key the template lists and the compose files default, missing from .env, +# falls back without a word. For TZ that means every cron runs on UTC. Only a +# warning, so a start never fails over it. +dev dev-local dev-lan up: env-check +env-check: + @missing=$$(grep -ohE '\$$\{[A-Z_]+:?-' docker/compose*.yml | sed 's/^..//; s/:\{0,1\}-$$//' | sort -u \ + | while read -r k; do \ + grep -q "^$$k=" .env.example && ! grep -q "^$$k=" .env && printf ' %s' "$$k"; \ + done); \ + [ -z "$$missing" ] || echo " ! .env lacks$$missing, so compose uses its defaults. See .env.example." + # First-time bootstrap. Idempotent: only secrets still set to `changethis` are # generated, so an existing .env keeps its values. init: ## First-time setup (.env + secrets + docker network) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 1f44773..47957e8 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -121,7 +121,8 @@ stack does. Production disables it entirely. Staging and production also refuse `changethis` values for the session key or administrator password. Set `TZ` in `.env` for schedules that should follow local time, such as -`Europe/Berlin`. +`Europe/Berlin`. `make up` and `make dev` warn when `.env` lacks it, or any other +key `.env.example` lists that Compose would otherwise default. ## Engine and resources From 4ae8535cbbb5f1f5aea6c9d3465ee556a79f6864 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:00:10 +0200 Subject: [PATCH 070/103] Draw each dashboard on a round panel's track in its own look The canvas states the look of the dashboard shown, so a neighbour beside it and the one a turn leaves wore that look too. Each page now restates its own look, theme and palette. The ring's paint moves from rules under [data-look] to variables each look sets on its root, which stop at the nearer root where a descendant rule reached into both. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/interface/dashboards.md | 2 +- .../src/components/Dashboard/RoundFace.tsx | 51 ++++++++++++++++--- .../src/components/Dashboard/ui/core/core.css | 43 +++++++++++++++- .../Dashboard/ui/fluksio/fluksio.css | 29 +++-------- .../components/Dashboard/ui/glass/glass.css | 40 ++++----------- .../Dashboard/ui/material/material.css | 28 ++-------- 6 files changed, 107 insertions(+), 86 deletions(-) diff --git a/docs/interface/dashboards.md b/docs/interface/dashboards.md index ffcf9f5..da58efe 100644 --- a/docs/interface/dashboards.md +++ b/docs/interface/dashboards.md @@ -330,7 +330,7 @@ dashboard's look's. | What | Moves | From → to | Timing | Input during it | |---|---|---|---|---| | Face paging (round) | the faces, side by side on a track | the track slides one face width S, the new face coming in from the side it lies on (the next from the right). A dot that skips faces brings its face in beside the one shown, so a turn is always one width | spring, from where the track is (a swipe's release speed carried on) | a new turn while one runs: the running one lands at once and the next starts | -| Dashboard paging (round panel) | the dashboard, and beside it the next one | slides S up (the next, from below) or down (the previous, from above), the next one drawn on the face it opens on; once landed, it is the dashboard shown, and its name pill shows for a moment. One opened by its address replaces the last at once | spring | further paging waits for the turn to land | +| Dashboard paging (round panel) | the dashboard, and beside it the next one | slides S up (the next, from below) or down (the previous, from above), the next one drawn on the face it opens on; once landed, it is the dashboard shown, and its name pill shows for a moment. Each one on the track wears its own look, theme and palette. The ground is the dashboard shown's and changes over as the turn begins, without a fade. One opened by its address replaces the last at once | spring | further paging waits for the turn to land | | Page dot | the current dot and the last | muted at 0.5 opacity ↔ text colour at 1 | 200 ms, cubic-bezier(0.4, 0, 0.2, 1) | retargets | | Dashboard name | the pill, on landing on another dashboard | opacity 0 → 1, held 1000 ms, 1 → 0 | in 200 ms, out 150 ms, cubic-bezier(0.4, 0, 0.2, 1) | — | | Ring value set by a notch, a key or the engine | the value arc's end, the handle (slider, selector, colour) | from where it is drawn to the new fraction of the sweep | spring | each notch retargets from where it is, at the speed it has | diff --git a/frontend/src/components/Dashboard/RoundFace.tsx b/frontend/src/components/Dashboard/RoundFace.tsx index fb9f84c..3832a60 100644 --- a/frontend/src/components/Dashboard/RoundFace.tsx +++ b/frontend/src/components/Dashboard/RoundFace.tsx @@ -15,6 +15,7 @@ import type { WidgetDef } from "@/client" import { slotsFor } from "@/components/Common/UplotChart" import { fadeIn } from "@/lib/motion" import { useShortcuts } from "@/lib/shortcuts" +import { cn } from "@/lib/utils" import { useClock } from "./ClockWidget" import type { Dashboard } from "./DashboardView" import { useBoundValue, useBoundValues } from "./dataContext" @@ -40,7 +41,7 @@ import { text, } from "./ui/core/config" import { TESTID } from "./ui/core/contract" -import { useLook } from "./ui/core/look" +import { LookProvider, useCanvasRoot, useLook } from "./ui/core/look" import { LOOK } from "./ui/core/motion" import { ANNOUNCE_MS, @@ -334,12 +335,7 @@ export function RoundPanel({ return ( // Keyed by name, so a neighbour becomes the one shown, and back, // without being drawn afresh: its own face memory and all. -
+ {editing ? ( view @@ -347,13 +343,52 @@ export function RoundPanel({ {view} )} -
+ ) })} ) } +/** + * One dashboard on a round panel's track, in its own look. + * + * The canvas states the look, theme and palette of the dashboard shown, so a + * neighbour beside it, or the one a turn is leaving, would wear those too. + * Each page restates its own instead. The ground stays the canvas's, which + * changes over as soon as the turn begins. + */ +function TrackPage({ + dashboard, + at, + size, + children, +}: { + dashboard: Dashboard + /** Its place on the track, in screens from the one shown. */ + at: number + size: number + children: React.ReactNode +}) { + // Touch belongs to the screen, so it is the canvas's whichever page this is. + const { touch } = useLook() + const root = useCanvasRoot(dashboard, touch) + return ( + +
+ {children} +
+
+ ) +} + /** A dashboard's icon and title: what a panel shows for a moment of a * dashboard it has just landed on. Opaque, so the face under it does not * read through the name. */ diff --git a/frontend/src/components/Dashboard/ui/core/core.css b/frontend/src/components/Dashboard/ui/core/core.css index 7303dbb..c49ae2e 100644 --- a/frontend/src/components/Dashboard/ui/core/core.css +++ b/frontend/src/components/Dashboard/ui/core/core.css @@ -425,7 +425,7 @@ ); --dui-reading: var(--foreground); --dui-reading-reached: var(--primary-foreground); - --dui-reading-on-handle: var(--foreground); + --dui-reading-on-handle: var(--dui-on-handle, var(--foreground)); } /* @@ -785,6 +785,47 @@ outline: none; } +/* + * The ring's paint. Each look states it on its root as these variables rather + * than in rules under `[data-look]`: a round panel draws a dashboard beside + * one in another look, and a descendant rule would reach into both, where an + * inherited value stops at the nearer root. Reset on every root, so a look + * that leaves one out takes the fallback rather than the outer root's. + */ +:where([data-look]) { + --dui-ring-track: initial; + --dui-ring-glow: initial; + --dui-ring-handle: initial; + --dui-ring-handle-edge: initial; + --dui-ring-handle-edge-width: initial; + --dui-ring-handle-shadow: initial; + --dui-ring-swatch: initial; + --dui-ring-swatch-width: initial; + --dui-on-handle: initial; +} + +.dui-ring-track { + stroke: var(--dui-ring-track, var(--muted)); +} + +.dui-ring-fill { + stroke: var(--primary); + filter: var(--dui-ring-glow, none); +} + +.dui-ring-handle { + fill: var(--dui-ring-handle, var(--primary)); + stroke: var(--dui-ring-handle-edge, none); + stroke-width: var(--dui-ring-handle-edge-width, 0); + filter: var(--dui-ring-handle-shadow, none); +} + +/* A colour's handle is filled with the colour, so only its edge is the look's. */ +.dui-ring-swatch { + stroke: var(--dui-ring-swatch, var(--card)); + stroke-width: var(--dui-ring-swatch-width, 3px); +} + .dui-ring-hit { pointer-events: stroke; cursor: pointer; diff --git a/frontend/src/components/Dashboard/ui/fluksio/fluksio.css b/frontend/src/components/Dashboard/ui/fluksio/fluksio.css index 150d871..f3a2795 100644 --- a/frontend/src/components/Dashboard/ui/fluksio/fluksio.css +++ b/frontend/src/components/Dashboard/ui/fluksio/fluksio.css @@ -21,6 +21,12 @@ [data-look="fluksio"] { --fx-radius: var(--radius-lg); --fx-field: var(--radius-md); + /* A round face's ring (`core.css`): the slider's handle, a ring of primary + around the page colour. */ + --dui-ring-handle: var(--background); + --dui-ring-handle-edge: var(--primary); + --dui-ring-handle-edge-width: 2px; + --dui-ring-handle-shadow: drop-shadow(0 1px 2px rgb(0 0 0 / 0.2)); } /* --- surfaces ------------------------------------------------------------ */ @@ -315,26 +321,3 @@ outline: none; box-shadow: 0 0 0 3px color-mix(in srgb, var(--ring) 50%, transparent); } - -/* --- a round dashboard's ring -------------------------------------------- */ - -[data-look="fluksio"] .dui-ring-track { - stroke: var(--muted); -} - -[data-look="fluksio"] .dui-ring-fill { - stroke: var(--primary); -} - -/* The slider's handle: a ring of primary around the page colour. */ -[data-look="fluksio"] .dui-ring-handle { - fill: var(--background); - stroke: var(--primary); - stroke-width: 2px; - filter: drop-shadow(0 1px 2px rgb(0 0 0 / 0.2)); -} - -[data-look="fluksio"] .dui-ring-swatch { - stroke: var(--card); - stroke-width: 3px; -} diff --git a/frontend/src/components/Dashboard/ui/glass/glass.css b/frontend/src/components/Dashboard/ui/glass/glass.css index ce34197..3b2f57a 100644 --- a/frontend/src/components/Dashboard/ui/glass/glass.css +++ b/frontend/src/components/Dashboard/ui/glass/glass.css @@ -29,6 +29,17 @@ /* A round face's press, which has no card behind it. See below. */ --gl-press: var(--gl-bg); --gl-press-shadow: var(--gl-inset); + /* A round face's ring (`core.css`): lit, as the gauge's arc is, under a + white handle, which a reading's dot on it shows dark against. */ + --dui-ring-track: var(--gl-track); + --dui-ring-glow: drop-shadow(0 0 4px var(--primary)); + --dui-ring-handle: #fff; + --dui-ring-handle-edge: var(--gl-border); + --dui-ring-handle-edge-width: 1px; + --dui-ring-handle-shadow: drop-shadow(0 2px 8px rgb(0 0 0 / 0.35)); + --dui-ring-swatch: rgb(255 255 255 / 0.85); + --dui-ring-swatch-width: 2px; + --dui-on-handle: rgb(0 0 0 / 0.75); } [data-look="glass"].light { @@ -433,32 +444,3 @@ outline: 2px solid var(--ring); outline-offset: 2px; } - -/* --- a round dashboard's ring -------------------------------------------- */ - -[data-look="glass"] .dui-ring-track { - stroke: var(--gl-track); -} - -/* Lit, as the gauge's arc is. */ -[data-look="glass"] .dui-ring-fill { - stroke: var(--primary); - filter: drop-shadow(0 0 4px var(--primary)); -} - -[data-look="glass"] .dui-ring-handle { - fill: #fff; - stroke: var(--gl-border); - stroke-width: 1px; - filter: drop-shadow(0 2px 8px rgb(0 0 0 / 0.35)); -} - -/* A reading's dot on the handle, which is white here in either theme. */ -[data-look="glass"] :is(.dui-ring, .dui-slider) { - --dui-reading-on-handle: rgb(0 0 0 / 0.75); -} - -[data-look="glass"] .dui-ring-swatch { - stroke: rgb(255 255 255 / 0.85); - stroke-width: 2px; -} diff --git a/frontend/src/components/Dashboard/ui/material/material.css b/frontend/src/components/Dashboard/ui/material/material.css index 22475bd..4ee9420 100644 --- a/frontend/src/components/Dashboard/ui/material/material.css +++ b/frontend/src/components/Dashboard/ui/material/material.css @@ -19,6 +19,10 @@ --m3-sc: color-mix(in oklab, var(--primary) 22%, var(--card)); --m3-outline: var(--border); --m3-radius: 1rem; + /* A round face's ring (`core.css`): a tonal track under a primary handle, + which a reading's dot on it shows light against. */ + --dui-ring-track: var(--m3-sc); + --dui-on-handle: var(--primary-foreground); } /* --- surfaces ----------------------------------------------------------- */ @@ -319,27 +323,3 @@ outline: 2px solid var(--primary); outline-offset: 2px; } - -/* --- a round dashboard's ring -------------------------------------------- */ - -[data-look="material"] .dui-ring-track { - stroke: var(--m3-sc); -} - -[data-look="material"] .dui-ring-fill { - stroke: var(--primary); -} - -[data-look="material"] .dui-ring-handle { - fill: var(--primary); -} - -/* A reading's dot on the handle, which is primary here. */ -[data-look="material"] :is(.dui-ring, .dui-slider) { - --dui-reading-on-handle: var(--primary-foreground); -} - -[data-look="material"] .dui-ring-swatch { - stroke: var(--card); - stroke-width: 3px; -} From 361b9d8e746c5b17e1a4baa7499373ab70629c5d Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:05:23 +0200 Subject: [PATCH 071/103] Grow the editor's canvas to reach widgets below its edge A widget that fell off the panel, after the canvas was made smaller, was drawn under the clip and could only be re-packed with Reflow. Editing, the canvas now extends to the lowest widget and scrolls, with the panel's own edge dashed and the area past it hatched, so the widget can be dragged back. The scale stays the panel's, and view mode is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/interface/dashboards.md | 4 ++ .../components/Dashboard/DashboardEditor.tsx | 7 ++- .../components/Dashboard/DashboardView.tsx | 53 +++++++++++++++++-- .../src/components/Dashboard/dashboard.css | 15 ++++++ 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/docs/interface/dashboards.md b/docs/interface/dashboards.md index da58efe..775a847 100644 --- a/docs/interface/dashboards.md +++ b/docs/interface/dashboards.md @@ -16,6 +16,10 @@ Edits stay in a draft; paired screens show the published version. On phones, tiles stack in one column and dragging is disabled, but settings remain editable. In edit mode, ⌘Z / Ctrl-Z undoes an edit and ⌘⇧Z / Ctrl-Shift-Z redoes it. +A widget can end up below the canvas, for example after the canvas was made +smaller. In edit mode the canvas then grows to reach it, under a dashed line +where the panel ends. Scroll down and drag it back, or use **Reflow**. + Controls can publish real values while you edit, so choose their targets with care. Your new Switch may already be flipping something! diff --git a/frontend/src/components/Dashboard/DashboardEditor.tsx b/frontend/src/components/Dashboard/DashboardEditor.tsx index ed4edae..8a3acb4 100644 --- a/frontend/src/components/Dashboard/DashboardEditor.tsx +++ b/frontend/src/components/Dashboard/DashboardEditor.tsx @@ -561,6 +561,11 @@ export function DashboardEditor({ // now falls past its bottom edge drawn under the clip. A round face has no // grid to fall off. const clipped = round ? [] : layout.filter((item) => item.y + item.h > rows) + // Editing, the canvas grows to reach them, so they can be dragged back. + const reach = + edit && clipped.length + ? Math.max(...clipped.map((item) => item.y + item.h)) + : undefined const shown = Math.max( 0, widgets.findIndex((widget) => widget.id === (selected ?? face)), @@ -669,7 +674,7 @@ export function DashboardEditor({ ) : null} ) : ( - + {(scale) => !edit ? ( diff --git a/frontend/src/components/Dashboard/DashboardView.tsx b/frontend/src/components/Dashboard/DashboardView.tsx index 3c15089..4495289 100644 --- a/frontend/src/components/Dashboard/DashboardView.tsx +++ b/frontend/src/components/Dashboard/DashboardView.tsx @@ -176,6 +176,7 @@ export function CanvasSurface({ dots, rail, touch, + reach, children, }: { dashboard: Dashboard @@ -185,6 +186,13 @@ export function CanvasSurface({ rail?: React.ReactNode /** Whether the panel this canvas hangs on is touched rather than pointed at. */ touch?: boolean + /** + * How many grid rows the arrangement reaches down to, while it is being + * arranged. A canvas they fall off grows to reach them, with its own edge + * still drawn, so they can be scrolled to and moved back. The scale stays + * the panel's. + */ + reach?: number children: (scale: number) => React.ReactNode }) { const ref = useRef(null) @@ -208,6 +216,17 @@ export function CanvasSurface({ const root = useCanvasRoot(dashboard, touch) const round = isRound(dashboard) const area = areaOf(dashboard, Boolean(rail)) + // How far past the bottom edge the rows reached go, margin included. + const beyond = reach + ? Math.max( + 0, + area.top + + reach * rowPitchOf(dashboard, Boolean(rail)) - + GRID_GAP + + GRID_INSET - + height, + ) + : 0 // The arrangement, held across the renders that are not about it. This // component reads the dashboard's own settings, so a bound one wakes it on // every message the flow sends — and calling a render prop again rebuilds @@ -219,7 +238,17 @@ export function CanvasSurface({ ) return ( -
+
{/* Measured first: a guessed scale would place the whole panel once and then move it. */} {scale > 0 ? ( @@ -238,7 +267,7 @@ export function CanvasSurface({ { ...root.style, width, - height, + height: height + beyond, // Round, nothing outside the circle is drawn. clipPath: round ? "circle(50%)" : undefined, left: (box.width - width * scale) / 2, @@ -255,14 +284,30 @@ export function CanvasSurface({ {/* Full bleed, and under everything: a background covers the panel, margin included. */} - {rail} + {beyond ? ( +
+ ) : null} + {/* Centred on the panel, not on a canvas grown past it. */} + {rail ? ( +
+ {rail} +
+ ) : null}
{/* Over the ground rather than under it: the dots are what is diff --git a/frontend/src/components/Dashboard/dashboard.css b/frontend/src/components/Dashboard/dashboard.css index e4496d0..2676c90 100644 --- a/frontend/src/components/Dashboard/dashboard.css +++ b/frontend/src/components/Dashboard/dashboard.css @@ -19,6 +19,21 @@ background-position: -1px -1px; } +/* + * Below the panel's bottom edge, while the editor has grown the canvas to + * reach widgets that fell off it: hatched, under a dashed line where the panel + * ends, so what it will not show reads as outside it. + */ +.canvas-beyond { + border-top: 3px dashed + color-mix(in srgb, var(--muted-foreground) 70%, transparent); + background: repeating-linear-gradient( + -45deg, + color-mix(in srgb, var(--muted-foreground) 14%, transparent) 0 3px, + transparent 3px 16px + ); +} + /* * View mode's grid. Column count is per dashboard (`--widget-cols`), so a wall * panel can be matched to its own width. It does not reflow: the grid lives on From a7fe925fdf60eb7f24718aab878e47795eca51c8 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:08:53 +0200 Subject: [PATCH 072/103] Draw the look's ground behind a stacked dashboard A phone stacks a dashboard instead of drawing it on a canvas, and the canvas was what carried the ground, so a stack had only the palette's colour. The panel and the editor now draw the look's ground behind the stack, held still while it scrolls. Glass's drift stops under reduced motion as it does on a canvas. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/interface/dashboards.md | 4 +++- frontend/src/components/Dashboard/DashboardEditor.tsx | 4 +++- frontend/src/components/Dashboard/PanelSurface.tsx | 8 +++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/interface/dashboards.md b/docs/interface/dashboards.md index 775a847..bb159e4 100644 --- a/docs/interface/dashboards.md +++ b/docs/interface/dashboards.md @@ -13,7 +13,9 @@ to a paired screen, such as a wall tablet. 4. **Publish** when the dashboard is ready. Edits stay in a draft; paired screens show the published version. On phones, -tiles stack in one column and dragging is disabled, but settings remain editable. +tiles stack in one column over the dashboard's ground (its background image, +or the Glass look's drifting colours) and dragging is disabled, but settings +remain editable. In edit mode, ⌘Z / Ctrl-Z undoes an edit and ⌘⇧Z / Ctrl-Shift-Z redoes it. A widget can end up below the canvas, for example after the canvas was made diff --git a/frontend/src/components/Dashboard/DashboardEditor.tsx b/frontend/src/components/Dashboard/DashboardEditor.tsx index 8a3acb4..28a3d17 100644 --- a/frontend/src/components/Dashboard/DashboardEditor.tsx +++ b/frontend/src/components/Dashboard/DashboardEditor.tsx @@ -42,6 +42,7 @@ import { handleError } from "@/utils" import { DashboardPalette } from "./DashboardPalette" import { areaOf, + CanvasGround, CanvasSurface, canvasOf, columnsOf, @@ -657,7 +658,8 @@ export function DashboardEditor({ )} style={root.style} > -
+ +
+ {/* The look's ground, held to the screen while the stack scrolls + over it, as it stays put under a canvas. */} +
+ +
{/* Padded here rather than by the route: the stack overflows this box, so only padding inside the overflow clears the rail. */} -
+
{rail ? ( From 3f0d6f05f7e0e1c8580b175dc236d0103af0b5e3 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:09:08 +0200 Subject: [PATCH 073/103] pre-commit CI installs the extras, so strict mypy finds the server-extra imports Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- .gitea/workflows/pre-commit.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/pre-commit.yml b/.gitea/workflows/pre-commit.yml index bef3351..a87007a 100644 --- a/.gitea/workflows/pre-commit.yml +++ b/.gitea/workflows/pre-commit.yml @@ -43,9 +43,11 @@ jobs: # settings, which read ../.env from backend/. run: cp .env.example .env # The hooks call `uv run ruff` from the repo root, so the workspace .venv - # (pyproject.toml → [tool.uv.workspace]) is what has to exist. + # (pyproject.toml → [tool.uv.workspace]) is what has to exist. The extras + # too, as `make lint-backend` does: strict mypy below checks call sites + # into what only the `server` extra installs. - name: Install backend dependencies - run: uv sync + run: uv sync --all-extras - name: Install frontend dependencies run: bun install --frozen-lockfile # The formatting hooks rewrite files; pre-commit exits non-zero when they From c5ce8cb49948d91f6147b0b918133d1415e26133 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:09:09 +0200 Subject: [PATCH 074/103] Backend CI drops the compose env and down -v notes left from Postgres Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- .gitea/workflows/test-backend.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.gitea/workflows/test-backend.yml b/.gitea/workflows/test-backend.yml index 43d8ef1..55854a9 100644 --- a/.gitea/workflows/test-backend.yml +++ b/.gitea/workflows/test-backend.yml @@ -16,14 +16,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -env: - # The compose files live in docker/, so a bare `docker compose` from the repo - # root finds nothing. With COMPOSE_FILE set, compose reads .env from the - # working directory — the same file the backend settings load as ../.env. - # compose.ci.yml drops the fixed container_names and the fixed 5432 — both - # daemon-global, so without it a second run on the same host cannot start. - COMPOSE_FILE: docker/compose.yml:docker/compose.dev.yml:docker/compose.ci.yml - jobs: test-backend: runs-on: ubuntu-latest @@ -34,10 +26,6 @@ jobs: matrix: python-version: ["3.12", "3.14"] env: - # Per run, so the `down -v` steps reach neither a dev stack nor a run - # still going. Unique names cannot be recycled, hence the always-on - # teardown below. - COMPOSE_PROJECT_NAME: fluksio-app-ci-backend-${{ github.run_id }} # uv would otherwise take a managed interpreter it already has over the # one set up below. UV_PYTHON: ${{ matrix.python-version }} From fb889a9e10adad9ed876c62bfd9dfcaebe300c71 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:10:36 +0200 Subject: [PATCH 075/103] make update warns when .env lacks a key compose would default, like make up Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- Makefile | 3 ++- docs/reference/configuration.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index be8cae8..0fb9b47 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ dev dev-utils dev-local dev-lan rebuild-frontend: export DOMAIN ?= localhost # A key the template lists and the compose files default, missing from .env, # falls back without a word. For TZ that means every cron runs on UTC. Only a -# warning, so a start never fails over it. +# warning, so a start never fails over it. `update` runs it after its pull. dev dev-local dev-lan up: env-check env-check: @missing=$$(grep -ohE '\$$\{[A-Z_]+:?-' docker/compose*.yml | sed 's/^..//; s/:\{0,1\}-$$//' | sort -u \ @@ -134,6 +134,7 @@ umami: ## Provision + start the optional Umami site-analytics service (idempote update: ## Pull, rebuild using the layer cache, and recreate changed containers git pull + @$(MAKE) --no-print-directory env-check $(COMPOSE_PROD_RUN) build $(COMPOSE_PROD_RUN) up -d --remove-orphans docker image prune -f diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 47957e8..149e8c9 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -121,8 +121,8 @@ stack does. Production disables it entirely. Staging and production also refuse `changethis` values for the session key or administrator password. Set `TZ` in `.env` for schedules that should follow local time, such as -`Europe/Berlin`. `make up` and `make dev` warn when `.env` lacks it, or any other -key `.env.example` lists that Compose would otherwise default. +`Europe/Berlin`. `make up`, `make update` and `make dev` warn when `.env` lacks +it, or any other key `.env.example` lists that Compose would otherwise default. ## Engine and resources From 17d58697c416cfb325619c21f0b455a533b57d4a Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:12:38 +0200 Subject: [PATCH 076/103] Stagger a rectangular panel's new dashboard by rail order Changing dashboard kept the grid mounted, so every new tile rose at once. The grid is drawn afresh per dashboard and its tiles come in row by row: from below, top row first, for one further down the rail, and from above, bottom row first, for one further up, as a round panel's dashboards slide. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/interface/dashboards.md | 2 +- .../components/Dashboard/DashboardView.tsx | 31 +++++++++++++------ .../src/components/Dashboard/PanelSurface.tsx | 9 ++++-- .../components/Dashboard/ui/core/motion.ts | 21 +++++++++---- frontend/src/routes/panel.$id.tsx | 13 +++++++- 5 files changed, 57 insertions(+), 19 deletions(-) diff --git a/docs/interface/dashboards.md b/docs/interface/dashboards.md index bb159e4..6f41cc0 100644 --- a/docs/interface/dashboards.md +++ b/docs/interface/dashboards.md @@ -347,7 +347,7 @@ dashboard's look's. | Value face, Value and Gauge tiles | the number | counts to the new value, written to its precision | spring | retargets | | Slider tile set elsewhere | the fill's end, the handle, the reading's dot | from where it is drawn to the new fraction | spring | a drag or a key moves it at once | | Gauge and Bar tiles | the value arc, each bar's fill | from where it is drawn to the new fraction | spring | retargets | -| Dashboard change (rectangular panel) | every tile | the old dashboard goes at once; each new tile opacity 0 → 1, y +8 → 0 | spring | — | +| Dashboard change (rectangular panel) | every tile | the old dashboard goes at once. The new tiles come in row by row, 30 ms apart, each opacity 0 → 1 and y +8 → 0 (Glass: +12, and scale 0.96 → 1): from below, top row first, for a dashboard further down the rail, and from above (y −8 → 0), bottom row first, for one further up | spring | — | | Rail | the current entry's marker | slides to the new entry | spring | retargets | With reduced motion asked for, nothing travels on its own: a page under the diff --git a/frontend/src/components/Dashboard/DashboardView.tsx b/frontend/src/components/Dashboard/DashboardView.tsx index 4495289..fc91f27 100644 --- a/frontend/src/components/Dashboard/DashboardView.tsx +++ b/frontend/src/components/Dashboard/DashboardView.tsx @@ -393,6 +393,7 @@ export function DashboardView({ editing, onDashboard, around, + turned, }: { dashboard: Dashboard renderWidget?: (widget: WidgetDef) => React.ReactNode @@ -407,6 +408,9 @@ export function DashboardView({ * previous (-1), and their names. */ onDashboard?: (by: number) => void around?: [string | undefined, string | undefined] + /** Which way along a panel's rail it was reached: down (1) or up (-1). + * Its tiles come in from that side. */ + turned?: number }) { const all = widgetsOf(dashboard) const round = isRound(dashboard) @@ -424,15 +428,15 @@ export function DashboardView({ ) } - // Stacked, the arrangement becomes a reading order, so it follows the rows - // the panel shows rather than the order widgets happened to be added in. - const widgets = stacked - ? [...all].sort((a, b) => { - const left = placement(a) - const right = placement(b) - return (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0) - }) - : all + // In the rows the panel shows rather than the order widgets happened to be + // added in: the reading order a stack follows, and the order tiles arrive + // in. A grid places each one itself, and one nobody arranged has them all + // at 0,0, which the sort leaves as they were. + const widgets = [...all].sort((a, b) => { + const left = placement(a) + const right = placement(b) + return (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0) + }) const columns = columnsOf(dashboard) const grid = ( @@ -449,11 +453,15 @@ export function DashboardView({ /> ) : ( )} @@ -480,6 +488,7 @@ function WidgetGrid({ columns, stacked, rail, + from, renderWidget, }: { dashboard: Dashboard @@ -487,6 +496,8 @@ function WidgetGrid({ columns: number stacked?: boolean rail?: boolean + /** The side the tiles come in from: below (1) or above (-1). */ + from: number renderWidget?: (widget: WidgetDef) => React.ReactNode }) { const { Frame } = useUi() @@ -496,6 +507,7 @@ function WidgetGrid({ className={cn("widget-grid", stacked && "widget-stacked")} data-placed={isPlaced(widgets) || undefined} variants={gridStagger} + custom={from} initial="hidden" animate="visible" style={ @@ -513,6 +525,7 @@ function WidgetGrid({ style={widgetStyle(widget, columns)} className="widget-cell" variants={LOOK[look].enter} + custom={from} transition={LOOK[look].spring} > {renderWidget ? ( diff --git a/frontend/src/components/Dashboard/PanelSurface.tsx b/frontend/src/components/Dashboard/PanelSurface.tsx index 39677f6..d30b2cc 100644 --- a/frontend/src/components/Dashboard/PanelSurface.tsx +++ b/frontend/src/components/Dashboard/PanelSurface.tsx @@ -30,6 +30,7 @@ export function PanelSurface({ touch, onDashboard, around, + turned, }: { dashboard: Dashboard /** A landscape arrangement scaled onto a phone comes out at about a fifth of @@ -45,6 +46,9 @@ export function PanelSurface({ * others instead, whose names are `around` it. */ onDashboard?: (by: number) => void around?: [string | undefined, string | undefined] + /** Which way along the rail this dashboard was reached: down (1) or up + * (-1). */ + turned?: number }) { const hasRail = Boolean(rail) const stack = stacked && !isRound(dashboard) @@ -57,9 +61,10 @@ export function PanelSurface({ rail={hasRail} onDashboard={onDashboard} around={around} + turned={turned} /> ), - [dashboard, hasRail, onDashboard, around], + [dashboard, hasRail, onDashboard, around, turned], ) return (
@@ -76,7 +81,7 @@ export function PanelSurface({ {/* Padded here rather than by the route: the stack overflows this box, so only padding inside the overflow clears the rail. */}
- +
{rail ? ( // No column to spare on a phone, so the rail is a row floating diff --git a/frontend/src/components/Dashboard/ui/core/motion.ts b/frontend/src/components/Dashboard/ui/core/motion.ts index cf70814..eb05953 100644 --- a/frontend/src/components/Dashboard/ui/core/motion.ts +++ b/frontend/src/components/Dashboard/ui/core/motion.ts @@ -21,7 +21,8 @@ import type { Look } from "../../settings" type LookMotion = { spring: Transition - /** How a widget arrives on the canvas. */ + /** How a widget arrives on the canvas. `custom` is the side it comes from, + * as `gridStagger` takes it. */ enter: Variants /** Seconds for one round of the ambient ground; 0 draws none. */ drift: number @@ -34,7 +35,7 @@ export const LOOK: Record = { // the way the rest of the product does. spring: { type: "spring", stiffness: 420, damping: 34, mass: 1 }, enter: { - hidden: { opacity: 0, y: 8 }, + hidden: (from = 1) => ({ opacity: 0, y: 8 * from }), visible: { opacity: 1, y: 0 }, }, drift: 0, @@ -42,7 +43,7 @@ export const LOOK: Record = { material: { spring: { type: "spring", stiffness: 550, damping: 38, mass: 1 }, enter: { - hidden: { opacity: 0, y: 8 }, + hidden: (from = 1) => ({ opacity: 0, y: 8 * from }), visible: { opacity: 1, y: 0 }, }, drift: 0, @@ -50,15 +51,23 @@ export const LOOK: Record = { glass: { spring: { type: "spring", stiffness: 320, damping: 22, mass: 0.8 }, enter: { - hidden: { opacity: 0, y: 12, scale: 0.96 }, + hidden: (from = 1) => ({ opacity: 0, y: 12 * from, scale: 0.96 }), visible: { opacity: 1, y: 0, scale: 1 }, }, drift: 46, }, } -/** Widgets arrive as a page rather than all at once. */ +/** + * Widgets arrive as a page rather than all at once, row by row. + * + * `custom` is the side they come from: below (1), as a round panel's next + * dashboard rises, or above (-1) for one further up a panel's rail, which + * brings the last row in first. + */ export const gridStagger: Variants = { hidden: {}, - visible: { transition: { staggerChildren: 0.03 } }, + visible: (from = 1) => ({ + transition: { staggerChildren: 0.03, staggerDirection: from < 0 ? -1 : 1 }, + }), } diff --git a/frontend/src/routes/panel.$id.tsx b/frontend/src/routes/panel.$id.tsx index 3030e8a..eaa3cde 100644 --- a/frontend/src/routes/panel.$id.tsx +++ b/frontend/src/routes/panel.$id.tsx @@ -1,6 +1,6 @@ import { keepPreviousData, useQuery } from "@tanstack/react-query" import { createFileRoute, redirect } from "@tanstack/react-router" -import { useCallback, useMemo } from "react" +import { useCallback, useMemo, useState } from "react" import type { Dashboard } from "@/components/Dashboard/DashboardView" import { PanelRail } from "@/components/Dashboard/PanelRail" @@ -84,6 +84,16 @@ function PanelRoute() { }, [panel, current, navigate], ) + // Which way along the rail the dashboard shown was reached, so its tiles + // come in from that side: further down it (1) from below, as a round + // panel's next dashboard rises, and further up (-1) from above. + const shown = dashboard?.name + const [turned, setTurned] = useState({ name: shown, by: 1 }) + if (shown !== turned.name) { + const by = + dashboards.indexOf(shown ?? "") - dashboards.indexOf(turned.name ?? "") + setTurned({ name: shown, by: by < 0 ? -1 : 1 }) + } // Said by the panel rather than by the dashboard: this is the screen, and // whether it is touched or pointed at is a fact about the screen. A document // hanging on a hallway tablet and on a desk browser is one document either @@ -131,6 +141,7 @@ function PanelRoute() { dashboard={dashboard as Dashboard} stacked={stacked} touch={touch} + turned={turned.by} onDashboard={rail && round ? step : undefined} around={rail && round ? around : undefined} // Drawn on the panel rather than beside it: a screen showing four From 0ac78764c87f6e9baf7da5eb0c22f3c5bffd6022 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:15:32 +0200 Subject: [PATCH 077/103] make test-frontend runs against a stack of its own The suite creates and deletes flows, dashboards and users, and ran against whatever stack was up, which is the dev instance's data. make test-stack starts the compose project fluksio-test (compose.test.yml over the local stack's files) with its own containers, volumes and image tags, served by the local Traefik on app.test.localhost and api.test.localhost, and leaves it running. The hostname is fixed, so it no longer comes off the running dev stack, and a LAN stack makes no difference. The suite's default origin is the test stack. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- Makefile | 67 ++++++++++++++++++++--------------- README.md | 2 +- development.md | 19 ++++++---- docker/compose.test.yml | 23 ++++++++++++ frontend/playwright.config.ts | 2 +- frontend/tests/config.ts | 3 +- frontend/tests/sweep.setup.ts | 6 ++-- 7 files changed, 82 insertions(+), 40 deletions(-) create mode 100644 docker/compose.test.yml diff --git a/Makefile b/Makefile index 0fb9b47..62cc4a0 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ # The workspace root delegates to these (see ../Makefile). .PHONY: init env-check dev-utils dev dev-local dev-lan rebuild-frontend up down update install dev-backend dev-frontend \ - generate-client sync-example test test-backend test-checks test-frontend soak bench-startup lint lint-backend \ - lint-frontend format-frontend umami build docs docs-serve clean help + generate-client sync-example test test-backend test-checks test-stack test-frontend soak \ + bench-startup lint lint-backend lint-frontend format-frontend umami build docs docs-serve clean help COMPOSE_ROOT := $(CURDIR) # Explicit project name keeps this stack isolated from the sibling website @@ -25,6 +25,13 @@ COMPOSE_LOCAL := $(COMPOSE_DEV) -f docker/compose.local.yml # Same stack, with the frontend also published on APP_PORT and proxying the API # there, for clients that cannot resolve app.$(DOMAIN). COMPOSE_LAN := $(COMPOSE_LOCAL) -f docker/compose.lan.yml +# The frontend suite's own stack, a project of its own behind the local Traefik +# (compose.test.yml). The hostname is fixed, never read from a file or the +# command line, so the suite can only ever reach this stack. +TEST_DOMAIN := test.localhost +COMPOSE_TEST := DOMAIN=$(TEST_DOMAIN) STACK_NAME=fluksio-test TAG=test \ + docker compose -p fluksio-test --env-file $(COMPOSE_ROOT)/.env -f docker/compose.yml \ + -f docker/compose.dev.yml -f docker/compose.local.yml -f docker/compose.test.yml help: ## Show available targets @awk 'BEGIN{FS=":.*?## "} /^[a-zA-Z_-]+:.*?##/ {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) @@ -141,6 +148,7 @@ update: ## Pull, rebuild using the layer cache, and recreate changed containers down: ## Stop all running containers -$(COMPOSE_LOCAL) down + -$(COMPOSE_TEST) down -$(COMPOSE_PROD_RUN) down # ── Development (local, no Docker) ─────────────────────────────── @@ -184,35 +192,38 @@ test-backend: ## Run backend tests (pytest + coverage) PW_VERSION = $(shell sed -n 's/.*"@playwright\/test": "[^0-9]*\([0-9.]*\)".*/\1/p' frontend/package.json | head -1) -# This suite creates and deletes flows, dashboards and users, so the hostname it -# is pointed at is not taken from any file: it is read off the running stack -# (the frontend container's own Traefik rule) and mapped onto the local Traefik -# by address, so DNS never decides. tests/guard.ts is the second line. The -# suite calls api. even on a LAN stack, so an api container recreated -# under another DOMAIN would answer every call with Traefik's 404. -test-frontend: ## Run frontend tests (Playwright e2e) against the local stack +# The suite creates and deletes flows, dashboards and users, so it runs against +# a stack of its own, never the dev instance. `up` only recreates what changed, +# and the stack is left running for the next run (and for a look at a failure +# on http://app.test.localhost). `make down` stops it. Its volumes are +# expendable: `docker compose -p fluksio-test down -v` starts it over. +test-stack: ## Start or update the frontend suite's own stack at http://app.test.localhost + @docker ps -q -f name='^fluksio-app-proxy-1$$' | grep -q . \ + || { echo " ✗ the local Traefik is not running, 'make dev' first"; exit 1; } + $(COMPOSE_TEST) up --build -d --wait redis backend frontend + +# The Playwright container, minus its network. tests/guard.ts refuses a public +# address on top of the fixed hostname. +PW_RUN = docker run --rm --ipc=host --user $$(id -u):$$(id -g) -e HOME=/tmp \ + -v $(COMPOSE_ROOT):/app -w /app/frontend \ + -e PLAYWRIGHT_BASE_URL=http://app.$(TEST_DOMAIN) \ + -e PLAYWRIGHT_API_URL=http://api.$(TEST_DOMAIN) \ + -e CI=$${CI:-1} \ + -e FIRST_SUPERUSER="$$(sed -n 's/^FIRST_SUPERUSER=//p' $(COMPOSE_ROOT)/.env | head -1)" \ + -e FIRST_SUPERUSER_PASSWORD="$$(sed -n 's/^FIRST_SUPERUSER_PASSWORD=//p' $(COMPOSE_ROOT)/.env | head -1)" +PW_IMAGE = mcr.microsoft.com/playwright:v$(PW_VERSION)-noble + +# The names are mapped onto the local Traefik by address, so DNS never decides. +# Twice, because Chromium pins *.localhost to loopback whatever /etc/hosts says. +test-frontend: test-stack ## Run frontend tests (Playwright e2e) against the test stack @ip=$$(docker network inspect proxy \ --format '{{range .Containers}}{{if eq .Name "fluksio-app-proxy-1"}}{{.IPv4Address}}{{end}}{{end}}' \ 2>/dev/null | cut -d/ -f1); \ - [ -n "$$ip" ] || { echo " ✗ proxy network or Traefik container not found — is the stack up?"; exit 1; }; \ - domain=$$(docker inspect fluksio-app 2>/dev/null \ - | grep -o 'Host(`app\.[^`]*`)' | head -1 | sed 's/Host(`app\.//;s/`)//'); \ - [ -n "$$domain" ] || { echo " ✗ no running app stack to test — 'make dev' first"; exit 1; }; \ - api=$$(docker inspect fluksio-api 2>/dev/null \ - | grep -o 'Host(`api\.[^`]*`)' | head -1 | sed 's/Host(`api\.//;s/`)//'); \ - [ "$$api" = "$$domain" ] || { echo " ✗ api container is on api.$${api:-?}, the app on app.$$domain — recreate the stack with 'make dev'"; exit 1; }; \ - docker run --rm --network proxy --ipc=host \ - --user $$(id -u):$$(id -g) -e HOME=/tmp \ - --add-host app.$$domain:$$ip --add-host api.$$domain:$$ip \ - -v $(COMPOSE_ROOT):/app -w /app/frontend \ - -e PLAYWRIGHT_BASE_URL=http://app.$$domain \ - -e PLAYWRIGHT_API_URL=http://api.$$domain \ - -e HOST_RESOLVER_RULES="MAP app.$$domain $$ip, MAP api.$$domain $$ip" \ - -e CI=$${CI:-1} \ - -e FIRST_SUPERUSER="$$(sed -n 's/^FIRST_SUPERUSER=//p' $(COMPOSE_ROOT)/.env | head -1)" \ - -e FIRST_SUPERUSER_PASSWORD="$$(sed -n 's/^FIRST_SUPERUSER_PASSWORD=//p' $(COMPOSE_ROOT)/.env | head -1)" \ - mcr.microsoft.com/playwright:v$(PW_VERSION)-noble \ - npx playwright test $(PLAYWRIGHT_ARGS) + [ -n "$$ip" ] || { echo " ✗ the local Traefik is not on the proxy network"; exit 1; }; \ + $(PW_RUN) --network proxy \ + --add-host app.$(TEST_DOMAIN):$$ip --add-host api.$(TEST_DOMAIN):$$ip \ + -e HOST_RESOLVER_RULES="MAP app.$(TEST_DOMAIN) $$ip, MAP api.$(TEST_DOMAIN) $$ip" \ + $(PW_IMAGE) npx playwright test $(PLAYWRIGHT_ARGS) # Load and chaos against a *running* stack, never part of `make test`: it # restarts this stack's containers. `SOAK_ARGS="--dry-run"` only looks. diff --git a/README.md b/README.md index e6ee9b9..1035146 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ make dev-frontend # Vite on :5173 ``` ```sh -make test # pytest + Playwright (the e2e half needs the stack up) +make test # pytest + Playwright (the e2e half on its own stack, behind make dev's Traefik) make lint # ruff + mypy + biome make generate-client # regenerate the frontend SDK from the OpenAPI schema ``` diff --git a/development.md b/development.md index a43215a..76582d2 100644 --- a/development.md +++ b/development.md @@ -129,12 +129,19 @@ SQLite file in a temp directory (deleted per session, so a run never touches dev data), its own superuser, and `ENVIRONMENT=local` with `DOMAIN=localhost`, so a checkout configured for a deployment cannot drag that deployment's configuration into the run. -**Frontend (e2e).** `make test-frontend` runs the suite inside the pinned -`mcr.microsoft.com/playwright:v-noble` image (version read from -`frontend/package.json`) on the `proxy` network, against a stack that must already be up. It -maps `app.$(DOMAIN)` and `api.$(DOMAIN)` onto the Traefik container's address twice, with -`--add-host` and with `HOST_RESOLVER_RULES`, because Chromium pins `*.localhost` to loopback -whatever `/etc/hosts` says. Narrow a run with +**Frontend (e2e).** `make test-frontend` runs the suite against a stack of its own, because +the suite creates and deletes flows, dashboards and users. `make test-stack`, which it runs +first, starts or updates the compose project `fluksio-test` (`docker/compose.test.yml` over the +local stack's files), with its own containers, volumes and image tags. The local Traefik serves +it on `app.test.localhost` and `api.test.localhost`, so `make dev` has to be up, in either +mode: the test stack never layers `compose.lan.yml` and publishes no port. It stays running +after a run for a look at a failure, `make down` stops it, and its volumes are expendable +(`docker compose -p fluksio-test down -v` starts it over). + +The suite runs inside the pinned `mcr.microsoft.com/playwright:v-noble` image +(version read from `frontend/package.json`) on the `proxy` network. It maps both names onto +the Traefik container's address twice, with `--add-host` and with `HOST_RESOLVER_RULES`, +because Chromium pins `*.localhost` to loopback whatever `/etc/hosts` says. Narrow a run with `make test-frontend PLAYWRIGHT_ARGS="--grep flows"`. The specs are `frontend/tests/*.spec.ts`; `playwright.config.ts` defines `setup` → diff --git a/docker/compose.test.yml b/docker/compose.test.yml new file mode 100644 index 0000000..9ad86d3 --- /dev/null +++ b/docker/compose.test.yml @@ -0,0 +1,23 @@ +# The frontend suite's own stack. `make test-stack` layers this on +# compose.local.yml under the project `fluksio-test`, with DOMAIN=test.localhost +# and STACK_NAME=fluksio-test. It shares the dev stack's Traefik and nothing +# else: its own containers, volumes, image tags and hostnames, so the flows, +# dashboards and users the suite creates never reach the dev instance's data. +services: + + # `container_name` ignores the project name, so the dev stack's names would + # collide. + redis: + container_name: !reset null + + backend: + container_name: !reset null + # compose.dev.yml also brings up the broker and the time-series database + # with it, which no spec talks to. + depends_on: !override + redis: + condition: service_healthy + restart: true + + frontend: + container_name: !reset null diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 7795f7b..fd9f2c7 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -28,7 +28,7 @@ export default defineConfig({ /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. Defaults to the - integrated stack (`make dev`), the only origin the API allows CORS from. + suite's own stack (`make test-stack`), never the dev instance's data. Point PLAYWRIGHT_BASE_URL elsewhere to test another running server; the API origin follows it (see tests/config.ts). */ baseURL: appUrl, diff --git a/frontend/tests/config.ts b/frontend/tests/config.ts index 6af784a..8ab661f 100644 --- a/frontend/tests/config.ts +++ b/frontend/tests/config.ts @@ -29,7 +29,8 @@ export const firstSuperuserPassword = getEnvVar("FIRST_SUPERUSER_PASSWORD") * instance. Both origins come from the same place instead: whatever the browser * is pointed at is what teardown may write to. */ -export const appUrl = process.env.PLAYWRIGHT_BASE_URL || "http://app.localhost" +export const appUrl = + process.env.PLAYWRIGHT_BASE_URL || "http://app.test.localhost" export const apiUrl = process.env.PLAYWRIGHT_API_URL || apiOrigin(appUrl) diff --git a/frontend/tests/sweep.setup.ts b/frontend/tests/sweep.setup.ts index d142452..466c1d4 100644 --- a/frontend/tests/sweep.setup.ts +++ b/frontend/tests/sweep.setup.ts @@ -11,9 +11,9 @@ import { apiUrl, firstSuperuser, firstSuperuserPassword } from "./config.ts" * rather than after one is what makes this work at all: it happens even when * the run that made the mess died. * - * The suite runs against whatever stack is up, which is somebody's own - * instance with real flows in it, so the predicate has to be incapable of - * matching one. A name qualifies only if it has the shape every fixture has — + * `make test-frontend` runs against a stack of its own, but the suite can be + * pointed at any (PLAYWRIGHT_BASE_URL), and that may be somebody's own instance + * with real flows in it, so the predicate has to be incapable of matching one. A name qualifies only if it has the shape every fixture has — * `test__`, plus an optional `_` for a second * artefact of the same spec — and only if the timestamp it carries is an hour * old, which nothing the run now starting is about to create can be. From 0da0c3875e061d1b7c617d646932909751c1b699 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:15:39 +0200 Subject: [PATCH 078/103] make test-webkit runs the drag specs in WebKit against the test stack WebKit takes *.localhost to loopback through GLib and ignores /etc/hosts, so the container shares Traefik's network namespace. The webkit project only exists with PLAYWRIGHT_WEBKIT=1, so test-frontend stays Chromium only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- Makefile | 9 ++++++++- development.md | 11 ++++++++--- frontend/playwright.config.ts | 24 ++++++++++++++++-------- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 62cc4a0..c6a1b7c 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # The workspace root delegates to these (see ../Makefile). .PHONY: init env-check dev-utils dev dev-local dev-lan rebuild-frontend up down update install dev-backend dev-frontend \ - generate-client sync-example test test-backend test-checks test-stack test-frontend soak \ + generate-client sync-example test test-backend test-checks test-stack test-frontend test-webkit soak \ bench-startup lint lint-backend lint-frontend format-frontend umami build docs docs-serve clean help COMPOSE_ROOT := $(CURDIR) @@ -225,6 +225,13 @@ test-frontend: test-stack ## Run frontend tests (Playwright e2e) against the te -e HOST_RESOLVER_RULES="MAP app.$(TEST_DOMAIN) $$ip, MAP api.$(TEST_DOMAIN) $$ip" \ $(PW_IMAGE) npx playwright test $(PLAYWRIGHT_ARGS) +# WebKit takes *.localhost to loopback through GLib and ignores /etc/hosts, so +# this run shares Traefik's network namespace, where loopback *is* Traefik. +# Only the drag specs, which is where the engines differ. +test-webkit: test-stack ## Run the drag specs in WebKit against the test stack + @$(PW_RUN) --network container:fluksio-app-proxy-1 -e PLAYWRIGHT_WEBKIT=1 \ + $(PW_IMAGE) npx playwright test --project=webkit $(PLAYWRIGHT_ARGS) + # Load and chaos against a *running* stack, never part of `make test`: it # restarts this stack's containers. `SOAK_ARGS="--dry-run"` only looks. soak: ## Run the soak/chaos harness (SOAK_ARGS="--minutes 30 --scenario redis --force") diff --git a/development.md b/development.md index 76582d2..cac3c93 100644 --- a/development.md +++ b/development.md @@ -144,10 +144,15 @@ the Traefik container's address twice, with `--add-host` and with `HOST_RESOLVER because Chromium pins `*.localhost` to loopback whatever `/etc/hosts` says. Narrow a run with `make test-frontend PLAYWRIGHT_ARGS="--grep flows"`. +`make test-webkit` runs the drag specs in WebKit against the same stack. WebKit takes +`*.localhost` to loopback through GLib and ignores `/etc/hosts`, so that container shares +Traefik's network namespace, where loopback is Traefik. Playwright's Linux WebKit reads a +`writing-mode: vertical-lr` range across, so a vertical slider's geometry is not asserted there. + The specs are `frontend/tests/*.spec.ts`; `playwright.config.ts` defines `setup` → -`chromium` and `mobile`. `auth.setup.ts` logs in once into `playwright/.auth/user.json`, -which both browser projects reuse; `mobile` runs `mobile.spec.ts` only, on a Pixel 5 at -393px. +`chromium` and `mobile`, plus `webkit` when `PLAYWRIGHT_WEBKIT=1` is set. `auth.setup.ts` logs +in once into `playwright/.auth/user.json`, which the browser projects reuse; `mobile` runs +`mobile.spec.ts` only, on a Pixel 5 at 393px. Both origins come from `PLAYWRIGHT_BASE_URL` and `PLAYWRIGHT_API_URL`, deliberately never from `VITE_API_URL` — that one belongs to the app build and in a deployment checkout it diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index fd9f2c7..0fc88dd 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -81,14 +81,22 @@ export default defineConfig({ // dependencies: ['setup'], // }, - // { - // name: 'webkit', - // use: { - // ...devices['Desktop Safari'], - // storageState: 'playwright/.auth/user.json', - // }, - // dependencies: ['setup'], - // }, + /* The drag specs in WebKit, where the engines differ. Opt-in, because + WebKit only reaches a *.localhost stack from inside Traefik's network + namespace, which is what `make test-webkit` runs in. */ + ...(process.env.PLAYWRIGHT_WEBKIT + ? [ + { + name: 'webkit', + use: { + ...devices['Desktop Safari'], + storageState: 'playwright/.auth/user.json', + }, + testMatch: /-drag\.spec\.ts/, + dependencies: ['setup'], + }, + ] + : []), // { // name: 'Mobile Safari', From 24d0190c5d76312a19d992a96c4c443d0e1a51f1 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:17:30 +0200 Subject: [PATCH 079/103] RemoteWorker.cancel makes no coroutine once the loop is closed run_coroutine_threadsafe refused it and it was never awaited, a RuntimeWarning under pytest 9. The cancel still logs that it could not reach the worker. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/remote.py | 5 +++++ backend/tests/flow/test_remote.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/backend/fluksio/flow/remote.py b/backend/fluksio/flow/remote.py index 30678ca..fcf638d 100644 --- a/backend/fluksio/flow/remote.py +++ b/backend/fluksio/flow/remote.py @@ -249,6 +249,11 @@ class RemoteWorker: def cancel(self, call_id: str) -> None: """Ask the worker to kill what it is running for this call.""" + # A closed loop refuses the coroutine and leaves it never awaited, so it + # is not made at all. + if self._loop.is_closed(): + logger.warning("Could not cancel '%s' on '%s'", call_id, self.name) + return try: asyncio.run_coroutine_threadsafe( self._send({"op": "cancel", "call_id": call_id}), self._loop diff --git a/backend/tests/flow/test_remote.py b/backend/tests/flow/test_remote.py index 16b6177..e872e5f 100644 --- a/backend/tests/flow/test_remote.py +++ b/backend/tests/flow/test_remote.py @@ -8,8 +8,10 @@ what happened rather than a wait that never ends. """ import asyncio +import gc import threading import time +import warnings from collections.abc import Iterator import pytest @@ -241,6 +243,21 @@ def test_cancelling_a_run_reaches_only_that_run(loop): assert [frame["call_id"] for frame in cancels] == ["run-a:flow.node"] +def test_a_cancel_after_the_loop_closed_leaves_no_coroutine_behind(): + closed = asyncio.new_event_loop() + closed.close() + worker = RemoteWorker( + name="gpu1", labels=[], send=FakeSocket().send_json, loop=closed + ) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + worker.cancel("run-a:flow.node") + gc.collect() + + assert not [w for w in caught if "never awaited" in str(w.message)] + + # ----------------------------------------------------------------------------- # Liveness, and what a heartbeat is evidence of # From d825acb768fa4245ac581db4063ce29e74d81e9e Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:26:20 +0200 Subject: [PATCH 080/103] LiveEdge pulse gaps: reset timer survives other producers, snapshots do not flash, flow: endpoints pulse as their node Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/src/components/Flow/EndpointNode.tsx | 7 +++++-- frontend/src/components/Flow/LiveEdge.tsx | 13 ++++++++++--- frontend/src/components/Flow/endpoints.ts | 15 ++++++++++++++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/Flow/EndpointNode.tsx b/frontend/src/components/Flow/EndpointNode.tsx index 44d8bdb..e76dbb2 100644 --- a/frontend/src/components/Flow/EndpointNode.tsx +++ b/frontend/src/components/Flow/EndpointNode.tsx @@ -4,7 +4,7 @@ import { memo, useEffect, useMemo, useRef, useState } from "react" import { useIsMobile } from "@/hooks/useMobile" import { cn } from "@/lib/utils" -import type { EndpointNodeData } from "./endpoints" +import { type EndpointNodeData, publisherOf } from "./endpoints" import { handleOffset, nodeHeight } from "./layout" import { useLiveValues } from "./liveStore" @@ -57,7 +57,10 @@ function EndpointNodeComponent({ id, data, selected }: NodeProps) { seen.current.set(name, live.ts) if (first || live.prior) continue // Published by this label, by the rule its own edges pulse on (LiveEdge). - if (provides.includes(name) && (!live.source || live.source.id === id)) { + if ( + provides.includes(name) && + (!live.source || live.source.id === publisherOf(id)) + ) { dir = "in" } else if (requires.includes(name)) { dir ??= "out" diff --git a/frontend/src/components/Flow/LiveEdge.tsx b/frontend/src/components/Flow/LiveEdge.tsx index 2ebd5b7..3a4940f 100644 --- a/frontend/src/components/Flow/LiveEdge.tsx +++ b/frontend/src/components/Flow/LiveEdge.tsx @@ -89,6 +89,11 @@ function LiveEdgeComponent({ const zoom = useStore((state) => state.transform[2]) const [pulsing, setPulsing] = useState(false) const lastTs = useRef(null) + // Owned by the pulse rather than by the effect: a value from another + // producer re-runs the effect too, and must not cancel a reset it does not + // replace, or the edge stays lit and the next pulse cannot restart. + const timer = useRef>(undefined) + useEffect(() => () => clearTimeout(timer.current), []) // Every rank sits further along the flow than the one before it, so an edge // whose target has ended up behind its source is one closing a cycle. @@ -113,13 +118,15 @@ function LiveEdgeComponent({ useEffect(() => { if (!live?.ts || live.ts === lastTs.current) return lastTs.current = live.ts + // A snapshot catching the page up is not a value passing now. + if (live.prior) return const from = live.source // No source at all is an older engine; pulse rather than go silent. if (from && from.id !== producerId) return setPulsing(true) - const timer = setTimeout(() => setPulsing(false), duration.pulse * 1000) - return () => clearTimeout(timer) - }, [live?.ts, live?.source, producerId]) + clearTimeout(timer.current) + timer.current = setTimeout(() => setPulsing(false), duration.pulse * 1000) + }, [live?.ts, live?.source, live?.prior, producerId]) return ( <> diff --git a/frontend/src/components/Flow/endpoints.ts b/frontend/src/components/Flow/endpoints.ts index 4ed68fb..d1abb3c 100644 --- a/frontend/src/components/Flow/endpoints.ts +++ b/frontend/src/components/Flow/endpoints.ts @@ -37,6 +37,19 @@ export function isEndpointNode(node: { id: string }): boolean { return PREFIXES.some((prefix) => node.id.startsWith(prefix)) } +/** + * Whom a value published through this endpoint is credited to. + * + * Every other kind publishes under its endpoint id, but a node in another + * flow publishes as itself — `other.node`, while its label here is + * `flow:other.node`. + */ +export function publisherOf(endpointId: string): string { + return endpointId.startsWith("flow:") + ? endpointId.slice("flow:".length) + : endpointId +} + /** * The flow's own boundary, as endpoints. * @@ -167,7 +180,7 @@ export function deriveEndpoints( data: { message, flow, - producerId: endpoint.id, + producerId: publisherOf(endpoint.id), } satisfies FlowEdgeData, }) } From af796f7a02b7e8ad88b4e8390a3de00cf0c9d30e Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:31:33 +0200 Subject: [PATCH 081/103] Flow canvas draws panel I/O ports as panel:: endpoints that open their panel in the Panels dialog Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/flows.py | 37 +++++++++++++++---- backend/tests/flow/test_endpoints.py | 34 +++++++++++++++++ docs/interface/dashboards.md | 4 +- docs/interface/flow-editor.md | 9 +++-- frontend/src/client/schemas.gen.ts | 9 +++-- frontend/src/client/types.gen.ts | 9 +++-- .../src/components/Common/GlobalSearch.tsx | 5 ++- .../src/components/Dashboard/PanelsDialog.tsx | 26 +++++++++++-- frontend/src/components/Flow/EndpointNode.tsx | 10 ++++- frontend/src/components/Flow/FlowEditor.tsx | 6 +++ frontend/src/components/Flow/endpoints.ts | 6 +-- .../src/routes/_layout/dashboards/index.tsx | 8 ++-- 12 files changed, 132 insertions(+), 31 deletions(-) diff --git a/backend/fluksio/api/routes/flows.py b/backend/fluksio/api/routes/flows.py index 4303578..37eb6eb 100644 --- a/backend/fluksio/api/routes/flows.py +++ b/backend/fluksio/api/routes/flows.py @@ -38,7 +38,7 @@ from fluksio.flow.controller import NODE_TYPES, FlowController from fluksio.flow.dashboards import DashboardStore from fluksio.flow.events import event_bus from fluksio.flow.messages import flow_of, qualify -from fluksio.flow.panels import messages_for +from fluksio.flow.panels import messages_for, read_config from fluksio.flow.pipeline import ValidationIssue from fluksio.flow.runs import RunRejected from fluksio.flow.schemas import ( @@ -78,18 +78,19 @@ ws_router = APIRouter(prefix="/flows", tags=["flows"]) class Endpoint(BaseModel): """Something wired into this flow that is not a node in it. - A dashboard control setting one of its messages, a tile showing one, or a - node in another flow on the far side of a dotted name. The canvas draws - these so a value never appears to come from nowhere — or worse, appears to - come from whichever node happens to be drawn as a producer. + A dashboard control setting one of its messages, a tile showing one, a + port on a panel's own hardware, or a node in another flow on the far side + of a dotted name. The canvas draws these so a value never appears to come + from nowhere — or worse, appears to come from whichever node happens to be + drawn as a producer. """ - #: dashboard or flow. + #: dashboard, panel or flow. kind: str #: Stable within its kind, and used as the canvas node id. id: str label: str - #: What sort of widget, or the node type in the other flow. + #: What sort of widget, the panel, or the node type in the other flow. detail: str = "" #: Messages of this flow it publishes, and ones it reads. provides: list[str] = [] @@ -155,6 +156,28 @@ def _endpoints(controller: FlowController, definition: FlowDef) -> list[Endpoint requires=binding["requires"], ) + # A panel's own hardware: an ``in`` port publishes like a control, an + # ``out`` port reads like a tile. The id is the source id the device + # publishes under, so its edges pulse for its values. + prefix = f"{flow}." + for panel in read_config().panels: + for port in panel.io: + if not port.message.startswith(prefix): + continue + key = f"panel:{panel.id}:{port.port}" + endpoint = found.setdefault( + key, + Endpoint( + kind="panel", + id=key, + label=port.port, + detail=panel.title or panel.id, + ), + ) + side = endpoint.provides if port.direction == "in" else endpoint.requires + if port.message not in side: + side.append(port.message) + def _flow_endpoint(key: str, detail: str) -> Endpoint: """One label per node on the far side, however many names reach it.""" return found.setdefault( diff --git a/backend/tests/flow/test_endpoints.py b/backend/tests/flow/test_endpoints.py index f448e81..dcf3bf6 100644 --- a/backend/tests/flow/test_endpoints.py +++ b/backend/tests/flow/test_endpoints.py @@ -10,8 +10,10 @@ from pathlib import Path import pytest from fluksio.api.routes.flows import _endpoints +from fluksio.core.config import settings from fluksio.flow.controller import FlowController from fluksio.flow.messages import DType, MessageSpec +from fluksio.flow.panels import PanelDef, PanelPort, PanelsConfig, write_config from fluksio.flow.schemas import FlowDef, NodeDef from fluksio.flow.store import FlowStore @@ -109,3 +111,35 @@ def test_the_wiring_index_is_read_once_per_store_revision(controller): controller.store.write_flow(FlowDef(name="shed", nodes=[])) _endpoints(controller, controller.store.read_flow("house")) assert reads == 2 + + +def test_a_panel_port_is_drawn_on_the_side_it_wires(controller, tmp_path, monkeypatch): + monkeypatch.setattr(settings, "PANELS_FILE", tmp_path / "panels.json") + write_config( + PanelsConfig( + panels=[ + PanelDef( + id="hall", + title="Hall", + io=[ + PanelPort(port="gpio.4", message="house.temp", direction="in"), + PanelPort( + port="backlight", message="house.temp", direction="out" + ), + PanelPort(port="mic", message="garage.open", direction="in"), + ], + ) + ] + ) + ) + + endpoints = { + one.id: one + for one in _endpoints(controller, controller.store.read_flow("house")) + } + + # The id is what the device publishes under, so its edges pulse for it. + assert endpoints["panel:hall:gpio.4"].provides == ["house.temp"] + assert endpoints["panel:hall:backlight"].requires == ["house.temp"] + assert endpoints["panel:hall:backlight"].detail == "Hall" + assert "panel:hall:mic" not in endpoints diff --git a/docs/interface/dashboards.md b/docs/interface/dashboards.md index 6f41cc0..3c99957 100644 --- a/docs/interface/dashboards.md +++ b/docs/interface/dashboards.md @@ -437,7 +437,9 @@ message: The credential may publish to its `in` messages even when a dashboard is locked, and may upload an artifact named after one (`PUT /artifacts?name=`), which is how a microphone sends a recording. -`out` messages are read only. +`out` messages are read only. The flow canvas draws each port as an endpoint, +an `in` port feeding the nodes that consume its message and an `out` port fed +by the nodes that produce it. !!! note "If the local link is wrong" diff --git a/docs/interface/flow-editor.md b/docs/interface/flow-editor.md index e13f3b1..4bbb2e0 100644 --- a/docs/interface/flow-editor.md +++ b/docs/interface/flow-editor.md @@ -9,10 +9,11 @@ The graph runs left to right on desktop and top to bottom on phones. - **Nodes** show their execution status. - **Edges** pulse when values pass through. Click one to inspect its payload. -- **Endpoints** show flow inputs, dashboard widgets and nodes in other flows - that exchange messages with this flow. A value passing lights one up in a - sweep: left to right when it sends into the flow, right to left when it reads - out of it. +- **Endpoints** show flow inputs, dashboard widgets, panel I/O ports and nodes + in other flows that exchange messages with this flow. A value passing lights + one up in a sweep: left to right when it sends into the flow, right to left + when it reads out of it. Click a panel port to open its panel in the Panels + dialog. Values update over a websocket. A connection banner tells you when the editor cannot reach the engine. diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index e780ce1..fe4797f 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -952,10 +952,11 @@ export const EndpointSchema = { title: 'Endpoint', description: `Something wired into this flow that is not a node in it. -A dashboard control setting one of its messages, a tile showing one, or a -node in another flow on the far side of a dotted name. The canvas draws -these so a value never appears to come from nowhere — or worse, appears to -come from whichever node happens to be drawn as a producer.` +A dashboard control setting one of its messages, a tile showing one, a +port on a panel's own hardware, or a node in another flow on the far side +of a dotted name. The canvas draws these so a value never appears to come +from nowhere — or worse, appears to come from whichever node happens to be +drawn as a producer.` } as const; export const EnrollBodySchema = { diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 7d42afa..2f5a75a 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -257,10 +257,11 @@ export type DType = 'float' | 'int' | 'str' | 'bool' | 'json' | 'series' | 'reco /** * Something wired into this flow that is not a node in it. * - * A dashboard control setting one of its messages, a tile showing one, or a - * node in another flow on the far side of a dotted name. The canvas draws - * these so a value never appears to come from nowhere — or worse, appears to - * come from whichever node happens to be drawn as a producer. + * A dashboard control setting one of its messages, a tile showing one, a + * port on a panel's own hardware, or a node in another flow on the far side + * of a dotted name. The canvas draws these so a value never appears to come + * from nowhere — or worse, appears to come from whichever node happens to be + * drawn as a producer. */ export type Endpoint = { kind: string; diff --git a/frontend/src/components/Common/GlobalSearch.tsx b/frontend/src/components/Common/GlobalSearch.tsx index 8007bee..0f7bbc7 100644 --- a/frontend/src/components/Common/GlobalSearch.tsx +++ b/frontend/src/components/Common/GlobalSearch.tsx @@ -182,7 +182,10 @@ export function GlobalSearch({ // Panels are managed in a dialog on the dashboards screen, which opens // itself when the address says so. case "panel": - return navigate({ to: "/dashboards", search: { panels: true } }) + return navigate({ + to: "/dashboards", + search: { panels: true, panel: entry.name }, + }) case "secret": return navigate({ to: "/secrets" }) case "module": diff --git a/frontend/src/components/Dashboard/PanelsDialog.tsx b/frontend/src/components/Dashboard/PanelsDialog.tsx index 6eba6e1..4750035 100644 --- a/frontend/src/components/Dashboard/PanelsDialog.tsx +++ b/frontend/src/components/Dashboard/PanelsDialog.tsx @@ -1,6 +1,6 @@ import { useMutation, useQuery } from "@tanstack/react-query" import { Plus, Trash2, X } from "lucide-react" -import { useState } from "react" +import { useEffect, useState } from "react" import { type ApiError, @@ -72,7 +72,7 @@ const objectOnly = (value: unknown) => * publish, so it lives in a dialog over the dashboards list instead of a page * of its own. Every change saves as it is made — there is no form to submit. */ -export function PanelsDialog() { +export function PanelsDialog({ focus }: { focus?: string } = {}) { const { data: config } = useQuery(panelsQueryOptions()) const { data: dashboards } = useQuery(dashboardsQueryOptions()) const { user } = useAuth() @@ -125,6 +125,15 @@ export function PanelsDialog() { const newId = slugify(name) const taken = panels.some((panel) => panel.id === newId) + // Arriving about one panel, from its port on a flow canvas, lands on it. + const focusLoaded = Boolean(config) && Boolean(focus) + useEffect(() => { + if (!focusLoaded) return + document + .querySelector(`[data-testid="panel-${focus}"]`) + ?.scrollIntoView({ block: "start" }) + }, [focusLoaded, focus]) + return ( @@ -146,6 +155,7 @@ export function PanelsDialog() { +
): Search => ({ panels: search.panels === true || search.panels === "true" || undefined, + // Which panel to scroll to, for a link that is about one of them. + panel: typeof search.panel === "string" ? search.panel : undefined, }), }) @@ -57,7 +59,7 @@ function Dashboards() { const [deleteOpen, setDeleteOpen] = useState(false) // Which screens exist is a question with an address, so anything can link to // it — the global search lands a panel here. - const { panels: panelsOpen } = Route.useSearch() + const { panels: panelsOpen, panel: panelFocus } = Route.useSearch() const setPanelsOpen = (open: boolean) => navigate({ to: "/dashboards", search: open ? { panels: true } : {} }) @@ -122,7 +124,7 @@ function Dashboards() { {/* Its own root rather than a nested one: which screens exist is a different question from which dashboards do. */} - + From c548031e94f8e32a790ffa6ad381b6534368b857 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:35:30 +0200 Subject: [PATCH 082/103] Following a flow: endpoint lands on the label leading back, selected, centred and with its edges highlighted Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/interface/flow-editor.md | 3 +- frontend/src/components/Flow/FlowEditor.tsx | 73 +++++++++++++++++---- 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/docs/interface/flow-editor.md b/docs/interface/flow-editor.md index 4bbb2e0..1cb2d9a 100644 --- a/docs/interface/flow-editor.md +++ b/docs/interface/flow-editor.md @@ -13,7 +13,8 @@ The graph runs left to right on desktop and top to bottom on phones. in other flows that exchange messages with this flow. A value passing lights one up in a sweep: left to right when it sends into the flow, right to left when it reads out of it. Click a panel port to open its panel in the Panels - dialog. + dialog. Click a node in another flow to open that flow on the label leading + back here, with its edges highlighted. Values update over a websocket. A connection banner tells you when the editor cannot reach the engine. diff --git a/frontend/src/components/Flow/FlowEditor.tsx b/frontend/src/components/Flow/FlowEditor.tsx index 59e338a..c6906b6 100644 --- a/frontend/src/components/Flow/FlowEditor.tsx +++ b/frontend/src/components/Flow/FlowEditor.tsx @@ -58,6 +58,7 @@ import { bindingsKey, boundaryKey, deriveEdges, + flowOf, portOf, qualify, } from "./deriveEdges" @@ -228,6 +229,15 @@ function CanvasBackground() { ) } +/** + * Whether arriving on this id selects it. A node does, and so does another + * flow's label, which is where a link from that flow lands; every other + * endpoint opens what it stands for instead. + */ +function landsOn(id: string): boolean { + return !isEndpointNode({ id }) || id.startsWith("flow:") +} + /** A name that does not collide with the nodes already on the canvas. */ function uniqueNodeId(existing: NodeDef_Input[], type: string): string { const taken = new Set(existing.map((node) => node.id)) @@ -276,8 +286,10 @@ function FlowEditorInner({ detail.definition.nodes ?? [], ) // An endpoint is never selected; arriving on one opens it instead (below). + // Another flow's label is the exception: it is where a link from that flow + // lands, so it is selected like a node. const [selectedId, setSelectedId] = useState( - focus && !isEndpointNode({ id: focus }) ? focus : null, + focus && landsOn(focus) ? focus : null, ) const [paletteOpen, setPaletteOpen] = useState(false) const [inspected, setInspected] = useState(null) @@ -474,9 +486,14 @@ function FlowEditorInner({ } }, [key]) - /** Where clicking an endpoint takes you: the thing it stands for. */ + /** + * Where clicking an endpoint takes you: the thing it stands for. + * + * `via` is the node here it is wired to. The other flow draws that node as + * a label of its own, which is where following the link lands. + */ const openEndpoint = useCallback( - (id: string) => { + (id: string, via?: string) => { const [kind, rest] = id.split(":", 2) if (kind === "input" || kind === "output") { // This flow's own boundary, declared in its settings. @@ -497,10 +514,11 @@ function FlowEditorInner({ navigate({ to: "/flows/$flowName", params: { flowName: (rest ?? "").split(".")[0] }, + search: { node: `flow:${via ? `${flowName}.${via}` : flowName}` }, }) } }, - [navigate], + [navigate, flowName], ) // React Flow measures a node once and keeps the size on it. An endpoint is @@ -563,6 +581,17 @@ function FlowEditorInner({ [key, flowName, detail.endpoints], ) + // Arrived from another flow: the label standing for the node the link left + // from, or failing that any label of that flow. Its edges are lit with it, + // so the connection that was followed is the one that reads first. + const landing = useMemo(() => { + if (!selectedId?.startsWith("flow:")) return null + const ids = external.nodes.map((node) => node.id) + if (ids.includes(selectedId)) return selectedId + const origin = flowOf(selectedId.slice("flow:".length)) + return ids.find((id) => id.startsWith(`flow:${origin}.`)) ?? null + }, [selectedId, external]) + const isMobile = useIsMobile() // Which way the graph runs. A phone has height to spare and no width, so it // reads top to bottom; everything else reads left to right. @@ -630,17 +659,24 @@ function FlowEditorInner({ .filter((node) => positions?.has(node.id)) .map((node) => ({ ...node, + selected: node.id === landing, position: positions?.get(node.id) ?? node.position, })), - [external, positions], + [external, positions, landing], ) // Marked after the layout has had the derived array, so selecting an edge // never sends the graph through the layout again. const shownEdges = useMemo( () => - edges.map((edge) => ({ ...edge, selected: selectedEdges.has(edge.id) })), - [edges, selectedEdges], + edges.map((edge) => ({ + ...edge, + selected: + selectedEdges.has(edge.id) || + edge.source === landing || + edge.target === landing, + })), + [edges, selectedEdges, landing], ) const shownNodes = useMemo( @@ -672,14 +708,15 @@ function FlowEditorInner({ // only changes the address. Seeded above for the first arrival, set here for // the ones after it. An endpoint — an input or output found by the search — // does what clicking its label does, which also covers one the canvas does - // not draw because a dashboard already accounts for it. + // not draw because a dashboard already accounts for it. Another flow's label + // is landed on instead, being where a link from that flow arrives. useEffect(() => { if (!focus) return - if (isEndpointNode({ id: focus })) { - openEndpoint(focus) - } else { + if (landsOn(focus)) { setFlowPanelOpen(false) setSelectedId(focus) + } else { + openEndpoint(focus) } }, [focus, openEndpoint]) @@ -705,8 +742,9 @@ function FlowEditorInner({ // Recorded as centred only once the fit below has run: arriving from // another screen selects before the layout has answered, and spending the // selection on the early returns is what fitted the whole flow instead. - const focus = - selectedId && selectedId !== centred.current ? selectedId : null + // Another flow's label is centred once it has been found on this canvas. + const target = selectedId?.startsWith("flow:") ? landing : selectedId + const focus = target && selectedId !== centred.current ? target : null // A phone's sheet covers the canvas outright, and so does the expanded // editor: there is no viewport to aim. if (editorExpanded) return @@ -732,6 +770,7 @@ function FlowEditorInner({ edges.length, positions, selectedId, + landing, panelOpen, editorExpanded, isMobile, @@ -1186,7 +1225,13 @@ function FlowEditorInner({ // An endpoint is somewhere else's: opening its panel here would // offer to edit a node this flow does not contain. if (!isDocumentNode(node)) { - openEndpoint(node.id) + const wire = edges.find( + (edge) => edge.source === node.id || edge.target === node.id, + ) + openEndpoint( + node.id, + wire && (wire.source === node.id ? wire.target : wire.source), + ) return } setFlowPanelOpen(false) From 749393034d30e795b1138e03ca499113f4b15ec5 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:42:28 +0200 Subject: [PATCH 083/103] Clicking a flow input label opens a popover with its declaration and current value, and a way into the flow settings Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/interface/flow-editor.md | 8 +- .../src/components/Flow/EdgeInspector.tsx | 4 +- frontend/src/components/Flow/FlowEditor.tsx | 35 ++++- .../src/components/Flow/InputInspector.tsx | 128 ++++++++++++++++++ 4 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 frontend/src/components/Flow/InputInspector.tsx diff --git a/docs/interface/flow-editor.md b/docs/interface/flow-editor.md index 1cb2d9a..d5c222f 100644 --- a/docs/interface/flow-editor.md +++ b/docs/interface/flow-editor.md @@ -12,9 +12,11 @@ The graph runs left to right on desktop and top to bottom on phones. - **Endpoints** show flow inputs, dashboard widgets, panel I/O ports and nodes in other flows that exchange messages with this flow. A value passing lights one up in a sweep: left to right when it sends into the flow, right to left - when it reads out of it. Click a panel port to open its panel in the Panels - dialog. Click a node in another flow to open that flow on the label leading - back here, with its edges highlighted. + when it reads out of it. +- **Clicking an endpoint** shows a flow input's type, starting value and + current value, with a pencil that opens the flow settings to edit it. A panel + port opens its panel in the Panels dialog. A node in another flow opens that + flow on the label leading back here, with its edges highlighted. Values update over a websocket. A connection banner tells you when the editor cannot reach the engine. diff --git a/frontend/src/components/Flow/EdgeInspector.tsx b/frontend/src/components/Flow/EdgeInspector.tsx index e47ec39..02d1a60 100644 --- a/frontend/src/components/Flow/EdgeInspector.tsx +++ b/frontend/src/components/Flow/EdgeInspector.tsx @@ -11,7 +11,7 @@ import { useLiveValue } from "./liveStore" import { MessageSparkline } from "./MessageSparkline" import { ValuePreview } from "./ValuePreview" -function relativeTime(ts: number | null | undefined): string { +export function relativeTime(ts: number | null | undefined): string { if (!ts) return "not seen yet" const seconds = Math.max(0, Math.round(Date.now() / 1000 - ts)) if (seconds < 5) return "just now" @@ -24,7 +24,7 @@ function relativeTime(ts: number | null | undefined): string { * A value short enough to share the summary row, or `null` for the objects and * arrays that need the payload view instead. */ -function formatScalar(value: unknown): string | null { +export function formatScalar(value: unknown): string | null { // Three decimals, trailing zeros dropped: enough precision to be useful, never // wide enough to wrap the row. if (typeof value === "number") return String(Number(value.toFixed(3))) diff --git a/frontend/src/components/Flow/FlowEditor.tsx b/frontend/src/components/Flow/FlowEditor.tsx index c6906b6..42a4bf6 100644 --- a/frontend/src/components/Flow/FlowEditor.tsx +++ b/frontend/src/components/Flow/FlowEditor.tsx @@ -73,6 +73,7 @@ import { import { FIT_VIEW, FIT_VIEW_PANEL, FlowDock } from "./FlowDock" import { FlowNode, type FlowNodeData } from "./FlowNode" import { FlowPanel } from "./FlowPanel" +import { InputInspector, type InspectedInput } from "./InputInspector" import { LiveEdge } from "./LiveEdge" import { type Direction, layoutGraph, type NodeShape } from "./layout" import { NodePanel } from "./NodePanel" @@ -293,6 +294,9 @@ function FlowEditorInner({ ) const [paletteOpen, setPaletteOpen] = useState(false) const [inspected, setInspected] = useState(null) + const [inspectedInput, setInspectedInput] = useState( + null, + ) // The edges are derived, so xyflow's own selection would be thrown away on // every rebuild: the ids live here instead and are marked on at render. const [selectedEdges, setSelectedEdges] = useState>( @@ -1221,9 +1225,26 @@ function FlowEditorInner({ onNodesDelete={(deleted) => deleteNodes(deleted.filter(isDocumentNode).map((node) => node.id)) } - onNodeClick={(_event, node) => { + onNodeClick={(event, node) => { // An endpoint is somewhere else's: opening its panel here would // offer to edit a node this flow does not contain. + // An input is read where it sits; editing it is one press on. + const declared = node.id.startsWith("input:") + ? (flowDoc.inputs ?? []).find( + (one) => + `input:${qualify(flowName, one.spec?.name ?? "")}` === + node.id, + ) + : undefined + if (declared) { + const { left, top, width, height } = + event.currentTarget.getBoundingClientRect() + setInspectedInput({ + declared, + rect: { left, top, width, height }, + }) + return + } if (!isDocumentNode(node)) { const wire = edges.find( (edge) => edge.source === node.id || edge.target === node.id, @@ -1244,6 +1265,7 @@ function FlowEditorInner({ setFlowPanelOpen(false) setEditorExpanded(false) setInspected(null) + setInspectedInput(null) }} onEdgeClick={(event, edge) => { const label = (id: string) => { @@ -1424,6 +1446,17 @@ function FlowEditorInner({ onUnbind={unbind} /> + setInspectedInput(null)} + onEdit={() => { + setInspectedInput(null) + setSelectedId(null) + setFlowPanelOpen(true) + }} + /> + void + onEdit: () => void +}) { + const spec = input?.declared.spec ?? {} + const name = spec.name ?? "" + const live = useLiveValue(input ? qualify(flow, name) : undefined) + + if (!input) return null + + const initial = input.declared.initial + const scalar = live === undefined ? null : formatScalar(live.value) + + return ( + !open && onClose()}> + + +
+ + + {input.declared.origin ?? "run"} input + + + + +
+
Type
+
{spec.dtype ?? "float"}
+
Starts at
+
+ {initial === null || initial === undefined ? ( + nothing + ) : ( + + )} +
+
Now
+
+ {live === undefined ? ( + nothing yet + ) : scalar === null ? ( + + ) : ( + + {scalar} + + )} + {live?.ts ? ( + + {relativeTime(live.ts)} + + ) : null} +
+
+ + {typeof live?.value === "number" || typeof live?.value === "boolean" ? ( +
+ +
+ ) : null} + + {live?.source && live.source.kind !== "node" ? ( +

+ Last set from {live.source.label} + {live.source.detail ? ` (${live.source.detail})` : ""}. +

+ ) : null} + + + ) +} From 2bea71db7f07316855e41cd323f21fdeede29ef9 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:49:26 +0200 Subject: [PATCH 084/103] Panels dialog saves a ready I/O port draft on close and keeps an unfinished one; one qualified message-name rule on both sides Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/messages.py | 19 +++ backend/fluksio/flow/panels.py | 10 +- backend/tests/flow/test_messages.py | 19 ++- docs/interface/dashboards.md | 5 +- .../src/components/Dashboard/PanelsDialog.tsx | 118 ++++++++++++++---- frontend/src/components/Flow/deriveEdges.ts | 13 ++ 6 files changed, 150 insertions(+), 34 deletions(-) diff --git a/backend/fluksio/flow/messages.py b/backend/fluksio/flow/messages.py index 3629e87..fc09483 100644 --- a/backend/fluksio/flow/messages.py +++ b/backend/fluksio/flow/messages.py @@ -12,6 +12,7 @@ from __future__ import annotations import json import math +import re from enum import Enum from typing import Any @@ -320,6 +321,24 @@ class MessageSpec(BaseModel): return hash((self.name, self.port)) +#: A message named with its flow: the flow's own name, then the name inside it. +#: The one rule for anywhere a name has to arrive qualified; the frontend holds +#: a copy (``QUALIFIED_MESSAGE`` in ``deriveEdges.ts``). +QUALIFIED_PATTERN = re.compile(r"[a-z][a-z0-9_]*(\.[A-Za-z0-9_]+)+") + +QUALIFIED_RULE = ( + "Name the message as flow.name: a lowercase flow name, a dot, " + "then letters, digits and underscores" +) + + +def validate_qualified(value: str) -> str: + """A qualified message name, or a ``ValueError`` saying what one looks like.""" + if not QUALIFIED_PATTERN.fullmatch(value): + raise ValueError(QUALIFIED_RULE) + return value + + def qualify(flow: str, name: str) -> str: """Resolve a message name against its flow namespace.""" if not name: diff --git a/backend/fluksio/flow/panels.py b/backend/fluksio/flow/panels.py index b573c99..26d4703 100644 --- a/backend/fluksio/flow/panels.py +++ b/backend/fluksio/flow/panels.py @@ -21,6 +21,7 @@ from pydantic import BaseModel, Field, field_validator from fluksio.core.config import settings from fluksio.flow.dashboards import DashboardDef, DashboardNotFound, DashboardStore +from fluksio.flow.messages import validate_qualified from fluksio.flow.schemas import _validate_name #: Free text the device interprets: ``gpio.4``, ``backlight``, ``mic``. @@ -51,14 +52,7 @@ class PanelPort(BaseModel): raise ValueError("Use lowercase letters, digits, underscores and dots") return value - @field_validator("message") - @classmethod - def _check_message(cls, value: str) -> str: - flow, _, name = value.partition(".") - if not name: - raise ValueError("Name the message with its flow, as flow.name") - _validate_name(flow) - return value + _check_message = field_validator("message")(validate_qualified) class PanelDef(BaseModel): diff --git a/backend/tests/flow/test_messages.py b/backend/tests/flow/test_messages.py index 8081dd7..2e11862 100644 --- a/backend/tests/flow/test_messages.py +++ b/backend/tests/flow/test_messages.py @@ -2,7 +2,13 @@ import json import pytest -from fluksio.flow.messages import DType, MessageSpec, flow_of, qualify +from fluksio.flow.messages import ( + DType, + MessageSpec, + flow_of, + qualify, + validate_qualified, +) def test_port_defaults_to_last_name_segment(): @@ -201,3 +207,14 @@ def test_a_value_that_refers_to_itself_does_not_hang_the_check(): loop: dict = {} loop["self"] = loop spec.check(loop) + + +@pytest.mark.parametrize("name", ["home.doorbell", "house.kitchen.temperature"]) +def test_a_qualified_name_names_its_flow(name): + assert validate_qualified(name) == name + + +@pytest.mark.parametrize("name", ["doorbell", "Home.x", "home.", "home.a-b", "home..x"]) +def test_anything_else_is_refused_with_the_rule(name): + with pytest.raises(ValueError, match="flow.name"): + validate_qualified(name) diff --git a/docs/interface/dashboards.md b/docs/interface/dashboards.md index 3c99957..86053db 100644 --- a/docs/interface/dashboards.md +++ b/docs/interface/dashboards.md @@ -425,13 +425,14 @@ screen serves none of them. ### The device's own I/O A panel can also act as a node. Under **I/O**, wire a port on the device to a -message: +message. A new port saves once it has a port name and a message, at the latest +when the dialog closes: | Field | Meaning | |---|---| | `port` | The device's name for it, such as `gpio.4`, `backlight`, `touch`, `mic` or `speaker`. Lowercase letters, digits, `_` and `.` | | `direction` | `in`: the device publishes to the message. `out`: the device follows it | -| `message` | A qualified `flow.name`. A flow must declare it, as for a control | +| `message` | A qualified `flow.name`, a lowercase flow name, a dot, then letters, digits and underscores. A flow must declare it, as for a control | | `config` | A JSON object passed to the device unread, such as `{"pull": "up", "debounce_ms": 50}` | The credential may publish to its `in` messages even when a dashboard is diff --git a/frontend/src/components/Dashboard/PanelsDialog.tsx b/frontend/src/components/Dashboard/PanelsDialog.tsx index 4750035..373209e 100644 --- a/frontend/src/components/Dashboard/PanelsDialog.tsx +++ b/frontend/src/components/Dashboard/PanelsDialog.tsx @@ -18,6 +18,7 @@ import { useSavePanels, useUnpairPanel, } from "@/components/Dashboard/queries" +import { qualifiedProblem } from "@/components/Flow/deriveEdges" import { JsonField } from "@/components/Flow/JsonField" import { InfoTip, PanelSection } from "@/components/Flow/SidePanel" import { Button } from "@/components/ui/button" @@ -60,6 +61,10 @@ const DIRECTIONS = [ const portOf = (value: string) => value.toLowerCase().replace(/[^a-z0-9_.]/g, "") +/** A new port the server will take: a name, and a message named with its flow. */ +const ready = (port: PanelPort) => + Boolean(port.port) && qualifiedProblem(port.message) === null + const objectOnly = (value: unknown) => value && typeof value === "object" && !Array.isArray(value) ? null @@ -97,6 +102,17 @@ export function PanelsDialog({ focus }: { focus?: string } = {}) { // Reading panels is any account's; every change to them is a superuser's. const canEdit = Boolean(user?.is_superuser) const [name, setName] = useState("") + // A port being added, per panel, until it is ready to save. Held here rather + // than in its row: the rows unmount when the dialog closes and this does + // not, so a port still missing its message is there again on reopening. + const [drafts, setDrafts] = useState>({}) + const setDraft = (id: string, draft: PanelPort | null) => + setDrafts((all) => { + const next = { ...all } + if (draft) next[id] = draft + else delete next[id] + return next + }) const panels = config?.panels ?? [] const known = dashboards?.data ?? [] @@ -122,6 +138,21 @@ export function PanelsDialog({ focus }: { focus?: string } = {}) { const replace = (id: string, panel: PanelDef) => write({ panels: panels.map((p) => (p.id === id ? panel : p)) }) + // Closing saves every new port that is ready, in one write. Its port field + // only commits on blur, and Escape closes the dialog without one. + const saveDrafts = () => { + const done = Object.keys(drafts).filter((id) => ready(drafts[id])) + if (!done.length) return + for (const id of done) setDraft(id, null) + write({ + panels: panels.map((panel) => + done.includes(panel.id) + ? { ...panel, io: [...(panel.io ?? []), drafts[panel.id]] } + : panel, + ), + }) + } + const newId = slugify(name) const taken = panels.some((panel) => panel.id === newId) @@ -135,7 +166,10 @@ export function PanelsDialog({ focus }: { focus?: string } = {}) { }, [focusLoaded, focus]) return ( - + Panels @@ -156,6 +190,8 @@ export function PanelsDialog({ focus }: { focus?: string } = {}) { key={panel.id} panel={panel} focused={panel.id === focus} + draft={drafts[panel.id] ?? null} + onDraft={(draft) => setDraft(panel.id, draft)} host={host} remoteHost={remoteHost} canEdit={canEdit} @@ -222,6 +258,8 @@ export function PanelsDialog({ focus }: { focus?: string } = {}) { function PanelRow({ panel, focused, + draft, + onDraft, host, remoteHost, canEdit, @@ -232,6 +270,9 @@ function PanelRow({ panel: PanelDef /** The one a link arrived about, marked so it is found at a glance. */ focused: boolean + /** A port being added to it, held by the dialog. */ + draft: PanelPort | null + onDraft: (draft: PanelPort | null) => void /** Where this instance answers, as it knows itself. */ host: string /** Where the portal serves this instance, when it is enrolled. */ @@ -411,7 +452,13 @@ function PanelRow({ )} - +
void onChange: (next: PanelDef) => void }) { const io = panel.io ?? [] - const [draft, setDraft] = useState(null) const listId = `ports-${panel.id}` const write = (next: PanelPort[]) => onChange({ ...panel, io: next }) const keepDraft = (next: PanelPort) => { - if (!next.port || !next.message) return setDraft(next) - setDraft(null) + if (!ready(next)) return onDraft(next) + onDraft(null) write([...io, next]) } @@ -574,7 +625,8 @@ function PanelIo({ listId={listId} testId={`io-${panel.id}-new`} onChange={keepDraft} - onRemove={() => setDraft(null)} + onType={(port) => onDraft({ ...draft, port })} + onRemove={() => onDraft(null)} /> ) : null} {canEdit && !draft ? ( @@ -583,7 +635,7 @@ function PanelIo({ size="sm-touch" className="justify-self-start" data-testid={`add-port-${panel.id}`} - onClick={() => setDraft({ port: "", message: "", direction: "in" })} + onClick={() => onDraft({ port: "", message: "", direction: "in" })} > Add port @@ -599,6 +651,7 @@ function IoRow({ listId, testId, onChange, + onType, onRemove, }: { row: PanelPort @@ -606,26 +659,44 @@ function IoRow({ listId: string testId: string onChange: (next: PanelPort) => void + /** + * A new port's name as it is typed, so closing the dialog before the field + * blurs still finds it. A saved port commits on blur instead. + */ + onType?: (port: string) => void onRemove: () => void }) { const name = row.port || "new port" + const problem = row.message ? qualifiedProblem(row.message) : null return (
- { - const port = portOf(event.currentTarget.value) - event.currentTarget.value = port || row.port - if (port && port !== row.port) onChange({ ...row, port }) - }} - /> + {onType ? ( + onType(portOf(event.currentTarget.value))} + onBlur={() => onChange(row)} + /> + ) : ( + { + const port = portOf(event.currentTarget.value) + event.currentTarget.value = port || row.port + if (port && port !== row.port) onChange({ ...row, port }) + }} + /> + )}
+ {problem ?

{problem}

: null}
) } diff --git a/frontend/src/components/Flow/deriveEdges.ts b/frontend/src/components/Flow/deriveEdges.ts index 6d4f028..07f5906 100644 --- a/frontend/src/components/Flow/deriveEdges.ts +++ b/frontend/src/components/Flow/deriveEdges.ts @@ -8,6 +8,19 @@ export function qualify(flow: string, name: string): string { return name.includes(".") ? name : `${flow}.${name}` } +/** + * A message named with its flow, as the server checks it wherever a name has + * to arrive qualified (`QUALIFIED_PATTERN` in `flow/messages.py`). + */ +const QUALIFIED_MESSAGE = /^[a-z][a-z0-9_]*(\.[A-Za-z0-9_]+)+$/ + +/** What is wrong with a name that has to be qualified, or null. */ +export function qualifiedProblem(name: string): string | null { + return QUALIFIED_MESSAGE.test(name) + ? null + : "Name the message as flow.name: a lowercase flow name, a dot, then letters, digits and underscores" +} + /** The flow a qualified message belongs to. */ export function flowOf(message: string): string { return message.split(".", 1)[0] From 5626ce841af9a0025d06776ff6082485ce9b7535 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 16:57:07 +0200 Subject: [PATCH 085/103] Home charts execution time per node type on a log scale, from a /observability/node-types rollup Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/observability.py | 63 ++++++++ .../tests/api/routes/test_observability.py | 56 +++++++ docs/interface/index.md | 6 + frontend/src/client/schemas.gen.ts | 29 ++++ frontend/src/client/sdk.gen.ts | 26 ++- frontend/src/client/types.gen.ts | 17 ++ .../src/components/Health/NodeTypeTimes.tsx | 153 ++++++++++++++++++ frontend/src/components/Health/queries.ts | 7 + frontend/src/routes/_layout/index.tsx | 6 +- 9 files changed, 360 insertions(+), 3 deletions(-) create mode 100644 frontend/src/components/Health/NodeTypeTimes.tsx diff --git a/backend/fluksio/api/routes/observability.py b/backend/fluksio/api/routes/observability.py index ad315f4..1374406 100644 --- a/backend/fluksio/api/routes/observability.py +++ b/backend/fluksio/api/routes/observability.py @@ -86,6 +86,17 @@ class FlowRollup(BaseModel): last_error_ts: float | None = None +class NodeTypeTiming(BaseModel): + """How long one kind of node takes to run, across every node of it.""" + + type: str + #: How many published nodes of this type ran inside the window. + nodes: int + executions: int + avg_ms: float + max_ms: float + + class RunRow(BaseModel): id: str flow: str @@ -428,6 +439,58 @@ def read_flow_rollups( ] +@router.get("/node-types", response_model=list[NodeTypeTiming]) +def read_node_type_timings( + session: SessionDep, + controller: FlowControllerDep, + hours: Annotated[int, Query(ge=0, le=8760)] = 24, +) -> Any: + """Execution time per node type, slowest first, to point at the hotspots. + + The rollups are kept per node, so the type is joined on from the published + flows. A node deleted since it ran has no type left to count under. + """ + hours = _window_hours(hours) + types = { + f"{flow.name}.{node.id}": node.type + for flow in controller.store.read_all() + for node in flow.nodes + } + statement = ( + sa_select( + col(MetricBucket.node), + func.sum(col(MetricBucket.executions)).label("executions"), + func.sum(col(MetricBucket.duration_sum_ms)).label("duration_sum_ms"), + func.max(col(MetricBucket.duration_max_ms)).label("max_ms"), + ) + .where(col(MetricBucket.bucket) >= _since(hours)) + .group_by(col(MetricBucket.node)) + ) + totals: dict[str, dict[str, float]] = {} + for row in session.execute(statement): + kind = types.get(row.node) + if kind is None or not row.executions: + continue + entry = totals.setdefault( + kind, {"nodes": 0, "executions": 0, "duration_sum_ms": 0.0, "max_ms": 0.0} + ) + entry["nodes"] += 1 + entry["executions"] += row.executions + entry["duration_sum_ms"] += row.duration_sum_ms + entry["max_ms"] = max(entry["max_ms"], row.max_ms) + timings = [ + NodeTypeTiming( + type=kind, + nodes=int(entry["nodes"]), + executions=int(entry["executions"]), + avg_ms=round(entry["duration_sum_ms"] / entry["executions"], 3), + max_ms=round(entry["max_ms"], 3), + ) + for kind, entry in totals.items() + ] + return sorted(timings, key=lambda timing: timing.avg_ms, reverse=True) + + @router.get("/runs", response_model=RunPage) def read_runs( session: SessionDep, diff --git a/backend/tests/api/routes/test_observability.py b/backend/tests/api/routes/test_observability.py index 77dafba..6a2984f 100644 --- a/backend/tests/api/routes/test_observability.py +++ b/backend/tests/api/routes/test_observability.py @@ -512,3 +512,59 @@ def test_a_superuser_can_flush_the_collector_now( ) ).all() assert len(written) == 1 + + +def test_execution_time_is_summed_per_node_type( + client: TestClient, superuser_token_headers: dict[str, str], db: Session +) -> None: + from fluksio.flow.schemas import FlowDef, NodeDef + + flow = "node_type_test" + controller = client.app.state.flow_controller + before = controller.store.read_all + controller.store.read_all = lambda: [ + FlowDef( + name=flow, + nodes=[ + NodeDef(id="a", type="python"), + NodeDef(id="b", type="python"), + NodeDef(id="tick", type="inject"), + ], + ) + ] + now = datetime.now(UTC).replace(second=0, microsecond=0) + for node, executions, total, peak in ( + ("a", 2, 30.0, 20.0), + ("b", 2, 10.0, 6.0), + ("tick", 4, 0.4, 0.2), + # Deleted since it ran: nothing to count it under. + ("gone", 1, 500.0, 500.0), + ): + db.add( + MetricBucket( + flow=flow, + node=f"{flow}.{node}", + bucket=now - timedelta(minutes=2), + executions=executions, + duration_sum_ms=total, + duration_max_ms=peak, + ) + ) + db.commit() + try: + rows = client.get( + f"{PREFIX}/node-types", headers=superuser_token_headers + ).json() + finally: + controller.store.read_all = before + + # Slowest first. + assert [row["type"] for row in rows] == ["python", "inject"] + assert rows[0] == { + "type": "python", + "nodes": 2, + "executions": 4, + "avg_ms": 10.0, + "max_ms": 20.0, + } + assert rows[1]["avg_ms"] == 0.1 diff --git a/docs/interface/index.md b/docs/interface/index.md index 7005ebd..ba86273 100644 --- a/docs/interface/index.md +++ b/docs/interface/index.md @@ -58,6 +58,12 @@ Health shows flow status, node load errors, queue depth and event-loop lag. After you have used batch runs, it also shows running and queued runs. A degraded instance lists the problems that need attention. +### Execution time per node type + +One bar per node type at its average time per run, slowest first, on a log +scale. Each row also shows how often the type ran and its slowest single run, +so a slow or busy kind of node stands out. + ### Activity Choose 1h, 6h, 24h or 7d to review executions and failures, recent runs, diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index fe4797f..4a9df04 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -2657,6 +2657,35 @@ export const NodeTypeInfoSchema = { description: 'A node type the editor can offer, with its parameter schema.' } as const; +export const NodeTypeTimingSchema = { + properties: { + type: { + type: 'string', + title: 'Type' + }, + nodes: { + type: 'integer', + title: 'Nodes' + }, + executions: { + type: 'integer', + title: 'Executions' + }, + avg_ms: { + type: 'number', + title: 'Avg Ms' + }, + max_ms: { + type: 'number', + title: 'Max Ms' + } + }, + type: 'object', + required: ['type', 'nodes', 'executions', 'avg_ms', 'max_ms'], + title: 'NodeTypeTiming', + description: 'How long one kind of node takes to run, across every node of it.' +} as const; + export const OAuthAuthorizeInfoSchema = { properties: { client_name: { diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index a5844ec..bbf17fa 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { AiReadAiConfigResponse, AiUpdateAiConfigData, AiUpdateAiConfigResponse, AiDeleteAiConfigResponse, AiListAiModelsData, AiListAiModelsResponse, AiReadAiStatusResponse, AiReadControlNodesResponse, AiReadChatResponse, AiChatData, AiChatResponse, AiClearChatResponse, AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsReadWebpushKeyResponse, AlertsAddWebpushSubscriptionData, AlertsAddWebpushSubscriptionResponse, AlertsRemoveWebpushSubscriptionData, AlertsRemoveWebpushSubscriptionResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, ArtifactsDeleteArtifactData, ArtifactsDeleteArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudReadMembersResponse, CloudLinkMemberData, CloudLinkMemberResponse, CloudRevokeMemberData, CloudRevokeMemberResponse, CloudDisconnectResponse, ConnectorsReadConnectorStateData, ConnectorsReadConnectorStateResponse, ConnectorsClearConnectorStateData, ConnectorsClearConnectorStateResponse, ConnectorsDeleteConnectorStateData, ConnectorsDeleteConnectorStateResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, FlowsIssueSocketTicketResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, ObservabilityFlushMetricsResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsImportRunsData, RunsImportRunsResponse, RunsReadRunsData, RunsReadRunsResponse, RunsDeleteRunsData, RunsDeleteRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadMetricNamesData, RunsReadMetricNamesResponse, RunsReadRunData, RunsReadRunResponse, RunsDeleteRunData, RunsDeleteRunResponse, RunsReadNodeOutputsData, RunsReadNodeOutputsResponse, RunsCancelRunData, RunsCancelRunResponse, RunsRetryRunData, RunsRetryRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SearchReadSearchIndexResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsReadConfigResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersReadTokensResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersRevokeTokenData, WorkersRevokeTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; +import type { AiReadAiConfigResponse, AiUpdateAiConfigData, AiUpdateAiConfigResponse, AiDeleteAiConfigResponse, AiListAiModelsData, AiListAiModelsResponse, AiReadAiStatusResponse, AiReadControlNodesResponse, AiReadChatResponse, AiChatData, AiChatResponse, AiClearChatResponse, AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsReadWebpushKeyResponse, AlertsAddWebpushSubscriptionData, AlertsAddWebpushSubscriptionResponse, AlertsRemoveWebpushSubscriptionData, AlertsRemoveWebpushSubscriptionResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, ArtifactsDeleteArtifactData, ArtifactsDeleteArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudReadMembersResponse, CloudLinkMemberData, CloudLinkMemberResponse, CloudRevokeMemberData, CloudRevokeMemberResponse, CloudDisconnectResponse, ConnectorsReadConnectorStateData, ConnectorsReadConnectorStateResponse, ConnectorsClearConnectorStateData, ConnectorsClearConnectorStateResponse, ConnectorsDeleteConnectorStateData, ConnectorsDeleteConnectorStateResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, FlowsIssueSocketTicketResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadNodeTypeTimingsData, ObservabilityReadNodeTypeTimingsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, ObservabilityFlushMetricsResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsImportRunsData, RunsImportRunsResponse, RunsReadRunsData, RunsReadRunsResponse, RunsDeleteRunsData, RunsDeleteRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadMetricNamesData, RunsReadMetricNamesResponse, RunsReadRunData, RunsReadRunResponse, RunsDeleteRunData, RunsDeleteRunResponse, RunsReadNodeOutputsData, RunsReadNodeOutputsResponse, RunsCancelRunData, RunsCancelRunResponse, RunsRetryRunData, RunsRetryRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SearchReadSearchIndexResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsReadConfigResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersReadTokensResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersRevokeTokenData, WorkersRevokeTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; export class AiService { /** @@ -1917,6 +1917,30 @@ export class ObservabilityService { }); } + /** + * Read Node Type Timings + * Execution time per node type, slowest first, to point at the hotspots. + * + * The rollups are kept per node, so the type is joined on from the published + * flows. A node deleted since it ran has no type left to count under. + * @param data The data for the request. + * @param data.hours + * @returns NodeTypeTiming Successful Response + * @throws ApiError + */ + public static readNodeTypeTimings(data: ObservabilityReadNodeTypeTimingsData = {}): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/observability/node-types', + query: { + hours: data.hours + }, + errors: { + 422: 'Validation Error' + } + }); + } + /** * Read Runs * Recent cascades, newest first, and how many there were in total. diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 2f5a75a..a599c09 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -919,6 +919,17 @@ export type NodeTypeInfo = { plugin?: (string | null); }; +/** + * How long one kind of node takes to run, across every node of it. + */ +export type NodeTypeTiming = { + type: string; + nodes: number; + executions: number; + avg_ms: number; + max_ms: number; +}; + /** * What the consent page shows, all of it validated server-side. */ @@ -2025,6 +2036,12 @@ export type ObservabilityReadFlowRollupsData = { export type ObservabilityReadFlowRollupsResponse = (Array); +export type ObservabilityReadNodeTypeTimingsData = { + hours?: number; +}; + +export type ObservabilityReadNodeTypeTimingsResponse = (Array); + export type ObservabilityReadRunsData = { flow?: (string | null); limit?: number; diff --git a/frontend/src/components/Health/NodeTypeTimes.tsx b/frontend/src/components/Health/NodeTypeTimes.tsx new file mode 100644 index 0000000..1a133fa --- /dev/null +++ b/frontend/src/components/Health/NodeTypeTimes.tsx @@ -0,0 +1,153 @@ +import { useQuery } from "@tanstack/react-query" + +import type { NodeTypeTiming } from "@/client" +import type { Range } from "@/components/Common/RangePicker" +import { nodeTypesQueryOptions } from "@/components/Flow/queries" +import { PANEL_SECTION } from "@/components/Flow/SidePanel" +import { Skeleton } from "@/components/ui/skeleton" +import { dur, si } from "@/lib/utils" +import { CARD, nodeTypeTimingsQueryOptions } from "./queries" + +/** + * One row's columns: the type, its bar, its average. A phone puts the bar on a + * line of its own under the other two, where it has the card's whole width. + */ +const COLUMNS = + "grid grid-cols-[minmax(0,1fr)_auto] gap-x-3 sm:grid-cols-[12rem_minmax(0,1fr)_5rem]" + +/** The fastest the axis reaches down to, in ms: a microsecond. */ +const FLOOR = 0.001 + +/** + * The axis, in whole decades of milliseconds: from below the fastest type to + * above the slowest, two decades at least so a bar has room to differ. + */ +function decades(timings: NodeTypeTiming[]): number[] { + const measured = timings.map((t) => t.avg_ms).filter((ms) => ms >= FLOOR) + if (!measured.length) return [] + const low = Math.floor(Math.log10(Math.min(...measured))) + const high = Math.max(Math.ceil(Math.log10(Math.max(...measured))), low + 2) + return Array.from({ length: high - low + 1 }, (_, i) => 10 ** (low + i)) +} + +/** Where a duration sits along the axis, as a percentage. */ +function at(ms: number, axis: number[]): number { + const low = Math.log10(axis[0]) + const high = Math.log10(axis[axis.length - 1]) + return ((Math.log10(ms) - low) / (high - low)) * 100 +} + +/** + * How long each kind of node takes per run, on a log scale. + * + * One bar per node type at its average, slowest first, so a hotspot is the top + * row. The scale is logarithmic because node types sit decades apart, a timer + * at microseconds and a model at seconds, and a linear one shows only the + * slowest. A type that ran without a measurable duration has no bar. + */ +export function NodeTypeTimes({ range }: { range: Range }) { + const { data, isPending } = useQuery(nodeTypeTimingsQueryOptions(range)) + const { data: types } = useQuery(nodeTypesQueryOptions()) + const titles = new Map((types ?? []).map((info) => [info.type, info.title])) + + const timings = data ?? [] + const axis = decades(timings) + + return ( +
+
+

Execution time per node type

+

+ average per run over the last {range.label}, log scale +

+
+ +
+ {isPending ? ( +
+ + +
+ ) : timings.length === 0 ? ( +

Nothing has run yet.

+ ) : ( +
+ {timings.map((timing) => ( + + ))} + {axis.length ? ( +
+
+ {axis.map((ms, index) => ( + + {dur(ms)} + + ))} +
+
+ ) : null} +
+ )} +
+
+ ) +} + +function Row({ + timing, + title, + axis, +}: { + timing: NodeTypeTiming + title: string + axis: number[] +}) { + const width = + timing.avg_ms >= FLOOR && axis.length ? at(timing.avg_ms, axis) : 0 + return ( +
+
+

{title}

+

+ {si(timing.executions)} runs · peak {dur(timing.max_ms)} +

+
+ {/* The decade lines run through every row, so together they read as + the grid; the bar starts at the axis's own floor. */} +
+ {axis.map((ms) => ( + + ))} + +
+ + {dur(timing.avg_ms)} + +
+ ) +} diff --git a/frontend/src/components/Health/queries.ts b/frontend/src/components/Health/queries.ts index 7f970c4..fecc76b 100644 --- a/frontend/src/components/Health/queries.ts +++ b/frontend/src/components/Health/queries.ts @@ -147,3 +147,10 @@ export const clock = (ts: number) => hour: "2-digit", minute: "2-digit", }) + +export const nodeTypeTimingsQueryOptions = (range: Range) => ({ + queryKey: ["observability", "node-types", range.hours] as const, + queryFn: () => + ObservabilityService.readNodeTypeTimings({ hours: range.hours }), + refetchInterval: REFRESH, +}) diff --git a/frontend/src/routes/_layout/index.tsx b/frontend/src/routes/_layout/index.tsx index 02b8251..794a04d 100644 --- a/frontend/src/routes/_layout/index.tsx +++ b/frontend/src/routes/_layout/index.tsx @@ -12,6 +12,7 @@ import { FlowTable } from "@/components/Health/FlowTable" import { HealthActivity } from "@/components/Health/HealthActivity" import { HealthOverview } from "@/components/Health/HealthOverview" import { LiveIndicator } from "@/components/Health/LiveIndicator" +import { NodeTypeTimes } from "@/components/Health/NodeTypeTimes" import { Card } from "@/components/ui/card" export const Route = createFileRoute("/_layout/")({ @@ -35,8 +36,8 @@ const HEADER = * than living at routes of their own. * * Top to bottom it is a widening lens: the whole instance as a graph, what - * has been built on it, whether it is well, then flow by flow and finally - * moment by moment. + * has been built on it, whether it is well, then flow by flow, node type by + * node type, and finally moment by moment. */ function Home() { // The health block's window: one choice, read by the tiles, the flow table, @@ -91,6 +92,7 @@ function Home() { +
) From 356b0331bfc4217cf506972f990b819839905b00 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:01:25 +0200 Subject: [PATCH 086/103] Docs: speech-to-text and text-to-speech recipes for a panel's mic and speaker faster-whisper and piper as plain Function nodes, the mic -> STT -> logic -> TTS -> speaker wiring, the Modules manifest they need (uv pip compile, av<19) and measured sizes, memory and CPU times. Modules docs now say uv pip sync installs no dependencies. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/code/nodes.md | 131 ++++++++++++++++++++++++++++++++++- docs/interface/dashboards.md | 3 +- docs/interface/operations.md | 6 +- 3 files changed, 134 insertions(+), 6 deletions(-) diff --git a/docs/code/nodes.md b/docs/code/nodes.md index cc86c81..f65c9a3 100644 --- a/docs/code/nodes.md +++ b/docs/code/nodes.md @@ -155,6 +155,130 @@ and the result plays there too, just smaller. The node needs `ffmpeg` on the [`WITH_FFMPEG=true`](../reference/configuration.md#image-options) in `.env`. Otherwise run the node on a [remote worker](workers.md) that has it. +### Speech to and from a panel + +openpanel publishes a 16 kHz mono WAV on its `mic` port after its wake word, +and plays any `audio` reference sent to its `speaker` port (WAV, MP3 or AAC). +Two Function nodes close that loop with local models. +[faster-whisper](https://github.com/SYSTRAN/faster-whisper) turns speech into +text and [piper](https://github.com/OHF-Voice/piper1-gpl) turns text into +speech. + +The listening node has an `audio` input port `audio` and a `str` output port +`text`: + +```python +from functools import cache +from pathlib import Path + +from faster_whisper import WhisperModel + +import fluksio + +MODELS = Path.home() / ".cache" / "voice" + + +@cache +def _model(size): + # Downloaded on the first call, then held by the worker between calls. + return WhisperModel( + size, device="cpu", compute_type="int8", download_root=str(MODELS) + ) + + +def process(audio, model="base", language="en"): + # vad_filter skips silence, which Whisper would otherwise hear as "You". + segments, _ = _model(model).transcribe( + fluksio.load_artifact(audio), language=language or None, vad_filter=True + ) + return {"text": " ".join(segment.text.strip() for segment in segments)} +``` + +The speaking node has a `str` input port `text` and an `audio` output port +`speech`: + +```python +import io +import wave +from functools import cache +from pathlib import Path + +from piper import PiperVoice +from piper.download_voices import download_voice + +import fluksio + +MODELS = Path.home() / ".cache" / "voice" + + +@cache +def _voice(name): + # Downloaded on the first call, then held by the worker between calls. + MODELS.mkdir(parents=True, exist_ok=True) + download_voice(name, MODELS) + return PiperVoice.load(MODELS / f"{name}.onnx") + + +def process(text, voice="en_US-lessac-medium"): + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wav: + _voice(voice).synthesize_wav(text, wav) + speech = fluksio.save_artifact(buffer.getvalue(), "speech.wav", "audio/wav") + return {"speech": speech} +``` + +`model`, `language` and `voice` are settings. `tiny` is faster than `base` and +less accurate. An empty `language` lets Whisper guess it for each clip. `voice` +takes any name from the [piper voices](https://huggingface.co/rhasspy/piper-voices), +such as `de_DE-thorsten-medium`. + +To wire the loop in a flow `home`: + +1. Declare a [flow input](../concepts/flows.md#flow-inputs) `heard` of type + `audio` with origin `panel`. +2. Under the panel's [I/O](../interface/dashboards.md#the-devices-own-io), wire + `mic` as `in` to `home.heard` and `speaker` as `out` to `home.reply`. +3. Bind the listening node's `audio` to `heard` and its `text` to `said`. +4. Have your own nodes read `said` and publish an answer as a `str`, such as + `answer`. +5. Bind the speaking node's `text` to `answer` and its `speech` to `reply`. + +Binding the speaking node's `text` straight to `said` makes the panel repeat +what it heard, which tests the loop before there is any logic in it. + +Both packages go into the node's environment, as described in +[Packages](#packages). A managed environment installs its manifest with +`uv pip sync`, which skips dependencies, so paste the fully resolved list into +**Modules**: + +```sh +printf 'faster-whisper\nav<19\npiper-tts\n' | uv pip compile --python-version 3.13 - +``` + +The Docker image runs Python 3.13. `av<19` holds PyAV back, since faster-whisper +1.2.1 cannot open files with PyAV 19. In your own venv or on a remote worker, +`pip install faster-whisper "av<19" piper-tts` is enough. + +The models land in `~/.cache/voice` on the machine running the node. The Docker +image keeps its home directory inside the container, so a recreated container +downloads them again. Point `MODELS` at a directory under `/data` to keep them +on the volume. An [attached worker](workers.md) starts a process per call and +loads the model each time. + +What it costs, measured on a 2016 server CPU (Xeon E5-2683 v4): + +| | Size | Memory | 3 s of speech, one core | All cores | +|---|---|---|---|---| +| Packages | 150 MB download, 460 MB installed | | | | +| Whisper `tiny` | 78 MB | 300 MB | 1.6 s | 0.7 s | +| Whisper `base` | 148 MB | 375 MB | 2.9 s | 1.2 s | +| piper `en_US-lessac-medium` | 63 MB | 250 MB | 0.6 s | 0.3 s | + +Each model downloads once, on its node's first call. Loading it takes about a +second for Whisper and four seconds for piper. Memory is the worker's peak while +it holds the model. These figures come from piper's own speech fed back into +Whisper. A room microphone is harder, on `tiny` most of all. + ## Printing `print` and `fluksio.logger.info(...)` are captured in the node logs. The @@ -220,13 +344,14 @@ environment. `NODE_VENV=managed` always requests one; the Docker image uses this setting. In managed mode, the Modules manifest is installed with `uv pip sync` and -versioned with the flows. Applying it refreshes local workers. The API accepts -the same manifest: +versioned with the flows. Sync installs only the packages listed, not their +dependencies, so list those too, for example as resolved by `uv pip compile`. +Applying it refreshes local workers. The API accepts the same manifest: ```sh curl -X POST "$FLUKSIO/modules/apply" -H "Authorization: Bearer $TOKEN" \ -H 'Content-Type: application/json' \ - -d '{"requirements": "numpy>=2\npandas\n"}' + -d '{"requirements": "numpy>=2\n"}' ``` Here `$FLUKSIO` is the API base URL ending in `/api/v1`, and `$TOKEN` is an diff --git a/docs/interface/dashboards.md b/docs/interface/dashboards.md index 86053db..c3a0bb8 100644 --- a/docs/interface/dashboards.md +++ b/docs/interface/dashboards.md @@ -420,7 +420,8 @@ to 640×360. Re-encode other clips with the [clip recipe](../code/nodes.md#makin The device decides what the panel's [I/O](#the-devices-own-io) port names mean. openpanel serves `gpio.`, `backlight`, `touch`, `mic` and `speaker`, lists each one's `config` in its README and ignores any other name. A browser -screen serves none of them. +screen serves none of them. The [speech recipe](../code/nodes.md#speech-to-and-from-a-panel) +turns `mic` into text and text into `speaker`. ### The device's own I/O diff --git a/docs/interface/operations.md b/docs/interface/operations.md index d332425..8a57d22 100644 --- a/docs/interface/operations.md +++ b/docs/interface/operations.md @@ -33,8 +33,10 @@ packages and Python version. There are two setups: - **Managed environment:** Docker uses a separate node environment. A superuser edits **Requirements** and presses **Apply**. Packages are synced - with `uv pip sync`; deleting a requirement removes it. The manifest is - versioned with the flows, and successful installs refresh the workers. + with `uv pip sync`; deleting a requirement removes it. Sync installs only + what is listed, so list the dependencies too, for example the output of + `uv pip compile`. The manifest is versioned with the flows, and successful + installs refresh the workers. - **Adopted environment:** a fresh pip installation inside your own virtual environment can use that environment for nodes too. Modules is read-only. Install dependencies there with pip or uv, then sync or restart Fluksio. From 50e6c033f12df8282426cf6e10cdc31887698ebf Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:06:20 +0200 Subject: [PATCH 087/103] Modules resolve the manifest with uv pip compile before syncing, so listed packages bring their dependencies Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/flow/modules.py | 74 +++++++++++++++++++----------- backend/tests/flow/test_modules.py | 35 ++++++++++++++ docs/code/nodes.md | 24 +++++----- docs/interface/operations.md | 9 ++-- 4 files changed, 98 insertions(+), 44 deletions(-) diff --git a/backend/fluksio/flow/modules.py b/backend/fluksio/flow/modules.py index cbb9598..ebee5a7 100644 --- a/backend/fluksio/flow/modules.py +++ b/backend/fluksio/flow/modules.py @@ -6,9 +6,12 @@ into a venv on the data volume rather than into the engine's environment: a pin here can never shadow — or be shadowed by — what the app itself runs on, and the worker processes that import them need nothing from the app. -``uv pip sync`` rather than install, so a line taken out of the manifest is -uninstalled. The manifest is written only after a sync succeeds, which is all -the rollback a failed resolve needs. +The manifest is resolved with ``uv pip compile`` first, so a line naming a +package brings its dependencies along, and a manifest that already pins +everything resolves to itself. The result goes to ``uv pip sync`` rather than +install, so a line taken out of the manifest is uninstalled. The manifest is +written only after a sync succeeds, which is all the rollback a failed resolve +needs. None of that applies to an *adopted* venv. A data scientist makes a venv, installs what they work with, and then installs Fluksio into it too — at which @@ -179,35 +182,52 @@ def sync(requirements: str) -> tuple[bool, str]: def _sync(requirements: str) -> tuple[bool, str]: - """The install itself, under `_sync_lock`.""" - manifest = "" + """The resolve and install, under `_sync_lock`.""" try: ensure_venv() - with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as handle: - handle.write(requirements) - manifest = handle.name - result = subprocess.run( - # Empty means empty: without the flag uv refuses to clear a venv, - # so deleting the last line would leave the package installed. - [ - uv_bin(), - "pip", - "sync", - "--allow-empty-requirements", - "--python", - venv_python(), - manifest, - ], - capture_output=True, - text=True, - timeout=SYNC_TIMEOUT, - ) + python = venv_python() + with tempfile.TemporaryDirectory() as directory: + manifest = Path(directory, "requirements.in") + lock = Path(directory, "requirements.txt") + manifest.write_text(requirements) + for command in ( + # The emit flags carry the manifest's index, find-links and + # binary options into the lock, which is all sync reads. + [ + uv_bin(), + "pip", + "compile", + "--quiet", + "--emit-index-url", + "--emit-find-links", + "--emit-build-options", + "--python", + python, + str(manifest), + "--output-file", + str(lock), + ], + # Empty means empty: without the flag uv refuses to clear a + # venv, so deleting the last line would leave the package + # installed. + [ + uv_bin(), + "pip", + "sync", + "--allow-empty-requirements", + "--python", + python, + str(lock), + ], + ): + result = subprocess.run( + command, capture_output=True, text=True, timeout=SYNC_TIMEOUT + ) + if result.returncode != 0: + break except (OSError, subprocess.SubprocessError) as exc: detail = str(getattr(exc, "stderr", None) or exc).strip() return False, f"uv could not run: {detail}" - finally: - if manifest: - Path(manifest).unlink(missing_ok=True) output = (result.stdout + result.stderr).strip() if result.returncode == 0: diff --git a/backend/tests/flow/test_modules.py b/backend/tests/flow/test_modules.py index 1d3d991..c2401cd 100644 --- a/backend/tests/flow/test_modules.py +++ b/backend/tests/flow/test_modules.py @@ -2,6 +2,7 @@ import shutil import subprocess +import zipfile from pathlib import Path import pytest @@ -77,6 +78,40 @@ def test_a_manifest_that_does_not_resolve_leaves_the_venv_alone(venv: Path): assert (venv / ".applied").read_text() == modules._digest("") +def _wheel(directory: Path, name: str, requires: str = "") -> None: + """A wheel with nothing in it but a name and, optionally, one dependency.""" + info = f"{name}-1.0.dist-info" + metadata = f"Metadata-Version: 2.1\nName: {name}\nVersion: 1.0\n" + if requires: + metadata += f"Requires-Dist: {requires}\n" + with zipfile.ZipFile(directory / f"{name}-1.0-py3-none-any.whl", "w") as wheel: + wheel.writestr(f"{name}/__init__.py", "") + wheel.writestr(f"{info}/METADATA", metadata) + wheel.writestr( + f"{info}/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + wheel.writestr(f"{info}/RECORD", "") + + +def test_a_listed_package_brings_its_dependencies( + venv: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + wheels = tmp_path / "wheels" + wheels.mkdir() + _wheel(wheels, "fluksio_top", requires="fluksio_dep") + _wheel(wheels, "fluksio_dep") + monkeypatch.setenv("UV_OFFLINE", "1") + + ok, output = modules.sync(f"--find-links {wheels}\nfluksio-top\n") + + assert ok, output + imported = subprocess.run( + [modules.venv_python(), "-c", "import fluksio_dep"], check=False + ) + assert imported.returncode == 0 + + # --------------------------------------------------------------------------- # Adopting the venv Fluksio was installed into # diff --git a/docs/code/nodes.md b/docs/code/nodes.md index f65c9a3..cee3609 100644 --- a/docs/code/nodes.md +++ b/docs/code/nodes.md @@ -247,17 +247,17 @@ Binding the speaking node's `text` straight to `said` makes the panel repeat what it heard, which tests the loop before there is any logic in it. Both packages go into the node's environment, as described in -[Packages](#packages). A managed environment installs its manifest with -`uv pip sync`, which skips dependencies, so paste the fully resolved list into -**Modules**: +[Packages](#packages). In a managed environment, list them in **Modules**: -```sh -printf 'faster-whisper\nav<19\npiper-tts\n' | uv pip compile --python-version 3.13 - +```text +faster-whisper +av<19 +piper-tts ``` -The Docker image runs Python 3.13. `av<19` holds PyAV back, since faster-whisper -1.2.1 cannot open files with PyAV 19. In your own venv or on a remote worker, -`pip install faster-whisper "av<19" piper-tts` is enough. +`av<19` holds PyAV back, since faster-whisper 1.2.1 cannot open files with +PyAV 19. In your own venv or on a remote worker, +`pip install faster-whisper "av<19" piper-tts` does the same. The models land in `~/.cache/voice` on the machine running the node. The Docker image keeps its home directory inside the container, so a recreated container @@ -343,10 +343,10 @@ already has a managed one. Without an existing venv, it creates a managed environment. `NODE_VENV=managed` always requests one; the Docker image uses this setting. -In managed mode, the Modules manifest is installed with `uv pip sync` and -versioned with the flows. Sync installs only the packages listed, not their -dependencies, so list those too, for example as resolved by `uv pip compile`. -Applying it refreshes local workers. The API accepts the same manifest: +In managed mode, the Modules manifest is resolved with `uv pip compile`, +installed with `uv pip sync` and versioned with the flows. Listing a package +brings its dependencies along. Applying it refreshes local workers. The API +accepts the same manifest: ```sh curl -X POST "$FLUKSIO/modules/apply" -H "Authorization: Bearer $TOKEN" \ diff --git a/docs/interface/operations.md b/docs/interface/operations.md index 8a57d22..9db94d8 100644 --- a/docs/interface/operations.md +++ b/docs/interface/operations.md @@ -32,11 +32,10 @@ the flow repository. Using references keeps credentials out of saved flows. packages and Python version. There are two setups: - **Managed environment:** Docker uses a separate node environment. A - superuser edits **Requirements** and presses **Apply**. Packages are synced - with `uv pip sync`; deleting a requirement removes it. Sync installs only - what is listed, so list the dependencies too, for example the output of - `uv pip compile`. The manifest is versioned with the flows, and successful - installs refresh the workers. + superuser edits **Requirements** and presses **Apply**. The list is resolved + with `uv pip compile` and synced with `uv pip sync`, so each package brings + its dependencies and deleting a requirement removes it. The manifest is + versioned with the flows, and successful installs refresh the workers. - **Adopted environment:** a fresh pip installation inside your own virtual environment can use that environment for nodes too. Modules is read-only. Install dependencies there with pip or uv, then sync or restart Fluksio. From cdded499b1118ad6aa0e7d7d431a9164e78b6615 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:07:22 +0200 Subject: [PATCH 088/103] LAN nginx proxies to the container name fluksio-api, which the test stack's backend does not carry Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docker/nginx-api-proxy.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/nginx-api-proxy.conf b/docker/nginx-api-proxy.conf index b09b499..d8ca29f 100644 --- a/docker/nginx-api-proxy.conf +++ b/docker/nginx-api-proxy.conf @@ -11,7 +11,10 @@ location ~ ^/(api|hooks) { # Docker's embedded DNS, re-resolved: nginx otherwise pins the backend's # address at startup and 502s for good once that container is recreated. resolver 127.0.0.11 valid=10s ipv6=off; - set $backend_upstream http://backend:8000; + # The container name, which only the dev API carries. The test stack's + # backend answers to the service name `backend` on the shared proxy + # network too. + set $backend_upstream http://fluksio-api:8000; proxy_pass $backend_upstream$request_uri; proxy_http_version 1.1; From 4a6bfc372918010ca67a929ec777f5d20f7db42a Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:07:39 +0200 Subject: [PATCH 089/103] bench-engine in the Makefile's .PHONY Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c6a1b7c..4477741 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ .PHONY: init env-check dev-utils dev dev-local dev-lan rebuild-frontend up down update install dev-backend dev-frontend \ generate-client sync-example test test-backend test-checks test-stack test-frontend test-webkit soak \ - bench-startup lint lint-backend lint-frontend format-frontend umami build docs docs-serve clean help + bench-startup bench-engine lint lint-backend lint-frontend format-frontend umami build docs docs-serve clean help COMPOSE_ROOT := $(CURDIR) # Explicit project name keeps this stack isolated from the sibling website From 59f069022f44a4100b3bc1c2d9b74261c9360e0c Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:08:09 +0200 Subject: [PATCH 090/103] development.md: only the CI jobs other than test-backend write .env Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- development.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/development.md b/development.md index cac3c93..797e7a5 100644 --- a/development.md +++ b/development.md @@ -184,8 +184,9 @@ contract between the two frontends, and that target is the only thing checking i ## CI -`.gitea/workflows/` runs four jobs, each writing `.env` from `.env.example` first: +`.gitea/workflows/` runs four jobs: `test-backend.yml` (pytest on Python 3.12 and 3.14, plus a check that both wheels build), `playwright.yml` (the e2e suite in two shards, one compose project each, the browser loading the SPA from the nginx `frontend` service), `pre-commit.yml` (the hooks above, plus strict mypy, which they do not cover) and -`test-compose.yml` (a smoke test that the production images actually come up). +`test-compose.yml` (a smoke test that the production images actually come up). All but +`test-backend.yml` write `.env` from `.env.example` first. The backend suite needs none. From 70af80f6f6e44782653592acf7ab39b61cdcbbac Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:10:36 +0200 Subject: [PATCH 091/103] One elapsed-time notation: ago() reads dur's units app-wide, health runs cell widened Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/src/components/Common/ConnectionNotice.tsx | 2 +- frontend/src/components/Flow/EdgeInspector.tsx | 7 ++----- frontend/src/components/Health/HealthActivity.tsx | 5 ++--- frontend/src/components/Health/HealthOverview.tsx | 3 +-- frontend/src/components/Health/queries.ts | 11 ----------- frontend/src/components/Runs/RunDetail.tsx | 3 +-- frontend/src/components/Runs/RunsScreen.tsx | 10 ++-------- frontend/src/lib/utils.ts | 10 ++++++++++ frontend/src/routes/_layout/workers.tsx | 10 ++-------- frontend/src/utils.ts | 2 +- 10 files changed, 22 insertions(+), 41 deletions(-) diff --git a/frontend/src/components/Common/ConnectionNotice.tsx b/frontend/src/components/Common/ConnectionNotice.tsx index f4675f7..03954d2 100644 --- a/frontend/src/components/Common/ConnectionNotice.tsx +++ b/frontend/src/components/Common/ConnectionNotice.tsx @@ -1,9 +1,9 @@ import { useEffect, useSyncExternalStore } from "react" -import { ago } from "@/components/Health/queries" import { connectionStore } from "@/lib/connectionStore" import { dismiss, notify } from "@/lib/notificationStore" import { isPortal } from "@/lib/portal" +import { ago } from "@/lib/utils" /** One card, replaced in place, rather than one per poll that confirms it. */ const ID = "connection-offline" diff --git a/frontend/src/components/Flow/EdgeInspector.tsx b/frontend/src/components/Flow/EdgeInspector.tsx index 02d1a60..6c2ec0f 100644 --- a/frontend/src/components/Flow/EdgeInspector.tsx +++ b/frontend/src/components/Flow/EdgeInspector.tsx @@ -6,6 +6,7 @@ import { Marquee } from "@/components/Common/Marquee" import { Button } from "@/components/ui/button" import { Popover, PopoverAnchor, PopoverContent } from "@/components/ui/popover" import useCustomToast from "@/hooks/useCustomToast" +import { ago } from "@/lib/utils" import { displayName } from "./deriveEdges" import { useLiveValue } from "./liveStore" import { MessageSparkline } from "./MessageSparkline" @@ -13,11 +14,7 @@ import { ValuePreview } from "./ValuePreview" export function relativeTime(ts: number | null | undefined): string { if (!ts) return "not seen yet" - const seconds = Math.max(0, Math.round(Date.now() / 1000 - ts)) - if (seconds < 5) return "just now" - if (seconds < 60) return `${seconds}s ago` - if (seconds < 3600) return `${Math.round(seconds / 60)}m ago` - return `${Math.round(seconds / 3600)}h ago` + return Date.now() / 1000 - ts < 5 ? "just now" : ago(ts) } /** diff --git a/frontend/src/components/Health/HealthActivity.tsx b/frontend/src/components/Health/HealthActivity.tsx index 2b4c0be..6386111 100644 --- a/frontend/src/components/Health/HealthActivity.tsx +++ b/frontend/src/components/Health/HealthActivity.tsx @@ -8,9 +8,8 @@ import { UplotChart } from "@/components/Common/UplotChart" import { useEngineEventMinute } from "@/components/Flow/liveStore" import { PANEL_SECTION } from "@/components/Flow/SidePanel" import { Button } from "@/components/ui/button" -import { cn, dur, si } from "@/lib/utils" +import { ago, cn, dur, si } from "@/lib/utils" import { - ago, auditQueryOptions, CARD, clock, @@ -358,7 +357,7 @@ export function HealthActivity({ range }: { range: Range }) { {dur(run.duration_ms)} - + {ago(run.started_at)}
diff --git a/frontend/src/components/Health/HealthOverview.tsx b/frontend/src/components/Health/HealthOverview.tsx index 61e2b06..81c5571 100644 --- a/frontend/src/components/Health/HealthOverview.tsx +++ b/frontend/src/components/Health/HealthOverview.tsx @@ -5,9 +5,8 @@ import { type Range, RangePicker } from "@/components/Common/RangePicker" import { PANEL_SECTION } from "@/components/Flow/SidePanel" import { runOverviewQueryOptions } from "@/components/Runs/queries" import { Badge } from "@/components/ui/badge" -import { cn, dur, si } from "@/lib/utils" +import { ago, cn, dur, si } from "@/lib/utils" import { - ago, CARD, failuresQueryOptions, flowRollupsQueryOptions, diff --git a/frontend/src/components/Health/queries.ts b/frontend/src/components/Health/queries.ts index fecc76b..81c269b 100644 --- a/frontend/src/components/Health/queries.ts +++ b/frontend/src/components/Health/queries.ts @@ -122,17 +122,6 @@ export const deadLetterQueryOptions = () => ({ refetchInterval: REFRESH, }) -/** How long ago something happened, in the coarsest unit that still says it. */ -export function ago(ts: string | number | null | undefined): string { - if (!ts) return "—" - const stamp = typeof ts === "number" ? ts * 1000 : Date.parse(ts) - const seconds = Math.max(0, (Date.now() - stamp) / 1000) - if (seconds < 90) return `${Math.round(seconds)}s ago` - if (seconds < 5400) return `${Math.round(seconds / 60)}m ago` - if (seconds < 172800) return `${Math.round(seconds / 3600)}h ago` - return `${Math.round(seconds / 86400)}d ago` -} - /** * The minute a timestamp belongs to, as unix seconds. * diff --git a/frontend/src/components/Runs/RunDetail.tsx b/frontend/src/components/Runs/RunDetail.tsx index 6a8b5c4..a26b8b0 100644 --- a/frontend/src/components/Runs/RunDetail.tsx +++ b/frontend/src/components/Runs/RunDetail.tsx @@ -5,7 +5,6 @@ import { useState } from "react" import type { ArtifactRow, RunNodeRow } from "@/client" import { ValuePreview } from "@/components/Flow/ValuePreview" -import { ago } from "@/components/Health/queries" import { Button } from "@/components/ui/button" import { Skeleton } from "@/components/ui/skeleton" import { @@ -17,7 +16,7 @@ import { TableRow, } from "@/components/ui/table" import useCustomToast from "@/hooks/useCustomToast" -import { cn, dur, si } from "@/lib/utils" +import { ago, cn, dur, si } from "@/lib/utils" import { MetricPicker, RunMetricChart, useMetricNames } from "./MetricChart" import { OpenInDashboard } from "./OpenInDashboard" import { diff --git a/frontend/src/components/Runs/RunsScreen.tsx b/frontend/src/components/Runs/RunsScreen.tsx index cdd4c95..18037c6 100644 --- a/frontend/src/components/Runs/RunsScreen.tsx +++ b/frontend/src/components/Runs/RunsScreen.tsx @@ -39,7 +39,7 @@ import { TableHeader, TableRow, } from "@/components/ui/table" -import { cn, dur } from "@/lib/utils" +import { ago, cn, dur } from "@/lib/utils" import { AxisPicker, MetricPicker, @@ -425,12 +425,6 @@ function ParamValue({ value }: { value: unknown }) { ) } -/** How long ago a moment was, in `dur`'s units so a row's two times agree. */ -function since(ts: unknown): string { - if (!ts) return "—" - return `${dur(Date.now() - Date.parse(String(ts)))} ago` -} - function RunsTable({ runs, search, @@ -611,7 +605,7 @@ function RunsTable({ `${dur(run.duration_ms)} ago` ) : ( <> - {since(run.created_at)} + {ago(String(run.created_at ?? ""))} | {run.duration_ms ? dur(run.duration_ms) : "—"} diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 3969708..5608c7d 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -81,3 +81,13 @@ export function dur(ms: number, digits = 3): string { ] return `${Number((rounded / factor).toPrecision(digits))} ${unit}` } + +/** + * How long ago a moment was, in `dur`'s notation: "4.52 min ago". A number is + * unix seconds, a string an ISO timestamp. + */ +export function ago(ts: string | number | null | undefined): string { + if (!ts) return "—" + const stamp = typeof ts === "number" ? ts * 1000 : Date.parse(ts) + return `${dur(Math.max(0, Date.now() - stamp))} ago` +} diff --git a/frontend/src/routes/_layout/workers.tsx b/frontend/src/routes/_layout/workers.tsx index 7abb438..c5a73d1 100644 --- a/frontend/src/routes/_layout/workers.tsx +++ b/frontend/src/routes/_layout/workers.tsx @@ -23,6 +23,7 @@ import { Input } from "@/components/ui/input" import { Label } from "@/components/ui/label" import useAuth from "@/hooks/useAuth" import useCustomToast from "@/hooks/useCustomToast" +import { ago } from "@/lib/utils" import { handleError } from "@/utils" export const Route = createFileRoute("/_layout/workers")({ @@ -40,13 +41,6 @@ const emptyFlavor = { name: "", cpus: 1, gpus: 0, ram: 2048, description: "" } /** Fresh enough to watch a node take a machine, slow enough to be free. */ const REFETCH_MS = 5000 -function ago(seconds: number): string { - const since = Date.now() / 1000 - seconds - if (since < 60) return `${Math.max(0, Math.round(since))}s ago` - if (since < 3600) return `${Math.round(since / 60)}m ago` - return `${Math.round(since / 3600)}h ago` -} - type Level = { total: number; free: number } /** "cpu 2/18" — what is in use, out of what there is. */ @@ -203,7 +197,7 @@ function Attached() { {worker.protocol} - {worker.behind ? "refused" : "seen"} {ago(worker.last_seen ?? 0)} + {worker.behind ? "refused" : "seen"} {ago(worker.last_seen)} diff --git a/frontend/src/utils.ts b/frontend/src/utils.ts index a7eb1eb..eb5f66c 100644 --- a/frontend/src/utils.ts +++ b/frontend/src/utils.ts @@ -1,7 +1,7 @@ import { AxiosError } from "axios" import type { ApiError } from "./client" -import { ago } from "./components/Health/queries" import { offlineDetail } from "./lib/connectionStore" +import { ago } from "./lib/utils" function extractErrorMessage(err: ApiError): string { // An unreachable instance is not a rejected action: say plainly that From a2ac9da32bee982d97a9822ca03c723dba5733fd Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:12:39 +0200 Subject: [PATCH 092/103] Live chart lets a held window go once its points have scrolled off Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/src/components/Common/UplotChart.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/Common/UplotChart.tsx b/frontend/src/components/Common/UplotChart.tsx index ea9aeb2..d4a9c97 100644 --- a/frontend/src/components/Common/UplotChart.tsx +++ b/frontend/src/components/Common/UplotChart.tsx @@ -466,9 +466,17 @@ export function UplotChart({ // is held the data goes in and the scales stay where they were put. Clearing // the window renders, which brings the next pass through here with the reset // back on: that is what puts the whole range back. + // + // A live chart keeps a count of points rather than a span, so a held window + // scrolls off the left as readings arrive. Once its last point has gone it + // is let go, or the chart would sit empty until somebody reset it. useEffect(() => { - if (!chart.current || plots.length === 0) return - chart.current.setData(table(plots), !zoomed.current) + const plot = chart.current + if (!plot || plots.length === 0) return + const data = table(plots) + const held = plot.scales.x.max + if (zoomed.current && held != null && held < data[0][0]) clearZoom() + plot.setData(data, !zoomed.current) }) // The canvas cannot follow a CSS variable, so a theme swap is a redraw. The From 9bd622a5b522b83b0c182832c86015741d014d3a Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:18:12 +0200 Subject: [PATCH 093/103] Assistant answers render as markdown react-markdown, lazy-loaded with the first answer (37 kB gzip chunk, main bundle unchanged). No raw HTML, images dropped, links open in a new tab. The system prompt asks for lists, inline code and fenced code blocks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/ai/agent.py | 4 +- bun.lock | 161 +++++++++++++++++++++ docs/code/agents.md | 5 +- frontend/package.json | 1 + frontend/src/components/Chat/ChatPanel.tsx | 15 +- frontend/src/components/Chat/Markdown.tsx | 57 ++++++++ 6 files changed, 237 insertions(+), 6 deletions(-) create mode 100644 frontend/src/components/Chat/Markdown.tsx diff --git a/backend/fluksio/ai/agent.py b/backend/fluksio/ai/agent.py index 123b9b3..df12bc0 100644 --- a/backend/fluksio/ai/agent.py +++ b/backend/fluksio/ai/agent.py @@ -39,7 +39,9 @@ Publish only when the user asks for the change to go live; otherwise leave \ the draft for review in the editor and say so. - list_node_types gives the node types and their parameters; get_node_source \ shows how a Python node is written before you write one. -- Keep answers short, in plain text without markdown. Name what you changed. +- Keep answers short and name what you changed. Answers render as markdown: \ +use lists, `inline code` and fenced code blocks where they help, and no \ +tables, images or HTML. The flow document `save_flow` takes, as a JSON schema: """ diff --git a/bun.lock b/bun.lock index b80c7dc..f825393 100644 --- a/bun.lock +++ b/bun.lock @@ -50,6 +50,7 @@ "react-grid-layout": "^2.2.4", "react-hook-form": "^7.68.0", "react-icons": "^5.5.0", + "react-markdown": "^10.1.0", "tailwind-merge": "^3.4.0", "tailwindcss": "^4.3.3", "uplot": "^1.6.32", @@ -493,10 +494,20 @@ "@types/d3-zoom": ["@types/d3-zoom@3.0.8", "", { "dependencies": { "@types/d3-interpolate": "*", "@types/d3-selection": "*" } }, "sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw=="], + "@types/debug": ["@types/debug@4.1.13", "", { "dependencies": { "@types/ms": "*" } }, "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw=="], + "@types/estree": ["@types/estree@1.0.9", "", {}, "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg=="], + "@types/estree-jsx": ["@types/estree-jsx@1.0.5", "", { "dependencies": { "@types/estree": "*" } }, "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg=="], + + "@types/hast": ["@types/hast@3.0.5", "", { "dependencies": { "@types/unist": "*" } }, "sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g=="], + "@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="], + "@types/mdast": ["@types/mdast@4.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="], + + "@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="], + "@types/node": ["@types/node@25.0.10", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-zWW5KPngR/yvakJgGOmZ5vTBemDoSqF3AcV/LrO5u5wTWyEAVVh+IT39G4gtyAkh3CtTZs8aX/yRM82OfzHJRg=="], "@types/react": ["@types/react@19.2.9", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-Lpo8kgb/igvMIPeNV2rsYKTgaORYdO1XGVZ4Qz3akwOj0ySGYMPlQWa8BaLn0G63D1aSaAQ5ldR06wCpChQCjA=="], @@ -505,6 +516,10 @@ "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="], + "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], + + "@ungap/structured-clone": ["@ungap/structured-clone@1.4.0", "", {}, "sha512-1mEZtMKPM09vDmQt5y7YvmN2+DFTP7Tg0EWXdic8/C6VRnpb33e4ghisCIE3WZjsE2N8mf+QV1Zqh7ZFYLWInQ=="], + "@vitejs/plugin-react-swc": ["@vitejs/plugin-react-swc@4.2.2", "", { "dependencies": { "@rolldown/pluginutils": "1.0.0-beta.47", "@swc/core": "^1.13.5" }, "peerDependencies": { "vite": "^4 || ^5 || ^6 || ^7" } }, "sha512-x+rE6tsxq/gxrEJN3Nv3dIV60lFflPj94c90b+NNo6n1QV1QQUTLoL0MpaOVasUZ0zqVBn7ead1B5ecx1JAGfA=="], "@xyflow/react": ["@xyflow/react@12.11.3", "", { "dependencies": { "@xyflow/system": "0.0.80", "classcat": "^5.0.3", "zustand": "^4.4.0" }, "peerDependencies": { "@types/react": ">=17", "@types/react-dom": ">=17", "react": ">=17", "react-dom": ">=17" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-G3jogHz2GWUtIOkhavUGno2YzY9u6fILIJBttfsBendb0/HWB90JG+sOTAvlIMEwyvq9zgy9V9ZQSwyQjR5QzQ=="], @@ -529,6 +544,8 @@ "babel-dead-code-elimination": ["babel-dead-code-elimination@1.0.12", "", { "dependencies": { "@babel/core": "^7.23.7", "@babel/parser": "^7.23.6", "@babel/traverse": "^7.23.7", "@babel/types": "^7.23.6" } }, "sha512-GERT7L2TiYcYDtYk1IpD+ASAYXjKbLTDPhBtYj7X1NuRMDTMtAx9kyBenub1Ev41lo91OHCKdmP+egTDmfQ7Ig=="], + "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], + "baseline-browser-mapping": ["baseline-browser-mapping@2.11.27", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-ElY12DaROGuan+lMmZ8Cvo/ZUbXPe7Enc/9VU/b1T3Kp4dwytRcNdR8DoSJN5SNJT/CuvcCA0DHDVmMOCePdRQ=="], "boring-avatars": ["boring-avatars@2.0.4", "", { "peerDependencies": { "react": ">=18.0.0", "react-dom": ">=18.0.0" } }, "sha512-xhZO/w/6aFmRfkaWohcl2NfyIy87gK5SBbys8kctZeTGF1Apjpv/10pfUuv+YEfVPkESU/h2Y6tt/Dwp+bIZPw=="], @@ -543,6 +560,16 @@ "caniuse-lite": ["caniuse-lite@1.0.30001814", "", {}, "sha512-/Uaf1lAzr59XcMpW0o96WoEfr+VXK2OX4U9AgFoiSHsVJ4HppnIFUjtYzsyDH2+tgANaQb2/oxYGwCPapN1FpA=="], + "ccount": ["ccount@2.0.1", "", {}, "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg=="], + + "character-entities": ["character-entities@2.0.2", "", {}, "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ=="], + + "character-entities-html4": ["character-entities-html4@2.1.0", "", {}, "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA=="], + + "character-entities-legacy": ["character-entities-legacy@3.0.0", "", {}, "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ=="], + + "character-reference-invalid": ["character-reference-invalid@2.0.1", "", {}, "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw=="], + "chokidar": ["chokidar@5.0.0", "", { "dependencies": { "readdirp": "^5.0.0" } }, "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw=="], "chownr": ["chownr@2.0.0", "", {}, "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="], @@ -561,6 +588,8 @@ "combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="], + "comma-separated-tokens": ["comma-separated-tokens@2.0.3", "", {}, "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg=="], + "commander": ["commander@13.0.0", "", {}, "sha512-oPYleIY8wmTVzkvQq10AEok6YcTC4sRUBl8F9gVuwchGVUCTbl/vhLTaQqutuuySYOsu8YTgV+OxKc/8Yvx+mQ=="], "confbox": ["confbox@0.1.8", "", {}, "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="], @@ -597,6 +626,8 @@ "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + "decode-named-character-reference": ["decode-named-character-reference@1.3.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q=="], + "default-browser": ["default-browser@5.4.0", "", { "dependencies": { "bundle-name": "^4.1.0", "default-browser-id": "^5.0.0" } }, "sha512-XDuvSq38Hr1MdN47EDvYtx3U0MTqpCEn+F6ft8z2vYDzMrvQhVp0ui9oQdqW3MvK3vqUETglt1tVGgjLuJ5izg=="], "default-browser-id": ["default-browser-id@5.0.1", "", {}, "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q=="], @@ -607,12 +638,16 @@ "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], + "dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="], + "destr": ["destr@2.0.5", "", {}, "sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA=="], "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], "detect-node-es": ["detect-node-es@1.1.0", "", {}, "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="], + "devlop": ["devlop@1.1.0", "", { "dependencies": { "dequal": "^2.0.0" } }, "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA=="], + "diff": ["diff@8.0.4", "", {}, "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw=="], "dompurify": ["dompurify@3.4.16", "", { "optionalDependencies": { "@types/trusted-types": "^2.0.7" } }, "sha512-sqo+pNp3qRhCIpbgRi1y8Tgk27Bo2Ry7w0dC1NBeNTdZChWjz9Xb/KOoZbRP/R6pQZ80Qw8YhXw13hWWBbMRnQ=="], @@ -639,6 +674,10 @@ "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], + "estree-util-is-identifier-name": ["estree-util-is-identifier-name@3.0.0", "", {}, "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg=="], + + "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], + "fast-equals": ["fast-equals@4.0.3", "", {}, "sha512-G3BSX9cfKttjr+2o1O22tYMLq0DPluZnYtq1rXumE1SpL/F/SLIfHx08WYQoWSIpeMYf8sRbJ8++71+v6Pnxfg=="], "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" } }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], @@ -681,12 +720,30 @@ "hasown": ["hasown@2.0.4", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A=="], + "hast-util-to-jsx-runtime": ["hast-util-to-jsx-runtime@2.3.6", "", { "dependencies": { "@types/estree": "^1.0.0", "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "comma-separated-tokens": "^2.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "hast-util-whitespace": "^3.0.0", "mdast-util-mdx-expression": "^2.0.0", "mdast-util-mdx-jsx": "^3.0.0", "mdast-util-mdxjs-esm": "^2.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0", "style-to-js": "^1.0.0", "unist-util-position": "^5.0.0", "vfile-message": "^4.0.0" } }, "sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg=="], + + "hast-util-whitespace": ["hast-util-whitespace@3.0.0", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw=="], + + "html-url-attributes": ["html-url-attributes@3.0.1", "", {}, "sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ=="], + "https-proxy-agent": ["https-proxy-agent@5.0.1", "", { "dependencies": { "agent-base": "6", "debug": "4" } }, "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA=="], + "inline-style-parser": ["inline-style-parser@0.2.7", "", {}, "sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA=="], + + "is-alphabetical": ["is-alphabetical@2.0.1", "", {}, "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ=="], + + "is-alphanumerical": ["is-alphanumerical@2.0.1", "", { "dependencies": { "is-alphabetical": "^2.0.0", "is-decimal": "^2.0.0" } }, "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw=="], + + "is-decimal": ["is-decimal@2.0.1", "", {}, "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A=="], + "is-docker": ["is-docker@3.0.0", "", { "bin": "cli.js" }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="], + "is-hexadecimal": ["is-hexadecimal@2.0.1", "", {}, "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg=="], + "is-inside-container": ["is-inside-container@1.0.0", "", { "dependencies": { "is-docker": "^3.0.0" }, "bin": "cli.js" }, "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA=="], + "is-plain-obj": ["is-plain-obj@4.1.0", "", {}, "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg=="], + "is-wsl": ["is-wsl@3.1.0", "", { "dependencies": { "is-inside-container": "^1.0.0" } }, "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw=="], "isbot": ["isbot@5.1.33", "", {}, "sha512-P4Hgb5NqswjkI0J1CM6XKXon/sxKY1SuowE7Qx2hrBhIwICFyXy54mfgB5eMHXsbe/eStzzpbIGNOvGmz+dlKg=="], @@ -727,6 +784,8 @@ "lodash": ["lodash@4.18.1", "", {}, "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q=="], + "longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="], + "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], "lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], @@ -739,6 +798,66 @@ "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], + "mdast-util-from-markdown": ["mdast-util-from-markdown@2.1.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "mdast-util-to-string": "^4.0.0", "micromark": "^4.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-KRLgmPaP2H1ul2bpoJkOS+9pAcDAAj9Nu1mqLnh8NLeJG5tlvj42TH80PIGBpROWlRXLk0lDrWNuZCFy/tsB4A=="], + + "mdast-util-mdx-expression": ["mdast-util-mdx-expression@2.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ=="], + + "mdast-util-mdx-jsx": ["mdast-util-mdx-jsx@3.2.0", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "devlop": "^1.1.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0", "parse-entities": "^4.0.0", "stringify-entities": "^4.0.0", "unist-util-stringify-position": "^4.0.0", "vfile-message": "^4.0.0" } }, "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q=="], + + "mdast-util-mdxjs-esm": ["mdast-util-mdxjs-esm@2.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg=="], + + "mdast-util-phrasing": ["mdast-util-phrasing@4.1.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "unist-util-is": "^6.0.0" } }, "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w=="], + + "mdast-util-to-hast": ["mdast-util-to-hast@13.2.1", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "@ungap/structured-clone": "^1.0.0", "devlop": "^1.0.0", "micromark-util-sanitize-uri": "^2.0.0", "trim-lines": "^3.0.0", "unist-util-position": "^5.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA=="], + + "mdast-util-to-markdown": ["mdast-util-to-markdown@2.2.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-character": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-2Jn/ADrHFFyyniBMi0bncTbJqU78K3/TEYvzvwrWWzD9MYPUa/Gm4r3nOeKFQg/PDM5BsbDYdc8ackW4NTY6NA=="], + + "mdast-util-to-string": ["mdast-util-to-string@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="], + + "micromark": ["micromark@4.0.3", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-edit-map": "^1.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-oGYfQzHSG5dOMovQcJ3fyTmZlWAWpi0XA0sJwJs+i6OT88o1+Jtw/8z0CmdowSLxGhhFK89rQ/oXph/wN02PNw=="], + + "micromark-core-commonmark": ["micromark-core-commonmark@2.0.4", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.1.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-edit-map": "^1.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.3" } }, "sha512-wxEeE8v8XVvOrxn1TZj74qYhAtTQsSqufHVS3uNVyT1MupXxhyaHk8/9xDNPh9BmL77QNDgWTkZ3RsYHB2ry7w=="], + + "micromark-factory-destination": ["micromark-factory-destination@2.0.1", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA=="], + + "micromark-factory-label": ["micromark-factory-label@2.0.1", "", { "dependencies": { "devlop": "^1.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg=="], + + "micromark-factory-space": ["micromark-factory-space@2.1.0", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-fS8hnLIjnjvdQIj39Geug8wWsR0HrYZ43KShKxNfwT7t2LOHo/LbWZEzEaSOjwtjdCsjoE6syHhonEzW0zv0+Q=="], + + "micromark-factory-title": ["micromark-factory-title@2.0.1", "", { "dependencies": { "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw=="], + + "micromark-factory-whitespace": ["micromark-factory-whitespace@2.0.1", "", { "dependencies": { "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ=="], + + "micromark-util-character": ["micromark-util-character@2.1.1", "", { "dependencies": { "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q=="], + + "micromark-util-chunked": ["micromark-util-chunked@2.0.1", "", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA=="], + + "micromark-util-classify-character": ["micromark-util-classify-character@2.0.1", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q=="], + + "micromark-util-combine-extensions": ["micromark-util-combine-extensions@2.0.1", "", { "dependencies": { "micromark-util-chunked": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg=="], + + "micromark-util-decode-numeric-character-reference": ["micromark-util-decode-numeric-character-reference@2.0.2", "", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw=="], + + "micromark-util-decode-string": ["micromark-util-decode-string@2.0.1", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "micromark-util-character": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-symbol": "^2.0.0" } }, "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ=="], + + "micromark-util-edit-map": ["micromark-util-edit-map@1.0.0", "", { "dependencies": { "micromark-util-types": "^2.0.0" } }, "sha512-Pa2ljlsEL6sVwFaYeyrOLSYbQt73JvGbPOYFq+9AElXiSnfI5Q4465RRZ1sHv7lDjDOnGRDDaqPXqClqWK/q1Q=="], + + "micromark-util-encode": ["micromark-util-encode@2.0.1", "", {}, "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw=="], + + "micromark-util-html-tag-name": ["micromark-util-html-tag-name@2.0.1", "", {}, "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA=="], + + "micromark-util-normalize-identifier": ["micromark-util-normalize-identifier@2.0.1", "", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q=="], + + "micromark-util-resolve-all": ["micromark-util-resolve-all@2.0.1", "", { "dependencies": { "micromark-util-types": "^2.0.0" } }, "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg=="], + + "micromark-util-sanitize-uri": ["micromark-util-sanitize-uri@2.0.1", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-symbol": "^2.0.0" } }, "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ=="], + + "micromark-util-subtokenize": ["micromark-util-subtokenize@2.1.0", "", { "dependencies": { "devlop": "^1.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA=="], + + "micromark-util-symbol": ["micromark-util-symbol@2.0.1", "", {}, "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q=="], + + "micromark-util-types": ["micromark-util-types@2.0.3", "", {}, "sha512-oxB2Ik03hI0gv+VNn9tnh1t1YEe9MDPptViAEgfdf3YQHsn0pzGTgCdlSCJXcwhqm8phaEuM7zeEu3QQzVBrPg=="], + "mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], "mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], @@ -779,6 +898,8 @@ "open": ["open@10.1.2", "", { "dependencies": { "default-browser": "^5.2.1", "define-lazy-prop": "^3.0.0", "is-inside-container": "^1.0.0", "is-wsl": "^3.1.0" } }, "sha512-cxN6aIDPz6rm8hbebcP7vrQNhvRcveZoJU72Y7vskh4oIm+BZwBECnx5nTmrlres1Qapvx27Qo1Auukpf8PKXw=="], + "parse-entities": ["parse-entities@4.0.2", "", { "dependencies": { "@types/unist": "^2.0.0", "character-entities-legacy": "^3.0.0", "character-reference-invalid": "^2.0.0", "decode-named-character-reference": "^1.0.0", "is-alphanumerical": "^2.0.0", "is-decimal": "^2.0.0", "is-hexadecimal": "^2.0.0" } }, "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw=="], + "pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="], "perfect-debounce": ["perfect-debounce@1.0.0", "", {}, "sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA=="], @@ -799,6 +920,8 @@ "prop-types": ["prop-types@15.8.1", "", { "dependencies": { "loose-envify": "^1.4.0", "object-assign": "^4.1.1", "react-is": "^16.13.1" } }, "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg=="], + "property-information": ["property-information@7.2.0", "", {}, "sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg=="], + "proxy-from-env": ["proxy-from-env@2.1.0", "", {}, "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA=="], "radix-ui": ["radix-ui@1.6.7", "", { "dependencies": { "@radix-ui/primitive": "1.1.7", "@radix-ui/react-accessible-icon": "1.1.15", "@radix-ui/react-accordion": "1.2.20", "@radix-ui/react-alert-dialog": "1.1.23", "@radix-ui/react-arrow": "1.1.15", "@radix-ui/react-aspect-ratio": "1.1.15", "@radix-ui/react-avatar": "1.2.6", "@radix-ui/react-checkbox": "1.3.11", "@radix-ui/react-collapsible": "1.1.20", "@radix-ui/react-collection": "1.1.15", "@radix-ui/react-compose-refs": "1.1.5", "@radix-ui/react-context": "1.2.2", "@radix-ui/react-context-menu": "2.3.7", "@radix-ui/react-dialog": "1.1.23", "@radix-ui/react-direction": "1.1.4", "@radix-ui/react-dismissable-layer": "1.1.19", "@radix-ui/react-dropdown-menu": "2.1.24", "@radix-ui/react-focus-guards": "1.1.6", "@radix-ui/react-focus-scope": "1.1.16", "@radix-ui/react-form": "0.1.16", "@radix-ui/react-hover-card": "1.1.23", "@radix-ui/react-label": "2.1.15", "@radix-ui/react-menu": "2.1.24", "@radix-ui/react-menubar": "1.1.24", "@radix-ui/react-navigation-menu": "1.2.22", "@radix-ui/react-one-time-password-field": "0.1.16", "@radix-ui/react-password-toggle-field": "0.1.11", "@radix-ui/react-popover": "1.1.23", "@radix-ui/react-popper": "1.3.7", "@radix-ui/react-portal": "1.1.17", "@radix-ui/react-presence": "1.1.10", "@radix-ui/react-primitive": "2.1.10", "@radix-ui/react-progress": "1.1.16", "@radix-ui/react-radio-group": "1.4.7", "@radix-ui/react-roving-focus": "1.1.19", "@radix-ui/react-scroll-area": "1.2.18", "@radix-ui/react-select": "2.3.7", "@radix-ui/react-separator": "1.1.15", "@radix-ui/react-slider": "1.4.7", "@radix-ui/react-slot": "1.3.3", "@radix-ui/react-switch": "1.3.7", "@radix-ui/react-tabs": "1.1.21", "@radix-ui/react-toast": "1.2.23", "@radix-ui/react-toggle": "1.1.18", "@radix-ui/react-toggle-group": "1.1.19", "@radix-ui/react-toolbar": "1.1.19", "@radix-ui/react-tooltip": "1.2.16", "@radix-ui/react-use-callback-ref": "1.1.4", "@radix-ui/react-use-controllable-state": "1.2.6", "@radix-ui/react-use-effect-event": "0.0.5", "@radix-ui/react-use-escape-keydown": "1.1.5", "@radix-ui/react-use-is-hydrated": "0.1.3", "@radix-ui/react-use-layout-effect": "1.1.4", "@radix-ui/react-use-size": "1.1.4", "@radix-ui/react-visually-hidden": "1.2.11" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-QBdhh1arIEUvPC0dQ5+nwWAxt7+N+oP/9jPwjJkGFoSk/sqxg32gJtSXGtFh8frAIcS6oC9cx2Q+7KYCQLOAeA=="], @@ -821,6 +944,8 @@ "react-is": ["react-is@16.13.1", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="], + "react-markdown": ["react-markdown@10.1.0", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "hast-util-to-jsx-runtime": "^2.0.0", "html-url-attributes": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "unified": "^11.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" }, "peerDependencies": { "@types/react": ">=18", "react": ">=18" } }, "sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ=="], + "react-remove-scroll": ["react-remove-scroll@2.7.2", "", { "dependencies": { "react-remove-scroll-bar": "^2.3.7", "react-style-singleton": "^2.2.3", "tslib": "^2.1.0", "use-callback-ref": "^1.3.3", "use-sidecar": "^1.1.3" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" } }, "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q=="], "react-remove-scroll-bar": ["react-remove-scroll-bar@2.3.8", "", { "dependencies": { "react-style-singleton": "^2.2.2", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q=="], @@ -831,6 +956,10 @@ "readdirp": ["readdirp@5.1.1", "", {}, "sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA=="], + "remark-parse": ["remark-parse@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-from-markdown": "^2.0.0", "micromark-util-types": "^2.0.0", "unified": "^11.0.0" } }, "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA=="], + + "remark-rehype": ["remark-rehype@11.1.2", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "mdast-util-to-hast": "^13.0.0", "unified": "^11.0.0", "vfile": "^6.0.0" } }, "sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw=="], + "resize-observer-polyfill": ["resize-observer-polyfill@1.5.1", "", {}, "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg=="], "rollup": ["rollup@4.64.0", "", { "dependencies": { "@types/estree": "1.0.9" }, "optionalDependencies": { "@napi-rs/lzma-linux-x64-gnu": "1.5.1", "@rollup/rollup-android-arm-eabi": "4.64.0", "@rollup/rollup-android-arm64": "4.64.0", "@rollup/rollup-darwin-arm64": "4.64.0", "@rollup/rollup-darwin-x64": "4.64.0", "@rollup/rollup-freebsd-arm64": "4.64.0", "@rollup/rollup-freebsd-x64": "4.64.0", "@rollup/rollup-linux-arm-gnueabihf": "4.64.0", "@rollup/rollup-linux-arm-musleabihf": "4.64.0", "@rollup/rollup-linux-arm64-gnu": "4.64.0", "@rollup/rollup-linux-arm64-musl": "4.64.0", "@rollup/rollup-linux-loong64-gnu": "4.64.0", "@rollup/rollup-linux-loong64-musl": "4.64.0", "@rollup/rollup-linux-ppc64-gnu": "4.64.0", "@rollup/rollup-linux-ppc64-musl": "4.64.0", "@rollup/rollup-linux-riscv64-gnu": "4.64.0", "@rollup/rollup-linux-riscv64-musl": "4.64.0", "@rollup/rollup-linux-s390x-gnu": "4.64.0", "@rollup/rollup-linux-x64-gnu": "4.64.0", "@rollup/rollup-linux-x64-musl": "4.64.0", "@rollup/rollup-openbsd-x64": "4.64.0", "@rollup/rollup-openharmony-arm64": "4.64.0", "@rollup/rollup-win32-arm64-msvc": "4.64.0", "@rollup/rollup-win32-ia32-msvc": "4.64.0", "@rollup/rollup-win32-x64-gnu": "4.64.0", "@rollup/rollup-win32-x64-msvc": "4.64.0", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-gjA/kFeDfffII8COoPsvLPvE16hWDTEc1+0QKQNKUkuV8iJjqsC4xXs8/FzuL0v9EV39kuxTbAgcFoTRyy2TGQ=="], @@ -849,8 +978,16 @@ "source-map-js": ["source-map-js@1.2.2", "", {}, "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw=="], + "space-separated-tokens": ["space-separated-tokens@2.0.2", "", {}, "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q=="], + "state-local": ["state-local@1.0.7", "", {}, "sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w=="], + "stringify-entities": ["stringify-entities@4.0.4", "", { "dependencies": { "character-entities-html4": "^2.0.0", "character-entities-legacy": "^3.0.0" } }, "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg=="], + + "style-to-js": ["style-to-js@1.1.21", "", { "dependencies": { "style-to-object": "1.0.14" } }, "sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ=="], + + "style-to-object": ["style-to-object@1.0.14", "", { "dependencies": { "inline-style-parser": "0.2.7" } }, "sha512-LIN7rULI0jBscWQYaSswptyderlarFkjQ+t79nzty8tcIAceVomEVlLzH5VP4Cmsv6MtKhs7qaAiwlcp+Mgaxw=="], + "tailwind-merge": ["tailwind-merge@3.4.0", "", {}, "sha512-uSaO4gnW+b3Y2aWoWfFpX62vn2sR3skfhbjsEnaBI81WD1wBLlHZe5sWf0AqjksNdYTbGBEd0UasQMT3SNV15g=="], "tailwindcss": ["tailwindcss@4.3.3", "", {}, "sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ=="], @@ -863,6 +1000,10 @@ "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], + "trim-lines": ["trim-lines@3.0.1", "", {}, "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg=="], + + "trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="], + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], "tw-animate-css": ["tw-animate-css@1.4.0", "", {}, "sha512-7bziOlRqH0hJx80h/3mbicLW7o8qLsH5+RaLR2t+OHM3D0JlWGODQKQ4cxbK7WlvmUxpcj6Kgu6EKqjrGFe3QQ=="], @@ -875,6 +1016,18 @@ "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], + "unified": ["unified@11.0.5", "", { "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", "devlop": "^1.0.0", "extend": "^3.0.0", "is-plain-obj": "^4.0.0", "trough": "^2.0.0", "vfile": "^6.0.0" } }, "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA=="], + + "unist-util-is": ["unist-util-is@6.0.1", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g=="], + + "unist-util-position": ["unist-util-position@5.0.0", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA=="], + + "unist-util-stringify-position": ["unist-util-stringify-position@4.0.0", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ=="], + + "unist-util-visit": ["unist-util-visit@5.1.0", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0", "unist-util-visit-parents": "^6.0.0" } }, "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg=="], + + "unist-util-visit-parents": ["unist-util-visit-parents@6.0.2", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0" } }, "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ=="], + "unplugin": ["unplugin@3.4.0", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "picomatch": "^4.0.7", "webpack-virtual-modules": "^0.6.2" }, "peerDependencies": { "@farmfe/core": "*", "@rsbuild/core": "*", "@rspack/core": "*", "bun-types-no-globals": "*", "esbuild": "*", "rolldown": "*", "rollup": "*", "unloader": "*", "vite": "*", "webpack": "*" }, "optionalPeers": ["@farmfe/core", "@rsbuild/core", "@rspack/core", "bun-types-no-globals", "esbuild", "rolldown", "rollup", "unloader", "vite", "webpack"] }, "sha512-9skdIFlCsPdFV7wUfZxNsFInlW+7nJmGu2gkTu0OUhF56aXGsHab9x52/QhdJ4lC7ZDPWTxbiC4ANqVlsuaW3w=="], "update-browserslist-db": ["update-browserslist-db@1.3.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ=="], @@ -887,6 +1040,10 @@ "use-sync-external-store": ["use-sync-external-store@1.6.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w=="], + "vfile": ["vfile@6.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "vfile-message": "^4.0.0" } }, "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q=="], + + "vfile-message": ["vfile-message@4.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw=="], + "vite": ["vite@7.3.7", "", { "dependencies": { "esbuild": "^0.27.0 || ^0.28.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-AHEzTORMbi7/7aXp7w7KZ5+DCUdSeLsJz0grY8xHpc9DHdYb0vQu6DqJfm2tOw3gDExNgPgnJTXxM6syYFHaCA=="], "webpack-virtual-modules": ["webpack-virtual-modules@0.6.2", "", {}, "sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ=="], @@ -899,6 +1056,8 @@ "zustand": ["zustand@4.5.7", "", { "dependencies": { "use-sync-external-store": "^1.2.2" }, "peerDependencies": { "@types/react": ">=16.8", "immer": ">=9.0.6", "react": ">=16.8" }, "optionalPeers": ["@types/react", "immer", "react"] }, "sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw=="], + "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], + "@tailwindcss/node/jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.11.3", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.3", "tslib": "^2.4.0" }, "bundled": true }, "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg=="], @@ -947,6 +1106,8 @@ "nypm/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + "parse-entities/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="], + "pkg-types/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], "playwright/fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], diff --git a/docs/code/agents.md b/docs/code/agents.md index 69080ae..0479858 100644 --- a/docs/code/agents.md +++ b/docs/code/agents.md @@ -54,8 +54,9 @@ A superuser configures an OpenAI-compatible provider under the dashboard's Assistant entry for signed-in users. The assistant uses the same tools with your own permissions. Draft edits -appear in the editor, and each answer includes a tool trace. Conversations -are stored per user on the instance; the bin button clears yours. +appear in the editor, and each answer renders as markdown with a tool trace. +Conversations are stored per user on the instance; the bin button clears +yours. `MCP_ENABLED` is only needed for external clients, so the built-in assistant works without it. diff --git a/frontend/package.json b/frontend/package.json index c85b5ae..791a0cb 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -57,6 +57,7 @@ "react-grid-layout": "^2.2.4", "react-hook-form": "^7.68.0", "react-icons": "^5.5.0", + "react-markdown": "^10.1.0", "tailwind-merge": "^3.4.0", "tailwindcss": "^4.3.3", "uplot": "^1.6.32", diff --git a/frontend/src/components/Chat/ChatPanel.tsx b/frontend/src/components/Chat/ChatPanel.tsx index 1932482..b2b0821 100644 --- a/frontend/src/components/Chat/ChatPanel.tsx +++ b/frontend/src/components/Chat/ChatPanel.tsx @@ -1,7 +1,7 @@ import { queryOptions, useQuery, useQueryClient } from "@tanstack/react-query" import { useRouterState } from "@tanstack/react-router" import { Bird, Bot, MessageCircle, Send, Square, Trash2 } from "lucide-react" -import { useEffect, useRef, useState } from "react" +import { lazy, Suspense, useEffect, useRef, useState } from "react" import { AiService, type ChatItem } from "@/client" import { flowKeys } from "@/components/Flow/queries" @@ -23,6 +23,9 @@ import { type ChatTab, chatStore, useChat } from "./chatStore" import { type ChatEvent, streamTurn } from "./stream" import { type Action, type ControlItem, useControl } from "./useControl" +/** The renderer arrives with the first answer, not with the app. */ +const Markdown = lazy(() => import("./Markdown")) + export const aiStatusQueryOptions = () => queryOptions({ queryKey: ["ai", "status"], @@ -382,8 +385,14 @@ function Message({ item }: { item: Item }) { ) case "assistant": return ( -
- {item.text} +
+ {item.text}
+ } + > + +
) case "error": diff --git a/frontend/src/components/Chat/Markdown.tsx b/frontend/src/components/Chat/Markdown.tsx new file mode 100644 index 0000000..b12783d --- /dev/null +++ b/frontend/src/components/Chat/Markdown.tsx @@ -0,0 +1,57 @@ +import ReactMarkdown, { type Components } from "react-markdown" + +/** + * An assistant's answer as markdown, compact and on the panel's text size. + * + * Raw HTML is never rendered, and images are dropped so an answer cannot make + * the browser fetch a URL on its own. A link opens in a new tab when clicked. + */ +const components: Components = { + a: ({ node: _, ...props }) => ( + + ), + ul: ({ node: _, ...props }) => ( +
    + ), + ol: ({ node: _, ...props }) => ( +
      + ), + h1: ({ node: _, ...props }) =>

      , + h2: ({ node: _, ...props }) =>

      , + h3: ({ node: _, ...props }) =>

      , + h4: ({ node: _, ...props }) =>

      , + blockquote: ({ node: _, ...props }) => ( +
      + ), + hr: () =>
      , + pre: ({ node: _, ...props }) => ( +
      +  ),
      +  code: ({ node: _, ...props }) => (
      +    
      +  ),
      +}
      +
      +export default function Markdown({ text }: { text: string }) {
      +  return (
      +    
      + + {text} + +
      + ) +} From 429100a0c7178de12530159ffd531ba77afb4cc1 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:27:17 +0200 Subject: [PATCH 094/103] Vendor stylesheets load in Tailwind's base layer through a vendor.css shim, so our overrides win outright; the editor's drag placeholder takes the 12% primary Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/src/components/Common/UplotChart.tsx | 7 +++---- .../src/components/Dashboard/DashboardEditor.tsx | 1 - frontend/src/components/Flow/BrainView.tsx | 1 - frontend/src/components/Flow/FlowEditor.tsx | 15 +++++++-------- frontend/src/index.css | 1 + frontend/src/vendor.css | 10 ++++++++++ 6 files changed, 21 insertions(+), 14 deletions(-) create mode 100644 frontend/src/vendor.css diff --git a/frontend/src/components/Common/UplotChart.tsx b/frontend/src/components/Common/UplotChart.tsx index d4a9c97..6be1676 100644 --- a/frontend/src/components/Common/UplotChart.tsx +++ b/frontend/src/components/Common/UplotChart.tsx @@ -1,10 +1,9 @@ import { useEffect, useLayoutEffect, useRef, useState } from "react" import uPlot from "uplot" -import "uplot/dist/uPlot.min.css" -// After uPlot's own sheet, and beside this component rather than in the -// dashboard chunk: a chart on Health or Home is styled without a dashboard -// having been visited first. +// Beside this component rather than in the dashboard chunk: a chart on Health +// or Home is styled without a dashboard having been visited first. uPlot's own +// sheet comes in through `vendor.css`. import "./uplot.css" import type { HistoryPoint } from "@/client" diff --git a/frontend/src/components/Dashboard/DashboardEditor.tsx b/frontend/src/components/Dashboard/DashboardEditor.tsx index 28a3d17..58604eb 100644 --- a/frontend/src/components/Dashboard/DashboardEditor.tsx +++ b/frontend/src/components/Dashboard/DashboardEditor.tsx @@ -9,7 +9,6 @@ import { type Position, setTransform, } from "react-grid-layout" -import "react-grid-layout/css/styles.css" import { type ApiError, diff --git a/frontend/src/components/Flow/BrainView.tsx b/frontend/src/components/Flow/BrainView.tsx index 31c61c0..8a7b7db 100644 --- a/frontend/src/components/Flow/BrainView.tsx +++ b/frontend/src/components/Flow/BrainView.tsx @@ -9,7 +9,6 @@ import { ReactFlowProvider, useReactFlow, } from "@xyflow/react" -import "@xyflow/react/dist/style.css" import { forceCollide, forceLink, diff --git a/frontend/src/components/Flow/FlowEditor.tsx b/frontend/src/components/Flow/FlowEditor.tsx index 42a4bf6..8ae0f69 100644 --- a/frontend/src/components/Flow/FlowEditor.tsx +++ b/frontend/src/components/Flow/FlowEditor.tsx @@ -1,3 +1,10 @@ +import { + useMutation, + useQuery, + useQueryClient, + useSuspenseQuery, +} from "@tanstack/react-query" +import { useNavigate } from "@tanstack/react-router" import { Background, BackgroundVariant, @@ -13,14 +20,6 @@ import { useStore, useUpdateNodeInternals, } from "@xyflow/react" -import "@xyflow/react/dist/style.css" -import { - useMutation, - useQuery, - useQueryClient, - useSuspenseQuery, -} from "@tanstack/react-query" -import { useNavigate } from "@tanstack/react-router" import { Workflow } from "lucide-react" import { motion } from "motion/react" import { useCallback, useEffect, useMemo, useRef, useState } from "react" diff --git a/frontend/src/index.css b/frontend/src/index.css index be747f8..4d5b59a 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -1,5 +1,6 @@ @import "tailwindcss"; @import "tw-animate-css"; +@import "./vendor.css"; @custom-variant dark (&:is(.dark *)); diff --git a/frontend/src/vendor.css b/frontend/src/vendor.css new file mode 100644 index 0000000..ecd6c4d --- /dev/null +++ b/frontend/src/vendor.css @@ -0,0 +1,10 @@ +/* + * The libraries' own stylesheets, in Tailwind's `base` layer. + * + * Imported from TS they were unlayered, so our overrides beat them only by + * loading later, and lost wherever a lazy chunk brought the library's sheet in + * after ours. In `base` every sheet of ours and every utility wins over them. + */ +@import "@xyflow/react/dist/style.css" layer(base); +@import "uplot/dist/uPlot.min.css" layer(base); +@import "react-grid-layout/css/styles.css" layer(base); From 58b92da8b0e2d915e7de1bcfceceae8a6db39ac5 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:37:56 +0200 Subject: [PATCH 095/103] Assistant streams its answer token by token ai/client.py streams the completion and puts the chunks back together (Assembler), covering OpenAI's piecewise tool-call arguments, Ollama's whole calls, OpenRouter's comment lines, unnumbered calls and in-stream errors. The turn emits a delta event per piece and a message per round as before, and the chat grows the last answer as deltas arrive. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/ai/agent.py | 16 ++- backend/fluksio/ai/client.py | 100 +++++++++++++++-- backend/fluksio/api/routes/ai.py | 5 +- backend/tests/test_ai.py | 124 ++++++++++++++++++++- docs/code/agents.md | 3 +- frontend/src/client/sdk.gen.ts | 5 +- frontend/src/components/Chat/ChatPanel.tsx | 21 +++- frontend/src/components/Chat/stream.ts | 1 + 8 files changed, 243 insertions(+), 32 deletions(-) diff --git a/backend/fluksio/ai/agent.py b/backend/fluksio/ai/agent.py index df12bc0..14ad091 100644 --- a/backend/fluksio/ai/agent.py +++ b/backend/fluksio/ai/agent.py @@ -119,8 +119,10 @@ async def run_turn( ) -> AsyncIterator[dict[str, Any]]: """One user turn, yielding progress events as it goes. - A round is appended to *messages* only once every tool in it has answered, - so a turn cut off part-way leaves a transcript the provider still accepts. + A round's text arrives as ``delta`` events while the model writes it, and + once more whole as ``message`` when the round is done. A round is appended + to *messages* only once every tool in it has answered, so a turn cut off + part-way leaves a transcript the provider still accepts. """ messages.append({"role": "user", "content": text}) tools = await tool_specs() @@ -128,18 +130,22 @@ async def run_turn( token = caller_authorization.set(authorization) try: for _ in range(config.max_rounds): + choice: dict[str, Any] = {} try: - response = await client.chat_completion( + async for part in client.stream_completion( api_base=config.api_base, api_key=config.api_key, model=config.model, messages=[system, *messages], tools=tools, - ) + ): + if isinstance(part, str): + yield {"type": "delta", "content": part} + else: + choice = part except (client.ProviderError, httpx.HTTPError) as exc: yield {"type": "error", "detail": f"The provider answered {exc}"} return - choice = response["choices"][0]["message"] content = choice.get("content") or "" calls = choice.get("tool_calls") or [] assistant: dict[str, Any] = {"role": "assistant", "content": content} diff --git a/backend/fluksio/ai/client.py b/backend/fluksio/ai/client.py index fd19e75..d9607a6 100644 --- a/backend/fluksio/ai/client.py +++ b/backend/fluksio/ai/client.py @@ -4,10 +4,14 @@ Chat completions with function tools is the dialect every hosted provider and every local server speaks — OpenAI, Anthropic's compatibility endpoint, OpenRouter, Ollama, vLLM — so one client covers them without a provider SDK. The admin types the URL; the model never chooses where a request goes. + +The call is streamed, so the answer reaches the browser as it is written. """ from __future__ import annotations +import json +from collections.abc import AsyncIterator from typing import Any from urllib.parse import urlparse @@ -39,7 +43,69 @@ def _detail(response: httpx.Response) -> str: return f"{response.status_code}: {text[:300]}" -async def chat_completion( +class Assembler: + """The streamed chunks put back together into the message a plain call returns. + + The providers stream the same ``data:`` lines and differ only in the + margins: OpenRouter interleaves ``:`` comment lines, a thinking model's + reasoning arrives in a field of its own and is left out, OpenAI and + Anthropic send a tool call's arguments in pieces where Ollama sends the call + whole, and some leave out the call's ``index``. + """ + + def __init__(self) -> None: + self.content = "" + self.calls: list[dict[str, Any]] = [] + + def feed(self, line: str) -> str: + """One line of the stream; returns the text it adds to the answer.""" + if not line.startswith("data:"): + return "" + data = line[5:].strip() + if not data or data == "[DONE]": + return "" + try: + chunk = json.loads(data) + except ValueError as exc: + raise ProviderError(f"sent {data[:300]}") from exc + error = chunk.get("error") + if error: + if isinstance(error, dict): + error = error.get("message", error) + raise ProviderError(str(error)[:300]) + choices = chunk.get("choices") or [{}] + delta = choices[0].get("delta") or {} + for piece in delta.get("tool_calls") or []: + self._merge(piece) + text: str = delta.get("content") or "" + self.content += text + return text + + def _merge(self, piece: dict[str, Any]) -> None: + index = piece.get("index", len(self.calls)) + while len(self.calls) <= index: + self.calls.append( + { + "id": "", + "type": "function", + "function": {"name": "", "arguments": ""}, + } + ) + call = self.calls[index] + call["id"] = piece.get("id") or call["id"] + function = piece.get("function") or {} + call["function"]["name"] = function.get("name") or call["function"]["name"] + call["function"]["arguments"] += function.get("arguments") or "" + + def message(self) -> dict[str, Any]: + return { + "role": "assistant", + "content": self.content or None, + "tool_calls": self.calls or None, + } + + +async def stream_completion( *, api_base: str, api_key: str, @@ -47,21 +113,33 @@ async def chat_completion( messages: list[dict[str, Any]], tools: list[dict[str, Any]], timeout: float = 120.0, -) -> dict[str, Any]: - """``POST {api_base}/chat/completions``, the whole answer at once.""" - body: dict[str, Any] = {"model": model, "messages": messages} +) -> AsyncIterator[str | dict[str, Any]]: + """``POST {api_base}/chat/completions`` streamed. + + Yields each piece of the answer's text as it arrives, then the whole + message. The timeout is per chunk, so a long answer is not cut off. + """ + body: dict[str, Any] = {"model": model, "messages": messages, "stream": True} if tools: body["tools"] = tools - async with httpx.AsyncClient(timeout=timeout) as http: - response = await http.post( + assembler = Assembler() + async with ( + httpx.AsyncClient(timeout=timeout) as http, + http.stream( + "POST", api_base.rstrip("/") + "/chat/completions", json=body, headers=_headers(api_key), - ) - if response.is_error: - raise ProviderError(_detail(response)) - result: dict[str, Any] = response.json() - return result + ) as response, + ): + if response.is_error: + await response.aread() + raise ProviderError(_detail(response)) + async for line in response.aiter_lines(): + text = assembler.feed(line) + if text: + yield text + yield assembler.message() async def list_models(api_base: str, api_key: str, timeout: float = 15.0) -> list[str]: diff --git a/backend/fluksio/api/routes/ai.py b/backend/fluksio/api/routes/ai.py index 8c93fd7..0cae07b 100644 --- a/backend/fluksio/api/routes/ai.py +++ b/backend/fluksio/api/routes/ai.py @@ -259,8 +259,9 @@ def chat( ) -> StreamingResponse: """One turn, streamed as server-sent events. - ``message``, ``tool_call`` and ``tool_result`` events as the model works, - ``error`` when something stops it, and ``done`` last either way. + ``delta`` as the model writes, ``message`` with a round's whole text, + ``tool_call`` and ``tool_result`` as it works, ``error`` when something + stops it, and ``done`` last either way. """ config = ai_config.load() if config is None or not config.ready: diff --git a/backend/tests/test_ai.py b/backend/tests/test_ai.py index 430df41..a13f63e 100644 --- a/backend/tests/test_ai.py +++ b/backend/tests/test_ai.py @@ -1,14 +1,14 @@ """The assistant: who may set it up, and that it acts as the person chatting.""" import json -from collections.abc import Iterator +from collections.abc import AsyncIterator, Iterator from pathlib import Path from typing import Any import pytest from fastapi.testclient import TestClient -from fluksio.ai import agent +from fluksio.ai import agent, client from fluksio.core.config import settings @@ -94,12 +94,16 @@ def test_a_turn_runs_the_tools_as_the_person_chatting( ] ) - async def fake_completion(**kwargs: Any) -> dict[str, Any]: + async def fake_completion(**kwargs: Any) -> AsyncIterator[str | dict[str, Any]]: seen.append(kwargs["messages"]) assert any(t["function"]["name"] == "save_flow" for t in kwargs["tools"]) - return {"choices": [{"message": next(answers)}]} + answer = next(answers) + if answer["content"]: + yield "Counted " + yield "them." + yield answer - monkeypatch.setattr(agent.client, "chat_completion", fake_completion) + monkeypatch.setattr(agent.client, "stream_completion", fake_completion) with client.stream( "POST", @@ -114,6 +118,8 @@ def test_a_turn_runs_the_tools_as_the_person_chatting( "start", "tool_call", "tool_result", + "delta", + "delta", "message", "done", ] @@ -136,3 +142,111 @@ def test_a_turn_runs_the_tools_as_the_person_chatting( } client.delete("/api/v1/ai/chat", headers=superuser_token_headers) assert client.get("/api/v1/ai/chat", headers=superuser_token_headers).json() == [] + + +# One answer per provider as it streams it. Ollama's text is recorded from a +# local server, the rest follows the shape each endpoint documents. +STREAMS = { + "ollama": """ +data: {"id":"chatcmpl-888","object":"chat.completion.chunk","model":"gemma3:4b-it-qat","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]} + +data: {"id":"chatcmpl-888","object":"chat.completion.chunk","model":"gemma3:4b-it-qat","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":" there"},"finish_reason":null}]} + +data: {"id":"chatcmpl-888","object":"chat.completion.chunk","model":"gemma3:4b-it-qat","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop"}]} + +data: [DONE] +""", + "ollama-tools": """ +data: {"id":"chatcmpl-12","object":"chat.completion.chunk","model":"qwen3","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"","reasoning":"The user wants"},"finish_reason":null}]} + +data: {"id":"chatcmpl-12","object":"chat.completion.chunk","model":"qwen3","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"","tool_calls":[{"id":"call_k2","index":0,"type":"function","function":{"name":"get_flow","arguments":"{\\"name\\":\\"demo\\"}"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-12","object":"chat.completion.chunk","model":"qwen3","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"tool_calls"}]} + +data: [DONE] +""", + "openai": """ +data: {"id":"chatcmpl-9","object":"chat.completion.chunk","model":"gpt-4.1","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_a","type":"function","function":{"name":"get_flow","arguments":""}}],"refusal":null},"finish_reason":null}]} + +data: {"id":"chatcmpl-9","object":"chat.completion.chunk","model":"gpt-4.1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\\"name\\""}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-9","object":"chat.completion.chunk","model":"gpt-4.1","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_b","type":"function","function":{"name":"list_flows","arguments":""}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-9","object":"chat.completion.chunk","model":"gpt-4.1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":":\\"demo\\"}"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-9","object":"chat.completion.chunk","model":"gpt-4.1","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{}"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-9","object":"chat.completion.chunk","model":"gpt-4.1","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} + +data: {"id":"chatcmpl-9","object":"chat.completion.chunk","model":"gpt-4.1","choices":[],"usage":{"prompt_tokens":9,"completion_tokens":12,"total_tokens":21}} + +data: [DONE] +""", + "anthropic": """ +data: {"id":"msg_01","object":"chat.completion.chunk","model":"claude-sonnet-4-5","choices":[{"index":0,"delta":{"role":"assistant","content":"Reading "},"finish_reason":null}]} + +data: {"id":"msg_01","object":"chat.completion.chunk","model":"claude-sonnet-4-5","choices":[{"index":0,"delta":{"content":"it."},"finish_reason":null}]} + +data: {"id":"msg_01","object":"chat.completion.chunk","model":"claude-sonnet-4-5","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"toolu_01","type":"function","function":{"name":"get_flow","arguments":""}}]},"finish_reason":null}]} + +data: {"id":"msg_01","object":"chat.completion.chunk","model":"claude-sonnet-4-5","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\\"name\\": \\"de"}}]},"finish_reason":null}]} + +data: {"id":"msg_01","object":"chat.completion.chunk","model":"claude-sonnet-4-5","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"mo\\"}"}}]},"finish_reason":null}]} + +data: {"id":"msg_01","object":"chat.completion.chunk","model":"claude-sonnet-4-5","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} + +data: [DONE] +""", + "openrouter": """ +: OPENROUTER PROCESSING + +data: {"id":"gen-1","provider":"Google","model":"google/gemini-2.5-flash","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"","reasoning":"Short answer."},"finish_reason":null}]} + +: OPENROUTER PROCESSING + +data: {"id":"gen-1","provider":"Google","model":"google/gemini-2.5-flash","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]} + +data: {"id":"gen-1","provider":"Google","model":"google/gemini-2.5-flash","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":" there"},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"completion_tokens":2,"total_tokens":6}} + +data: [DONE] +""", + # Gemini's compatibility endpoint sends each call whole and unnumbered. + "gemini": """ +data: {"choices":[{"delta":{"role":"assistant","tool_calls":[{"id":"c1","type":"function","function":{"name":"get_flow","arguments":"{\\"name\\":\\"demo\\"}"}},{"id":"c2","type":"function","function":{"name":"list_flows","arguments":"{}"}}]},"finish_reason":"tool_calls","index":0}],"object":"chat.completion.chunk"} + +data: [DONE] +""", +} + +GET_DEMO = {"name": "get_flow", "arguments": '{"name":"demo"}'} +LIST = {"name": "list_flows", "arguments": "{}"} + + +@pytest.mark.parametrize( + ("provider", "content", "calls"), + [ + ("ollama", "Hello there", []), + ("ollama-tools", None, [GET_DEMO]), + ("openai", None, [GET_DEMO, LIST]), + ("anthropic", "Reading it.", [{**GET_DEMO, "arguments": '{"name": "demo"}'}]), + ("openrouter", "Hello there", []), + ("gemini", None, [GET_DEMO, LIST]), + ], +) +def test_a_streamed_answer_comes_back_whole( + provider: str, content: str | None, calls: list[dict[str, str]] +) -> None: + assembler = client.Assembler() + pieces = [assembler.feed(line) for line in STREAMS[provider].splitlines()] + message = assembler.message() + assert "".join(pieces) == (content or "") + assert message["content"] == content + assert [c["function"] for c in message["tool_calls"] or []] == calls + assert all(c["id"] and c["type"] == "function" for c in message["tool_calls"] or []) + + +def test_an_error_inside_the_stream_stops_it() -> None: + assembler = client.Assembler() + line = 'data: {"error":{"code":502,"message":"Provider disconnected"},"choices":[{"index":0,"delta":{"content":""},"finish_reason":"error"}]}' + with pytest.raises(client.ProviderError, match="Provider disconnected"): + assembler.feed(line) diff --git a/docs/code/agents.md b/docs/code/agents.md index 0479858..363299c 100644 --- a/docs/code/agents.md +++ b/docs/code/agents.md @@ -54,7 +54,8 @@ A superuser configures an OpenAI-compatible provider under the dashboard's Assistant entry for signed-in users. The assistant uses the same tools with your own permissions. Draft edits -appear in the editor, and each answer renders as markdown with a tool trace. +appear in the editor. Each answer streams in as the model writes it and +renders as markdown with a tool trace. Conversations are stored per user on the instance; the bin button clears yours. `MCP_ENABLED` is only needed for external clients, so the built-in assistant diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index bbf17fa..8e727a1 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -115,8 +115,9 @@ export class AiService { * Chat * One turn, streamed as server-sent events. * - * ``message``, ``tool_call`` and ``tool_result`` events as the model works, - * ``error`` when something stops it, and ``done`` last either way. + * ``delta`` as the model writes, ``message`` with a round's whole text, + * ``tool_call`` and ``tool_result`` as it works, ``error`` when something + * stops it, and ``done`` last either way. * @param data The data for the request. * @param data.requestBody * @returns unknown Successful Response diff --git a/frontend/src/components/Chat/ChatPanel.tsx b/frontend/src/components/Chat/ChatPanel.tsx index b2b0821..63ef012 100644 --- a/frontend/src/components/Chat/ChatPanel.tsx +++ b/frontend/src/components/Chat/ChatPanel.tsx @@ -175,10 +175,12 @@ export function ChatPanel() { .finally(() => setLoaded(true)) }, [open, loaded, available]) - // biome-ignore lint/correctness/useExhaustiveDependencies: scroll when the list grows, not on every render. + const tail = shown[shown.length - 1] + const tailLength = tail && "text" in tail ? tail.text.length : 0 + // biome-ignore lint/correctness/useExhaustiveDependencies: scroll when the list or a streamed answer grows, not on every render. useEffect(() => { bottom.current?.scrollIntoView({ block: "end" }) - }, [shown.length, waiting]) + }, [shown.length, tailLength, waiting]) const send = async () => { const text = draft.trim() @@ -190,11 +192,18 @@ export function ChatPanel() { controller.current = abort const onEvent = (event: ChatEvent) => { switch (event.type) { + // A round's text grows in the last item and is settled by `message`; + // a tool call or the next turn always comes between two rounds. + case "delta": case "message": - setItems((prev) => [ - ...prev, - { kind: "assistant", text: event.content }, - ]) + setItems((prev) => { + const last = prev[prev.length - 1] + if (last?.kind !== "assistant") + return [...prev, { kind: "assistant", text: event.content }] + const text = + event.type === "delta" ? last.text + event.content : event.content + return [...prev.slice(0, -1), { kind: "assistant", text }] + }) break case "tool_call": setItems((prev) => [ diff --git a/frontend/src/components/Chat/stream.ts b/frontend/src/components/Chat/stream.ts index bc8b276..6a0eea1 100644 --- a/frontend/src/components/Chat/stream.ts +++ b/frontend/src/components/Chat/stream.ts @@ -3,6 +3,7 @@ import { apiToken } from "@/lib/portal" export type ChatEvent = | { type: "start" } + | { type: "delta"; content: string } | { type: "message"; content: string } | { type: "tool_call"; name: string; arguments: Record } | { type: "tool_result"; name: string; ok: boolean; error?: string } From 868a10d94bee138333ca1bf4be2ded631cccd901 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:42:59 +0200 Subject: [PATCH 096/103] Namespace the subtree theme scopes as .theme-light/.theme-dark: dark: follows the nearest scope, portalled dashboard menus and tooltips restate it Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- DESIGN.md | 2 +- .../src/components/Dashboard/settings.tsx | 5 ++-- .../src/components/Dashboard/ui/core/look.tsx | 28 +++++++++++-------- .../Dashboard/ui/core/theme.check.ts | 2 +- .../src/components/Dashboard/ui/core/theme.ts | 2 +- .../Dashboard/ui/fluksio/Controls.tsx | 10 +++++-- .../Dashboard/ui/glass/Controls.tsx | 10 +++++-- .../components/Dashboard/ui/glass/glass.css | 2 +- .../Dashboard/ui/material/Controls.tsx | 10 +++++-- .../Dashboard/ui/material/Surfaces.tsx | 10 ++++++- frontend/src/components/Flow/BrainView.tsx | 5 ++-- frontend/src/components/Flow/FlowEditor.tsx | 5 ++-- frontend/src/components/Flow/flow.css | 8 +++--- frontend/src/index.css | 26 +++++++++++------ 14 files changed, 78 insertions(+), 47 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 19f4551..706c2c5 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -8,7 +8,7 @@ any UI work. ## Sync rule (read before touching tokens) -The `@theme inline`, `:root`, and `.dark` blocks in `app/frontend/src/index.css` +The `@theme inline`, `:root`, and `.theme-dark` blocks in `app/frontend/src/index.css` and `index/src/index.css` **must be kept byte-identical**, and `frontend/src/lib/motion.ts` is duplicated verbatim. Any palette / radius / elevation / motion-token change is made in *both* repos in the same change. diff --git a/frontend/src/components/Dashboard/settings.tsx b/frontend/src/components/Dashboard/settings.tsx index 18cd319..0abd77f 100644 --- a/frontend/src/components/Dashboard/settings.tsx +++ b/frontend/src/components/Dashboard/settings.tsx @@ -139,8 +139,9 @@ export function useDashboardBackground( * The class that themes a dashboard's own surface, or `""` to follow the app. * * A class rather than the root, because inside the app shell this must not - * flip the chrome around it. `.light` and `.dark` both redefine the tokens on - * whatever carries them (`index.css`), so either direction works on a subtree. + * flip the chrome around it. `.theme-light` and `.theme-dark` both redefine the + * tokens on whatever carries them (`index.css`), and `dark:` follows the + * nearest of them, so either direction works on a subtree. * * A palette decides this for itself: its first colour *is* the ground, so * whether the surface is light or dark is a fact about the palette rather than diff --git a/frontend/src/components/Dashboard/ui/core/look.tsx b/frontend/src/components/Dashboard/ui/core/look.tsx index eb009ad..cd6a55f 100644 --- a/frontend/src/components/Dashboard/ui/core/look.tsx +++ b/frontend/src/components/Dashboard/ui/core/look.tsx @@ -6,9 +6,10 @@ * below reads them from there: the tokens by inheritance, the look and the * touch flag through this context. * - * The style is carried in the context as well as on the element, because a - * menu is portalled to `body` and lands outside the canvas. A surface that - * floats out of the tree re-states the tokens rather than borrowing the app's. + * The style and the theme class are carried in the context as well as on the + * element, because a menu is portalled to `body` and lands outside the canvas. + * A surface that floats out of the tree re-states them rather than borrowing + * the app's. */ import { createContext, useContext, useMemo } from "react" @@ -28,6 +29,8 @@ type Canvas = { touch: boolean /** The palette's tokens, or `{}` when the dashboard names none. */ style: React.CSSProperties + /** The theme scope class, `theme-light` or `theme-dark`. */ + theme: string } /** A dashboard that names no palette, as one object rather than one per render. */ @@ -37,6 +40,7 @@ const LookContext = createContext({ look: "material", touch: false, style: {}, + theme: "", }) /** The look everything under here is drawn in. */ @@ -46,10 +50,10 @@ export const useLook = () => useContext(LookContext) * What the canvas root carries: the theme class, the palette, and the two * attributes the looks' stylesheets key off. * - * The class is always resolved to `light` or `dark`, rather than left empty - * for a dashboard that follows the device: the sets state their own colours - * per theme, and a chart canvas has to be told which one it is drawing in. - * Restating the app's own resolved theme changes nothing when they agree. + * The class is always resolved to `theme-light` or `theme-dark`, rather than + * left empty for a dashboard that follows the device: the sets state their own + * colours per theme, and a chart canvas has to be told which one it is drawing + * in. Restating the app's own resolved theme changes nothing when they agree. * * `touch` is passed in rather than read off the document, because it belongs to * the screen rather than to the dashboard: the same document may hang on a @@ -73,7 +77,7 @@ export function useCanvasRoot( [roles], ) return { - className: stated || (resolvedTheme === "dark" ? "dark" : "light"), + className: `theme-${stated || resolvedTheme}`, style, "data-look": look, "data-touch": touch ? "" : undefined, @@ -101,13 +105,13 @@ export function LookProvider({ const root = useCanvasRoot(dashboard, touch) const look = root["data-look"] const touched = root["data-touch"] !== undefined - const { style } = root + const { style, className: theme } = root // A bound setting wakes this component on every message its flow sends, and // what the widgets below actually react to is the value — so it changes only - // when one of the three things it carries does. + // when one of the four things it carries does. const value = useMemo( - () => ({ look, touch: touched, style }), - [look, touched, style], + () => ({ look, touch: touched, style, theme }), + [look, touched, style, theme], ) return {children} } diff --git a/frontend/src/components/Dashboard/ui/core/theme.check.ts b/frontend/src/components/Dashboard/ui/core/theme.check.ts index 8c9a420..79739b8 100644 --- a/frontend/src/components/Dashboard/ui/core/theme.check.ts +++ b/frontend/src/components/Dashboard/ui/core/theme.check.ts @@ -121,7 +121,7 @@ assert.deepEqual( ) assert.deepEqual( rolesOf(["#0a0a0a"])?.series.slice(1), - slots(block(".dark {")), + slots(block(".theme-dark {")), "a dark dashboard continues with the dark --chart-2…8", ) diff --git a/frontend/src/components/Dashboard/ui/core/theme.ts b/frontend/src/components/Dashboard/ui/core/theme.ts index 622397e..9c705c2 100644 --- a/frontend/src/components/Dashboard/ui/core/theme.ts +++ b/frontend/src/components/Dashboard/ui/core/theme.ts @@ -10,7 +10,7 @@ * * The result is written onto the canvas as the plain token variables the app * already themes with (`--background`, `--card`, `--primary`, …), which is the - * same mechanism `.light` and `.dark` use on a subtree. Nothing here touches + * same mechanism `.theme-light` and `.theme-dark` use on a subtree. Nothing here touches * `index.css`: that file's token blocks are byte-identical with the website's * and `make design-check` holds them there. * diff --git a/frontend/src/components/Dashboard/ui/fluksio/Controls.tsx b/frontend/src/components/Dashboard/ui/fluksio/Controls.tsx index efad0ef..d89901b 100644 --- a/frontend/src/components/Dashboard/ui/fluksio/Controls.tsx +++ b/frontend/src/components/Dashboard/ui/fluksio/Controls.tsx @@ -154,8 +154,9 @@ export function Select({ onChange, }: SelectProps) { // The menu is portalled to `body`, which is outside the canvas — so it - // re-states the dashboard's own colours rather than borrowing the app's. - const { style } = useLook() + // re-states the dashboard's own theme and colours rather than borrowing the + // app's. + const { style, theme } = useLook() return ( {options.map((option) => ( diff --git a/frontend/src/components/Dashboard/ui/glass/Controls.tsx b/frontend/src/components/Dashboard/ui/glass/Controls.tsx index 0103ed1..6661c59 100644 --- a/frontend/src/components/Dashboard/ui/glass/Controls.tsx +++ b/frontend/src/components/Dashboard/ui/glass/Controls.tsx @@ -172,8 +172,9 @@ export function Select({ onChange, }: SelectProps) { // The menu is portalled to `body`, which is outside the canvas — so it - // re-states the dashboard's own colours rather than borrowing the app's. - const { style } = useLook() + // re-states the dashboard's own theme and colours rather than borrowing the + // app's. + const { style, theme } = useLook() return ( {options.map((option) => ( diff --git a/frontend/src/components/Dashboard/ui/glass/glass.css b/frontend/src/components/Dashboard/ui/glass/glass.css index 3b2f57a..d4098e4 100644 --- a/frontend/src/components/Dashboard/ui/glass/glass.css +++ b/frontend/src/components/Dashboard/ui/glass/glass.css @@ -42,7 +42,7 @@ --dui-on-handle: rgb(0 0 0 / 0.75); } -[data-look="glass"].light { +[data-look="glass"].theme-light { --gl-bg: rgb(255 255 255 / 0.5); --gl-bg-strong: rgb(255 255 255 / 0.7); --gl-border: rgb(255 255 255 / 0.75); diff --git a/frontend/src/components/Dashboard/ui/material/Controls.tsx b/frontend/src/components/Dashboard/ui/material/Controls.tsx index f5fdc58..7861c4a 100644 --- a/frontend/src/components/Dashboard/ui/material/Controls.tsx +++ b/frontend/src/components/Dashboard/ui/material/Controls.tsx @@ -199,8 +199,9 @@ export function Select({ onChange, }: SelectProps) { // The menu is portalled to `body`, which is outside the canvas — so it - // re-states the dashboard's own colours rather than borrowing the app's. - const { style } = useLook() + // re-states the dashboard's own theme and colours rather than borrowing the + // app's. + const { style, theme } = useLook() return ( {options.map((option) => ( diff --git a/frontend/src/components/Dashboard/ui/material/Surfaces.tsx b/frontend/src/components/Dashboard/ui/material/Surfaces.tsx index 1b39084..6d2f342 100644 --- a/frontend/src/components/Dashboard/ui/material/Surfaces.tsx +++ b/frontend/src/components/Dashboard/ui/material/Surfaces.tsx @@ -23,6 +23,7 @@ import type { RailProps, } from "../core/contract" import { railGeometry, TESTID } from "../core/contract" +import { useLook } from "../core/look" import { LOOK } from "../core/motion" /** Material's ground is the surface colour itself; only an image is drawn. */ @@ -39,6 +40,8 @@ export function Backdrop({ image }: BackdropProps) { } function Issue({ issue }: { issue: string }) { + // Portalled to `body`, so it re-states the dashboard's theme and colours. + const { style, theme } = useLook() return ( @@ -49,7 +52,12 @@ function Issue({ issue }: { issue: string }) { data-testid={TESTID.issue} /> - {issue} + + {issue} + ) } diff --git a/frontend/src/components/Flow/BrainView.tsx b/frontend/src/components/Flow/BrainView.tsx index 8a7b7db..82ab43d 100644 --- a/frontend/src/components/Flow/BrainView.tsx +++ b/frontend/src/components/Flow/BrainView.tsx @@ -219,9 +219,8 @@ function BrainCanvas() { nodesConnectable={false} elementsSelectable={false} proOptions={{ hideAttribution: true }} - // React Flow stamps this on the wrapper as a class, and the app's own - // token scopes are named `.light` / `.dark` (index.css) — so leaving it - // unset re-themes the whole canvas to light inside a dark shell. + // React Flow's own defaults, for whatever flow.css does not route + // through the tokens, follow the theme too. colorMode={resolvedTheme} fitView fitViewOptions={FIT} diff --git a/frontend/src/components/Flow/FlowEditor.tsx b/frontend/src/components/Flow/FlowEditor.tsx index 8ae0f69..239f751 100644 --- a/frontend/src/components/Flow/FlowEditor.tsx +++ b/frontend/src/components/Flow/FlowEditor.tsx @@ -1284,9 +1284,8 @@ function FlowEditorInner({ nodeTypes={nodeTypes} edgeTypes={edgeTypes} proOptions={{ hideAttribution: true }} - // React Flow stamps this on the wrapper as a class, and the app's own - // token scopes are named `.light` / `.dark` (index.css) — so leaving it - // unset re-themes the whole canvas to light inside a dark shell. + // React Flow's own defaults, for whatever flow.css does not route + // through the tokens, follow the theme too. colorMode={resolvedTheme} fitView fitViewOptions={FIT_VIEW} diff --git a/frontend/src/components/Flow/flow.css b/frontend/src/components/Flow/flow.css index aa26557..24d478d 100644 --- a/frontend/src/components/Flow/flow.css +++ b/frontend/src/components/Flow/flow.css @@ -2,10 +2,10 @@ * React Flow, routed through the design tokens. * * Scoped to `.react-flow` rather than added to index.css, so the canvas needs no - * tokens of its own. The `colorMode` prop is passed all the same: React Flow - * stamps it on the wrapper as a class, and `.light` / `.dark` are the app's own - * token scopes — an unset one defaults to `light` and re-themes the whole canvas - * inside a dark shell. See the root DESIGN-GUIDELINES.md → Shells. + * tokens of its own. The `colorMode` prop is passed all the same, so React + * Flow's own defaults for anything not routed here follow the theme. The class + * it stamps on the wrapper cannot re-theme the canvas: the app's token scopes + * are `:root.dark` and `.theme-light` / `.theme-dark` (index.css). */ .react-flow { diff --git a/frontend/src/index.css b/frontend/src/index.css index 4d5b59a..732804f 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -2,14 +2,19 @@ @import "tw-animate-css"; @import "./vendor.css"; -@custom-variant dark (&:is(.dark *)); +/* `dark:` follows the nearest theme scope: inside the dark document or a + `.theme-dark` subtree, unless a `.theme-light` subtree sits in between. */ +@custom-variant dark (&:is(:where(:root.dark, .theme-dark) *):not( + :where(:root.dark, .theme-dark) .theme-light * + )); /* * Self-hosted font, duplicated verbatim across the fluksio and website repos (a * sync point like `lib/motion.ts`; see the root DESIGN-GUIDELINES.md, * Typography). Kept OUT of the byte-identical `@theme inline` / `:root` / - * `.dark` blocks. Inter is used for both body (set on `body` in `@layer base`) - * and the display face applied via the `.font-display` class below. + * `.theme-dark` blocks. Inter is used for both body (set on `body` in + * `@layer base`) and the display face applied via the `.font-display` class + * below. */ @font-face { font-family: "Inter"; @@ -106,11 +111,13 @@ * `--primary`, the rest are hues that clear 3:1 against `--card` and stay * apart under colour blindness. See DESIGN-GUIDELINES.md → Colour tokens. */ -/* `.light` is the symmetric half of `.dark` below: both are plain classes, so - either themes a subtree as well as the whole document — which is what lets a - dashboard forced to one theme sit inside a shell on the other. `:root` is - listed second so the parity grep keeps its anchor line. */ -.light, +/* `.theme-light` and `.theme-dark` theme a subtree, which is what lets a + dashboard forced to one theme sit inside a shell on the other. The document + is themed by the bare `.dark` the theme provider puts on ``, matched + only there as `:root.dark`, so a library that stamps `light` or `dark` on its + own wrapper cannot re-theme what is inside it. `:root` is listed second so + the parity grep keeps its anchor line. */ +.theme-light, :root { --background: #ffffff; --foreground: #333232; @@ -152,7 +159,8 @@ --chart-8: #b32474; } -.dark { +:root.dark, +.theme-dark { --background: #0a0a0a; --foreground: #f5f5f5; --card: #1a1a1a; From 92c562a7d5efc2cd6860e387d071e4e15d683234 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 17:48:52 +0200 Subject: [PATCH 097/103] Brain force layout runs in a worker: 300 ticks took ~75 ms cold, ~30 ms warm on the Demo's render thread Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/src/components/Flow/BrainView.tsx | 75 ++++++++----------- .../src/components/Flow/brainLayout.worker.ts | 54 +++++++++++++ 2 files changed, 87 insertions(+), 42 deletions(-) create mode 100644 frontend/src/components/Flow/brainLayout.worker.ts diff --git a/frontend/src/components/Flow/BrainView.tsx b/frontend/src/components/Flow/BrainView.tsx index 82ab43d..15f1657 100644 --- a/frontend/src/components/Flow/BrainView.tsx +++ b/frontend/src/components/Flow/BrainView.tsx @@ -9,15 +9,6 @@ import { ReactFlowProvider, useReactFlow, } from "@xyflow/react" -import { - forceCollide, - forceLink, - forceManyBody, - forceSimulation, - forceX, - forceY, - type SimulationNodeDatum, -} from "d3-force" import { motion } from "motion/react" import { useEffect, useMemo, useState } from "react" @@ -27,6 +18,8 @@ import { useIsMobile } from "@/hooks/useMobile" import { scaleIn } from "@/lib/motion" import { BrainEdge, type BrainEdgeData } from "./BrainEdge" import { BrainNode, type BrainNodeData } from "./BrainNode" +import type { LayoutRequest, Seed } from "./brainLayout.worker" +import BrainLayout from "./brainLayout.worker?worker" import "./flow.css" import { liveStore } from "./liveStore" import { graphQueryOptions } from "./queries" @@ -48,21 +41,22 @@ const SIZE_MAX = 96 */ const LABEL_ROOM = 26 -type Placed = SimulationNodeDatum & { id: string; size: number; room: number } +type Placed = Seed & { size: number } + +type Layout = { nodes: Node[]; edges: Edge[] } + +const EMPTY: Layout = { nodes: [], edges: [] } /** - * Where the neurons sit: a force layout run to rest once and then frozen. + * Where the neurons start: a force layout (`brainLayout.worker.ts`) then runs + * them to rest once, and they stay frozen there. * * A live simulation would keep nudging nodes under the pointer while someone is * panning, and a graph that never stops moving is unreadable. Starting from a * fixed ellipse rather than d3's own random phyllotaxis also means the same * flows lay out the same way twice. - * - * `forceX`/`forceY` rather than `forceCenter`: centering only translates the - * whole thing, so unconnected flows — which is most of them — would push each - * other apart forever with nothing pulling back. */ -function build(graph: BrainGraph): { nodes: Node[]; edges: Edge[] } { +function seed(graph: BrainGraph): LayoutRequest & { placed: Placed[] } { const degree = new Map() for (const edge of graph.edges ?? []) { degree.set(edge.source, (degree.get(edge.source) ?? 0) + 1) @@ -89,29 +83,11 @@ function build(graph: BrainGraph): { nodes: Node[]; edges: Edge[] } { source: edge.source, target: edge.target, })) + return { placed, links } +} - forceSimulation(placed) - .force( - "link", - forceLink(links) - .id((node) => (node as Placed).id) - .distance(130), - ) - .force("charge", forceManyBody().strength(-320).distanceMax(500)) - .force( - "collide", - forceCollide((node) => node.room), - ) - // Centring pulls nearly three times as hard vertically as horizontally, so - // the graph compresses into a band roughly twice as wide as it is tall - // instead of the square an even pull settles into. Coming out of the - // simulation rather than out of a transform keeps the circles round and the - // edges the length the link force asked for. - .force("x", forceX(0).strength(0.05)) - .force("y", forceY(0).strength(0.14)) - .stop() - .tick(300) - +/** The settled neurons as React Flow nodes, and the edges between them. */ +function assemble(graph: BrainGraph, placed: Placed[]): Layout { // An edge draws itself from the two centres outwards, so it needs the radius // to find the rims — and the size that fixes it is only known here. const radius = new Map(placed.map((node) => [node.id, node.size / 2])) @@ -165,10 +141,25 @@ function BrainCanvas() { // anywhere re-fetches this and the layout runs again. const { data } = useQuery(graphQueryOptions()) - const { nodes, edges } = useMemo( - () => build(data ?? { nodes: [], edges: [] }), - [data], - ) + // The previous layout stays up until the next one has settled. + const [{ nodes, edges }, setLayout] = useState(EMPTY) + useEffect(() => { + if (!data) return + const { placed, links } = seed(data) + const worker = new BrainLayout() + worker.onmessage = ({ + data: settled, + }: MessageEvent<[number, number][]>) => { + settled.forEach(([x, y], index) => { + placed[index].x = x + placed[index].y = y + }) + setLayout(assemble(data, placed)) + worker.terminate() + } + worker.postMessage({ placed, links } satisfies LayoutRequest) + return () => worker.terminate() + }, [data]) // A name is revealed on hover, which a finger does not have. On a phone the // first tap says which neuron this is and the second follows it — kept out diff --git a/frontend/src/components/Flow/brainLayout.worker.ts b/frontend/src/components/Flow/brainLayout.worker.ts new file mode 100644 index 0000000..2651245 --- /dev/null +++ b/frontend/src/components/Flow/brainLayout.worker.ts @@ -0,0 +1,54 @@ +/** + * The brain's force layout, off the render thread. + * + * Run to rest in one go, it took several frames on the main thread: about + * 75 ms on a page's first layout and 30 ms after for the Demo's 29 neurons, + * and 250 ms at five times that. Answers with where each neuron settled, in + * the order they came. + * + * `forceX`/`forceY` rather than `forceCenter`: centering only translates the + * whole thing, so unconnected flows — which is most of them — would push each + * other apart forever with nothing pulling back. + */ +import { + forceCollide, + forceLink, + forceManyBody, + forceSimulation, + forceX, + forceY, + type SimulationNodeDatum, +} from "d3-force" + +export type Seed = SimulationNodeDatum & { id: string; room: number } + +export type LayoutRequest = { + placed: Seed[] + links: { source: string; target: string }[] +} + +self.onmessage = ({ data }: MessageEvent) => { + const { placed, links } = data + forceSimulation(placed) + .force( + "link", + forceLink(links) + .id((node) => (node as Seed).id) + .distance(130), + ) + .force("charge", forceManyBody().strength(-320).distanceMax(500)) + .force( + "collide", + forceCollide((node) => node.room), + ) + // Centring pulls nearly three times as hard vertically as horizontally, so + // the graph compresses into a band roughly twice as wide as it is tall + // instead of the square an even pull settles into. Coming out of the + // simulation rather than out of a transform keeps the circles round and the + // edges the length the link force asked for. + .force("x", forceX(0).strength(0.05)) + .force("y", forceY(0).strength(0.14)) + .stop() + .tick(300) + self.postMessage(placed.map((node) => [node.x ?? 0, node.y ?? 0])) +} From 762e7fb1fee3d557217a2ed2e65980d4c11dc05f Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 18:19:19 +0200 Subject: [PATCH 098/103] Header of process and node ports follow each other both ways: a panel edit rewrites only the def line of edited code, a code save reads the header back into input ports and settings (draft and response); Space types in the node editor again Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- backend/fluksio/api/routes/flows.py | 50 ++++++- backend/fluksio/flow/controller.py | 87 ++++++++++++ backend/fluksio/flow/schemas.py | 11 ++ backend/fluksio/mcp/server.py | 4 +- backend/tests/api/routes/test_flows.py | 27 ++++ backend/tests/flow/test_node_settings.py | 65 ++++++++- docs/code/nodes.md | 18 +++ frontend/src/client/schemas.gen.ts | 86 ++++++++++++ frontend/src/client/sdk.gen.ts | 6 +- frontend/src/client/types.gen.ts | 24 +++- frontend/src/components/Flow/FlowEditor.tsx | 30 +++- frontend/src/components/Flow/NodePanel.tsx | 58 ++++---- frontend/src/components/Flow/header.check.ts | 79 +++++++++++ frontend/src/components/Flow/header.ts | 140 +++++++++++++++++++ frontend/src/components/Flow/queries.ts | 8 +- 15 files changed, 645 insertions(+), 48 deletions(-) create mode 100644 frontend/src/components/Flow/header.check.ts create mode 100644 frontend/src/components/Flow/header.ts diff --git a/backend/fluksio/api/routes/flows.py b/backend/fluksio/api/routes/flows.py index 37eb6eb..af7fda6 100644 --- a/backend/fluksio/api/routes/flows.py +++ b/backend/fluksio/api/routes/flows.py @@ -34,7 +34,7 @@ from fluksio.api.deps import ( from fluksio.api.socket_auth import TICKET_TTL_S, issue_ticket, socket_token from fluksio.core.db import engine from fluksio.flow.artifacts import is_reference -from fluksio.flow.controller import NODE_TYPES, FlowController +from fluksio.flow.controller import NODE_TYPES, FlowController, follow_header from fluksio.flow.dashboards import DashboardStore from fluksio.flow.events import event_bus from fluksio.flow.messages import flow_of, qualify @@ -54,6 +54,7 @@ from fluksio.flow.schemas import ( MessageValue, NodeDef, NodeSource, + NodeSourceSaved, NodeStatusPublic, NodeTypeInfo, ) @@ -672,9 +673,41 @@ def read_node_source( ) +def _follow_header( + controller: FlowController, name: str, node_id: str, code: str +) -> FlowDef | None: + """Bring a function node's ports and settings in line with its header. + + The draft is what the editor is on, so that is what moves, and only for a + node with its own code: a shared one serves other flows with other ports, + and a synced flow's ports are declared in its repository. Returns the + draft when it changed. + """ + definition = controller.store.read_flow(name, draft=True) + node = next((n for n in definition.nodes if n.id == node_id), None) + spec = NODE_TYPES.get(node.type) if node is not None else None + if ( + node is None + or spec is None + or not spec.free_params + or node.source_ref + or definition.origin + ): + return None + followed = follow_header(node, code) + if followed == node: + return None + definition.nodes = [followed if n.id == node_id else n for n in definition.nodes] + try: + return controller.store.write_draft(definition, definition.version) + except StaleVersion: + # Someone saved in between; the next save of this code tries again. + return None + + @router.put( "/{name}/nodes/{node_id}/source", - response_model=NodeStatusPublic, + response_model=NodeSourceSaved, dependencies=[Depends(get_code_author)], ) async def save_node_source( @@ -694,6 +727,10 @@ async def save_node_source( it — and that one is live immediately rather than waiting for a publish, because the copy is not any single flow's to hold back. Being live, it is audited like a publish. + + A node with its own code takes its input ports and settings from the + header of ``process`` (see `follow_header`), and the answer carries the + node when that changed it. """ definition = _read_flow(controller, name) ref = _source_ref(definition, node_id) @@ -708,6 +745,9 @@ async def save_node_source( await run_in_threadpool( controller.store.write_node_source, name, node_id, source.code, True ) + followed = await run_in_threadpool( + _follow_header, controller, name, node_id, source.code + ) node_def = next((n for n in definition.nodes if n.id == node_id), None) device = ( node_def.device @@ -717,10 +757,14 @@ async def save_node_source( error = await run_in_threadpool( controller.compile_check, name, node_id, source.code, device ) - return NodeStatusPublic( + return NodeSourceSaved( id=f"{name}.{node_id}", status="error" if error else "active", error=error, + node=next((n for n in followed.nodes if n.id == node_id), None) + if followed + else None, + version=followed.version if followed else None, ) diff --git a/backend/fluksio/flow/controller.py b/backend/fluksio/flow/controller.py index 8c8bf1c..ba631d1 100644 --- a/backend/fluksio/flow/controller.py +++ b/backend/fluksio/flow/controller.py @@ -1837,6 +1837,93 @@ def _signature_gap( ) +def follow_header(node: NodeDef, code: str) -> NodeDef: + """The node with its input ports and settings read off ``def process(...)``. + + The inverse of the canvas's scaffold, which writes the input ports first, + in their order, and the settings after them. A parameter already declared + keeps its kind. A new one is a setting when it has a default, which becomes + its value, or when it follows a setting, and an input port otherwise. One + name swapped for one other is a rename, so the port keeps its message and + the setting its value. A declared name the header no longer has is dropped. + + A header this cannot read for certain leaves the node as it is: a syntax + error, no top-level ``def process``, a decorator, ``*args``, ``**kwargs``, + a bare ``*`` or ``/``. + """ + try: + tree = ast.parse(code) + except SyntaxError: + return node + defs = [ + entry + for entry in tree.body + if isinstance(entry, ast.FunctionDef) and entry.name == "process" + ] + if not defs: + return node + args = defs[-1].args + if ( + defs[-1].decorator_list + or args.vararg + or args.kwarg + or args.posonlyargs + or args.kwonlyargs + ): + return node + # Defaults belong to the last parameters, so they line up from the end. + defaults: list[ast.expr | None] = [None] * ( + len(args.args) - len(args.defaults) + ) + list(args.defaults) + header = [ + (arg.arg, default) + for arg, default in zip(args.args, defaults, strict=True) + if arg.arg not in RESERVED_SETTINGS + ] + + def written(name: str) -> bool: + return name.isascii() and name.isidentifier() + + ports = {spec.port: spec for spec in node.requires if written(spec.port)} + settings = { + key: value + for key, value in node.params.items() + if key not in RESERVED_SETTINGS and written(key) + } + names = [name for name, _ in header] + gone = [name for name in [*ports, *settings] if name not in names] + new = [name for name in names if name not in ports and name not in settings] + renamed = {new[0]: gone[0]} if len(gone) == len(new) == 1 else {} + + requires: list[MessageSpec] = [] + params = {key: value for key, value in node.params.items() if key not in settings} + after_setting = False + for name, default in header: + was = renamed.get(name, name) + if was in ports: + requires.append(ports[was].model_copy(update={"port": name})) + elif was in settings or default is not None or after_setting: + params[name] = settings[was] if was in settings else _literal(default) + after_setting = True + else: + requires.append(MessageSpec(port=name)) + # Ports the header cannot name, such as a row not named yet, stay. + requires += [spec for spec in node.requires if not written(spec.port)] + return node.model_copy(update={"requires": requires, "params": params}) + + +def _literal(default: ast.expr | None) -> Any: + """A parameter's default as a setting's value, or empty when it is not JSON.""" + if default is None: + return "" + try: + value = ast.literal_eval(default) + json.dumps(value) + except (ValueError, TypeError, SyntaxError, RecursionError): + return "" + return value + + def with_settings( function: Callable[..., Any], params: dict[str, Any] ) -> Callable[..., Any]: diff --git a/backend/fluksio/flow/schemas.py b/backend/fluksio/flow/schemas.py index 00d5f1d..afe4397 100644 --- a/backend/fluksio/flow/schemas.py +++ b/backend/fluksio/flow/schemas.py @@ -343,6 +343,17 @@ class NodeStatusPublic(BaseModel): last_error_ts: float | None = None +class NodeSourceSaved(NodeStatusPublic): + """Whether saved code loads, and what its header changed. + + `node` is set when the parameters of ``process`` added, dropped or renamed + an input port or setting. The draft moved on to `version` with it. + """ + + node: NodeDef | None = None + version: int | None = None + + class MessageValue(BaseModel): """The last payload seen on a message.""" diff --git a/backend/fluksio/mcp/server.py b/backend/fluksio/mcp/server.py index 5c7b5a8..4cffc76 100644 --- a/backend/fluksio/mcp/server.py +++ b/backend/fluksio/mcp/server.py @@ -208,7 +208,9 @@ async def save_node_source(name: str, node_id: str, code: str) -> Any: """Save a node's Python source and report whether it compiles. A node defines ``process(...)``, taking one argument per input port and - one per setting, and returns a dict keyed by output port. + one per setting, and returns a dict keyed by output port. A parameter added + or dropped there adds or drops the input port or setting in the draft, + which moves its version on: read the flow again before saving it. """ return await _call( "PUT", f"/flows/{_seg(name)}/nodes/{_seg(node_id)}/source", json={"code": code} diff --git a/backend/tests/api/routes/test_flows.py b/backend/tests/api/routes/test_flows.py index 692a3a2..a83a281 100644 --- a/backend/tests/api/routes/test_flows.py +++ b/backend/tests/api/routes/test_flows.py @@ -557,6 +557,33 @@ async def test_the_live_socket_ends_when_its_client_goes( assert parked == [] +def test_saving_a_header_moves_the_node_ports_in_the_draft( + client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + saved = client.put( + f"{PREFIX}/header", headers=superuser_token_headers, json=a_flow("header") + ).json() + + response = client.put( + f"{PREFIX}/header/nodes/logger/source", + headers=superuser_token_headers, + json={"code": "def process(reading, level, gain=2):\n return {}\n"}, + ).json() + + assert [spec["port"] for spec in response["node"]["requires"]] == [ + "reading", + "level", + ] + assert response["node"]["params"] == {"gain": 2} + draft = client.get(f"{PREFIX}/header", headers=superuser_token_headers).json() + assert draft["definition"]["version"] == response["version"] + assert response["version"] == saved["definition"]["version"] + 1 + logger = next(n for n in draft["definition"]["nodes"] if n["id"] == "logger") + assert logger["params"] == {"gain": 2} + + client.delete(f"{PREFIX}/header", headers=superuser_token_headers) + + def test_a_shared_node_in_use_is_retired_only_with_detach( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: diff --git a/backend/tests/flow/test_node_settings.py b/backend/tests/flow/test_node_settings.py index 6fabb30..96a2650 100644 --- a/backend/tests/flow/test_node_settings.py +++ b/backend/tests/flow/test_node_settings.py @@ -9,7 +9,7 @@ from pathlib import Path import pytest -from fluksio.flow.controller import FlowController +from fluksio.flow.controller import FlowController, follow_header from fluksio.flow.messages import DType, MessageSpec from fluksio.flow.pipeline import Pipeline from fluksio.flow.schemas import FlowDef, NodeDef @@ -148,3 +148,66 @@ def test_a_shared_body_is_checked_against_each_user(store: FlowStore): assert [n.status for n in nodes] == ["error"] assert "'level' is an input of 'scale'" in (nodes[0].error or "") + + +def _scaffold_header(node: NodeDef) -> str: + """The header the canvas writes: input ports in order, then settings.""" + names = [spec.port for spec in node.requires] + [ + key for key in node.params if key != "synchronous" + ] + return f"def process({', '.join(names)}):\n return {{}}\n" + + +def test_a_scaffold_header_reads_back_as_the_node_it_came_from(): + node = _flow( + [*READING, MessageSpec(name="garden.level", port="depth")], + factor=3, + synchronous=True, + ).nodes[0] + + assert follow_header(node, _scaffold_header(node)) == node + + +def test_a_header_declares_ports_and_settings_that_read_back_the_same(): + header = "def process(reading, level, gain=1.5, mode='eco'):\n return {}\n" + + node = follow_header(_flow([]).nodes[0], header) + + assert [spec.port for spec in node.requires] == ["reading", "level"] + assert node.params == {"gain": 1.5, "mode": "eco"} + assert _scaffold_header(node).split(":")[0] == ( + "def process(reading, level, gain, mode)" + ) + + +def test_a_renamed_parameter_keeps_the_message_and_a_dropped_one_goes(): + node = _flow(READING, factor=3, offset=1).nodes[0] + + renamed = follow_header(node, "def process(value, factor, offset): pass\n") + dropped = follow_header(node, "def process(reading, factor): pass\n") + + assert renamed.requires == [READING[0].model_copy(update={"port": "value"})] + assert renamed.params == {"factor": 3, "offset": 1} + assert dropped.requires == READING + assert dropped.params == {"factor": 3} + + +def test_a_new_parameter_after_a_setting_is_a_setting(): + node = _flow(READING, factor=3).nodes[0] + + followed = follow_header(node, "def process(reading, factor, offset): pass\n") + + assert followed.requires == READING + assert followed.params == {"factor": 3, "offset": ""} + + +def test_a_header_it_cannot_read_for_certain_leaves_the_node(): + node = _flow(READING, factor=3).nodes[0] + + for code in ( + "def process(reading, factor", + "def process(**kw): pass\n", + "@cache\ndef process(value): pass\n", + "def run(value): pass\n", + ): + assert follow_header(node, code) == node diff --git a/docs/code/nodes.md b/docs/code/nodes.md index cee3609..4b6dec3 100644 --- a/docs/code/nodes.md +++ b/docs/code/nodes.md @@ -29,6 +29,24 @@ Inside a node, `import fluksio` exposes reporting and artifact helpers. It does not expose the engine's internal modules. Other imports come from the node's Python environment; see [Packages](#packages). +## The header follows the ports + +On the canvas, the parameters of `process` and the node's input ports and +settings are kept in step, in both directions. + +- Changing ports or settings in the node panel rewrites the + `def process(...)` line and leaves the rest of the code as it is. Input + ports come first, in their order, then the settings. A parameter that stays + keeps its annotation and default. +- Saving the code reads that line back. A name the node already has keeps + its kind. A new parameter becomes a setting when it has a default, which + becomes its value, or when it comes after a setting. Any other new + parameter becomes an input port. Removing a parameter removes its port or + setting, and renaming a single one renames the port (keeping its message) + or the setting (keeping its value). +- Shared code, code from `fluksio sync`, and a header with a decorator, + `*args`, `**kwargs`, a bare `*` or `/` are left alone in both directions. + ## Producing values over time Yield dictionaries to publish outputs before the node finishes: diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 4a9df04..222746c 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -2543,6 +2543,92 @@ export const NodeSourceSchema = { description: 'The Python source of a node.' } as const; +export const NodeSourceSavedSchema = { + properties: { + id: { + type: 'string', + title: 'Id' + }, + status: { + type: 'string', + title: 'Status', + default: 'active' + }, + error: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Error' + }, + health: { + type: 'string', + enum: ['ok', 'degraded', 'down'], + title: 'Health', + default: 'ok' + }, + health_detail: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Health Detail' + }, + last_error: { + type: 'string', + title: 'Last Error', + default: '' + }, + last_error_ts: { + anyOf: [ + { + type: 'number' + }, + { + type: 'null' + } + ], + title: 'Last Error Ts' + }, + node: { + anyOf: [ + { + '$ref': '#/components/schemas/NodeDef-Output' + }, + { + type: 'null' + } + ] + }, + version: { + anyOf: [ + { + type: 'integer' + }, + { + type: 'null' + } + ], + title: 'Version' + } + }, + type: 'object', + required: ['id'], + title: 'NodeSourceSaved', + description: `Whether saved code loads, and what its header changed. + +\`node\` is set when the parameters of \`\`process\`\` added, dropped or renamed +an input port or setting. The draft moved on to \`version\` with it.` +} as const; + export const NodeStatusPublicSchema = { properties: { id: { diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index 8e727a1..221429d 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -1116,11 +1116,15 @@ export class FlowsService { * it — and that one is live immediately rather than waiting for a publish, * because the copy is not any single flow's to hold back. Being live, it is * audited like a publish. + * + * A node with its own code takes its input ports and settings from the + * header of ``process`` (see `follow_header`), and the answer carries the + * node when that changed it. * @param data The data for the request. * @param data.name * @param data.nodeId * @param data.requestBody - * @returns NodeStatusPublic Successful Response + * @returns NodeSourceSaved Successful Response * @throws ApiError */ public static saveNodeSource(data: FlowsSaveNodeSourceData): CancelablePromise { diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index a599c09..f825088 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -888,6 +888,26 @@ export type NodeSource = { missing?: boolean; }; +/** + * Whether saved code loads, and what its header changed. + * + * `node` is set when the parameters of ``process`` added, dropped or renamed + * an input port or setting. The draft moved on to `version` with it. + */ +export type NodeSourceSaved = { + id: string; + status?: string; + error?: (string | null); + health?: 'ok' | 'degraded' | 'down'; + health_detail?: (string | null); + last_error?: string; + last_error_ts?: (number | null); + node?: (NodeDef_Output | null); + version?: (number | null); +}; + +export type health = 'ok' | 'degraded' | 'down'; + /** * Whether a node loaded, and how its connection is doing. */ @@ -901,8 +921,6 @@ export type NodeStatusPublic = { last_error_ts?: (number | null); }; -export type health = 'ok' | 'degraded' | 'down'; - /** * A node type the editor can offer, with its parameter schema. */ @@ -1836,7 +1854,7 @@ export type FlowsSaveNodeSourceData = { requestBody: NodeSource; }; -export type FlowsSaveNodeSourceResponse = (NodeStatusPublic); +export type FlowsSaveNodeSourceResponse = (NodeSourceSaved); export type FlowsShareNodeData = { name: string; diff --git a/frontend/src/components/Flow/FlowEditor.tsx b/frontend/src/components/Flow/FlowEditor.tsx index 239f751..34c286f 100644 --- a/frontend/src/components/Flow/FlowEditor.tsx +++ b/frontend/src/components/Flow/FlowEditor.tsx @@ -273,6 +273,7 @@ function FlowEditorInner({ flush, conflict, resolveConflict, + adopt, mutation: saving, } = useAutosave(flowName) const publish = usePublish(flowName) @@ -855,13 +856,34 @@ function FlowEditorInner({ }) const sourceMutation = useMutation({ - mutationFn: ({ nodeId, code }: { nodeId: string; code: string }) => - FlowsService.saveNodeSource({ + mutationFn: async ({ nodeId, code }: { nodeId: string; code: string }) => { + // The server reads the ports off the header and compares them with its + // draft, so the ports on the canvas have to be there first. + await flush() + return FlowsService.saveNodeSource({ name: flowName, nodeId, requestBody: { code }, - }), - onSuccess: () => { + }) + }, + onSuccess: (saved) => { + const followed = saved.node + if (followed && saved.version) { + // The header changed the ports in the draft: take them, and the + // version they moved it to, so the next save is not a conflict. + adopt(saved.version) + commit( + (latest.current.nodes ?? []).map((node) => + node.id === followed.id + ? { + ...node, + requires: followed.requires, + params: followed.params, + } + : node, + ), + ) + } queryClient.invalidateQueries({ queryKey: ["flows", flowName] }) }, }) diff --git a/frontend/src/components/Flow/NodePanel.tsx b/frontend/src/components/Flow/NodePanel.tsx index b5cdffc..f09bfe4 100644 --- a/frontend/src/components/Flow/NodePanel.tsx +++ b/frontend/src/components/Flow/NodePanel.tsx @@ -49,6 +49,13 @@ import useCustomToast from "@/hooks/useCustomToast" import { inCodeEditor, useShortcuts } from "@/lib/shortcuts" import { cn } from "@/lib/utils" import { DtypeValue } from "./FlowBoundary" +import { + IDENTIFIER, + portName, + processArgs, + RESERVED_PARAMS, + withHeader, +} from "./header" import { JsonField } from "./JsonField" import { MessageSparkline } from "./MessageSparkline" import { @@ -92,12 +99,6 @@ const NO_SECRET = "__none__" const SECTION = PANEL_SECTION -/** Settings the engine reads itself, so they are not the author's to name. */ -const RESERVED_PARAMS = new Set(["synchronous"]) - -/** A setting reaches the function by name, so the name has to be one. */ -const IDENTIFIER = /^[A-Za-z_]\w*$/ - /** Sentinels for the resources dropdown, which has two options that are not sizes. */ const NO_RESOURCES = "__shared__" const CUSTOM_RESOURCES = "__custom__" @@ -582,6 +583,8 @@ function FreeParamsForm({ // remount its row and take the half-typed value with it.
      - spec.port || (spec.name ?? "").split(".").pop() || "" - /** * Whether a port was named by hand rather than after its message. Nothing * stores that: a port whose name is not the one its message gives it is one. @@ -1376,25 +1375,11 @@ const ownsName = (spec: MessageSpec) => Boolean(spec.port) && spec.port !== portName({ name: spec.name }) /** - * A `process` that takes this node's inputs and settings, and returns its - * outputs. - * - * Both arrive by name, so both are arguments — the ports first, in the order - * they are declared, then the settings. + * A `process` that takes this node's inputs and settings (`processArgs`), and + * returns its outputs. */ function scaffoldFor(node: NodeDef_Input): string { - const args = [ - ...new Set( - [ - ...(node.requires ?? []).map(portName), - ...Object.keys(node.params ?? {}).filter( - (name) => !RESERVED_PARAMS.has(name), - ), - ] - // Anything else cannot be a keyword argument, so it cannot be one. - .filter((name) => IDENTIFIER.test(name)), - ), - ] + const args = processArgs(node) const returns = (node.provides ?? []) .filter((spec) => portName(spec)) .map((spec) => `"${portName(spec)}": ${PLACEHOLDER[spec.dtype ?? "float"]}`) @@ -1470,11 +1455,12 @@ function PanelBody({ } /** - * Change the node, and keep an untouched scaffold in step with its ports. + * Change the node, and keep its code in step with its ports and settings. * - * Only code that is still exactly what this panel generates is rewritten — - * one edit of your own and it is yours, ports or no ports. Shared code is - * never rewritten either: the other flows using it have their own ports. + * Code that is still exactly what this panel generates is generated again, + * outputs included. Code of your own only has the header of `process` + * rewritten, and the body is left as it is. Shared code is never rewritten: + * the other flows using it have their own ports. */ const editNode = (next: NodeDef_Input) => { onChange(next) @@ -1487,8 +1473,10 @@ function PanelBody({ current === undefined ) return - const wanted = scaffoldFor(next) - if (current !== wanted && SCAFFOLD_SHAPE.test(current)) editCode(wanted) + const wanted = SCAFFOLD_SHAPE.test(current) + ? scaffoldFor(next) + : withHeader(current, processArgs(next)) + if (current !== wanted) editCode(wanted) } /** @@ -1705,7 +1693,9 @@ function PanelBody({ {CODE_LOCKED}

      )} -
      + {/* `nokey`: Monaco types through a plain div (EditContext), which + the canvas takes for itself, so Space would pan instead. */} +
      diff --git a/frontend/src/components/Flow/header.check.ts b/frontend/src/components/Flow/header.check.ts new file mode 100644 index 0000000..bebc171 --- /dev/null +++ b/frontend/src/components/Flow/header.check.ts @@ -0,0 +1,79 @@ +/** + * The header of `process` kept in step with a node's ports and settings, + * checked. The server's half, reading them back, is `follow_header` in + * `backend/tests/flow/test_node_settings.py`. + * + * cd frontend && bun run src/components/Flow/header.check.ts + */ + +import assert from "node:assert/strict" + +import { processArgs, readHeader, withHeader } from "./header" + +const BODY = ` # the body stays exactly as written + return {"out": reading * gain} +` +const code = (params: string) => + `import math\n\n\ndef process(${params}) -> dict:\n${BODY}` + +// Inputs in order, then settings; the engine's own setting is not one. +assert.deepEqual( + processArgs({ + id: "n", + requires: [{ name: "house.reading" }, { name: "x", port: "level" }], + params: { gain: 2, synchronous: true }, + }), + ["reading", "level", "gain"], +) + +// Config to header and back: the names written are the names read. +const written = withHeader(code("reading, gain=2.0"), [ + "reading", + "level", + "gain", +]) +assert.equal(written, code("reading, level, gain=2.0")) +assert.deepEqual( + readHeader(written)?.params.map((param) => param.name), + ["reading", "level", "gain"], +) + +// A rename keeps the annotation; a dropped name goes; the body never moves. +assert.equal( + withHeader(code("reading: float, gain=2.0"), ["value", "gain"]), + code("value: float, gain=2.0"), +) +assert.equal(withHeader(code("reading, gain"), ["reading"]), code("reading")) + +// A new setting without a default cannot follow one with a default. +assert.equal( + withHeader(code("reading, gain=2.0"), ["reading", "gain", "offset"]), + code("reading, offset, gain=2.0"), +) + +// Nothing to change is no rewrite, not even of the spacing. +assert.equal( + withHeader(code("reading ,gain"), ["reading", "gain"]), + code("reading ,gain"), +) + +// Commas inside a default or a string are not parameter boundaries. +assert.deepEqual( + readHeader(code('reading, sep=", ", sizes=(1, 2)'))?.params.map( + (param) => param.name, + ), + ["reading", "sep", "sizes"], +) + +// What the server will not read is not rewritten either. +for (const params of [ + "*values", + "reading, **settings", + "reading, /", + "a, # why\n b", +]) + assert.equal(withHeader(code(params), ["other"]), code(params)) +const decorated = `@cache\n${code("reading").slice("import math\n\n\n".length)}` +assert.equal(withHeader(decorated, ["other"]), decorated) + +console.log("header: ok") diff --git a/frontend/src/components/Flow/header.ts b/frontend/src/components/Flow/header.ts new file mode 100644 index 0000000..30b459d --- /dev/null +++ b/frontend/src/components/Flow/header.ts @@ -0,0 +1,140 @@ +/** + * The header of a function node's `process`, and keeping it in step with the + * node's ports and settings. + * + * The other direction, reading ports and settings back off a header, is the + * server's: `follow_header` runs on every save of the code. + */ + +import type { MessageSpec, NodeDef_Input } from "@/client" + +/** Settings the engine reads itself, so they are not the author's to name. */ +export const RESERVED_PARAMS = new Set(["synchronous"]) + +/** A setting reaches the function by name, so the name has to be one. */ +export const IDENTIFIER = /^[A-Za-z_]\w*$/ + +/** The name the node function sees; the engine derives it the same way. */ +export const portName = (spec: MessageSpec) => + spec.port || (spec.name ?? "").split(".").pop() || "" + +/** + * The parameters `process` takes for this node. + * + * Inputs and settings both arrive by name, so both are parameters, the inputs + * first in the order they are declared, then the settings. + */ +export function processArgs(node: NodeDef_Input): string[] { + return [ + ...new Set( + [ + ...(node.requires ?? []).map(portName), + ...Object.keys(node.params ?? {}).filter( + (name) => !RESERVED_PARAMS.has(name), + ), + ] + // Anything else cannot be a keyword argument, so it cannot be one. + .filter((name) => IDENTIFIER.test(name)), + ), + ] +} + +type Param = { name: string; text: string; hasDefault: boolean } + +/** Where the parameters of the last top-level `def process(` sit. */ +const PROCESS_DEF = /^def process\(/gm + +/** + * The parameters of `process` as written, and the span they take up. + * + * Null for a header this cannot read for certain, the same ones the server + * leaves alone: no top-level `def process`, a decorator, `*args`, `**kwargs`, + * a bare `*` or `/`, or a comment inside the parentheses. + */ +export function readHeader( + code: string, +): { from: number; to: number; params: Param[] } | null { + // Python keeps the last definition, so that is the one that runs. + const match = [...code.matchAll(PROCESS_DEF)].pop() + if (match?.index === undefined) return null + const above = code.slice(0, match.index).trimEnd() + if (above.slice(above.lastIndexOf("\n") + 1).startsWith("@")) return null + + const from = match.index + match[0].length + const params: Param[] = [] + let start = from + let depth = 0 + let hasDefault = false + const take = (end: number) => { + const text = code.slice(start, end).trim() + const name = /^[A-Za-z_]\w*/.exec(text)?.[0] + params.push({ name: name ?? "", text, hasDefault }) + start = end + 1 + hasDefault = false + return Boolean(name) + } + for (let i = from; i < code.length; i++) { + const char = code[i] + if (char === '"' || char === "'") { + const quote = code.startsWith(char.repeat(3), i) ? char.repeat(3) : char + let j = i + quote.length + while (j < code.length && !code.startsWith(quote, j)) + j += code[j] === "\\" ? 2 : 1 + if (j >= code.length) return null + i = j + quote.length - 1 + } else if (char === "#") { + return null + } else if ("([{".includes(char)) { + depth++ + } else if (")]}".includes(char) && depth > 0) { + depth-- + } else if (char === ")") { + // A trailing comma leaves nothing after it, which is not a parameter. + if (code.slice(start, i).trim() && !take(i)) return null + return { from, to: i, params } + } else if (depth === 0 && char === ",") { + if (!take(i)) return null + } else if (depth === 0 && char === "=") { + hasDefault = true + } + } + return null +} + +/** + * `code` with `process` taking `args`, and every other byte as it was. + * + * A parameter that stays keeps its annotation and default, and so does one + * renamed (one name gone, one new). Python wants the parameters without a + * default first, so a new one goes ahead of any that have one. A header this + * cannot read is left alone. + */ +export function withHeader(code: string, args: string[]): string { + const header = readHeader(code) + if (!header) return code + if (header.params.map((param) => param.name).join() === args.join()) + return code + const written = new Map(header.params.map((param) => [param.name, param])) + const gone = header.params.filter((param) => !args.includes(param.name)) + const fresh = args.filter((name) => !written.has(name)) + if (gone.length === 1 && fresh.length === 1) { + const [was] = gone + written.set(fresh[0], { + ...was, + name: fresh[0], + text: fresh[0] + was.text.slice(was.name.length), + }) + } + const params = args.map( + (name) => written.get(name) ?? { name, text: name, hasDefault: false }, + ) + const ordered = [ + ...params.filter((param) => !param.hasDefault), + ...params.filter((param) => param.hasDefault), + ] + return ( + code.slice(0, header.from) + + ordered.map((param) => param.text).join(", ") + + code.slice(header.to) + ) +} diff --git a/frontend/src/components/Flow/queries.ts b/frontend/src/components/Flow/queries.ts index 66a748d..8c82c78 100644 --- a/frontend/src/components/Flow/queries.ts +++ b/frontend/src/components/Flow/queries.ts @@ -172,6 +172,8 @@ export function useAutosave(name: string): { flush: () => Promise conflict: boolean resolveConflict: (mode: "theirs" | "mine") => Promise + /** Take a version the server moved the draft to by itself. */ + adopt: (next: number) => void mutation: UseMutationResult } { const queryClient = useQueryClient() @@ -283,5 +285,9 @@ export function useAutosave(name: string): { } }, [flush]) - return { save, flush, conflict, resolveConflict, mutation } + const adopt = useCallback((next: number) => { + version.current = next + }, []) + + return { save, flush, conflict, resolveConflict, adopt, mutation } } From 271be7aae7dbd66599a00a14f7de8893778f3517 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 18:19:34 +0200 Subject: [PATCH 099/103] A round face's press is not clipped by the face body, so Glass light's disc shadow falls soft instead of ending in hard edges Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/src/components/Dashboard/ui/core/core.css | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frontend/src/components/Dashboard/ui/core/core.css b/frontend/src/components/Dashboard/ui/core/core.css index c49ae2e..e7d01f4 100644 --- a/frontend/src/components/Dashboard/ui/core/core.css +++ b/frontend/src/components/Dashboard/ui/core/core.css @@ -717,6 +717,12 @@ line-height: 1.35; } +/* A press is a disc of a fixed size, as tall as the body, so there is no text + to clip and its shadow falls outside the body by design. */ +.dui-face-body:has(> .dui-face-press) { + overflow: visible; +} + .dui-face-body .dui-clock { font-size: 72px; line-height: 1; From ee3ed6a2ec4c15f662effe2c95367de4c34645d3 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 18:19:41 +0200 Subject: [PATCH 100/103] A dashboard's portalled select menu restates its look, so a Glass menu blurs what is under it (the colour wheel no longer shows through) and a Material menu gets its tonal fill back Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/src/components/Dashboard/ui/fluksio/Controls.tsx | 8 +++++--- frontend/src/components/Dashboard/ui/glass/Controls.tsx | 8 +++++--- .../src/components/Dashboard/ui/material/Controls.tsx | 8 +++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/frontend/src/components/Dashboard/ui/fluksio/Controls.tsx b/frontend/src/components/Dashboard/ui/fluksio/Controls.tsx index d89901b..f689e98 100644 --- a/frontend/src/components/Dashboard/ui/fluksio/Controls.tsx +++ b/frontend/src/components/Dashboard/ui/fluksio/Controls.tsx @@ -154,9 +154,10 @@ export function Select({ onChange, }: SelectProps) { // The menu is portalled to `body`, which is outside the canvas — so it - // re-states the dashboard's own theme and colours rather than borrowing the - // app's. - const { style, theme } = useLook() + // re-states the dashboard's own theme, colours and look rather than + // borrowing the app's. The look carries the variables its menu is painted + // with: without it a glass menu has no blur, and the tile under it shows. + const { style, theme, look } = useLook() return ( Date: Thu, 15 Oct 2026 18:23:40 +0200 Subject: [PATCH 101/103] useIsMobile reads the width on the first render, so a phone's flow nodes no longer draw desktop handles first and React Flow cannot measure those: the Music and House history edges stop changing between loads Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- frontend/src/hooks/useMobile.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/frontend/src/hooks/useMobile.ts b/frontend/src/hooks/useMobile.ts index 2b0fe1d..4ee2dcf 100644 --- a/frontend/src/hooks/useMobile.ts +++ b/frontend/src/hooks/useMobile.ts @@ -3,7 +3,13 @@ import * as React from "react" const MOBILE_BREAKPOINT = 768 export function useIsMobile() { - const [isMobile, setIsMobile] = React.useState(undefined) + // Read on the first render, not after it: a node on the flow canvas draws + // its handles for the direction this returns, and React Flow measures them + // once. Starting at `false` drew them for a desktop first, so whether an + // edge met its handle depended on which of the two React measured. + const [isMobile, setIsMobile] = React.useState( + () => window.innerWidth < MOBILE_BREAKPOINT, + ) React.useEffect(() => { const mql = window.matchMedia(`(max-width: ${MOBILE_BREAKPOINT - 1}px)`) @@ -15,5 +21,5 @@ export function useIsMobile() { return () => mql.removeEventListener("change", onChange) }, []) - return !!isMobile + return isMobile } From bcd2173a65d53ecbf2529084cba31fbb449869e2 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 18:24:38 +0200 Subject: [PATCH 102/103] Edges follow ELK's routes: layoutGraph returns each edge's start, bends, end and chip point, LiveEdge draws them square with rounded corners, moved onto the drawn handles, and keeps the curve for an edge whose node moved since; a feedback edge clears the nodes it passes Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/interface/flow-editor.md | 4 +- frontend/src/components/Flow/FlowEditor.tsx | 33 ++-- frontend/src/components/Flow/LiveEdge.tsx | 174 ++++++++++++++++--- frontend/src/components/Flow/deriveEdges.ts | 3 + frontend/src/components/Flow/layout.check.ts | 158 +++++++++++++++-- frontend/src/components/Flow/layout.ts | 128 ++++++++++++-- 6 files changed, 432 insertions(+), 68 deletions(-) diff --git a/docs/interface/flow-editor.md b/docs/interface/flow-editor.md index d5c222f..e188169 100644 --- a/docs/interface/flow-editor.md +++ b/docs/interface/flow-editor.md @@ -8,7 +8,9 @@ the connections and arranges the canvas for you. The graph runs left to right on desktop and top to bottom on phones. - **Nodes** show their execution status. -- **Edges** pulse when values pass through. Click one to inspect its payload. +- **Edges** are routed around the nodes, a loop back to an earlier node + included, and pulse when values pass through. Click one to inspect its + payload. - **Endpoints** show flow inputs, dashboard widgets, panel I/O ports and nodes in other flows that exchange messages with this flow. A value passing lights one up in a sweep: left to right when it sends into the flow, right to left diff --git a/frontend/src/components/Flow/FlowEditor.tsx b/frontend/src/components/Flow/FlowEditor.tsx index 34c286f..0ee46dc 100644 --- a/frontend/src/components/Flow/FlowEditor.tsx +++ b/frontend/src/components/Flow/FlowEditor.tsx @@ -57,6 +57,7 @@ import { bindingsKey, boundaryKey, deriveEdges, + type FlowEdgeData, flowOf, portOf, qualify, @@ -74,7 +75,12 @@ import { FlowNode, type FlowNodeData } from "./FlowNode" import { FlowPanel } from "./FlowPanel" import { InputInspector, type InspectedInput } from "./InputInspector" import { LiveEdge } from "./LiveEdge" -import { type Direction, layoutGraph, type NodeShape } from "./layout" +import { + type Direction, + type Layout, + layoutGraph, + type NodeShape, +} from "./layout" import { NodePanel } from "./NodePanel" import { RunDialog } from "./RunDialog" import "./flow.css" @@ -93,7 +99,7 @@ import { useFlowSocket } from "./useFlowSocket" const nodeTypes = { flow: FlowNode, [ENDPOINT_TYPE]: EndpointNode } /** - * Where every node sits, once the engine has said so. + * Where every node sits and how every edge runs, once the engine has said so. * * The layout is a chunk of its own and answers asynchronously, so a canvas has * a moment before it knows where anything goes. It draws nothing at all until @@ -107,18 +113,15 @@ function useLayout( direction: Direction, shapes: Map, shapeKey: string, -): Map | null { - const [placed, setPlaced] = useState | null>(null) +): Layout | null { + const [placed, setPlaced] = useState(null) // biome-ignore lint/correctness/useExhaustiveDependencies: the shape key is the dependency; the arrays are rebuilt every render. useEffect(() => { let stale = false - layoutGraph(ids, edges, direction, shapes).then((positions) => { + layoutGraph(ids, edges, direction, shapes).then((layout) => { // A graph edited while the last one was still being laid out has already // asked for another; that one is the answer, not this. - if (!stale) setPlaced(positions) + if (!stale) setPlaced(layout) }) return () => { stale = true @@ -644,7 +647,8 @@ function FlowEditorInner({ ] as const }), ]) - const positions = useLayout(ids, edges, direction, shapes, shapeKey) + const layout = useLayout(ids, edges, direction, shapes, shapeKey) + const positions = layout?.positions ?? null /** * The endpoints, placed. @@ -670,17 +674,22 @@ function FlowEditorInner({ ) // Marked after the layout has had the derived array, so selecting an edge - // never sends the graph through the layout again. + // never sends the graph through the layout again. The route rides along + // the same way. const shownEdges = useMemo( () => edges.map((edge) => ({ ...edge, + data: { + ...(edge.data as FlowEdgeData), + route: layout?.routes.get(edge.id), + }, selected: selectedEdges.has(edge.id) || edge.source === landing || edge.target === landing, })), - [edges, selectedEdges, landing], + [edges, selectedEdges, landing, layout], ) const shownNodes = useMemo( diff --git a/frontend/src/components/Flow/LiveEdge.tsx b/frontend/src/components/Flow/LiveEdge.tsx index 3a4940f..33b46b6 100644 --- a/frontend/src/components/Flow/LiveEdge.tsx +++ b/frontend/src/components/Flow/LiveEdge.tsx @@ -12,6 +12,7 @@ import { describeArtifact, isRef } from "@/lib/media" import { duration } from "@/lib/motion" import { cn } from "@/lib/utils" import type { FlowEdgeData } from "./deriveEdges" +import type { Point } from "./layout" import { useLiveValue } from "./liveStore" /** Below this zoom the value chips would be unreadable, so they step aside. */ @@ -20,22 +21,129 @@ const CHIP_MIN_ZOOM = 0.5 /** How far a feedback edge stands off the ranks it has to get back past. */ const BOW = 60 +/** The radius a routed edge turns its corners with. */ +const CORNER = 10 + /** - * The path a feedback edge takes. + * The route the layout found, met up with the handles React Flow drew. + * + * The layout reserves a card's widest and a label's widest, and either is + * narrower when its text is, so a handle on screen can sit beside the port the + * route was found for. Each end moves onto its handle and takes the bend next + * to it along, which keeps every segment square. A straight run between two + * handles that no longer line up gets a step halfway. + * + * Null for a route that does not alternate along and across the flow, which + * is all this knows how to move. + */ +function stitch( + route: Point[], + from: Point, + to: Point, + vertical: boolean, +): Point[] | null { + // [along the flow, across it] and back. + const split = (p: Point) => (vertical ? [p.y, p.x] : [p.x, p.y]) + const join = (along: number, across: number): Point => + vertical ? { x: across, y: along } : { x: along, y: across } + + // ELK can hand back a corner that is not one: two runs in the same line. + const points = route.filter((p, i) => { + const [a, b] = [route[i - 1], route[i + 1]] + return ( + !a || !b || ((a.x !== p.x || b.x !== p.x) && (a.y !== p.y || b.y !== p.y)) + ) + }) + for (let i = 1; i < points.length; i += 1) { + const [a0, c0] = split(points[i - 1]) + const [a1, c1] = split(points[i]) + // Odd segments run along the flow, even ones across it. + if (i % 2 ? c0 !== c1 : a0 !== a1) return null + } + if (points.length % 2) return null + + const [fromAlong, fromAcross] = split(from) + const [toAlong, toAcross] = split(to) + if (points.length === 2) { + if (fromAcross === toAcross) return [from, to] + const halfway = (fromAlong + toAlong) / 2 + return [from, join(halfway, fromAcross), join(halfway, toAcross), to] + } + const bends = points.slice(1, -1) + const first = bends[0] + const last = bends[bends.length - 1] + bends[0] = join(split(first)[0], fromAcross) + bends[bends.length - 1] = join(split(last)[0], toAcross) + return [from, ...bends, to] +} + +/** A square route with its corners rounded off. */ +function roundedPath(points: Point[]): string { + const towards = (from: Point, to: Point, by: number): Point => { + const length = Math.hypot(to.x - from.x, to.y - from.y) || 1 + return { + x: from.x + ((to.x - from.x) * by) / length, + y: from.y + ((to.y - from.y) * by) / length, + } + } + let path = `M${points[0].x},${points[0].y}` + for (let i = 1; i < points.length - 1; i += 1) { + const [before, corner, after] = [points[i - 1], points[i], points[i + 1]] + // Never more than half of either run, or two corners would overlap. + const radius = Math.min( + CORNER, + Math.hypot(corner.x - before.x, corner.y - before.y) / 2, + Math.hypot(after.x - corner.x, after.y - corner.y) / 2, + ) + const enter = towards(corner, before, radius) + const leave = towards(corner, after, radius) + path += ` L${enter.x},${enter.y} Q${corner.x},${corner.y} ${leave.x},${leave.y}` + } + const end = points[points.length - 1] + return `${path} L${end.x},${end.y}` +} + +/** The point of a polyline closest to `to`. */ +function nearest(points: Point[], to: Point): Point { + let best = points[0] + let gap = Number.POSITIVE_INFINITY + for (let i = 1; i < points.length; i += 1) { + const [a, b] = [points[i - 1], points[i]] + const [dx, dy] = [b.x - a.x, b.y - a.y] + const t = Math.max( + 0, + Math.min( + 1, + ((to.x - a.x) * dx + (to.y - a.y) * dy) / (dx * dx + dy * dy || 1), + ), + ) + const at = { x: a.x + t * dx, y: a.y + t * dy } + const distance = Math.hypot(at.x - to.x, at.y - to.y) + if (distance < gap) [best, gap] = [at, distance] + } + return best +} + +/** + * React Flow reads a handle's place out of the DOM through the zoom, which + * leaves noise in the last digits — enough for one flow to draw two different + * paths on two loads. Half a pixel is as fine as a handle is ever placed. + */ +const half = (value: number) => Math.round(value * 2) / 2 + +/** + * The path a feedback edge takes when the layout has no route for it. * * The layout ranks a flow in the direction it runs, so an edge closing a cycle * is the one that goes the other way — a widget setting the window a node * upstream of it reads. A curve drawn straight between those two handles is * drawn straight through everything between them. This one leaves its handle * the way every other edge does, swings out into a lane beside the graph, and - * comes back into its target from the far side, so what it passes it passes - * around. + * comes back into its target from the far side. * - * ponytail: how far out the lane sits is read off how far back the edge has to - * travel, not off what is actually in the way — this stands a cycle apart from - * the flow rather than guaranteeing it clears every node. The layout engine - * routes these properly and React Flow draws its own curves; taking its bend - * points is the real fix, and a bigger one. + * How far out the lane sits is read off how far back the edge has to travel, + * not off what is in the way, which is good enough for the moment between an + * edit and the layout answering it. */ function cyclePath( sourceX: number, @@ -75,6 +183,8 @@ function formatValue(value: unknown): string { function LiveEdgeComponent({ id, + source, + target, sourceX, sourceY, targetX, @@ -84,9 +194,20 @@ function LiveEdgeComponent({ data, selected, }: EdgeProps) { - const { message, producerId, stream } = (data ?? {}) as FlowEdgeData + const { message, producerId, stream, route } = (data ?? {}) as FlowEdgeData const live = useLiveValue(message) const zoom = useStore((state) => state.transform[2]) + // The route holds while both ends are where the layout put them. One that + // has moved since — dragged, or placed by a layout this edge is not from yet + // — takes the plain curve until the next layout routes it again. + const placed = useStore((state) => { + if (!route) return false + const at = (node: string, where: Point) => { + const now = state.nodeLookup.get(node)?.internals.positionAbsolute + return now?.x === where.x && now?.y === where.y + } + return at(source, route.from) && at(target, route.to) + }) const [pulsing, setPulsing] = useState(false) const lastTs = useRef(null) // Owned by the pulse rather than by the effect: a value from another @@ -95,21 +216,34 @@ function LiveEdgeComponent({ const timer = useRef>(undefined) useEffect(() => () => clearTimeout(timer.current), []) + const vertical = sourcePosition === Position.Bottom + const square = + placed && route + ? stitch( + route.points, + { x: half(sourceX), y: half(sourceY) }, + { x: half(targetX), y: half(targetY) }, + vertical, + ) + : null // Every rank sits further along the flow than the one before it, so an edge // whose target has ended up behind its source is one closing a cycle. - const vertical = sourcePosition === Position.Bottom const backwards = vertical ? targetY < sourceY : targetX < sourceX + const chip = square && route ? nearest(square, route.label) : null - const [path, labelX, labelY] = backwards - ? cyclePath(sourceX, sourceY, targetX, targetY, vertical) - : getBezierPath({ - sourceX, - sourceY, - sourcePosition, - targetX, - targetY, - targetPosition, - }) + const [path, labelX, labelY] = + square && chip + ? [roundedPath(square), chip.x, chip.y] + : backwards + ? cyclePath(sourceX, sourceY, targetX, targetY, vertical) + : getBezierPath({ + sourceX, + sourceY, + sourcePosition, + targetX, + targetY, + targetPosition, + }) // Restart the stroke animation whenever a newer message lands — but only // for the producer that actually published it. A message can have several diff --git a/frontend/src/components/Flow/deriveEdges.ts b/frontend/src/components/Flow/deriveEdges.ts index 07f5906..d72a253 100644 --- a/frontend/src/components/Flow/deriveEdges.ts +++ b/frontend/src/components/Flow/deriveEdges.ts @@ -1,6 +1,7 @@ import type { Edge } from "@xyflow/react" import type { MessageSpec, NodeDef_Input } from "@/client" +import type { Route } from "./layout" /** Resolve a message name against its flow: bare names belong to this flow. */ export function qualify(flow: string, name: string): string { @@ -44,6 +45,8 @@ export type FlowEdgeData = { producerId: string /** The producing port publishes repeatedly during one run, not once at its end. */ stream?: boolean + /** How the layout routed it, once it has. */ + route?: Route [key: string]: unknown } diff --git a/frontend/src/components/Flow/layout.check.ts b/frontend/src/components/Flow/layout.check.ts index d0a9953..d96ff15 100644 --- a/frontend/src/components/Flow/layout.check.ts +++ b/frontend/src/components/Flow/layout.check.ts @@ -51,14 +51,22 @@ const one = (port: string) => ({ requires: [port], provides: [port] }) // A chain lays out along the flow, an edge at a time. const chain = await layoutGraph( ["a", "b"], - [{ source: "a", target: "b", sourceHandle: "v", targetHandle: "v" }], + [ + { + id: "a>b", + source: "a", + target: "b", + sourceHandle: "v", + targetHandle: "v", + }, + ], "LR", new Map([ ["a", one("v")], ["b", one("v")], ]), ) -const [ca, cb] = [chain.get("a"), chain.get("b")] +const [ca, cb] = [chain.positions.get("a"), chain.positions.get("b")] assert.ok(ca && cb) // Two single-port nodes wired to each other have nothing to step around, so // the edge between them is level — which is what a straight edge means here. @@ -69,14 +77,22 @@ assert.ok(cb.x > ca.x) // itself. const again = await layoutGraph( ["a", "b"], - [{ source: "a", target: "b", sourceHandle: "v", targetHandle: "v" }], + [ + { + id: "a>b", + source: "a", + target: "b", + sourceHandle: "v", + targetHandle: "v", + }, + ], "LR", new Map([ ["a", one("v")], ["b", one("v")], ]), ) -assert.deepEqual([...again], [...chain]) +assert.deepEqual(again, chain) /** * The crossing this whole thing exists for. @@ -89,8 +105,20 @@ assert.deepEqual([...again], [...chain]) const fan = await layoutGraph( ["a", "b", "c"], [ - { source: "a", target: "c", sourceHandle: "out", targetHandle: "low" }, - { source: "b", target: "c", sourceHandle: "out", targetHandle: "high" }, + { + id: "a>c", + source: "a", + target: "c", + sourceHandle: "out", + targetHandle: "low", + }, + { + id: "b>c", + source: "b", + target: "c", + sourceHandle: "out", + targetHandle: "high", + }, ], "LR", new Map([ @@ -99,7 +127,7 @@ const fan = await layoutGraph( ["c", { requires: ["high", "low"], provides: [] }], ]), ) -const [fa, fb] = [fan.get("a"), fan.get("b")] +const [fa, fb] = [fan.positions.get("a"), fan.positions.get("b")] assert.ok(fa && fb) assert.ok( fb.y < fa.y, @@ -111,8 +139,20 @@ assert.ok( const parallel = await layoutGraph( ["a", "b"], [ - { source: "a", target: "b", sourceHandle: "p1", targetHandle: "q1" }, - { source: "a", target: "b", sourceHandle: "p2", targetHandle: "q2" }, + { + id: "1", + source: "a", + target: "b", + sourceHandle: "p1", + targetHandle: "q1", + }, + { + id: "2", + source: "a", + target: "b", + sourceHandle: "p2", + targetHandle: "q2", + }, ], "LR", new Map([ @@ -120,7 +160,7 @@ const parallel = await layoutGraph( ["b", { requires: ["q1", "q2"], provides: [] }], ]), ) -const [pa, pb] = [parallel.get("a"), parallel.get("b")] +const [pa, pb] = [parallel.positions.get("a"), parallel.positions.get("b")] assert.ok(pa && pb) assert.ok(pb.x > pa.x + 220, "the pair still ranks one after the other") @@ -128,14 +168,25 @@ assert.ok(pb.x > pa.x + 220, "the pair still ranks one after the other") // a node does — 192 against 220, plus the rank gap either side of the chip. const labelled = await layoutGraph( ["input:x", "n"], - [{ source: "input:x", target: "n", sourceHandle: "x", targetHandle: "x" }], + [ + { + id: "input:x>n", + source: "input:x", + target: "n", + sourceHandle: "x", + targetHandle: "x", + }, + ], "LR", new Map([ ["input:x", { requires: [], provides: ["x"], endpoint: true }], ["n", { requires: ["x"], provides: [] }], ]), ) -const [ex, en] = [labelled.get("input:x"), labelled.get("n")] +const [ex, en] = [ + labelled.positions.get("input:x"), + labelled.positions.get("n"), +] assert.ok(ex && en) assert.equal(ex.x, 40, "the graph starts at its margin") assert.equal( @@ -155,9 +206,22 @@ assert.equal( const spine = await layoutGraph( ["start", "mid", "end", "dashboard:panel:v"], [ - { source: "start", target: "mid", sourceHandle: "v", targetHandle: "v" }, - { source: "mid", target: "end", sourceHandle: "w", targetHandle: "w" }, { + id: "start>mid", + source: "start", + target: "mid", + sourceHandle: "v", + targetHandle: "v", + }, + { + id: "mid>end", + source: "mid", + target: "end", + sourceHandle: "w", + targetHandle: "w", + }, + { + id: "start>dashboard:panel:v", source: "start", target: "dashboard:panel:v", sourceHandle: "v", @@ -172,7 +236,7 @@ const spine = await layoutGraph( ["dashboard:panel:v", { requires: ["v"], provides: [], endpoint: true }], ]), ) -const [s1, s2] = [spine.get("start"), spine.get("mid")] +const [s1, s2] = [spine.positions.get("start"), spine.positions.get("mid")] assert.ok(s1 && s2) assert.equal( s1.x, @@ -184,6 +248,7 @@ assert.equal( const wide = await layoutGraph( ["src", "s0", "s1", "s2", "s3", "s4", "s5"], ["s0", "s1", "s2", "s3", "s4", "s5"].map((sink) => ({ + id: sink, source: "src", target: sink, sourceHandle: "v", @@ -195,7 +260,7 @@ const wide = await layoutGraph( ), ) const rows = new Map() -for (const [, at] of wide) { +for (const [, at] of wide.positions) { rows.set(Math.round(at.y), (rows.get(Math.round(at.y)) ?? 0) + 1) } for (const [y, count] of rows) { @@ -205,4 +270,65 @@ for (const [y, count] of rows) { ) } +/** + * Every edge comes back routed, port to port, square, and around what is in + * the way. `back` closes a cycle over `mid`: drawn straight between its two + * handles it would run through `mid`, which is what the route exists to avoid. + */ +for (const direction of ["LR", "TB"] as const) { + const ids = ["start", "mid", "end"] + const loop = await layoutGraph( + ids, + [ + { + id: "on", + source: "start", + target: "mid", + sourceHandle: "v", + targetHandle: "v", + }, + { + id: "off", + source: "mid", + target: "end", + sourceHandle: "v", + targetHandle: "v", + }, + { + id: "back", + source: "end", + target: "start", + sourceHandle: "v", + targetHandle: "v", + }, + ], + direction, + new Map(ids.map((id) => [id, one("v")])), + ) + const back = loop.routes.get("back") + const end = loop.positions.get("end") + const mid = loop.positions.get("mid") + assert.ok(back && end && mid && loop.routes.size === 3) + // One port on each side, so it sits halfway along: the bottom or the right. + assert.deepEqual( + back.points[0], + direction === "TB" + ? { x: end.x + 110, y: end.y + NODE_H } + : { x: end.x + 220, y: end.y + NODE_H / 2 }, + ) + for (let i = 1; i < back.points.length; i += 1) { + const [a, b] = [back.points[i - 1], back.points[i]] + assert.ok(a.x === b.x || a.y === b.y, `a slanted run in ${direction}`) + const [left, right] = [Math.min(a.x, b.x), Math.max(a.x, b.x)] + const [top, bottom] = [Math.min(a.y, b.y), Math.max(a.y, b.y)] + assert.ok( + right <= mid.x || + left >= mid.x + 220 || + bottom <= mid.y || + top >= mid.y + NODE_H, + `the feedback edge runs through the node it passes in ${direction}`, + ) + } +} + console.log("layout: ok") diff --git a/frontend/src/components/Flow/layout.ts b/frontend/src/components/Flow/layout.ts index 9f645cb..d540c72 100644 --- a/frontend/src/components/Flow/layout.ts +++ b/frontend/src/components/Flow/layout.ts @@ -1,7 +1,7 @@ import type { ElkNode, ElkPort } from "elkjs/lib/elk-api" /** - * Where the nodes of a flow go. + * Where the nodes of a flow go, and how the edges between them run. * * Nothing on this canvas is placed by hand: a flow is a graph the editor draws, * not a picture someone arranges. That is the design decision — a canvas nobody @@ -128,12 +128,35 @@ export type NodeShape = { } export type LayoutEdge = { + id: string source: string target: string sourceHandle?: string | null targetHandle?: string | null } +export type Point = { x: number; y: number } + +/** + * The way the layout routed an edge: from the source port through its bends to + * the target port, square to the ranks. `from` and `to` are where its two + * nodes were placed, so a drawing can tell when either has moved since. + */ +export type Route = { + points: Point[] + /** Where the room reserved for the edge's value chip is centred. */ + label: Point + from: Point + to: Point +} + +export type Layout = { + /** Each node's top-left corner. */ + positions: Map + /** By edge id. An edge whose ends the layout did not know has none. */ + routes: Map +} + const EMPTY: NodeShape = { requires: [], provides: [] } /** @@ -152,6 +175,14 @@ function sizeOf(shape: NodeShape, direction: Direction) { } } +/** The edges the layout can place: both of their nodes are in it. */ +function wiredOf(ids: string[], edges: LayoutEdge[]): LayoutEdge[] { + const known = new Set(ids) + return edges.filter( + (edge) => known.has(edge.source) && known.has(edge.target), + ) +} + /** A wire the layout adds and nobody draws: "put this one a rank further on". */ type Wrap = [string, string] @@ -224,10 +255,7 @@ function graphOf( } }) - const known = new Set(ids) - const wired = edges.filter( - (edge) => known.has(edge.source) && known.has(edge.target), - ) + const wired = wiredOf(ids, edges) const isEndpoint = (id: string) => shapes.get(id)?.endpoint === true // An edge whose handle the node never declared — a half-bound spec — is hung // off the node itself rather than dropped. @@ -269,8 +297,8 @@ function graphOf( : { "org.eclipse.elk.layered.priority.straightness": "10" }, labels: [ { - // Nothing reads this — `LiveEdge` draws the chip itself — but a - // label with no text at all is one ELK reserves no room for. + // Only where it lands is read — `LiveEdge` draws the chip itself — + // but a label with no text at all is one ELK reserves no room for. text: " ", width: across ? LABEL_W : 0, height: LABEL_H, @@ -301,10 +329,7 @@ function graphOf( * is the y its nodes share. They do share it: every node is `NODE_H` tall in * this direction, so nothing in a band is offset against its neighbours. */ -function wrapWideRanks( - placed: Map, - ids: string[], -): Wrap[] { +function wrapWideRanks(placed: Map, ids: string[]): Wrap[] { const ranks = new Map() for (const id of ids) { const at = placed.get(id) @@ -340,7 +365,7 @@ function elk() { } /** - * Lay the graph out and return each node's top-left corner. + * Lay the graph out: each node's top-left corner, and the route of each edge. * * `shapes` is what the document says each node is — its ports in the order * they are drawn in, and whether it is an endpoint label. Anything left out is @@ -351,20 +376,20 @@ export async function layoutGraph( edges: LayoutEdge[], direction: Direction, shapes: Map = new Map(), -): Promise> { +): Promise { const engine = await elk() const place = async (wrap: Wrap[]) => { const laid = await engine.layout( graphOf(ids, edges, direction, shapes, wrap), ) - const placed = new Map() + const placed = new Map() for (const child of laid.children ?? []) { placed.set(child.id, { x: child.x ?? 0, y: child.y ?? 0 }) } - return placed + return { placed, laid } } - let placed = await place([]) + let { placed, laid } = await place([]) // Running downwards, a rank wider than the screen is the one thing the // layout can still do something about. Wrapping one rank pushes whatever was @@ -377,10 +402,75 @@ export async function layoutGraph( const more = wrapWideRanks(placed, ids) if (!more.length) break wrap.push(...more) - placed = await place(wrap) + ;({ placed, laid } = await place(wrap)) } } - // ELK places top-left corners already, which is what React Flow wants. - return new Map(ids.map((id) => [id, placed.get(id) ?? { x: 0, y: 0 }])) + // ELK places top-left corners already, which is what React Flow wants, and + // routes every edge in the same frame: they all hang off the root. + const positions = new Map( + ids.map((id) => [id, placed.get(id) ?? { x: 0, y: 0 }]), + ) + const wired = wiredOf(ids, edges) + // How far across the nodes reach, for `longestRun`. + let [left, right] = [Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY] + for (const [id, at] of positions) { + left = Math.min(left, at.x) + right = Math.max( + right, + at.x + sizeOf(shapes.get(id) ?? EMPTY, direction).width, + ) + } + const routes = new Map() + for (const edge of laid.edges ?? []) { + const index = Number(edge.id.slice(1)) + const section = edge.sections?.[0] + // The wrap wires are `w`; only the ones somebody draws are `e`. + if (!edge.id.startsWith("e") || !section) continue + const label = edge.labels?.[0] + const { source, target, id } = wired[index] + const points = [ + section.startPoint, + ...(section.bendPoints ?? []), + section.endPoint, + ] + const room = label + ? { + x: (label.x ?? 0) + (label.width ?? 0) / 2, + y: (label.y ?? 0) + (label.height ?? 0) / 2, + } + : section.startPoint + routes.set(id, { + points, + label: + direction === "TB" && (room.x < left || room.x > right) + ? (longestRun(points) ?? room) + : room, + from: positions.get(source) ?? { x: 0, y: 0 }, + to: positions.get(target) ?? { x: 0, y: 0 }, + }) + } + return { positions, routes } +} + +/** + * The middle of a route's longest run across the flow, running downwards. + * + * Where the chip of an edge goes when ELK put its label outside every node's + * column, on a lane round the side of the graph. A label is given no width in + * this direction (see {@link LABEL_W}), so a chip centred on that lane would + * hang half of itself past the edge of the fit. On a run across, its width + * lies along the line. + */ +function longestRun(points: Point[]): Point | null { + let best: Point | null = null + let longest = 0 + for (let i = 1; i < points.length; i += 1) { + const [a, b] = [points[i - 1], points[i]] + const length = Math.abs(b.x - a.x) + if (a.y !== b.y || length <= longest) continue + longest = length + best = { x: (a.x + b.x) / 2, y: a.y } + } + return best } From 8f2b66b9f97e08f3bfaf28b46c23faacd94ef721 Mon Sep 17 00:00:00 2001 From: stroblme Date: Thu, 15 Oct 2026 18:32:44 +0200 Subject: [PATCH 103/103] A phone's wrapped flow stacks into one column: when a graph is still wider than three nodes after wrapping, a last pass centres every rank, so wrapped rows sit under the row above (Demo house 3721 to 947px across at 390, crossings 295 to 263) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx --- docs/interface/flow-editor.md | 4 +- frontend/src/components/Flow/layout.check.ts | 32 +++++++++ frontend/src/components/Flow/layout.ts | 68 ++++++++++++++++---- 3 files changed, 92 insertions(+), 12 deletions(-) diff --git a/docs/interface/flow-editor.md b/docs/interface/flow-editor.md index e188169..56d2620 100644 --- a/docs/interface/flow-editor.md +++ b/docs/interface/flow-editor.md @@ -5,7 +5,9 @@ the connections and arranges the canvas for you. ## The canvas -The graph runs left to right on desktop and top to bottom on phones. +The graph runs left to right on desktop and top to bottom on phones. On a +phone, at most two nodes sit side by side and the rest wrap onto the rows +below, so a large flow grows downwards in one column. - **Nodes** show their execution status. - **Edges** are routed around the nodes, a loop back to an earlier node diff --git a/frontend/src/components/Flow/layout.check.ts b/frontend/src/components/Flow/layout.check.ts index d96ff15..8876f68 100644 --- a/frontend/src/components/Flow/layout.check.ts +++ b/frontend/src/components/Flow/layout.check.ts @@ -270,6 +270,38 @@ for (const [y, count] of rows) { ) } +/** + * A hub read by eight readers, each feeding a sink of its own: the shape of + * the Demo's house and its dashboards. Wrapped two abreast with straight + * edges, each row landed where its edges pulled it and the graph staircased + * to 1400px across. Centred, the rows stack into one column. + */ +const readers = Array.from({ length: 8 }, (_, i) => `r${i}`) +const hub = await layoutGraph( + ["hub", ...readers, ...readers.map((r) => `${r}.sink`)], + readers.flatMap((r) => [ + { id: r, source: "hub", target: r, sourceHandle: "v", targetHandle: "v" }, + { + id: `${r}.sink`, + source: r, + target: `${r}.sink`, + sourceHandle: "v", + targetHandle: "v", + }, + ]), + "TB", + new Map( + ["hub", ...readers, ...readers.map((r) => `${r}.sink`)].map((id) => [ + id, + one("v"), + ]), + ), +) +const hubX = [...hub.positions.values()].map((at) => at.x) +const across = Math.max(...hubX) + 220 - Math.min(...hubX) +// Three nodes and their gaps: one more than the two a row holds. +assert.ok(across <= 3 * 260, `a wrapped graph runs ${across}px across`) + /** * Every edge comes back routed, port to port, square, and around what is in * the way. `back` closes a cycle over `mid`: drawn straight between its two diff --git a/frontend/src/components/Flow/layout.ts b/frontend/src/components/Flow/layout.ts index d540c72..61ca7cb 100644 --- a/frontend/src/components/Flow/layout.ts +++ b/frontend/src/components/Flow/layout.ts @@ -114,6 +114,27 @@ const ABREAST = 2 */ const WRAP_PASSES = 6 +/** + * How wide a wrapped graph may still be before its ranks are centred. + * + * Wrapping keeps a rank to `ABREAST` nodes, but the layout straightens edges, + * so each wrapped row lands where its edges pull it: the rows staircase + * sideways, and a big flow ended up thousands of pixels across on a phone. + * Past one node more than the rows allow, every rank is centred instead, which + * stacks the rows into one column. Below it the straight edges are worth more, + * which is the case for a flow that wraps a rank or two. + */ +const TOO_WIDE = (ABREAST + 1) * (NODE_W + NODE_GAP) + +/** + * Room for a value chip when the ranks are centred, running downwards. + * + * Centring packs a rank's chips together where the edges meet in the middle, + * so with no width at all they piled onto each other. This is a number's + * worth: a longer value overlaps its neighbour rather than widening the graph. + */ +const CENTRED_LABEL_W = 60 + /** * What a node is, as far as the layout is concerned. * @@ -200,6 +221,7 @@ function graphOf( direction: Direction, shapes: Map, wrap: Wrap[], + centred: boolean, ): ElkNode { const across = direction === "LR" // A port is identified by the side it is on as well as by its name: a node @@ -279,6 +301,16 @@ function graphOf( // The default, set out loud: the same flow has to lay out the same way // every time it is opened. "org.eclipse.elk.randomSeed": "1", + // See `TOO_WIDE`. Every rank is centred on one axis, so a wrapped row + // sits under the row above rather than where its edges pull it. What + // bends the edges then takes room down the channels between the ranks, + // so the runs there sit closer together. + ...(centred + ? { + "org.eclipse.elk.layered.nodePlacement.strategy": "SIMPLE", + "org.eclipse.elk.layered.spacing.edgeEdgeBetweenLayers": "6", + } + : {}), }, children, edges: [ @@ -300,7 +332,7 @@ function graphOf( // Only where it lands is read — `LiveEdge` draws the chip itself — // but a label with no text at all is one ELK reserves no room for. text: " ", - width: across ? LABEL_W : 0, + width: across ? LABEL_W : centred ? CENTRED_LABEL_W : 0, height: LABEL_H, layoutOptions: { "org.eclipse.elk.edgeLabels.inline": "true" }, }, @@ -378,9 +410,9 @@ export async function layoutGraph( shapes: Map = new Map(), ): Promise { const engine = await elk() - const place = async (wrap: Wrap[]) => { + const place = async (wrap: Wrap[], centred = false) => { const laid = await engine.layout( - graphOf(ids, edges, direction, shapes, wrap), + graphOf(ids, edges, direction, shapes, wrap, centred), ) const placed = new Map() for (const child of laid.children ?? []) { @@ -404,6 +436,10 @@ export async function layoutGraph( wrap.push(...more) ;({ placed, laid } = await place(wrap)) } + const [left, right] = extentOf(placed, shapes, direction) + if (wrap.length && right - left > TOO_WIDE) { + ;({ placed, laid } = await place(wrap, true)) + } } // ELK places top-left corners already, which is what React Flow wants, and @@ -413,14 +449,7 @@ export async function layoutGraph( ) const wired = wiredOf(ids, edges) // How far across the nodes reach, for `longestRun`. - let [left, right] = [Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY] - for (const [id, at] of positions) { - left = Math.min(left, at.x) - right = Math.max( - right, - at.x + sizeOf(shapes.get(id) ?? EMPTY, direction).width, - ) - } + const [left, right] = extentOf(positions, shapes, direction) const routes = new Map() for (const edge of laid.edges ?? []) { const index = Number(edge.id.slice(1)) @@ -453,6 +482,23 @@ export async function layoutGraph( return { positions, routes } } +/** The leftmost and rightmost x any node reaches. */ +function extentOf( + placed: Map, + shapes: Map, + direction: Direction, +): [number, number] { + let [left, right] = [Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY] + for (const [id, at] of placed) { + left = Math.min(left, at.x) + right = Math.max( + right, + at.x + sizeOf(shapes.get(id) ?? EMPTY, direction).width, + ) + } + return [left, right] +} + /** * The middle of a route's longest run across the flow, running downwards. *