diff --git a/Makefile b/Makefile index e3df5de..291035a 100644 --- a/Makefile +++ b/Makefile @@ -105,11 +105,13 @@ down: ## Stop all running containers # Run `make dev-backend` and `make dev-frontend` in two separate terminals. install: ## Install all dependencies (backend + frontend) - cd backend && uv sync + # Every extra: the suite exercises the connectors that moved into + # `fluksio[server]`, and strict mypy checks their call sites. + cd backend && uv sync --all-extras cd frontend && bun install dev-backend: ## Start the FastAPI backend with hot-reload (local) - cd backend && uv run fastapi dev fluksio/main.py + cd backend && uv run --all-extras fastapi dev fluksio/main.py dev-frontend: ## Start the Vite dev server (local) cd frontend && bun dev @@ -130,7 +132,7 @@ test: test-backend test-frontend ## Run all tests (backend + frontend) test-backend: ## Run backend tests (pytest + coverage) # Its own SQLite file in a temp directory (tests/__init__.py), so this needs # nothing running and touches no development data. - cd backend && uv run bash scripts/test.sh + cd backend && uv run --all-extras bash scripts/test.sh PW_VERSION = $(shell sed -n 's/.*"@playwright\/test": "[^0-9]*\([0-9.]*\)".*/\1/p' frontend/package.json | head -1) @@ -181,9 +183,9 @@ build: ## Build the fluksio and fluksio-worker wheels into dist/ lint: lint-backend lint-frontend ## Run all linters lint-backend: ## Lint backend with ruff + mypy - cd backend && uv run ruff check . - cd backend && uv run ruff format --check . - cd backend && uv run mypy fluksio + cd backend && uv run --all-extras ruff check . + cd backend && uv run --all-extras ruff format --check . + cd backend && uv run --all-extras mypy fluksio cd worker && uv run --no-project --with mypy mypy fluksio_worker lint-frontend: ## Lint frontend with biome diff --git a/backend/Dockerfile b/backend/Dockerfile index 27041f8..93f978e 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -31,7 +31,7 @@ ENV NODE_VENV=managed RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=uv.lock,target=uv.lock \ --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - uv sync --frozen --no-install-workspace --package fluksio + uv sync --frozen --no-install-workspace --package fluksio --extra server COPY ./backend/scripts /app/backend/scripts @@ -52,7 +52,7 @@ COPY ./worker /app/worker RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=uv.lock,target=uv.lock \ --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - uv sync --frozen --package fluksio + uv sync --frozen --package fluksio --extra server # Connectors are ordinary installed packages found through the # `fluksio.node_types` entry point. `make connectors` builds them into here; diff --git a/backend/fluksio/api/main.py b/backend/fluksio/api/main.py index ba56eaa..a4e1bdf 100644 --- a/backend/fluksio/api/main.py +++ b/backend/fluksio/api/main.py @@ -15,6 +15,7 @@ from fluksio.api.routes import ( panels, private, runs, + search, secrets, users, utils, @@ -38,6 +39,7 @@ api_router.include_router(observability.router) api_router.include_router(runs.router) api_router.include_router(artifacts.router) api_router.include_router(workers.router) +api_router.include_router(search.router) # Remote access through a portal. Always mounted; with no enrolment the # endpoints only ever report that there is none. api_router.include_router(cloud.router) diff --git a/backend/fluksio/api/routes/observability.py b/backend/fluksio/api/routes/observability.py index f578563..54b6176 100644 --- a/backend/fluksio/api/routes/observability.py +++ b/backend/fluksio/api/routes/observability.py @@ -176,10 +176,16 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any: paused = set(controller.paused_flows()) entries = list(controller.loaded.values()) errored = [e for e in entries if e.status is NodeStatus.ERROR] + unhealthy = [e for e in entries if e.health == "down"] if quarantined: problems.append(f"{len(quarantined)} flow(s) quarantined") if errored: problems.append(f"{len(errored)} node(s) failed to load") + if unhealthy: + problems.append( + f"{len(unhealthy)} node(s) down: " + f"{', '.join(sorted(e.id for e in unhealthy))}" + ) # What the canvas flags on a flow — a dependency loop, an input nothing # feeds — stops that flow running just as surely as a node that will not @@ -219,7 +225,11 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any: "quarantined": len(quarantined), "invalid": len(invalid), }, - nodes={"total": len(entries), "error": len(errored)}, + nodes={ + "total": len(entries), + "error": len(errored), + "unhealthy": len(unhealthy), + }, queue=queue, loop_lag=( watchdog.snapshot() diff --git a/backend/fluksio/api/routes/runs.py b/backend/fluksio/api/routes/runs.py index 3cd9b46..bea6b67 100644 --- a/backend/fluksio/api/routes/runs.py +++ b/backend/fluksio/api/routes/runs.py @@ -8,20 +8,22 @@ or to listen on the flow socket, which carries its start and finish. import csv import io import json +import time from collections.abc import Iterator from datetime import UTC, datetime from itertools import groupby from typing import Any, Literal -from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.concurrency import run_in_threadpool from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field, model_validator -from sqlalchemy import func +from sqlalchemy import delete, func from sqlalchemy import select as sa_select from sqlmodel import Session, col, select from fluksio.api.deps import CurrentUser, SessionDep, get_current_user +from fluksio.flow.events import event_bus from fluksio.flow.messages import requalify from fluksio.flow.runs import RunRejected, RunService, new_run_id from fluksio.flow.store import FlowNotFound @@ -102,6 +104,10 @@ class ArtifactRow(BaseModel): digest: str size: int media_type: str + #: What it was called where it was written, so something downloading it can + #: give it that name back rather than the message's. Absent when the node + #: never said one. + filename: str | None = None class RunRow(BaseModel): @@ -420,25 +426,6 @@ def _dig(record: dict[str, Any], path: str) -> Any: return value -def _varying(runs: list[Run]) -> list[str]: - """The inputs that differ across these runs — the axis of a sweep. - - What a reader comparing arms wants as columns, compared leaf by leaf: two - configurations differing in one field give that field as a column rather - than two blobs that are not the same. Under two runs nothing can differ, - and a table of one run with none of its inputs in it is not worth reading, - so all of them are kept. - """ - keys = sorted({path for run in runs for path, _ in _leaves(run.params)}) - if len(runs) < 2: - return keys - return [ - key - for key in keys - if len({json.dumps(_dig(run.params, key), sort_keys=True) for run in runs}) > 1 - ] - - def _scored(runs: list[Run]) -> list[str]: """A run's final numbers: every number its declared outputs carry, however deep it sits. A flag is not a number, and neither is a label.""" @@ -555,9 +542,9 @@ def export_runs( """One row per run: what it was given, what it scored, what code it ran. The arm-comparison table. Inputs are columns rather than one JSON blob — - by default the ones that vary across the selection, which is the sweep - axis; ``params`` names them instead. ``metrics`` narrows the final numbers - to a few of a run's declared outputs. + every input the selection recorded, so the schema is the same whichever + runs are asked for; ``params`` narrows it. ``metrics`` narrows the final + numbers to a few of a run's declared outputs. Both take dotted paths into a record a node returned: ``metrics=final_metrics.train_loss,test_metrics.known.perfect`` selects @@ -565,7 +552,9 @@ def export_runs( depth. """ runs = _selected(session, flow, status, group, ids, since, until) - inputs = [part for part in params.split(",") if part] or _varying(runs) + inputs = [part for part in params.split(",") if part] or sorted( + {path for run in runs for path, _ in _leaves(run.params)} + ) scores = [part for part in metrics.split(",") if part] or _scored(runs) columns = [ *RUN_COLUMNS, @@ -613,6 +602,56 @@ async def cancel_run(run_id: str, request: Request, session: SessionDep) -> Any: return run +@router.delete("/{run_id}", status_code=204) +def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response: + """Forget a run and everything hanging off it. + + The same four statements ``_forget_runs`` uses when a flow goes: the run + tables carry a plain string ``run_id`` and no foreign key, so nothing + cascades on its own. ``flow_run``, ``metric_minute`` and ``engine_event`` + stay — they are the observability record and are pruned on their own window. + + A live run is refused rather than raced: the driver writes its nodes back + when it finishes, and those rows would arrive for a run that no longer + exists. Cancel it first. + + Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage + cache, so a later run loses hits this one would have served. And a node + restored from this run points here through ``cached_from`` — ``_series`` + already reads a missing source as an empty curve, which is what ``NO_CURVE`` + explains on the screen. The artifact bytes need no help: ``sweep_artifacts`` + keeps whatever a ``run_artifact`` row or a live message still names, so + dropping the rows is enough and the hourly sweep reclaims the blobs. + """ + run = session.get(Run, run_id) + if run is None: + raise HTTPException(status_code=404, detail="No such run") + if run.status in ("running", "queued"): + raise HTTPException( + status_code=409, + detail=( + f"Run {run_id} is {run.status}. Cancel it, or wait for it to " + "finish, before deleting it." + ), + ) + flow = run.flow + session.execute(delete(RunNode).where(col(RunNode.run_id) == run_id)) + session.execute(delete(RunMetric).where(col(RunMetric.run_id) == run_id)) + session.execute(delete(RunArtifact).where(col(RunArtifact.run_id) == run_id)) + session.execute(delete(Run).where(col(Run.id) == run_id)) + session.commit() + event_bus.publish( + { + "type": "audit", + "action": f"deleted run {run_id}", + "flow": flow, + "user": user.email, + "ts": time.time(), + } + ) + return Response(status_code=204) + + def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]: """A run's numbers, including the ones a cached node points at. diff --git a/backend/fluksio/api/routes/search.py b/backend/fluksio/api/routes/search.py new file mode 100644 index 0000000..2cdd5c5 --- /dev/null +++ b/backend/fluksio/api/routes/search.py @@ -0,0 +1,156 @@ +"""One index of everything in this installation worth jumping to by name.""" + +from typing import Any, Literal + +from fastapi import APIRouter, Depends, Request +from fastapi.concurrency import run_in_threadpool +from pydantic import BaseModel + +from fluksio.api.deps import ( + CurrentUser, + DashboardStoreDep, + FlowControllerDep, + get_current_user, +) +from fluksio.api.routes.alerts import read_config as read_alerts_config +from fluksio.flow import modules, panels +from fluksio.flow.controller import FlowController +from fluksio.flow.dashboards import DashboardNotFound, DashboardStore +from fluksio.flow.secrets import get_secrets +from fluksio.flow.store import FlowNotFound + +# A wall panel never reaches this route: ``deps._panel_may`` is a whitelist that +# ends in a 403, and a whole-installation index is the opposite of what a screen +# on a wall is allowed to read. +router = APIRouter( + prefix="/search", tags=["search"], dependencies=[Depends(get_current_user)] +) + +Category = Literal[ + "flow", + "node", + "dashboard", + "widget", + "panel", + "secret", + "module", + "worker", + "alert", +] + + +class SearchEntry(BaseModel): + """One thing somebody might be looking for. + + Deliberately not a route: where a category lands is the frontend's business, + and it already owns the router. This says what the thing is and what it is + called, which is all the matching needs. + """ + + category: Category + #: The id the frontend routes on. + name: str + #: Human title, often empty — a flow is usually only its name. + title: str = "" + #: The flow a node sits in, or the dashboard a widget sits on. + parent: str = "" + #: Node type, widget type, channel kind. + kind: str = "" + + +def _build( + controller: FlowController, dashboards: DashboardStore, hub: Any, secrets: bool +) -> list[SearchEntry]: + """Read every store once. Blocking: disk and git throughout. + + # ponytail: rebuilt per call. Key it on ``controller.store.revision`` if a + # store large enough to feel it ever shows up in a profile. + """ + entries: list[SearchEntry] = [] + + for name in controller.store.list_flows(): + try: + flow = controller.store.read_flow(name, draft=True) + except FlowNotFound: + continue + entries.append(SearchEntry(category="flow", name=flow.name, title=flow.title)) + entries.extend( + SearchEntry( + category="node", + name=node.id, + title=node.title, + parent=flow.name, + kind=node.type, + ) + for node in flow.nodes + ) + + for summary in dashboards.list(): + try: + dashboard = dashboards.read(summary.name, draft=True) + except DashboardNotFound: + continue + entries.append( + SearchEntry( + category="dashboard", name=dashboard.name, title=dashboard.title + ) + ) + entries.extend( + SearchEntry( + category="widget", + name=widget.id, + title=widget.title, + parent=dashboard.name, + kind=widget.type, + ) + for widget in dashboard.widgets + ) + + entries.extend( + SearchEntry(category="panel", name=panel.id, title=panel.title) + for panel in panels.read_config().panels + ) + + if secrets: + entries.extend( + SearchEntry(category="secret", name=name) for name in get_secrets().list() + ) + + entries.extend( + SearchEntry(category="module", name=package.name, kind=package.version) + for package in modules.info(controller.store).packages + ) + + entries.extend( + SearchEntry(category="worker", name=worker.name) + for worker in (hub.workers() if hub is not None else []) + ) + + entries.extend( + SearchEntry(category="alert", name=channel.name, kind=channel.kind) + for channel in read_alerts_config().channels + ) + + return entries + + +@router.get("/", response_model=list[SearchEntry]) +async def read_search_index( + current_user: CurrentUser, + request: Request, + controller: FlowControllerDep, + dashboards: DashboardStoreDep, +) -> Any: + """Everything searchable, for the client to match against as it is typed. + + The whole index rather than a query: it is a few hundred short rows for an + installation of any ordinary size, so one fetch when the panel opens beats a + round trip per keystroke — and the client already has a matcher. + + Secrets are named only to a superuser, which is who ``/secrets`` answers to. + """ + # Absent when remote workers are switched off — not a reason to fail a search. + hub = getattr(request.app.state, "worker_hub", None) + return await run_in_threadpool( + _build, controller, dashboards, hub, current_user.is_superuser + ) diff --git a/backend/fluksio/cli.py b/backend/fluksio/cli.py index b476ea1..20f43df 100644 --- a/backend/fluksio/cli.py +++ b/backend/fluksio/cli.py @@ -17,10 +17,12 @@ from __future__ import annotations import argparse import copy +import json import os import secrets import socket import sys +from collections.abc import Callable from pathlib import Path from typing import Any @@ -80,6 +82,39 @@ def _mention_other_installation(data_dir: Path) -> None: _say(" `fluksio serve --global` runs that one instead.") +def _mention_undeclared_cards() -> None: + """Say when a stored flow asks for a card this engine does not have. + + Cards are declared rather than detected, so an engine told nothing has + none — and a node asking for one is clamped to zero and runs beside every + other, which on a GPU is the deadlock the declaration exists to prevent. + The placer says so once it happens, into the log; this says it while + somebody is still reading the terminal. + """ + from fluksio.core.config import settings + + if settings.FLOW_GPUS: + return + from fluksio.flow.runs import required_resources + from fluksio.flow.store import FlowStore + + store = FlowStore(settings.FLOWS_DIR) + asking = [] + for name in store.list_flows(): + try: + needs = required_resources(store.read_flow(name)) + except Exception: + # A flow that will not parse is the engine's to complain about. + continue + if (needs or {}).get("gpus"): + asking.append(name) + if not asking: + return + named = ", ".join(sorted(asking)[:3]) + (" …" if len(asking) > 3 else "") + _say(f" Cards 0 declared, but {named} asks for one.") + _say(" Nothing detects them: `--gpus N` says how many are here.") + + def _warn_if_networked(path: Path) -> None: """A cluster's $HOME is often NFS, and SQLite's WAL does not work there.""" try: @@ -228,9 +263,26 @@ CONCURRENCY_FLAGS = { "max_workers": "FLOW_MAX_WORKERS", "max_cascades": "FLOW_MAX_CASCADES", "max_runs": "FLOW_MAX_RUNS", + "gpus": "FLOW_GPUS", } +def _at_least(minimum: int) -> Callable[[str], int]: + """A flag's value, checked here rather than by the settings. + + These are written into the environment before the settings are built, so a + number they refuse dies inside a pydantic import with no flag named in it. + """ + + def parse(text: str) -> int: + value = int(text) + if value < minimum: + raise argparse.ArgumentTypeError(f"is {value}, needs at least {minimum}") + return value + + return parse + + #: What `serve` listens on when nobody says. Taken often enough — another #: engine, another framework's dev server — that dying on it is the first #: thing a zero-config start would hit. @@ -240,6 +292,85 @@ DEFAULT_PORT = 8000 PORT_TRIES = 20 +#: Where a serving engine records itself, beside the data it is serving. Read +#: to tell "this directory's engine is already up" from "something else has the +#: port", which are the two ways a second `serve` fails to be what was wanted. +PIDFILE = "serve.pid" + + +def write_pidfile(data_dir: Path, port: int) -> Path: + """Record which process is serving this directory, and where.""" + path = data_dir / PIDFILE + path.write_text(json.dumps({"pid": os.getpid(), "port": port})) + return path + + +def read_pidfile(data_dir: Path) -> dict[str, int] | None: + """The engine serving this directory, if one still is. + + A process killed outright leaves the file behind, so the pid is checked + rather than believed — a stale file is the same as no file. + """ + try: + record = json.loads((data_dir / PIDFILE).read_text()) + pid, port = int(record["pid"]), int(record["port"]) + except (OSError, ValueError, KeyError, TypeError): + return None + try: + os.kill(pid, 0) + except (OSError, ProcessLookupError): + return None + return {"pid": pid, "port": port} + + +def _token_for(data_dir: Path) -> str: + """The credential this directory's last engine wrote, if it wrote one.""" + from fluksio.sdk.client import config_path + + try: + return str(json.loads(config_path(data_dir).read_text()).get("token", "")) + except (OSError, ValueError, AttributeError): + return "" + + +def probe_engine(url: str, token: str, client: Any = None) -> str: + """Who is on this port: ``ours``, ``foreign``, or ``other``. + + ``ours`` means an engine serving *this* data directory, which is what + makes stopping it something this command may offer. The proof is the + token: it is signed with this directory's secret key, so an engine that + accepts it is one reading this directory's database. A Fluksio belonging + to another installation answers the health check and refuses the token, + and is only ever named — never stopped from here. + """ + import logging + + import httpx + + # Two requests nobody asked for, on a start that is otherwise quiet until + # uvicorn's own banner. httpx logs every one of them at INFO. + noisy = logging.getLogger("httpx") + was = noisy.level + noisy.setLevel(logging.WARNING) + http = client or httpx.Client(timeout=2.0) + try: + health = http.get(f"{url}/api/v1/utils/health-check/") + if health.status_code != 200: + return "other" + # No token at all is asked unauthenticated: `Bearer ` is not a legal + # header value, and a Fluksio this directory cannot prove is its own + # is foreign — which is the answer that never stops anything. + auth = {"Authorization": f"Bearer {token}"} if token else {} + answer = http.get(f"{url}/api/v1/observability/summary", headers=auth) + return "ours" if answer.status_code == 200 else "foreign" + except Exception: + return "other" + finally: + noisy.setLevel(was) + if client is None: + http.close() + + def _free_port(host: str, start: int) -> int: """The first port from ``start`` that nothing is listening on. @@ -258,6 +389,14 @@ def _free_port(host: str, start: int) -> int: def cmd_serve(args: argparse.Namespace) -> int: + # At a terminal this is a dashboard with the engine as a child of it. The + # import is here rather than at the top because it is only ever needed on + # that path, and `serve` in a container must not pay for it. + if not args.plain and sys.stdout.isatty() and sys.stdin.isatty(): + from fluksio.tui import run_tui + + return run_tui(args) + data_dir = _data_dir(args.data_dir, args.shared) for flag, name in CONCURRENCY_FLAGS.items(): value = getattr(args, flag, None) @@ -294,15 +433,34 @@ def cmd_serve(args: argparse.Namespace) -> int: from fluksio.flow import modules from fluksio.main import app + # The client talks to this engine, and 0.0.0.0 is not an address to talk + # to — it is a statement about which interfaces to listen on. + reachable = "127.0.0.1" if args.host in ("0.0.0.0", "::", "") else args.host + port = args.port if port is None: port = _free_port(args.host, DEFAULT_PORT) if port != DEFAULT_PORT: - _say(f"Port {DEFAULT_PORT} is in use; serving on {port} instead.") + # Moving off the port quietly makes starting a second engine for + # one directory look like it worked. Two of them on one SQLite + # file is not a supported shape — two *installations* on one + # machine is — so the one already up is named instead. + url = f"http://{reachable}:{DEFAULT_PORT}" + who = probe_engine(url, _token_for(data_dir)) + if who == "ours": + running = read_pidfile(data_dir) + where = f" (pid {running['pid']})" if running else "" + _say(f"An engine for {data_dir} is already serving at {url}{where}.") + _say(" fluksio status talks to it; stop it to start another.") + return 0 + if who == "foreign": + _say( + f"Port {DEFAULT_PORT} holds another installation's Fluksio; " + f"serving on {port} instead." + ) + else: + _say(f"Port {DEFAULT_PORT} is in use; serving on {port} instead.") - # The client talks to this engine, and 0.0.0.0 is not an address to talk - # to — it is a statement about which interfaces to listen on. - reachable = "127.0.0.1" if args.host in ("0.0.0.0", "::", "") else args.host url = f"http://{reachable}:{port}" token_path = _sign_in(admin_id, url, data_dir) @@ -329,17 +487,22 @@ def cmd_serve(args: argparse.Namespace) -> int: _say(" fluksio enroll ") _say(f" Signed in as {admin_email}") _say(f" token in {token_path}") + _mention_undeclared_cards() _mention_other_installation(data_dir) # One process: it holds the flow engine, and a second worker would be a # second engine — duplicated subscriptions, cron ticks and webhooks. - uvicorn.run( - app, - host=args.host, - port=port, - log_level=args.log_level, - log_config=_log_config(args.log_level), - ) + pidfile = write_pidfile(data_dir, port) + try: + uvicorn.run( + app, + host=args.host, + port=port, + log_level=args.log_level, + log_config=_log_config(args.log_level), + ) + finally: + pidfile.unlink(missing_ok=True) return 0 @@ -426,25 +589,37 @@ def _parser() -> argparse.ArgumentParser: ) serve.add_argument( "--max-runs", - type=int, + type=_at_least(1), default=None, metavar="N", help="batch runs driven at once (default 4, FLOW_MAX_RUNS)", ) serve.add_argument( "--max-cascades", - type=int, + type=_at_least(1), default=None, metavar="N", help="cascades in flight at once (default 4, FLOW_MAX_CASCADES)", ) serve.add_argument( "--max-workers", - type=int, + type=_at_least(1), default=None, metavar="N", help="python worker processes (default 4, FLOW_MAX_WORKERS)", ) + serve.add_argument( + "--plain", + action="store_true", + help="the log stream rather than the dashboard (the default with no terminal)", + ) + serve.add_argument( + "--gpus", + type=_at_least(0), + default=None, + metavar="N", + help="GPUs on this machine a node may be given (default 0, FLOW_GPUS)", + ) serve.set_defaults(func=cmd_serve) enroll = subparsers.add_parser( diff --git a/backend/fluksio/core/config.py b/backend/fluksio/core/config.py index 1e8729b..91497fc 100644 --- a/backend/fluksio/core/config.py +++ b/backend/fluksio/core/config.py @@ -9,6 +9,7 @@ from pydantic import ( BeforeValidator, EmailStr, HttpUrl, + PositiveInt, computed_field, model_validator, ) @@ -97,15 +98,19 @@ class Settings(BaseSettings): # rather than a person, and it can refresh unattended. MCP_TOKEN_EXPIRE_MINUTES: int = 60 MCP_REFRESH_EXPIRE_DAYS: int = 30 - FLOW_MAX_WORKERS: int = 4 + # The three below are all pool sizes, so 0 says neither "none" nor + # "unlimited" — it is a pool that cannot be built and an engine that would + # accept no work. Rejected here rather than quietly read as the default, + # because a limit somebody set and did not get is the worse surprise. + FLOW_MAX_WORKERS: PositiveInt = 4 # How many cascades may be in flight at once. Sustained throughput is this # over the mean cascade time, so an installation whose nodes wait on the # network rather than on a CPU wants it higher than the core count. - FLOW_MAX_CASCADES: int = 4 + FLOW_MAX_CASCADES: PositiveInt = 4 # How many batch runs are driven at once. A different limit from the one # above: a run drives a whole graph, and its nodes are bounded by the worker # pool rather than by cascade slots. A sweep is what this governs. - FLOW_MAX_RUNS: int = 4 + FLOW_MAX_RUNS: PositiveInt = 4 # How long a python node may be silent before its worker is killed, unless # the node sets its own. 0, the default, disables it: a dead worker still # fails fast, and a slow one is left to finish. Set it where silence means diff --git a/backend/fluksio/flow/connector.py b/backend/fluksio/flow/connector.py index ac49e4b..1cfe897 100644 --- a/backend/fluksio/flow/connector.py +++ b/backend/fluksio/flow/connector.py @@ -86,7 +86,14 @@ class ConnectorNode(Node): description="Seconds between polls; 0 polls never.", ) - __slots__ = ("config", "_poll_task", "_stop_event", "_last_published", "_artifacts") + __slots__ = ( + "config", + "_poll_task", + "_stop_event", + "_last_published", + "_artifacts", + "_down", + ) def __init__(self, **kwargs: Any) -> None: super().__init__(f=self._dispatch, **kwargs) @@ -95,6 +102,7 @@ class ConnectorNode(Node): self._stop_event: asyncio.Event | None = None self._last_published: dict[str, Any] = {} self._artifacts: ArtifactStore | None = None + self._down = False def _dispatch(self, params: dict[str, Any], **ports: Any) -> dict[str, Any] | None: """The scheduler's entry point. Settings are already on ``self.config``.""" @@ -161,11 +169,7 @@ class ConnectorNode(Node): return self._stop_event.set() if self._poll_task is not None: - self._poll_task.cancel() - try: - await self._poll_task - except (asyncio.CancelledError, Exception): # noqa: B014 - shutting down - pass + await self._cancel_task(self._poll_task) self._poll_task = None self._stop_event = None self._last_published = {} @@ -175,24 +179,29 @@ class ConnectorNode(Node): Only changed ports are published: a device polled every few seconds is usually saying the same thing, and every publication wakes everything - downstream of it. + downstream of it. What is remembered is what was *published*, not what + the poll returned — a publication that raised is retried next tick + rather than counting as said. """ while not (self._stop_event and self._stop_event.is_set()): try: values = await self.poll() - self.report_health("ok") changed = { port: value for port, value in (values or {}).items() if self._last_published.get(port, object()) != value } if changed: - self._last_published.update(changed) # inject runs the graph, which is blocking work. await asyncio.to_thread(self.inject, changed) + self._last_published.update(changed) + self.report_health("ok") + self._down = False except asyncio.CancelledError: break except Exception as exc: - logger.warning("Connector '%s' failed to poll: %s", self.id, exc) + if not self._down: + logger.warning("Connector '%s' failed to poll: %s", self.id, exc) + self._down = True self.report_health("down", f"{type(exc).__name__}: {exc}") await asyncio.sleep(self.config.poll_interval) diff --git a/backend/fluksio/flow/controller.py b/backend/fluksio/flow/controller.py index d883416..1ffce99 100644 --- a/backend/fluksio/flow/controller.py +++ b/backend/fluksio/flow/controller.py @@ -916,12 +916,12 @@ class FlowController: placer, pool = self.placer, self.workers def call(**kwargs: Any) -> Any: - with placer.claim( + with placer.claim( # type: ignore[union-attr] wanted, device=device, policy=policy, node=node_id, run=run_id ) as (target, allocation): env = derive_env(wanted, allocation) if target.worker is None: - return pool.for_env(env).run( + return pool.for_env(env).run( # type: ignore[union-attr] owner, local, code, @@ -1224,8 +1224,31 @@ class FlowController: } ) + def _health_issues(self, flow: str | None = None) -> list[ValidationIssue]: + """Nodes that are running but not working, as issues on their flow. + + Not part of `self.issues`: that list is what a build found, and this is + what is happening now. Derived on read, so a node reporting itself well + again clears it with nothing to remember. + """ + return [ + ValidationIssue( + code="node_unhealthy", + message=( + f"Node '{entry.id.rpartition('.')[2]}' is down: " + f"{entry.health_detail or 'no detail given'}" + ), + flow=entry.flow, + node=entry.id, + ) + for entry in self.loaded.values() + if entry.health == "down" and (flow is None or entry.flow == flow) + ] + def flow_issues(self, flow: str) -> list[ValidationIssue]: - return [issue for issue in self.issues if not issue.flow or issue.flow == flow] + return [ + issue for issue in self.issues if not issue.flow or issue.flow == flow + ] + self._health_issues(flow) def preview(self, name: str) -> Preview: """Build a flow's unpublished draft without deploying it. diff --git a/backend/fluksio/flow/executor.py b/backend/fluksio/flow/executor.py index 8711df1..cbbef7a 100644 --- a/backend/fluksio/flow/executor.py +++ b/backend/fluksio/flow/executor.py @@ -46,6 +46,11 @@ DELAYED_INTERVAL_S = 1.0 #: whose nodes wait on a network rather than a CPU may want more of them — #: `FLOW_MAX_CASCADES` is where that is said. MAX_CASCADES = 4 +#: Node threads, unless the service is given a number. Both this and the one +#: above are taken as written: only ``None`` means "nobody said", so a number +#: that reached here is one somebody chose, and an unusable one is the pool's +#: ``ValueError`` rather than a silent 4. +MAX_WORKERS = 4 # How long a reload waits for claimed work to finish before rebuilding anyway. DRAIN_TIMEOUT_S = 10.0 # Work waiting in the stream, undelivered. A burst is normal — the pool claims @@ -68,7 +73,7 @@ class ExecutionService: ) -> None: self.queue = queue self._events = events - self.max_cascades = max_cascades or MAX_CASCADES + self.max_cascades = MAX_CASCADES if max_cascades is None else max_cascades self._pipeline: Pipeline | None = None self._stop = threading.Event() # Set when a deadline moves closer, so the timer thread stops waiting @@ -82,7 +87,8 @@ class ExecutionService: # Entry ids claimed and still running, under _inflight_lock. self._active: set[str] = set() self.node_pool = ThreadPoolExecutor( - max_workers=max_workers or 4, thread_name_prefix="node" + max_workers=MAX_WORKERS if max_workers is None else max_workers, + thread_name_prefix="node", ) self._cascade_pool = ThreadPoolExecutor( max_workers=self.max_cascades, thread_name_prefix="cascade" diff --git a/backend/fluksio/flow/nodes/base.py b/backend/fluksio/flow/nodes/base.py index e959070..b46a57d 100644 --- a/backend/fluksio/flow/nodes/base.py +++ b/backend/fluksio/flow/nodes/base.py @@ -212,6 +212,22 @@ class Node: return None return asyncio.create_task(factory()) + @staticmethod + async def _cancel_task(task: asyncio.Task[None]) -> None: + """Stop an unsupervised loop and wait for it to be gone. + + `wait` keeps whatever the task raises on its way out to itself, and + lets a cancellation aimed at *this* coroutine through — the + `except CancelledError` around `await task` it replaces swallowed that, + which left whoever asked for the teardown unkillable. The same trap + `Supervisor._cancel` documents. + """ + task.cancel() + await asyncio.wait([task]) + if not task.cancelled(): + # Retrieved so a crash on the way out is not reported at exit. + task.exception() + def report_health(self, status: str, detail: str | None = None) -> None: """Say how this node's connection is doing: ok, degraded or down.""" if self._on_health is not None: diff --git a/backend/fluksio/flow/nodes/delay.py b/backend/fluksio/flow/nodes/delay.py index ac3698a..0cc1cf3 100644 --- a/backend/fluksio/flow/nodes/delay.py +++ b/backend/fluksio/flow/nodes/delay.py @@ -211,11 +211,7 @@ class DelayNode(Node): self._stop_cron.set() if self._cron_task is not None: - self._cron_task.cancel() - try: - await self._cron_task - except asyncio.CancelledError: - pass + await self._cancel_task(self._cron_task) self._cron_task = None self._stop_cron = None diff --git a/backend/fluksio/flow/nodes/influx.py b/backend/fluksio/flow/nodes/influx.py index 3b192a0..08cb4b8 100644 --- a/backend/fluksio/flow/nodes/influx.py +++ b/backend/fluksio/flow/nodes/influx.py @@ -14,6 +14,22 @@ from fluksio.flow.nodes.base import Node, NodeResult logger = logging.getLogger(__name__) +def _influxdb() -> Any: + """The client library, which is a `fluksio[server]` extra. + + Imported per use rather than at module level, because the node type is + registered at boot and an installation with no InfluxDB behind it should + not have to carry the library to start. + """ + try: + import influxdb_client + except ImportError: + raise RuntimeError( + "the influxdb node needs the server extra: pip install 'fluksio[server]'" + ) from None + return influxdb_client + + class InfluxDbNode(Node): """ InfluxDB node for writing to and reading from InfluxDB. @@ -58,6 +74,7 @@ class InfluxDbNode(Node): - ``bucket`` (str): Bucket name (required) - ``write_precision`` (str): Write precision ("ns", "us", "ms", "s"), default "ms" - ``query_range`` (str): Default time range for queries, e.g., "-1h", "-24h" + - ``timeout`` (float): Deadline for a request in seconds (default: 10.0) - ``writes`` (dict): Write configurations keyed by message name, each with: - ``measurement`` (str): Measurement name to write to - ``field`` (str): Field name to write (default: "value") @@ -154,6 +171,7 @@ class InfluxDbNode(Node): "bucket", "write_precision", "query_range", + "timeout", "writes", "queries", "_write_client", @@ -169,6 +187,14 @@ class InfluxDbNode(Node): bucket: str write_precision: str = "ms" query_range: str = "-1h" + # Bounds every request to the server. Without one the client falls + # back to its own default, which no flow can see or change. Held in + # seconds like every other node; the client counts in milliseconds. + timeout: float = Field( + default=10.0, + gt=0, + description="Give up on a query or write after this many seconds.", + ) # Per-port write and query configuration. writes: dict[str, dict[str, Any]] = {} queries: dict[str, dict[str, Any]] = {} @@ -201,6 +227,7 @@ class InfluxDbNode(Node): self.bucket = cfg.bucket self.write_precision = cfg.write_precision self.query_range = cfg.query_range + self.timeout = cfg.timeout self.writes = cfg.writes self.queries = cfg.queries @@ -279,13 +306,18 @@ class InfluxDbNode(Node): :returns: ``{"rows": [...], **echo}``. :rtype: dict """ - from influxdb_client import InfluxDBClient + InfluxDBClient = _influxdb().InfluxDBClient flux = str(request["flux"]) echo = {key: value for key, value in request.items() if key != "flux"} logger.info("Running Flux for node '%s': %s", self.name, flux) - with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client: + with InfluxDBClient( + url=self.url, + token=self.token, + org=self.org, + timeout=int(self.timeout * 1000), + ) as client: tables = client.query_api().query(flux, org=self.org) rows = [ @@ -322,11 +354,18 @@ class InfluxDbNode(Node): - Dict with "value" and "tags" keys: Value written with merged tags :type data: dict """ - from influxdb_client import InfluxDBClient, Point, WritePrecision - from influxdb_client.client.write_api import SYNCHRONOUS + influxdb_client = _influxdb() + InfluxDBClient = influxdb_client.InfluxDBClient + Point, WritePrecision = influxdb_client.Point, influxdb_client.WritePrecision + SYNCHRONOUS = influxdb_client.client.write_api.SYNCHRONOUS try: - with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client: + with InfluxDBClient( + url=self.url, + token=self.token, + org=self.org, + timeout=int(self.timeout * 1000), + ) as client: write_api = client.write_api(write_options=SYNCHRONOUS) precision_map = { @@ -417,12 +456,17 @@ class InfluxDbNode(Node): :returns: Dict of port name to queried value. :rtype: dict """ - from influxdb_client import InfluxDBClient + InfluxDBClient = _influxdb().InfluxDBClient results = {} try: - with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client: + with InfluxDBClient( + url=self.url, + token=self.token, + org=self.org, + timeout=int(self.timeout * 1000), + ) as client: query_api = client.query_api() for spec in self.output_ports: diff --git a/backend/fluksio/flow/nodes/mqtt.py b/backend/fluksio/flow/nodes/mqtt.py index f7d9ef2..80ffaef 100644 --- a/backend/fluksio/flow/nodes/mqtt.py +++ b/backend/fluksio/flow/nodes/mqtt.py @@ -19,9 +19,21 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -# Deep enough to ride out a broker hiccup, shallow enough that a publisher -# which cannot keep up drops old values instead of growing without bound. -PUBLISH_QUEUE_SIZE = 256 + +def _aiomqtt() -> Any: + """The client library, which is a `fluksio[server]` extra. + + Imported per use rather than at module level, because the node type is + registered at boot and an installation that talks to no broker should not + have to carry the library to start. + """ + try: + import aiomqtt + except ImportError: + raise RuntimeError( + "the mqtt node needs the server extra: pip install 'fluksio[server]'" + ) from None + return aiomqtt def topic_matches(filter_: str, topic: str) -> bool: @@ -83,6 +95,9 @@ class MqttNode(Node): - ``qos`` (int): Quality of Service level 0, 1, or 2 (default: 0) - ``retain`` (bool): Retain flag for published messages (default: False) - ``keepalive`` (int): Keepalive interval in seconds (default: 60) + - ``timeout`` (float): Deadline for a broker operation in seconds + (default: 10.0) + - ``publish_queue_size`` (int): Publisher backlog depth (default: 256) :type params: dict :param name: Optional name for the node. :type name: str | None @@ -151,6 +166,8 @@ class MqttNode(Node): "qos", "retain", "keepalive", + "timeout", + "publish_queue_size", "json_keys", "_topic_to_ports", "_wildcards", @@ -175,6 +192,26 @@ class MqttNode(Node): qos: int = 0 retain: bool = False keepalive: int = 60 + # Bounds every broker operation: subscribe, publish, and the + # disconnect acknowledgement on the way out. Without one a client + # whose socket died waits for that ack forever, and the task never + # finishes unwinding. Brokers differ, so it is per node. + timeout: float = Field( + default=10.0, + gt=0, + description="Give up on a broker operation after this many seconds.", + ) + # Deep enough to ride out a broker hiccup, shallow enough that a + # publisher which cannot keep up drops old values instead of growing + # without bound. A node that bursts wants more than one that trickles. + publish_queue_size: int = Field( + default=256, + gt=0, + description=( + "How many payloads may wait for the broker. Past this the oldest " + "is dropped and the node reports degraded." + ), + ) # Which key to lift out of a JSON object payload. A device that wraps # its reading — Victron's ``{"value": 5}`` — is otherwise a Python node # per port. One key for every port, or a per-port mapping. @@ -247,6 +284,8 @@ class MqttNode(Node): self.qos = cfg.qos self.retain = cfg.retain self.keepalive = cfg.keepalive + self.timeout = cfg.timeout + self.publish_queue_size = cfg.publish_queue_size # Runtime state self._subscription_task: asyncio.Task[None] | None = None @@ -359,7 +398,7 @@ class MqttNode(Node): A dropped connection raises, and the supervisor decides when to reconnect — the same arrangement the subscriber uses. """ - import aiomqtt + aiomqtt = _aiomqtt() queue = self._publish_queue if queue is None: @@ -372,6 +411,7 @@ class MqttNode(Node): password=self.password, identifier=self.client_id, keepalive=self.keepalive, + timeout=self.timeout, ) as client: self.report_health("ok") while True: @@ -384,7 +424,7 @@ class MqttNode(Node): async def _publish_once(self, data: dict[str, Any]) -> None: """Connect, publish, disconnect — the unstarted node's path.""" - import aiomqtt + aiomqtt = _aiomqtt() async with aiomqtt.Client( hostname=self.broker_host, @@ -393,6 +433,7 @@ class MqttNode(Node): password=self.password, identifier=self.client_id, keepalive=self.keepalive, + timeout=self.timeout, ) as client: await self._publish_with(client, data) @@ -452,7 +493,7 @@ class MqttNode(Node): """Run the task that owns this node's connection to the broker.""" if self._publish_queue is not None: return - self._publish_queue = asyncio.Queue(maxsize=PUBLISH_QUEUE_SIZE) + self._publish_queue = asyncio.Queue(maxsize=self.publish_queue_size) self._loop = asyncio.get_running_loop() self._publisher_task = self._run_supervised("mqtt-out", self._publisher_loop) @@ -461,11 +502,7 @@ class MqttNode(Node): if self._publish_queue is None: return if self._publisher_task is not None: - self._publisher_task.cancel() - try: - await self._publisher_task - except (asyncio.CancelledError, Exception): # noqa: B014 - shutting down - pass + await self._cancel_task(self._publisher_task) self._publisher_task = None self._publish_queue = None self._loop = None @@ -515,11 +552,7 @@ class MqttNode(Node): self._stop_event.set() if self._subscription_task is not None: - self._subscription_task.cancel() - try: - await self._subscription_task - except asyncio.CancelledError: - pass + await self._cancel_task(self._subscription_task) self._subscription_task = None self._stop_event = None @@ -553,7 +586,7 @@ class MqttNode(Node): """ import json - import aiomqtt + aiomqtt = _aiomqtt() if not (self._stop_event and self._stop_event.is_set()): try: @@ -564,6 +597,7 @@ class MqttNode(Node): password=self.password, identifier=self.client_id, keepalive=self.keepalive, + timeout=self.timeout, ) as client: # Subscribe to every unique topic for topic in self._topic_to_ports: diff --git a/backend/fluksio/flow/pipeline.py b/backend/fluksio/flow/pipeline.py index cd0516a..ba6a6a6 100644 --- a/backend/fluksio/flow/pipeline.py +++ b/backend/fluksio/flow/pipeline.py @@ -59,6 +59,7 @@ class ValidationIssue(BaseModel): "unauthenticated_hook", "self_loop_needs_initial", "missing_source", + "node_unhealthy", ] message: str flow: str = "" diff --git a/backend/fluksio/flow/placement.py b/backend/fluksio/flow/placement.py index 9b70dc6..da1111a 100644 --- a/backend/fluksio/flow/placement.py +++ b/backend/fluksio/flow/placement.py @@ -215,9 +215,18 @@ class Placer: said = [shape[2] for shape in capable if shape[2]] ram = min(ram, max(said)) if said and ram else ram if (cpus, gpus, ram) != (wanted.cpus, wanted.gpus, wanted.ram or 0): + # Cards are not detected, so a machine that has one still reports + # none until it is told — which reads as "no GPU here" to a node + # that then runs unserialised beside every other one. + hint = ( + "; no machine here declares a GPU — `fluksio serve --gpus N` " + "(or FLOW_GPUS) says how many this one has" + if wanted.gpus and not gpus + else "" + ) logger.warning( "%s asked for %d cpu(s), %d gpu(s) and %s MB; " - "the largest machine here can give %d, %d and %s", + "the largest machine here can give %d, %d and %s%s", node or "a node", wanted.cpus, wanted.gpus, @@ -225,6 +234,7 @@ class Placer: cpus, gpus, ram or "no stated", + hint, ) return cpus, gpus, ram diff --git a/backend/fluksio/flow/runs.py b/backend/fluksio/flow/runs.py index 39f4260..e8a18f1 100644 --- a/backend/fluksio/flow/runs.py +++ b/backend/fluksio/flow/runs.py @@ -769,7 +769,9 @@ class RunService: # the isolation it wants, minus surviving the process. self._state_factory = state_factory or (lambda _ns: MemoryState()) self.engine_name = f"{socket.gethostname()}-{os.getpid()}"[:64] - self.parallel = max(1, parallel) + # Taken as written: clamping a 0 up to 1 would hide a limit somebody + # set, and the pool below rejects an unusable one loudly anyway. + self.parallel = parallel self._pool = ThreadPoolExecutor( max_workers=self.parallel, thread_name_prefix="run" ) @@ -844,6 +846,26 @@ class RunService: # reference a python caller would have passed and every later reader — # the digest, the cache, the run detail — sees one spelling. params = resolve_references(flow, params, self._artifacts) + # What the run actually starts from, not only what was passed: an input + # left out takes its declared value, and a row that records `{}` cannot + # say which. Folded literally — an initial is a value from the + # definition, never a reference to resolve. + declared = { + one.spec.name: one.initial for one in flow.inputs if one.initial is not None + } + # The run's own seed fills an input of that name, outranking what the + # flow declares and outranked by one passed as a parameter — the order + # `seed_values` applies, moved to where the record is written. + if seed is not None and any(one.spec.name == "seed" for one in flow.inputs): + declared["seed"] = seed + params = {**declared, **params} + # And back the other way, so the run-level column holds the seed the + # run actually used however it arrived. Otherwise `--seed 1` fills one + # column and a declared seed the other, and that is the single field an + # export still has to coalesce. + resolved_seed = params.get("seed") + if isinstance(resolved_seed, int) and not isinstance(resolved_seed, bool): + seed = resolved_seed # Checked here rather than in the driver: a caller who mistyped a # parameter should be told now, not by a run that fails in a minute. seed_values(flow, params, seed) @@ -1174,6 +1196,7 @@ class RunService: self._finish(run_id, status, reason, result, duration) run.status = status self._publish(run, "run_finished") + self._release_cards(run) # Its values were only ever this run's; nothing reads them once it # has a result. On Redis the namespace would expire anyway. if state is not None and status != "error": @@ -1182,6 +1205,29 @@ class RunService: except Exception: logger.warning("Could not clear state of run %s", run_id) + def _release_cards(self, run: Run) -> None: + """Hand a GPU run's device memory back when the run is over. + + The accountant frees the card the moment the node returns, but the + worker that ran on it is kept warm and a library that preallocated + most of the VRAM never gives it up — so the next process to want the + card found it taken by one sitting idle. + """ + # ponytail: retires every CUDA pool rather than the ones this run used, + # which needs no bookkeeping — a concurrent GPU run's busy worker only + # dies when it returns, which is when its own memory should go back + # anyway. What it costs is the warm worker of a *live* flow's GPU node. + # Track the pools per run if that ever matters. + if not (run.needs or {}).get("gpus"): + return + pool = getattr(self.controller, "workers", None) + if pool is None: + return + try: + pool.retire_gpu_children() + except Exception: + logger.warning("Could not retire the GPU workers of run %s", run.id) + def _record_node(self, run_id: str, outcome: NodeOutcome) -> None: outputs = _cacheable(outcome) row = RunNode( diff --git a/backend/fluksio/flow/workers.py b/backend/fluksio/flow/workers.py index 06180bf..4abf517 100644 --- a/backend/fluksio/flow/workers.py +++ b/backend/fluksio/flow/workers.py @@ -321,6 +321,24 @@ class PythonWorkerPool: for child in children: child.respawn_all() + def retire_gpu_children(self) -> None: + """Retire the pools holding a card, so the VRAM goes back. + + A library like JAX takes most of the device when it imports and never + releases it, so a warm worker that has run one such node is a held + card — and warm is the point of a pool, so nothing retires it. At the + end of a run there is something to key on: the environments carrying a + GPU assignment are exactly the pools that ran on one. + """ + with self._lock: + children = [ + child + for key, child in self._children.items() + if any(name == "CUDA_VISIBLE_DEVICES" for name, _ in key) + ] + for child in children: + child.respawn_all() + def _drain(self) -> list[_Worker | None]: slots = [] while True: diff --git a/backend/fluksio/main.py b/backend/fluksio/main.py index 2edba0c..8b271c7 100644 --- a/backend/fluksio/main.py +++ b/backend/fluksio/main.py @@ -4,7 +4,6 @@ import logging from collections.abc import AsyncIterator from contextlib import AbstractAsyncContextManager, asynccontextmanager -import sentry_sdk from fastapi import FastAPI, Request from fastapi.concurrency import run_in_threadpool from fastapi.responses import JSONResponse @@ -51,6 +50,10 @@ def custom_generate_unique_id(route: APIRoute) -> str: if settings.SENTRY_DSN and settings.ENVIRONMENT != "local": + # Imported here rather than at the top: it is a `fluksio[server]` extra, so + # a pip install without one has no sentry to import — and no DSN either. + import sentry_sdk + # `enable_tracing` was removed in sentry-sdk 2.x; this is what it meant. sentry_sdk.init(dsn=str(settings.SENTRY_DSN), traces_sample_rate=1.0) diff --git a/backend/fluksio/sdk/__init__.py b/backend/fluksio/sdk/__init__.py index e74e5f6..b845c6b 100644 --- a/backend/fluksio/sdk/__init__.py +++ b/backend/fluksio/sdk/__init__.py @@ -388,11 +388,6 @@ def _check_signature(spec: NodeSpec) -> None: fn, where = spec.fn, f"node '{spec.id}'" if inspect.iscoroutinefunction(fn): raise SyncError(f"{where}: async functions cannot be nodes") - if fn.__module__ == "__main__": - raise SyncError( - f"{where}: {fn.__name__}() is defined in a script run directly, so the " - "generated node could not import it — put it in an importable module" - ) generator = inspect.isgeneratorfunction(fn) if generator and spec.single: raise SyncError( @@ -551,6 +546,21 @@ def _check(spec: NodeSpec, mode: str) -> dict[str, Any]: } +def _refuse_main(spec: NodeSpec) -> None: + """A node the generated body would have no way to import. + + Refused where the body is written rather than where the decorator is: a + module defining nodes is then still runnable as a script, which is what a + `__main__` self-check beside them needs. + """ + if spec.fn.__module__ == "__main__": + raise SyncError( + f"node '{spec.id}': {spec.fn.__name__}() is defined in a script run " + "directly, so the generated node could not import it — put it in an " + "importable module" + ) + + def _code_of(spec: NodeSpec) -> dict[str, Any]: """What this node's function calls into, which its shim does not say. @@ -559,6 +569,7 @@ def _code_of(spec: NodeSpec) -> dict[str, Any]: side is the only one that can work it out at all: it has imported the code, and the engine never does. """ + _refuse_main(spec) files = reached(spec.fn) return {"code_files": files, "code_digest": digest_of(files)} @@ -762,6 +773,7 @@ def _shim(spec: NodeSpec) -> str: simply happens to be generated, which is why it says so and says where the real thing is. """ + _refuse_main(spec) fn = spec.fn where = inspect.getsourcefile(fn) or fn.__module__ repo = import_root(fn) diff --git a/backend/fluksio/sdk/cli.py b/backend/fluksio/sdk/cli.py index e203360..bc18d3b 100644 --- a/backend/fluksio/sdk/cli.py +++ b/backend/fluksio/sdk/cli.py @@ -19,6 +19,7 @@ import sys import time from collections.abc import Callable, Iterable, Iterator from contextlib import contextmanager, nullcontext +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -136,8 +137,8 @@ def _import(root: str, dotted: str, expect: Path | None = None) -> None: f" {actual}\n {expect}\n" "Python keeps one module per name, and a node's generated body " "imports by that name, so the second would run the first's code. " - "Put an `__init__.py` in each directory — they become " - f"'.{dotted}' and stop colliding — or rename one of the files." + "Sync the directory they are both under — each then imports as " + "'.' — or rename one of the files." ) @@ -211,7 +212,14 @@ def discover(targets: list[str], keep_going: bool = False) -> list[Flow]: if entry.is_dir(): _import_package(entry) else: - _import(*_module_of(entry), entry) + # Named for where it sits under the directory being synced, + # so `dev/s1/study.py` imports as `s1.study` and one + # `study.py` per study collides with nothing. No + # `__init__.py` needed: the directories in between are + # namespace packages. A file at the root keeps its bare + # name, which is what it has always had. + dotted = ".".join(entry.relative_to(path).with_suffix("").parts) + _import(str(path), dotted, entry) except SyncError: # A name collision is never somebody else's problem: it would # put the wrong file behind a node. @@ -439,33 +447,52 @@ def _ask_params(definition: dict[str, Any]) -> dict[str, Any]: def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]: """Turn `--lr 0.05` into a typed parameter, using the flow's own inputs.""" types = _input_types(definition) - params: dict[str, Any] = {} + # Collected as written and typed afterwards, so a name this flow does not + # have is refused by name rather than by whatever its value failed to parse + # as: `--param lr=0.002` is a sweep's spelling, and said so it reads as an + # input called `param` holding unparseable json. + raw: dict[str, str | bool] = {} pending: str | None = None for token in rest: if token.startswith("--"): if pending is not None: # A flag with no value is a flag: `--resume` means true. - params[pending] = True + raw[pending] = True name, sep, value = token[2:].partition("=") # Only the name is spelled with dashes; a value may hold one, and # `--lr=1e-4` is the case that says so. pending = name.replace("-", "_") if sep: - params[pending] = _coerce(value, types.get(pending, "json")) + raw[pending] = value pending = None continue if pending is None: raise SyncError(f"unexpected argument '{token}'") - params[pending] = _coerce(token, types.get(pending, "json")) + raw[pending] = token pending = None if pending is not None: - params[pending] = True - unknown = sorted(set(params) - set(types)) + raw[pending] = True + unknown = sorted(set(raw) - set(types)) if unknown: + hint = "" + if unknown[0] == "param": + hint = ( + " — one value is `-- `; several is a sweep: " + "`fluksio sweep --param name=v1,v2`" + ) raise SyncError( f"'{unknown[0]}' is not an input of this flow (it takes " - f"{', '.join(sorted(types)) or 'none'})" + f"{', '.join(sorted(types)) or 'none'}){hint}" ) + params: dict[str, Any] = {} + for key, written in raw.items(): + if isinstance(written, bool): + params[key] = written + continue + try: + params[key] = _coerce(written, types[key]) + except ValueError as exc: + raise SyncError(f"'{key}' takes {types[key]}: {exc}") from exc return params @@ -778,7 +805,8 @@ def _status_screen(client: Client) -> Any: ) + Text( f"{str(row.get('flow', '')):<16}" - f"{(row.get('duration_ms') or 0) / 1000:7.1f}s", + f"{(row.get('duration_ms') or 0) / 1000:7.1f}s" + f" {_ago(row.get('finished_at') or row.get('created_at')):>9}", style="dim", ) ) @@ -790,6 +818,7 @@ def _status_screen(client: Client) -> Any: ) parts.append( Text(f" {where} ", style="red") + + Text(f"{_ago(event.get('ts')):>9} ", style="dim") + Text(str(event.get("detail", ""))[:100], style="dim") ) return Group(*parts) @@ -831,9 +860,9 @@ def cmd_status(args: argparse.Namespace) -> int: #: value is read. PARAMS_WIDTH = 80 -#: What the columns before the inputs take: the id, status, flow, duration and -#: stamp, with their spacing. -LISTING_WIDTH = 84 +#: What the columns before the inputs take: the id, status, flow, duration, +#: age and stamp, with their spacing. +LISTING_WIDTH = 95 def _declared(client: Client, flows: Iterable[str]) -> dict[str, dict[str, Any]]: @@ -857,6 +886,28 @@ def _declared(client: Client, flows: Iterable[str]) -> dict[str, dict[str, Any]] return known +def _ago(stamp: Any) -> str: + """How long ago something happened, in the notation the screens use. + + A duration and an age read together, so they are spelled the same way: + seconds, then minutes, hours, days. + """ + if not stamp: + return "" + try: + then = datetime.fromisoformat(str(stamp)) + except ValueError: + return "" + if then.tzinfo is None: + # Everything the engine records is UTC; only some spellings say so. + then = then.replace(tzinfo=UTC) + seconds = max((datetime.now(UTC) - then).total_seconds(), 0) + for span, unit in ((86400, "d"), (3600, "h"), (60, "min")): + if seconds >= span: + return f"{seconds / span:.0f}{unit} ago" + return f"{seconds:.0f}s ago" + + def _stamp(row: dict[str, Any]) -> str: """What code a run ran: the commit, whether it was dirty, and the digest. @@ -899,11 +950,49 @@ def cmd_runs(args: argparse.Namespace) -> int: params = params[: room - 3] + "..." _say( f"{row['id']} {_status(row['status'], 9)} {row['flow']:<16} " - f"{row['duration_ms'] / 1000:7.1f}s {_stamp(row):<22} {params}" + f"{row['duration_ms'] / 1000:7.1f}s {_ago(row.get('created_at')):>9} " + f"{_stamp(row):<22} {params}" ) return 0 +def _artifacts(client: Client, args: argparse.Namespace) -> int: + """List a run's files, or write one of them here.""" + handle = RunHandle(client, args.run_id, client.run(args.run_id)) + rows = handle.artifacts + if not args.name: + if not rows: + _say("This run produced no artifacts.") + return 0 + for row in rows: + named = row.get("filename") or "" + _say( + f"{str(row['name']):<24} {row['size']:>10} B " + f"{row.get('media_type', ''):<24} {named}" + ) + return 0 + data = handle.download(args.name) + # The name it was written under reads better than the message's, which is + # chosen for the graph; `--out` beats both. + match = next((row for row in rows if row.get("name") == args.name), {}) + out = Path(args.out or match.get("filename") or args.name) + out.write_bytes(data) + _say(f"{out} {len(data)} bytes") + return 0 + + +def cmd_artifacts(args: argparse.Namespace) -> int: + try: + with _client_for(args, retries=0) as client: + return _artifacts(client, args) + except KeyError as exc: + return _fail(str(exc.args[0])) + except (SyncError, ApiError) as exc: + return _fail(str(exc)) + except httpx.HTTPError as exc: + return _unreachable(exc) + + def cmd_flavors(args: argparse.Namespace) -> int: """The named sizes a node can ask for.""" try: @@ -1267,6 +1356,21 @@ def add_parsers(subparsers: Any) -> None: with_engine(parser, local=True) parser.set_defaults(func=cmd_runs) + parser = subparsers.add_parser( + "artifacts", help="the files a run produced; name one to download it" + ) + parser.add_argument("run_id") + parser.add_argument("name", nargs="?", default="") + parser.add_argument( + "-o", + "--out", + default="", + metavar="PATH", + help="where to write it (default: the name it was saved under)", + ) + with_engine(parser, local=True) + parser.set_defaults(func=cmd_artifacts) + parser = subparsers.add_parser( "flavors", help="the named resource sizes a node can ask for" ) @@ -1375,7 +1479,7 @@ def add_parsers(subparsers: Any) -> None: metavar="A,B", help=( "the inputs to put in columns, dotted into a record " - "(default: the ones that vary)" + "(default: every input the runs recorded)" ), ) sub.add_argument( diff --git a/backend/fluksio/sdk/client.py b/backend/fluksio/sdk/client.py index d0dc25d..1f5dc2f 100644 --- a/backend/fluksio/sdk/client.py +++ b/backend/fluksio/sdk/client.py @@ -428,9 +428,9 @@ class Client: ) -> list[dict[str, Any]]: """One row per run: its inputs as columns, its final numbers, its code. - The arm-comparison table. The inputs kept are the ones that vary - across the selection unless ``params`` names them, which is the axis a - sweep is read along. + The arm-comparison table. Every input the selection recorded is a + column, so the schema does not depend on which runs were asked for; + ``params`` narrows it to the axis a sweep is read along. """ query: dict[str, Any] = {} if params: diff --git a/backend/fluksio/tui.py b/backend/fluksio/tui.py new file mode 100644 index 0000000..a02223a --- /dev/null +++ b/backend/fluksio/tui.py @@ -0,0 +1,342 @@ +"""The dashboard `fluksio serve` opens at a terminal. + +A supervisor, not an engine: it starts `serve --plain` as a child and talks to +it over the same HTTP API every other client uses. So the engine is a process +that can be stopped and started under the screen watching it, an engine +somebody else started can be adopted rather than duplicated, and quitting the +dashboard is not the same as stopping the engine. + +Nothing of the engine is imported here — that lives in the child. +""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import subprocess +import sys +from pathlib import Path +from typing import Any + +from textual import work +from textual.app import App, ComposeResult +from textual.containers import Horizontal, Vertical +from textual.screen import ModalScreen +from textual.widgets import Button, DataTable, Footer, Header, Input, Label, RichLog + +from fluksio.cli import ( + DEFAULT_PORT, + DEFAULT_PORTAL, + _data_dir, + _token_for, + probe_engine, + read_pidfile, +) +from fluksio.sdk.cli import WATCH_INTERVAL_S, _status_screen +from fluksio.sdk.client import Client, config_path + +#: How long to keep asking the child for the credential it writes on the way +#: up, before giving up and letting the panels say so. +STARTUP_TRIES = 60 + + +def child_argv(argv: list[str]) -> list[str]: + """The same command, told to serve without a dashboard of its own. + + Every flag is passed through: whatever `serve` was asked for is what the + engine under this screen is running with. + """ + passed = [flag for flag in argv if flag != "--plain"] + return [sys.executable, "-m", "fluksio.cli", *passed, "--plain"] + + +class Enroll(ModalScreen[tuple[str, str] | None]): + """The claim code a portal minted, and which portal minted it.""" + + BINDINGS = [("escape", "dismiss(None)", "cancel")] + + def compose(self) -> ComposeResult: + with Vertical(id="enroll"): + yield Label("Pair this installation with a portal") + yield Input(placeholder="claim code", id="code") + yield Input(value=DEFAULT_PORTAL, id="portal") + with Horizontal(): + yield Button("Enroll", variant="primary", id="go") + yield Button("Cancel", id="cancel") + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "cancel": + self.dismiss(None) + return + code = self.query_one("#code", Input).value.strip() + portal = self.query_one("#portal", Input).value.strip() + self.dismiss((code, portal or DEFAULT_PORTAL) if code else None) + + +class ServeApp(App[int]): + """One screen: how the engine is, what has run, and what it is saying.""" + + CSS = """ + #status { height: auto; padding: 0 1; } + #runs { height: 2fr; } + #log { height: 1fr; border-top: solid $panel; } + #enroll { width: 60; height: auto; padding: 1 2; background: $surface; } + """ + + BINDINGS = [ + ("q", "quit", "quit (engine keeps running)"), + ("s", "stop_start", "stop/start"), + ("r", "restart", "restart"), + ("c", "cancel_run", "cancel run"), + ("e", "enroll", "enroll"), + ] + + def __init__(self, args: argparse.Namespace) -> None: + super().__init__() + self.args = args + self.data_dir: Path = _data_dir(args.data_dir, args.shared) + self.child: subprocess.Popen[str] | None = None + #: The pid of an engine this dashboard did not start. Only ever one + #: whose data directory is this one — the probe is what proves it. + self.adopted: int | None = None + self.client: Client | None = None + self.url = "" + + # -- layout --------------------------------------------------------------- + + def compose(self) -> ComposeResult: + yield Header() + yield RichLog(id="status", markup=True, wrap=True) + table: DataTable[str] = DataTable(id="runs", cursor_type="row") + yield table + yield RichLog(id="log", markup=False, highlight=False, max_lines=2000) + yield Footer() + + def on_mount(self) -> None: + self.title = f"fluksio — {self.data_dir}" + table = self.query_one("#runs", DataTable) + table.add_columns("run", "status", "flow", "took", "params") + self.start_engine(first=True) + self.set_interval(WATCH_INTERVAL_S, self.refresh_panels) + + # -- the engine under the screen ------------------------------------------ + + def note(self, message: str) -> None: + self.query_one("#log", RichLog).write(message) + + def start_engine(self, first: bool = False) -> None: + """Adopt whatever is already serving this directory, or start one.""" + host = self.args.host + reachable = "127.0.0.1" if host in ("0.0.0.0", "::", "") else host + wanted = self.args.port or DEFAULT_PORT + url = f"http://{reachable}:{wanted}" + who = probe_engine(url, _token_for(self.data_dir)) if first else "other" + + if who == "ours": + running = read_pidfile(self.data_dir) + self.adopted = running["pid"] if running else None + self.url = url + named = f" (pid {self.adopted})" if self.adopted else "" + self.note(f"Adopted the engine already serving this directory{named}.") + if self.adopted is None: + self.note("It wrote no pidfile, so this screen cannot stop it.") + self.connect() + return + if who == "foreign": + self.note(f"Port {wanted} holds another installation's Fluksio.") + + self.child = subprocess.Popen( # noqa: S603 + child_argv(sys.argv[1:]), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env={**os.environ, "PYTHONUNBUFFERED": "1"}, + ) + self.tail_child(self.child) + self.await_engine() + + @work(thread=True, exclusive=False) + def tail_child(self, child: subprocess.Popen[str]) -> None: + """The engine's own output, which is why a second terminal was needed.""" + if child.stdout is None: + return + for line in child.stdout: + self.call_from_thread(self.note, line.rstrip()) + self.call_from_thread(self.note, f"The engine stopped ({child.wait()}).") + + @work(thread=True, exclusive=True, group="startup") + def await_engine(self) -> None: + """Wait for the child to say where it landed, then talk to it. + + The port is the child's to choose — it moves off a taken one — and + `client.json` is where it says which it took. + """ + import time + + for _ in range(STARTUP_TRIES): + if self.child is not None and self.child.poll() is not None: + return + try: + stored = json.loads(config_path(self.data_dir).read_text()) + except (OSError, ValueError): + stored = {} + if stored.get("url") and stored.get("token"): + self.url = str(stored["url"]) + self.call_from_thread(self.connect) + return + time.sleep(0.5) + self.call_from_thread(self.note, "The engine did not come up.") + + def connect(self) -> None: + stored = config_path(self.data_dir) + try: + config = json.loads(stored.read_text()) + self.client = Client( + url=str(config["url"]), token=str(config["token"]), retries=0 + ) + except (OSError, ValueError, KeyError) as exc: + self.note(f"No credential to talk to the engine with: {exc}") + return + self.url = self.client.url + self.refresh_panels() + + def engine_pid(self) -> int | None: + return self.child.pid if self.child is not None else self.adopted + + def stop_engine(self) -> None: + if self.child is not None: + self.note(f"Stopping the engine (pid {self.child.pid}).") + self.child.terminate() + self.child = None + elif self.adopted is not None: + self.note(f"Stopping the adopted engine (pid {self.adopted}).") + try: + os.kill(self.adopted, signal.SIGTERM) + except OSError as exc: + self.note(f"Could not stop it: {exc}") + self.adopted = None + self.client = None + + # -- what the panels show ------------------------------------------------- + + def refresh_panels(self) -> None: + if self.client is not None: + self.read_engine() + + @work(thread=True, exclusive=True, group="poll") + def read_engine(self) -> None: + client = self.client + if client is None: + return + try: + screen = _status_screen(client) + rows = client.runs(limit=20) + except Exception as exc: + self.call_from_thread(self.show_offline, exc) + return + self.call_from_thread(self.show, screen, rows) + + def show(self, screen: Any, rows: list[dict[str, Any]]) -> None: + status = self.query_one("#status", RichLog) + status.clear() + status.write(screen) + table = self.query_one("#runs", DataTable) + table.clear() + for row in rows: + table.add_row( + str(row.get("id", ""))[-8:], + str(row.get("status", "")), + str(row.get("flow", "")), + f"{(row.get('duration_ms') or 0) / 1000:.1f}s", + json.dumps(row.get("params") or {})[:60], + key=str(row.get("id", "")), + ) + + def show_offline(self, exc: Exception) -> None: + status = self.query_one("#status", RichLog) + status.clear() + which = "starting" if self.engine_pid() is not None else "not running" + status.write(f"[yellow]The engine is {which}.[/] ({type(exc).__name__})") + + # -- keys ----------------------------------------------------------------- + + def action_stop_start(self) -> None: + if self.engine_pid() is not None: + self.stop_engine() + else: + self.start_engine() + + def action_restart(self) -> None: + self.stop_engine() + self.start_engine() + + def action_cancel_run(self) -> None: + table = self.query_one("#runs", DataTable) + if self.client is None or not table.row_count: + return + # The cell holds the tail of the id, which is what fits; the row's key + # is the whole of it, which is what the engine is asked about. + key = table.coordinate_to_cell_key(table.cursor_coordinate).row_key + if key.value: + self.cancel_run(str(key.value)) + + @work(thread=True) + def cancel_run(self, run_id: str) -> None: + client = self.client + if client is None: + return + try: + client.cancel(run_id) + except Exception as exc: + self.call_from_thread(self.note, f"Could not cancel {run_id}: {exc}") + return + self.call_from_thread(self.note, f"Cancelled {run_id}.") + self.call_from_thread(self.refresh_panels) + + def action_enroll(self) -> None: + self.push_screen(Enroll(), self.enrolled) + + def enrolled(self, answer: tuple[str, str] | None) -> None: + if answer is None: + return + code, portal = answer + self.run_enroll(code, portal) + + @work(thread=True) + def run_enroll(self, code: str, portal: str) -> None: + """`fluksio enroll`, as its own process for the same reason serve is. + + A running engine picks the configuration up on its own; this screen + only has to report what the command said. + """ + done = subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "fluksio.cli", + "enroll", + code, + "--portal", + portal, + "--data-dir", + str(self.data_dir), + ], + capture_output=True, + text=True, + ) + for line in (done.stdout + done.stderr).splitlines(): + self.call_from_thread(self.note, line) + + +def run_tui(args: argparse.Namespace) -> int: + """Open the dashboard, and say what is still running when it closes.""" + app = ServeApp(args) + app.run() + pid, url = app.engine_pid(), app.url + if pid is not None: + print(f"The engine is still running: pid {pid} at {url or 'its port'}.") + print(f" fluksio serve reattaches to it; kill {pid} stops it.") + return 0 diff --git a/backend/fluksio/utils.py b/backend/fluksio/utils.py index d49532b..702a744 100644 --- a/backend/fluksio/utils.py +++ b/backend/fluksio/utils.py @@ -4,7 +4,6 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any -import emails # type: ignore import jwt from jinja2 import Template from jwt.exceptions import InvalidTokenError @@ -37,6 +36,12 @@ def send_email( html_content: str = "", ) -> None: assert settings.emails_enabled, "no provided configuration for email variables" + try: + import emails # type: ignore + except ImportError: + raise RuntimeError( + "sending mail needs the server extra: pip install 'fluksio[server]'" + ) from None message = emails.Message( subject=subject, html=html_content, diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 3ea0651..7158390 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fluksio" -version = "0.1.4" +version = "0.1.4+dev" description = "Node-based automation engine: flows, dashboards, batch runs" readme = "README.md" license = "AGPL-3.0-or-later" @@ -25,15 +25,12 @@ dependencies = [ "fastapi[standard]<1.0.0,>=0.114.2", "python-multipart<1.0.0,>=0.0.7", "email-validator<3.0.0.0,>=2.1.0.post1", - "tenacity<9.0.0,>=8.2.3", "pydantic>2.0", - "emails<1.0,>=0.6", "jinja2<4.0.0,>=3.1.4", "alembic<2.0.0,>=1.12.1", "httpx<1.0.0,>=0.25.1", "sqlmodel<1.0.0,>=0.0.21", "pydantic-settings<3.0.0,>=2.2.1", - "sentry-sdk[fastapi]>=2.20.0", "pyjwt<3.0.0,>=2.8.0", "pwdlib[argon2,bcrypt]>=0.3.0", "numpy>=2.2.6", @@ -43,8 +40,6 @@ dependencies = [ # copied onto other people's machines and stays dependency-free. "orjson>=3.10", "cryptography>=44.0.0", - "aiomqtt>=2.0.0", - "influxdb-client[async]>=1.40.0", "croniter>=1.3.0", "mcp>=1.29,<2", "fluksio-worker>=0.1,<0.2", @@ -55,6 +50,9 @@ dependencies = [ # `fluksio status` draws with it. Already here underneath fastapi's CLI, # named because a command that depends on it should say so. "rich>=13", + # `fluksio serve` opens a dashboard with it at a terminal. Pure python and + # mostly rich underneath, which is already here. + "textual>=1.0", ] [project.optional-dependencies] @@ -62,6 +60,17 @@ dependencies = [ # extra rather than a dependency: csv and jsonl need nothing, and pyarrow is # tens of megabytes for whoever wants dtypes kept. parquet = ["pyarrow>=17"] +# What a deployment has and a laptop does not: the device connectors, outbound +# mail and error reporting. The engine, the CLI and every python node work +# without them — `pip install fluksio` in a data-science environment is the +# case this exists for, and lxml and the aiohttp stack are most of its wait. +# Each import is guarded and names this extra. The image installs it. +server = [ + "emails<1.0,>=0.6", + "sentry-sdk[fastapi]>=2.20.0", + "aiomqtt>=2.0.0", + "influxdb-client[async]>=1.40.0", +] [project.urls] Homepage = "https://fluksio.com" @@ -146,6 +155,8 @@ ignore = [ # It talks to whoever ran it; that is what a command line is. "fluksio/cli.py" = ["T201"] "fluksio/sdk/cli.py" = ["T201"] +# What it prints is the line left in the terminal after the dashboard closes. +"fluksio/tui.py" = ["T201"] # Node functions take `params` whether or not they use it — that is the # contract the engine calls them with. "fluksio/flow/nodes.py" = ["ARG001", "ARG002"] diff --git a/backend/tests/api/routes/test_dashboards.py b/backend/tests/api/routes/test_dashboards.py new file mode 100644 index 0000000..609c376 --- /dev/null +++ b/backend/tests/api/routes/test_dashboards.py @@ -0,0 +1,88 @@ +"""Dashboards over HTTP: saving one, which is also how the first one is made.""" + +from fastapi.testclient import TestClient + +from fluksio.core.config import settings + +PREFIX = f"{settings.API_V1_STR}/dashboards" + + +def a_dashboard(name: str) -> dict: + return { + "name": name, + "title": "Hall", + "icon": "gauge", + "widgets": [ + { + "id": "temperature", + "type": "stat", + "title": "Temperature", + "layout": {"lg": {"x": 0, "y": 0, "w": 3, "h": 2}}, + "config": {"message": "house.temperature", "dtype": "float"}, + } + ], + "settings": {"theme": {"value": "dark", "message": "", "dtype": "str"}}, + "version": 0, + } + + +def test_a_save_creates_a_dashboard_that_does_not_exist_yet( + client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + """A first draft, not a 404: creating one *is* saving it at version 0.""" + body = a_dashboard("put_creates") + + response = client.put( + f"{PREFIX}/put_creates", headers=superuser_token_headers, json=body + ) + + assert response.status_code == 200, response.text + saved = response.json() + assert saved["version"] == 1 and saved["has_draft"] is True + # A draft alone: nothing was published, so no panel can be shown it. + assert ( + client.get(f"{PREFIX}/put_creates", headers=superuser_token_headers).status_code + == 404 + ) + draft = client.get( + f"{PREFIX}/put_creates?draft=true", headers=superuser_token_headers + ) + assert draft.json()["widgets"] == body["widgets"] + + +def test_a_save_of_an_existing_dashboard_keeps_what_it_did_not_change( + client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + seeded = a_dashboard("put_updates") + first = client.put( + f"{PREFIX}/put_updates", headers=superuser_token_headers, json=seeded + ).json() + + renamed = {**first, "widgets": [{**first["widgets"][0], "title": "Outside"}]} + second = client.put( + f"{PREFIX}/put_updates", headers=superuser_token_headers, json=renamed + ) + + assert second.status_code == 200, second.text + stored = second.json() + assert stored["version"] == first["version"] + 1 + assert stored["widgets"][0]["title"] == "Outside" + # Everything the edit did not name is still what was first written. + assert stored["icon"] == seeded["icon"] + assert stored["settings"] == seeded["settings"] + assert stored["widgets"][0]["layout"] == seeded["widgets"][0]["layout"] + assert stored["widgets"][0]["config"] == seeded["widgets"][0]["config"] + + +def test_a_save_based_on_a_version_someone_moved_past_is_refused( + client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + body = a_dashboard("put_conflicts") + client.put(f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body) + + stale = client.put( + f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body + ) + + assert stale.status_code == 409 + assert stale.json()["detail"]["current_version"] == 1 diff --git a/backend/tests/api/routes/test_observability.py b/backend/tests/api/routes/test_observability.py index 010714f..088b7c6 100644 --- a/backend/tests/api/routes/test_observability.py +++ b/backend/tests/api/routes/test_observability.py @@ -127,6 +127,37 @@ def test_a_flow_that_cannot_run_makes_the_summary_degraded( assert not any("hooky" in problem for problem in body["problems"]) +def test_a_down_node_makes_the_summary_degraded( + client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + """A connector that cannot reach its device is not a flow that cannot run. + + It is counted on its own, so the flow keeps running and "invalid" stays + about validation. + """ + from fluksio.flow.controller import LoadedNode + + controller = client.app.state.flow_controller + before = controller.loaded + controller.loaded = { + "house.owm": LoadedNode( + id="house.owm", + flow="house", + health="down", + health_detail="ConnectionError: name resolution failed", + ) + } + try: + body = client.get(f"{PREFIX}/summary", headers=superuser_token_headers).json() + finally: + controller.loaded = before + + assert body["status"] == "degraded" + assert body["nodes"]["unhealthy"] == 1 + assert any("down" in problem for problem in body["problems"]) + assert body["flows"]["invalid"] == 0 + + def test_the_history_reads_back( client: TestClient, superuser_token_headers: dict[str, str], db: Session ) -> None: diff --git a/backend/tests/api/routes/test_runs.py b/backend/tests/api/routes/test_runs.py index 9fe41ba..423d9ee 100644 --- a/backend/tests/api/routes/test_runs.py +++ b/backend/tests/api/routes/test_runs.py @@ -400,6 +400,97 @@ def test_a_repeated_submit_returns_the_run_it_already_made(): assert again.id == "dedup-1" +class _OneFlow: + """A controller that has exactly one flow and no engine behind it.""" + + def __init__(self, flow): + self.store = self + self._flow = flow + + def read_flow(self, name, draft=False): + return self._flow + + def head(self): + return "" + + +class _Collect: + def __init__(self): + self.items = [] + + def add(self, item): + self.items.append(item) + + +def test_a_run_records_the_inputs_it_actually_starts_from(): + """An input left out takes its declared value, and the row says so. + + `params = {}` could not tell a run that took every default from one + submitted with those same numbers spelled out — and an export of the + first had a blank cell where its `lr` should be. + """ + flow = FlowDef( + name="study", + mode="batch", + inputs=[ + FlowInput(spec=MessageSpec(name="lr", dtype=DType.FLOAT), initial=0.01), + FlowInput(spec=MessageSpec(name="epochs", dtype=DType.INT)), + ], + ) + service = RunService(controller=_OneFlow(flow), queue=_Collect()) + made = [] + try: + defaulted = service.submit("study", {"epochs": 5}) + made.append(defaulted.id) + assert defaulted.params == {"lr": 0.01, "epochs": 5} + + # Spelling out the declared value is the same run, and now reads as it. + spelled = service.submit("study", {"lr": 0.01, "epochs": 5}) + made.append(spelled.id) + assert spelled.params_digest == defaulted.params_digest + finally: + with Session(db_engine) as session: + for run in session.exec(select(Run).where(col(Run.id).in_(made))).all(): + session.delete(run) + session.commit() + + +def test_the_seed_is_recorded_the_same_way_however_it_arrived(): + """One field an export should not have to coalesce two columns for. + + `--seed 1` fills the run's own column; a flow declaring a `seed` input + fills the parameter. Both are the seed the run used, so both are written. + """ + flow = FlowDef( + name="seeded", + mode="batch", + inputs=[FlowInput(spec=MessageSpec(name="seed", dtype=DType.INT), initial=42)], + ) + service = RunService(controller=_OneFlow(flow), queue=_Collect()) + made = [] + try: + passed = service.submit("seeded", {}, seed=1) + made.append(passed.id) + assert (passed.seed, passed.params) == (1, {"seed": 1}) + + # Nothing passed: the declared value is the seed it ran with, and the + # run-level column says so rather than staying empty. + defaulted = service.submit("seeded", {}) + made.append(defaulted.id) + assert (defaulted.seed, defaulted.params) == (42, {"seed": 42}) + + # A parameter still outranks the run's own seed, as it always has — + # and the column follows it rather than reporting the one that lost. + both = service.submit("seeded", {"seed": 7}, seed=1) + made.append(both.id) + assert (both.seed, both.params) == (7, {"seed": 7}) + finally: + with Session(db_engine) as session: + for run in session.exec(select(Run).where(col(Run.id).in_(made))).all(): + session.delete(run) + session.commit() + + def test_a_key_nobody_used_submits_normally( client, superuser_token_headers, monkeypatch ): @@ -432,6 +523,99 @@ def test_overview_is_not_read_as_a_run_id(client, superuser_token_headers): assert isinstance(answer.json(), list) +# ----------------------------------------------------------------------------- +# Deleting a run +# +# The route owns the four statements; what these guard is that it takes the +# children with it and refuses a run the driver is still writing to. +# ----------------------------------------------------------------------------- + + +@pytest.fixture +def deletable_run(): + """One finished run with a node, a number and an artifact row hanging off it.""" + run_id = "del-1" + with Session(db_engine) as session: + session.add( + Run(id=run_id, flow="deleted", status="ok", created_at=datetime.now(UTC)) + ) + session.add(RunNode(run_id=run_id, node="deleted.a", status="ok")) + session.add(RunMetric(run_id=run_id, name="deleted.loss", step=0, value=1.0)) + session.add( + RunArtifact( + run_id=run_id, + name="deleted.out", + filename="out.bin", + node="a", + digest="d" * 64, + size=7, + ) + ) + session.commit() + yield run_id + with Session(db_engine) as session: + run = session.get(Run, run_id) + if run is not None: + session.delete(run) + session.commit() + + +def test_deleting_a_run_takes_its_children_with_it( + client, superuser_token_headers, deletable_run +): + """No foreign key cascades here, so the route has to do it itself.""" + answer = client.delete( + f"{settings.API_V1_STR}/runs/{deletable_run}", headers=superuser_token_headers + ) + + assert answer.status_code == 204 + with Session(db_engine) as session: + assert session.get(Run, deletable_run) is None + for table in (RunNode, RunMetric, RunArtifact): + left = session.exec( + select(table).where(col(table.run_id) == deletable_run) + ).all() + assert left == [], f"{table.__name__} rows outlived the run" + + +def test_deleting_a_run_that_is_not_there_is_a_404(client, superuser_token_headers): + answer = client.delete( + f"{settings.API_V1_STR}/runs/nope-1", headers=superuser_token_headers + ) + + assert answer.status_code == 404 + + +def test_a_running_run_is_refused_rather_than_raced(client, superuser_token_headers): + """The driver writes its nodes back at the end; they would have no run.""" + run_id = "del-live" + with Session(db_engine) as session: + session.add( + Run( + id=run_id, + flow="deleted", + status="running", + created_at=datetime.now(UTC), + ) + ) + session.commit() + try: + answer = client.delete( + f"{settings.API_V1_STR}/runs/{run_id}", headers=superuser_token_headers + ) + + assert answer.status_code == 409 + assert "Cancel it" in answer.json()["detail"] + with Session(db_engine) as session: + assert session.get(Run, run_id) is not None + finally: + with Session(db_engine) as session: + run = session.get(Run, run_id) + if run is not None: + session.delete(run) + session.commit() + + # ----------------------------------------------------------------------------- # A cached node's curve # @@ -723,10 +907,10 @@ def test_an_export_strides_each_series_and_names_its_run( assert all(steps == [0, 2] for steps in curves.values()) -def test_an_exported_run_row_carries_the_inputs_that_vary( +def test_an_exported_run_row_carries_every_recorded_input( client, superuser_token_headers, exported ): - """The sweep axis becomes columns; what every run shares stays out of them.""" + """Every input is a column, so the schema does not move with the selection.""" def export(**extra): answer = client.get( @@ -740,12 +924,15 @@ def test_an_exported_run_row_carries_the_inputs_that_vary( rows = _lines(export(format="jsonl")) assert [row["id"] for row in rows] == ["exp-1", "exp-0"] assert {row["param.lr"] for row in rows} == {0.1, 0.01} - # `epochs` is the same on both runs, so it is not what they differ by, and - # neither is the model inside the config — but the depth beside it is. - assert "param.epochs" not in rows[0] - assert "param.config.model" not in rows[0] + # `epochs` is the same on both runs and stays a column anyway: which runs + # were asked for is not something a downstream filter should have to know. + assert rows[0]["param.epochs"] == 10 + assert rows[0]["param.config.model"] == "mlp" assert {row["param.config.depth"] for row in rows} == {1, 2} - assert "param.epochs" in _lines(export(format="jsonl", params="epochs"))[0] + + narrowed = _lines(export(format="jsonl", params="epochs"))[0] + assert "param.epochs" in narrowed + assert "param.lr" not in narrowed # A number inside a record is a column of its own, however deep; a string # is not one of the run's numbers wherever it sits. @@ -762,7 +949,7 @@ def test_an_exported_run_row_carries_the_inputs_that_vary( header = export().text.splitlines()[0] assert header.startswith("id,flow,status,") assert header.endswith( - "param.config.depth,param.lr," + "param.config.depth,param.config.model,param.epochs,param.lr," "metric.acc,metric.final_metrics.train_loss," "metric.test_metrics.known.perfect" ) diff --git a/backend/tests/api/routes/test_search.py b/backend/tests/api/routes/test_search.py new file mode 100644 index 0000000..160282c --- /dev/null +++ b/backend/tests/api/routes/test_search.py @@ -0,0 +1,80 @@ +"""The one index the global search matches against.""" + +from fastapi.testclient import TestClient + +from fluksio.core.config import settings + +PREFIX = f"{settings.API_V1_STR}/search" +FLOWS = f"{settings.API_V1_STR}/flows" +DASHBOARDS = f"{settings.API_V1_STR}/dashboards" +SECRETS = f"{settings.API_V1_STR}/secrets" + + +def test_search_requires_authentication(client: TestClient) -> None: + assert client.get(f"{PREFIX}/").status_code == 401 + + +def test_index_reaches_inside_flows_and_dashboards( + client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + """A node and a widget are the point: neither is on any list endpoint.""" + client.put( + f"{FLOWS}/searchable", + headers=superuser_token_headers, + json={ + "name": "searchable", + "title": "Searchable", + "nodes": [{"id": "sensor", "type": "python", "title": "Hall sensor"}], + }, + ) + client.put( + f"{DASHBOARDS}/hall", + headers=superuser_token_headers, + json={ + "name": "hall", + "title": "Hall", + "widgets": [ + { + "id": "temperature", + "type": "stat", + "title": "Temperature", + "layout": {"lg": {"x": 0, "y": 0, "w": 3, "h": 2}}, + "config": {"message": "hall.temperature", "dtype": "float"}, + } + ], + "version": 0, + }, + ) + + entries = client.get(f"{PREFIX}/", headers=superuser_token_headers).json() + # Keyed on the parent too: an id is only unique within the document it is + # in, and the other suites seed their own `sensor` and `temperature`. + found = { + (entry["category"], entry["parent"], entry["name"]): entry for entry in entries + } + + assert found[("flow", "", "searchable")]["title"] == "Searchable" + assert found[("node", "searchable", "sensor")]["title"] == "Hall sensor" + assert found[("node", "searchable", "sensor")]["kind"] == "python" + assert found[("dashboard", "", "hall")]["title"] == "Hall" + assert found[("widget", "hall", "temperature")]["title"] == "Temperature" + assert found[("widget", "hall", "temperature")]["kind"] == "stat" + + +def test_secrets_are_named_only_to_a_superuser( + client: TestClient, + superuser_token_headers: dict[str, str], + normal_user_token_headers: dict[str, str], +) -> None: + client.put( + f"{SECRETS}/broker_password", + headers=superuser_token_headers, + json={"value": "hunter2"}, + ) + + def secrets(headers: dict[str, str]) -> set[str]: + entries = client.get(f"{PREFIX}/", headers=headers).json() + return {e["name"] for e in entries if e["category"] == "secret"} + + assert "broker_password" in secrets(superuser_token_headers) + assert secrets(normal_user_token_headers) == set() diff --git a/backend/tests/flow/test_connector.py b/backend/tests/flow/test_connector.py index ed44191..4abde58 100644 --- a/backend/tests/flow/test_connector.py +++ b/backend/tests/flow/test_connector.py @@ -104,6 +104,40 @@ def test_a_failing_poll_reports_down_and_keeps_going(): assert health[-1][0] == "ok" +def test_an_undeclared_port_keeps_failing_until_the_node_declares_it(): + """A publication that raised is retried, not remembered as published. + + The loop remembers what it published. If it remembered what it read, a + value the node cannot publish would be skipped on the next poll, the poll + would succeed, and the node would go back to reporting itself healthy with + its port still dark. + """ + + class Chatty(Sensor): + """Reads a port it never declared.""" + + async def poll(self) -> dict[str, Any]: + self.polls += 1 + return {"reading": 21.5, "lat": 48.1} + + health: list[tuple[str, str | None]] = [] + node = Chatty( + provides=[MessageSpec(name="reading", dtype=DType.FLOAT)], + params={"poll_interval": 0.01}, + ) + node.assign_flow("demo", "sensor") + node._on_health = lambda _node, status, detail: health.append((status, detail)) + pipeline = Pipeline(nodes=[node]) + + run_briefly(node) + + assert pipeline.state.get("demo.reading") is None + assert health[-1][0] == "down" + assert "NodeOutputError" in (health[-1][1] or "") + # Still failing on the last poll, not just the first. + assert len([entry for entry in health if entry[0] == "down"]) > 1 + + class Actuator(ConnectorNode): """A connector that commands something instead of reading it.""" diff --git a/backend/tests/flow/test_node_teardown.py b/backend/tests/flow/test_node_teardown.py new file mode 100644 index 0000000..a1b284c --- /dev/null +++ b/backend/tests/flow/test_node_teardown.py @@ -0,0 +1,50 @@ +"""Tearing a node down must not swallow a cancellation meant for the caller.""" + +import asyncio + +import pytest + +from fluksio.flow.connector import ConnectorNode +from fluksio.flow.nodes import DelayNode + + +async def stubborn() -> None: + """A loop whose shutdown does not answer the first cancellation.""" + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + await asyncio.sleep(3600) + + +def test_stop_cron_lets_the_callers_cancellation_through(): + async def scenario() -> None: + node = DelayNode(params={"cron": "* * * * *"}) + node._stop_cron = asyncio.Event() + node._cron_task = asyncio.create_task(stubborn()) + + stopping = asyncio.create_task(node.stop_cron()) + await asyncio.sleep(0.05) # let it reach the await on the cron task + stopping.cancel() + with pytest.raises(asyncio.CancelledError): + await stopping + + node._cron_task.cancel() + + asyncio.run(scenario()) + + +def test_connector_stop_lets_the_callers_cancellation_through(): + async def scenario() -> None: + node = ConnectorNode() + node._stop_event = asyncio.Event() + node._poll_task = asyncio.create_task(stubborn()) + + stopping = asyncio.create_task(node.stop()) + await asyncio.sleep(0.05) # let it reach the await on the poll task + stopping.cancel() + with pytest.raises(asyncio.CancelledError): + await stopping + + node._poll_task.cancel() + + asyncio.run(scenario()) diff --git a/backend/tests/flow/test_node_types.py b/backend/tests/flow/test_node_types.py index 7cc1dc1..056593c 100644 --- a/backend/tests/flow/test_node_types.py +++ b/backend/tests/flow/test_node_types.py @@ -197,6 +197,47 @@ def test_a_flux_request_is_run_rather_than_written(monkeypatch): assert out == {"answer": {"rows": [], "range_s": 3600}} +def test_the_configured_timeout_reaches_the_influx_client(monkeypatch): + """The client counts in milliseconds; the param is seconds like its peers.""" + import influxdb_client + + from fluksio.flow.nodes import InfluxDbNode + + seen: dict = {} + + class FakeClient: + def __init__(self, **kwargs): + seen.update(kwargs) + + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + def query_api(self): + return self + + def query(self, *_, **__): + return [] + + monkeypatch.setattr(influxdb_client, "InfluxDBClient", FakeClient) + + node = InfluxDbNode( + provides=[MessageSpec(name="answer", dtype=DType.JSON)], + params={ + "url": "http://influx", + "token": "t", + "org": "o", + "bucket": "b", + "timeout": 2.5, + }, + ) + node._run_flux({"flux": 'from(bucket: "b")'}) + + assert seen["timeout"] == 2500 + + def test_a_falsy_return_is_a_mistake_not_silence(): """Only None means "nothing to publish".""" from fluksio.flow.nodes import Node diff --git a/backend/tests/flow/test_placement.py b/backend/tests/flow/test_placement.py index d5edeca..73dabf5 100644 --- a/backend/tests/flow/test_placement.py +++ b/backend/tests/flow/test_placement.py @@ -129,7 +129,7 @@ def test_a_preferred_label_falls_back_here_and_is_still_accounted(loop): assert placer.local.snapshot()["cpus"]["free"] == 1 -def test_asking_for_more_than_anything_has_gets_what_there_is(loop): +def test_asking_for_more_than_anything_has_gets_what_there_is(loop, caplog): """A flow written on a cluster still has to run on a laptop.""" placer = placer_over(cpus=2) @@ -138,6 +138,10 @@ def test_asking_for_more_than_anything_has_gets_what_there_is(loop): assert allocation.cpus == 2 assert allocation.gpus == () + # Cards are declared, not detected, so a machine that has one reads as + # having none until it is told — and the warning is where that is noticed. + assert "fluksio serve --gpus" in caplog.text + def test_what_is_clamped_to_is_a_machine_that_exists(loop): """Each dimension taken separately can describe a machine nobody has. diff --git a/backend/tests/flow/test_senders.py b/backend/tests/flow/test_senders.py index 0f1ee11..655066a 100644 --- a/backend/tests/flow/test_senders.py +++ b/backend/tests/flow/test_senders.py @@ -89,3 +89,65 @@ def test_a_string_goes_on_the_wire_bare(): asyncio.run(node._publish_with(client, {"plug": "ON", "level": 60})) assert client.published == [("actor/plug", "ON"), ("light/level", "60")] + + +def test_the_configured_timeout_reaches_the_broker_client(monkeypatch): + """Without one, a dead socket makes the disconnect ack wait forever.""" + import aiomqtt + + seen: dict = {} + + class FakeClient: + def __init__(self, **kwargs): + seen.update(kwargs) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + return False + + async def publish(self, *_, **__): + return None + + monkeypatch.setattr(aiomqtt, "Client", FakeClient) + + node = MqttNode( + requires=[MessageSpec(name="setpoint", port="setpoint", dtype=DType.FLOAT)], + params={"topic": {"setpoint": "heating/setpoint"}, "timeout": 2.5}, + ) + asyncio.run(node._publish_once({"setpoint": 21.0})) + + assert seen["timeout"] == 2.5 + + +def test_the_configured_backlog_reaches_the_publish_queue(monkeypatch): + """The depth is read when the queue is built, so it has to be per node.""" + import aiomqtt + + class FakeClient: + def __init__(self, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + return False + + monkeypatch.setattr(aiomqtt, "Client", FakeClient) + + node = MqttNode( + requires=[MessageSpec(name="setpoint", port="setpoint", dtype=DType.FLOAT)], + params={"topic": {"setpoint": "heating/setpoint"}, "publish_queue_size": 8}, + ) + node.assign_flow("heating", "out") + + async def scenario() -> int: + await node.start_publisher() + assert node._publish_queue is not None + size = node._publish_queue.maxsize + await node.stop_publisher() + return size + + assert asyncio.run(scenario()) == 8 diff --git a/backend/tests/flow/test_unhealthy_node_issue.py b/backend/tests/flow/test_unhealthy_node_issue.py new file mode 100644 index 0000000..45e44fa --- /dev/null +++ b/backend/tests/flow/test_unhealthy_node_issue.py @@ -0,0 +1,42 @@ +"""A node that loaded but is not working shows up as an issue on its flow. + +Health used to go nowhere: the connector reported it, the controller stored it, +and no screen ever asked. These cover the derivation that closes that gap. +""" + +from pathlib import Path + +from fluksio.flow.controller import FlowController, LoadedNode +from fluksio.flow.store import FlowStore + + +def a_controller(tmp_path: Path) -> FlowController: + controller = FlowController(FlowStore(tmp_path / "flows")) + controller.loaded["house.owm"] = LoadedNode( + id="house.owm", + flow="house", + health="down", + health_detail="ConnectionError: name resolution failed", + ) + return controller + + +def test_a_down_node_is_an_issue_on_its_flow(tmp_path: Path) -> None: + controller = a_controller(tmp_path) + + issues = controller.flow_issues("house") + + assert [issue.code for issue in issues] == ["node_unhealthy"] + assert issues[0].node == "house.owm" + assert "name resolution failed" in issues[0].message + # Not advisory: the canvas has to mark the node. + assert not issues[0].advisory + assert controller.flow_issues("other") == [] + + +def test_the_issue_clears_when_the_node_reports_itself_well(tmp_path: Path) -> None: + controller = a_controller(tmp_path) + + controller.loaded["house.owm"].health = "ok" + + assert controller.flow_issues("house") == [] diff --git a/backend/tests/flow/test_workers.py b/backend/tests/flow/test_workers.py index 4d8c9a0..335c708 100644 --- a/backend/tests/flow/test_workers.py +++ b/backend/tests/flow/test_workers.py @@ -69,6 +69,24 @@ def test_a_node_returns_its_value_and_what_it_printed(pool, capsys): assert "seen 21" in capsys.readouterr().out +def test_a_node_can_log_through_the_module_it_imports(pool, capsys): + """The SDK exports `logger`; inside a worker `fluksio` is the reporter. + + Without it, `fluksio.logger.info(...)` died with AttributeError — after + the training it was reporting on had already succeeded. + """ + result = run( + pool, + "import fluksio\n\n\n" + "def process(value):\n" + " fluksio.logger.info('tuned %s', value)\n" + " return {'out': value}\n", + value=7, + ) + assert result == {"out": 7} + assert "tuned 7" in capsys.readouterr().out + + def test_a_failure_keeps_its_class_and_points_at_the_node(pool): with pytest.raises(Exception) as caught: run(pool, "def process():\n raise ValueError('bad input')\n") @@ -667,6 +685,19 @@ def test_retiring_workers_reaches_the_children(pool): assert child._generation > before +def test_retiring_the_cards_leaves_the_other_pools_warm(pool): + """A library that preallocated the card only gives it back by dying.""" + card = pool.for_env({"CUDA_VISIBLE_DEVICES": "0"}) + threads = pool.for_env({"OMP_NUM_THREADS": "2"}) + before = (card._generation, threads._generation) + + pool.retire_gpu_children() + + assert card._generation > before[0] + # Nothing to hand back, so nothing pays a cold start for it. + assert threads._generation == before[1] + + def test_cancelling_reaches_a_node_running_in_a_child(pool): child = pool.for_env({"OMP_NUM_THREADS": "2"}) started = threading.Event() diff --git a/backend/tests/sdk/test_build.py b/backend/tests/sdk/test_build.py index 3f21d11..834d91a 100644 --- a/backend/tests/sdk/test_build.py +++ b/backend/tests/sdk/test_build.py @@ -6,7 +6,7 @@ from fluksio.flow.schemas import FlowDef from fluksio.sdk import MARKER, Flow, Port, SyncError, node, use # The functions live here rather than in each test: a node's module is what the -# generated body imports, and `__main__` is refused for exactly that reason. +# generated body imports, and `__main__` is refused at sync for that reason. @node(provides=[Port("dataset", "artifact"), Port("rows", "int")]) @@ -316,3 +316,27 @@ def test_a_negative_timeout_is_refused(): def test_a_zero_timeout_means_no_limit(): decorated = node(requires=["a"], timeout=0)(one_default) assert decorated.__fluksio__.timeout == 0 + + +def test_a_node_in_a_script_run_directly_is_refused_at_sync_not_at_import(): + """A study module has to be runnable as a script for a self-check. + + The refusal belongs where the body is generated: the decorator hands the + function back untouched, so `python study.py` declares its nodes, calls + them and checks itself. What cannot be done is syncing them, because + nothing could import `__main__`. + """ + + def probe(rows=1): + return {"score": float(rows)} + + probe.__module__ = "__main__" + decorated = node(provides=[Port("score", "float")])(probe) + # Declared and callable, which is the whole point of running the file. + assert decorated(rows=2) == {"score": 2.0} + + flow = Flow("mainflow", nodes=[decorated], outputs=["score"]) + with pytest.raises(SyncError, match="run directly"): + flow.document() + with pytest.raises(SyncError, match="run directly"): + flow.shims() diff --git a/backend/tests/test_cli.py b/backend/tests/test_cli.py index 4cad80f..15fb523 100644 --- a/backend/tests/test_cli.py +++ b/backend/tests/test_cli.py @@ -88,6 +88,24 @@ def test_run_arguments_are_typed_by_the_flow_they_are_for() -> None: _params(definition, ["--nonesuch", "1"]) +def test_a_sweeps_param_spelling_is_refused_by_name() -> None: + """`run --param lr=0.002` is a name this flow has not got, and says so.""" + import pytest + + from fluksio.sdk import SyncError + from fluksio.sdk.cli import _params + + definition = {"inputs": [{"spec": {"name": "lr", "dtype": "float"}}]} + + # Not a JSONDecodeError over `lr=0.002`, which is what reading the value + # before the name used to give. + with pytest.raises(SyncError, match="sweep --param"): + _params(definition, ["--param", "lr=0.002"]) + + with pytest.raises(SyncError, match="'lr' takes float"): + _params(definition, ["--lr", "fast"]) + + def test_serve_uses_the_installation_the_directory_belongs_to( tmp_path: Path, monkeypatch ) -> None: @@ -372,6 +390,43 @@ def test_the_metric_names_are_asked_for_rather_than_guessed() -> None: assert _list_names(Engine(), args) == 0 +def test_a_runs_artifact_is_listed_and_downloaded(tmp_path, monkeypatch) -> None: + """`save_artifact` had no counterpart: the bytes were API-only.""" + from fluksio.cli import _parser + from fluksio.sdk.cli import _artifacts + + row = { + "name": "weights", + "node": "fit", + "digest": "sha256:abc", + "size": 3, + "media_type": "application/octet-stream", + "filename": "weights.npz", + } + + class Engine: + def run(self, run_id): + assert run_id == "r-1" + return {"id": run_id, "status": "ok", "artifacts": [row]} + + def download(self, digest): + assert digest == "sha256:abc" + return b"abc" + + parser = _parser() + monkeypatch.chdir(tmp_path) + + assert _artifacts(Engine(), parser.parse_args(["artifacts", "r-1"])) == 0 + + # Written under the name the node saved it as, not the message's. + assert _artifacts(Engine(), parser.parse_args(["artifacts", "r-1", "weights"])) == 0 + assert (tmp_path / "weights.npz").read_bytes() == b"abc" + + args = parser.parse_args(["artifacts", "r-1", "weights", "-o", "here.bin"]) + assert _artifacts(Engine(), args) == 0 + assert (tmp_path / "here.bin").read_bytes() == b"abc" + + def test_an_engine_without_the_route_is_named_rather_than_404() -> None: """A client ships ahead of the engine; a flat 404 does not say so.""" from fluksio.sdk.cli import _too_old @@ -417,8 +472,38 @@ def test_a_study_in_a_subfolder_is_found(tmp_path) -> None: } +def test_a_study_per_directory_imports_under_its_own_name(tmp_path) -> None: + """One `study.py` per folder is a layout people have, and it works. + + Named for where each sits under the directory being synced, so nothing + collides and no `__init__.py` has to be added — which would break the + bare `from study import ...` a test beside it does. + """ + import sys + + from fluksio.sdk.cli import discover + + for study in ("s1", "s2"): + (tmp_path / "dev" / study).mkdir(parents=True) + (tmp_path / "dev" / study / "study.py").write_text(f"VALUE = {study!r}\n") + + try: + discover([str(tmp_path / "dev")]) + assert sys.modules["s1.study"].VALUE == "s1" + assert sys.modules["s2.study"].VALUE == "s2" + finally: + for name in ("s1.study", "s2.study", "s1", "s2"): + sys.modules.pop(name, None) + sys.path[:] = [entry for entry in sys.path if entry != str(tmp_path / "dev")] + + def test_two_files_of_one_name_are_refused(tmp_path) -> None: - """Python keeps one module per name, and a node's body imports by it.""" + """Python keeps one module per name, and a node's body imports by it. + + Unreachable from one sync of a directory now that a file is named for + where it sits; this is the spelling that still gets there — two files + named on the command line, each rooted at its own directory. + """ import pytest from fluksio.sdk import SyncError @@ -435,6 +520,77 @@ def test_two_files_of_one_name_are_refused(tmp_path) -> None: _import(*_module_of(second), second) +def test_how_long_ago_reads_like_a_duration() -> None: + """A failure with no time on it says nothing about whether it is current.""" + from datetime import UTC, datetime, timedelta + + from fluksio.sdk.cli import _ago + + def then(**delta): + return (datetime.now(UTC) - timedelta(**delta)).isoformat() + + assert _ago(then(seconds=5)) == "5s ago" + assert _ago(then(minutes=3)) == "3min ago" + assert _ago(then(hours=2)) == "2h ago" + assert _ago(then(days=3)) == "3d ago" + # What the engine stores is UTC whether or not the spelling says so. + assert _ago(datetime.now(UTC).replace(tzinfo=None).isoformat()) == "0s ago" + assert _ago(None) == "" + + +def test_serve_says_when_a_flow_wants_a_card_nobody_declared( + tmp_path, monkeypatch, capsys +) -> None: + """The clamp warning goes to the log; this is said while someone is reading. + + Cards are declared rather than detected, so a fresh install that forgets + `--gpus` clamps a GPU node to zero and runs them all at once. + """ + from fluksio import cli + from fluksio.core.config import settings + from fluksio.flow.schemas import FlowDef, NodeDef, Resources + from fluksio.flow.store import FlowStore + + store = FlowStore(tmp_path / "flows") + store.write_flow( + FlowDef( + name="finetune", + mode="batch", + nodes=[NodeDef(id="fit", type="python", resources=Resources(gpus=1))], + ) + ) + monkeypatch.setattr(settings, "FLOWS_DIR", tmp_path / "flows") + + monkeypatch.setattr(settings, "FLOW_GPUS", 0) + cli._mention_undeclared_cards() + said = capsys.readouterr().out + assert "finetune asks for one" in said + assert "--gpus" in said + + # Told how many there are, it has nothing to say. + monkeypatch.setattr(settings, "FLOW_GPUS", 1) + cli._mention_undeclared_cards() + assert capsys.readouterr().out == "" + + +def test_a_serve_limit_is_refused_as_a_flag_not_as_a_traceback(capsys) -> None: + """These are written into the environment before the settings are built.""" + import pytest + + from fluksio.cli import _parser + + parser = _parser() + assert parser.parse_args(["serve", "--max-workers", "2"]).max_workers == 2 + # A machine may genuinely have no card, so zero is a number here. + assert parser.parse_args(["serve", "--gpus", "0"]).gpus == 0 + assert parser.parse_args(["serve"]).gpus is None + + for flag, value in (("--max-workers", "0"), ("--gpus", "-1")): + with pytest.raises(SystemExit): + parser.parse_args(["serve", flag, value]) + assert "at least" in capsys.readouterr().err + + def test_run_and_sweep_take_what_to_sync() -> None: from fluksio.cli import _parser @@ -446,6 +602,98 @@ def test_run_and_sweep_take_what_to_sync() -> None: assert parser.parse_args(["sweep", "train"]).sync == [] +def test_the_dashboard_runs_the_engine_as_a_child_of_itself(monkeypatch) -> None: + """At a terminal `serve` is a dashboard; the engine is a plain serve. + + Every flag is passed through, so what the child runs with is what serve + was asked for — and `--plain` is what stops it opening a second one. + """ + import sys + + from fluksio import cli + from fluksio.tui import child_argv + + argv = child_argv(["serve", "--port", "8123", "--gpus", "1"]) + assert argv[:3] == [sys.executable, "-m", "fluksio.cli"] + assert argv[3:] == ["serve", "--port", "8123", "--gpus", "1", "--plain"] + # Already plain: told once, not twice. + assert child_argv(["serve", "--plain"])[3:] == ["serve", "--plain"] + + opened: list[str] = [] + monkeypatch.setattr( + "fluksio.tui.run_tui", lambda args: opened.append("tui") or 0, raising=False + ) + monkeypatch.setattr(sys.stdout, "isatty", lambda: True, raising=False) + monkeypatch.setattr(sys.stdin, "isatty", lambda: True, raising=False) + + parser = cli._parser() + assert cli.cmd_serve(parser.parse_args(["serve"])) == 0 + assert opened == ["tui"] + + # `--plain` goes past it, which is what the child and every container does. + # Nothing else of serve runs here, so it fails on the data directory it is + # given rather than opening a dashboard. + opened.clear() + monkeypatch.setattr( + cli, "_data_dir", lambda *a, **k: (_ for _ in ()).throw(SystemExit(3)) + ) + with __import__("pytest").raises(SystemExit): + cli.cmd_serve(parser.parse_args(["serve", "--plain"])) + assert opened == [] + + +def test_a_serving_engine_records_itself_until_it_stops(tmp_path) -> None: + """A pid nobody is running is the same as no pidfile at all.""" + import os + + from fluksio.cli import read_pidfile, write_pidfile + + assert read_pidfile(tmp_path) is None + + written = write_pidfile(tmp_path, 8000) + assert read_pidfile(tmp_path) == {"pid": os.getpid(), "port": 8000} + + # Killed outright: the file outlives the process it names. + written.write_text('{"pid": 2147483646, "port": 8000}') + assert read_pidfile(tmp_path) is None + + written.write_text("not json") + assert read_pidfile(tmp_path) is None + + +def test_who_holds_the_port_is_told_apart_by_the_token() -> None: + """Only this directory's own engine may be reported as already up. + + The token is signed with this directory's secret key, so an engine that + accepts it is one reading this directory's database. Another + installation's Fluksio answers the health check and refuses it. + """ + import httpx + + from fluksio.cli import probe_engine + + def engine(health: int, summary: int): + def handle(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/health-check/"): + return httpx.Response(health) + return httpx.Response(summary) + + return httpx.Client(transport=httpx.MockTransport(handle)) + + with engine(200, 200) as client: + assert probe_engine("http://x", "t", client) == "ours" + with engine(200, 401) as client: + assert probe_engine("http://x", "t", client) == "foreign" + # A directory with no credential yet cannot prove anything is its own, and + # `Bearer ` is not a legal header value — so it asks without one. + with engine(200, 401) as client: + assert probe_engine("http://x", "", client) == "foreign" + # Somebody else's dev server, or nothing listening at all. + with engine(404, 404) as client: + assert probe_engine("http://x", "t", client) == "other" + assert probe_engine("http://127.0.0.1:1", "t") == "other" + + def test_serve_moves_off_a_port_that_is_taken() -> None: """A first start should not die on somebody else's dev server.""" import socket diff --git a/docker/compose.yml b/docker/compose.yml index e554fbb..75f5e03 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -75,6 +75,13 @@ services: image: '${DOCKER_IMAGE_BACKEND?Variable not set}:${TAG-latest}' container_name: fluksio-api restart: always + # PID 1 that reaps orphans. A python node's worker processes outlive the + # process holding their handle whenever that one goes without stopping the + # pool -- which `--reload` does on every source edit -- and reparent onto + # PID 1, which is the app itself and waits for nobody else's children. The + # container filled up with zombie `python`. Declared here rather than in + # compose.dev.yml because an init closes the whole class, not just reload. + init: true security_opt: - no-new-privileges:true networks: diff --git a/docs/code/cli.md b/docs/code/cli.md index 03c6aa9..9f8b0dc 100644 --- a/docs/code/cli.md +++ b/docs/code/cli.md @@ -7,13 +7,19 @@ pip install fluksio Installs the engine and the `fluksio` command. Python 3.12 or newer, Linux or macOS. +The MQTT and InfluxDB connectors, outbound mail and error reporting are +`pip install 'fluksio[server]'` — a deployment talking to devices wants them, +and a laptop waiting on them to install does not. Everything else, the engine +and every python node included, is in the plain install; a node type whose +library is missing says which extra to add when one is actually built. + There is a second, smaller distribution — `fluksio-worker` — for a machine that should only *run nodes* for an engine elsewhere. It has none of the engine in it. See [Remote workers](workers.md). The command is two things at once: `serve`, `enroll` and `worker` *are* an -installation, while `login`, `sync`, `run`, `runs`, `sweep` and `status` talk to -one that may be anywhere. +installation, while `login`, `sync`, `run`, `runs`, `artifacts`, `sweep` and +`status` talk to one that may be anywhere. ## Where an installation lives @@ -39,22 +45,41 @@ On the first start it creates an admin account and prints its password **once**. Nothing else has to be running: no database server, no message broker, no Docker. +At a terminal this opens a dashboard with the engine running under it; see +[below](#the-dashboard). `--plain` prints the log stream instead, which is +also what happens with no terminal — in a container, under systemd, or in CI. + The default port moves out of the way when something already has it — 8001, 8002, and so on — and says which one it took; the URL written to `client.json` is the one it is actually on. A port you *asked* for is never moved off: `--port 9000` on a taken 9000 fails, because something else is there and you named it. +What it will *not* do is start a second engine for the same installation. If +the port is held by an engine already serving this directory, it says so and +stops — one SQLite database wants one engine. Another installation's Fluksio +on that port is named, and the move happens as usual. + | Option | Default | What it does | |---|---|---| | `--data-dir PATH` | `./.fluksio` (or `$FLUKSIO_HOME`) | where this installation keeps everything | | `--host HOST` | `127.0.0.1` | what to bind | | `--port PORT` | `8000`, or the next free one | what to listen on | +| `--plain` | off at a terminal | the log stream rather than the dashboard | | `--log-level LEVEL` | `info` | uvicorn's log level | | `--admin-email ADDR` | `admin@example.com` | the account created on first run | | `--admin-password PW` | generated | set it instead of having one generated | | `--enroll CODE` | — | pair with a portal as part of coming up | | `--portal URL` | — | the portal `--enroll` redeems at | +| `--max-runs N` | 4 | batch runs driven at once (`FLOW_MAX_RUNS`) | +| `--max-cascades N` | 4 | cascades in flight at once (`FLOW_MAX_CASCADES`) | +| `--max-workers N` | 4 | python worker processes (`FLOW_MAX_WORKERS`) | +| `--gpus N` | 0 | GPUs on this machine a node may be given (`FLOW_GPUS`) | + +Cards are declared rather than detected — asking a vendor's tooling would make +one dependency two — so a machine with a GPU reports none until `--gpus` says +otherwise, and a node asking for one is clamped to zero and runs alongside +every other. `--gpus 1` is what serialises them. `--enroll` with `--portal` is the one-command setup: it pairs before the engine starts, so the connection is dialled as part of coming up rather than needing a @@ -88,6 +113,29 @@ Fluksio 0.1.0 — data in /home/you/.fluksio An enrolled installation says which portal it is on instead, and notes that the dashboard is served from there rather than here. +### The dashboard + +At a terminal, `serve` draws the health overview, the recent runs, and the +engine's own log in a pane below — the output above is in there, not replaced +by it. + +| Key | What it does | +|---|---| +| `q` | close the dashboard. **The engine keeps running**, and the pid is printed | +| `s` | stop the engine, or start it again | +| `r` | restart it | +| `c` | cancel the run the cursor is on | +| `e` | pair with a portal, without leaving the screen | + +The engine is a child process rather than a thread, which is what makes those +possible — and what makes `q` a way out of the screen rather than a way to +stop the engine. Running `fluksio serve` again reattaches to it. + +An engine started elsewhere is adopted rather than duplicated, and can be +stopped from here only when it is this installation's own: both the pidfile +beside the data and a token this directory's key signed have to agree. Another +installation's engine is named and left alone. + ## `fluksio enroll` Pairs an existing installation with a portal. @@ -160,12 +208,14 @@ file path, because the shim has to import the same way. A plain directory is walked all the way down, so one folder per study — `fluksio sync dev` over `dev/s1_baseline/study.py` — needs no naming. Hidden -directories, `__pycache__`, `node_modules` and virtualenvs are left alone. Two -files that would import under the same name are refused rather than -silently collapsed into one: Python keeps one module per name, and a node's -generated body imports by that name, so `dev/s1/study.py` and `dev/s2/study.py` -need an `__init__.py` each — making them `s1.study` and `s2.study` — or -different filenames. +directories, `__pycache__`, `node_modules` and virtualenvs are left alone. + +Each file is imported under the name its path spells beneath the directory +being synced, so `dev/s1/study.py` and `dev/s2/study.py` are `s1.study` and +`s2.study` and a `study.py` per study collides with nothing. No `__init__.py` +is needed — the directories in between are namespace packages — which leaves a +bare `from study import ...` in a test beside it working. A file at the top of +what is synced keeps its plain name. | Flag | What it does | |---|---| @@ -206,7 +256,9 @@ flow nothing changed in, so what the walk costs is importing the others. is not free, and `--no-sync` skips it entirely. Flags that are not its own are the flow's inputs, typed by what the flow -declares them as. `--wait` blocks until the run +declares them as — so a name the flow has not got is refused by name, and +`--param lr=0.002` is told that one value is `--lr 0.002` and several is a +[sweep](#fluksio-sweep). `--wait` blocks until the run finishes and exits non-zero if it failed. `--follow` waits as well, and prints the numbers the run reports as they arrive: @@ -309,10 +361,10 @@ engine is the command itself, so nothing changes under it. fluksio runs [--flow train] [--limit 20] ``` -The runs an engine has recorded, newest first: id, status, flow, duration, the -commit of the repository it came from, and the inputs it was given. Statuses -are coloured when a terminal is reading the output — `ok` green, `error` red, -`cached` cyan. +The runs an engine has recorded, newest first: id, status, flow, duration, how +long ago it was submitted, the commit of the repository it came from, and the +inputs it was given. Statuses are coloured when a terminal is reading the +output — `ok` green, `error` red, `cached` cyan. Only the inputs that *differ from what the flow declares* are shown, and they are clamped to what is left of the terminal's width — a run that took the @@ -359,18 +411,21 @@ point of *each* curve. `export runs` is the wide one: a row per run with its inputs as columns, its final numbers, its status, its duration and the commit and digest of the code it ran. -The inputs that become columns are the ones that **vary** across the selected -runs — the axis of the sweep, which is what a comparison is read along — -unless `--params lr,seed` names them. `--metrics` narrows the final numbers -the same way. +Every input the selected runs recorded becomes a column, so the schema does +not move with the selection and a filter written against one export keeps +working on the next; `--params lr,seed` narrows it to the axis a comparison is +read along. `--metrics` narrows the final numbers the same way. + +An input left out of a submit is recorded at the value the flow declares for +it, so every row says what it was actually run with rather than leaving the +cell blank. A node usually returns a record rather than a scalar, so both sides take dotted paths into one: `--metrics final_metrics.train_loss,test_metrics.known.perfect` selects three fields rather than two blobs, and `--params model.ansatz` does the same for an input. The defaults reach the same depth — every number a -result carries becomes a column wherever it sits, and inputs are compared -leaf by leaf, so two configurations differing in one field give that field -rather than two records that are merely not equal. +result carries becomes a column wherever it sits, and a record's inputs are +taken leaf by leaf rather than as one blob. Metric names are flow-qualified — a node of `train` writing `train_loss` records `train.train_loss` — so `--list` prints the names the selected runs @@ -387,6 +442,21 @@ stream. In a notebook, `Client.export_metrics()` and `Client.export_runs()` answer the same rows as a list of dicts, which `pandas.DataFrame` takes directly. +### `fluksio artifacts` + +```sh +fluksio artifacts 1758042000123-9f2ab41c +fluksio artifacts 1758042000123-9f2ab41c weights -o model.npz +``` + +The files a run produced — what `fluksio.save_artifact(...)` wrote, and any +artifact a node returned. Named alone it lists them: the message each left on, +its size, its media type and the filename the node gave it. Name one and it is +written here, under that filename unless `-o` says otherwise. + +The message name is the one to pass, since it is what addresses the bytes; +`--local` reads them from this directory without an engine served. + ## What lives in the data directory ```text diff --git a/docs/code/connectors.md b/docs/code/connectors.md index c9df560..967a710 100644 --- a/docs/code/connectors.md +++ b/docs/code/connectors.md @@ -54,7 +54,7 @@ Two things to know about this: make dev-frontend # Vite on :5173 ``` -Restart the backend and your node type appears in the add-node palette (⌘K), +Restart the backend and your node type appears in the add-node palette (⌘P), labelled with the package it came from. ## Write the node diff --git a/docs/code/nodes.md b/docs/code/nodes.md index 249151b..ab5bf12 100644 --- a/docs/code/nodes.md +++ b/docs/code/nodes.md @@ -110,6 +110,18 @@ Because the address is the content's hash, a sweep whose fifty configs share one preprocessed input stores it once, and a reference stays valid wherever the store is reachable from — including on another machine. +`fluksio artifacts RUN NAME` is how one comes back out at a terminal. + +### `fluksio.logger` + +A node's `print` is kept as that node's logs, and so is anything on +`fluksio.logger` — the same logger the SDK exports at top level, so code that +runs both inside a node and outside one says it the same way: + +```python +fluksio.logger.info("resuming from epoch %d", start) +``` + ### Media Say what the bytes are and the port can be typed for them: diff --git a/docs/concepts/flows.md b/docs/concepts/flows.md index a73db87..9aae75f 100644 --- a/docs/concepts/flows.md +++ b/docs/concepts/flows.md @@ -163,10 +163,13 @@ to: | `self_loop_needs_initial` | a node reads a message it also writes, with no starting value | | `node_error` | the node's code did not load: a syntax error, a missing import | | `unauthenticated_hook` | advisory — a webhook with no shared secret is open to anyone | +| `node_unhealthy` | the node loaded but is not working: a connector that cannot reach its device, or whose last publication failed | -A flow with any of these except the advisory one does not run. The health -summary on Home counts them, so "why is nothing happening?" has an answer that -does not involve reading logs. +A flow with any of these except the advisory one and `node_unhealthy` does not +run — a node reporting itself down is a live condition, not a build error, so +the rest of the flow keeps going and the issue clears by itself once the node +reports well again. The health summary on Home counts them, so "why is nothing +happening?" has an answer that does not involve reading logs. ## What happens at runtime diff --git a/docs/getting-started/data-science.md b/docs/getting-started/data-science.md index d156971..c688512 100644 --- a/docs/getting-started/data-science.md +++ b/docs/getting-started/data-science.md @@ -249,6 +249,11 @@ stops your own code from running. - **The declaration is checked against the function.** A port with no matching parameter, or a parameter that is neither port nor setting, is an error when the module is imported — not when the node is first called. +- **The decorator hands the function back untouched.** `fit(dataset, lr=0.05)` + is an ordinary call, and the module is an ordinary script — so a + `if __name__ == "__main__":` check beside the nodes runs. Syncing one + defined in a script *run as* `__main__` is what cannot work, since the + generated body would have no name to import it by. !!! note "Where a `yield` cannot reach" @@ -294,6 +299,22 @@ nothing else is given that card while it runs. string whose contents depend on the version you have installed, so writing it for you would silently replace whatever you had put there. +The engine has to be told how many cards it has — `fluksio serve --gpus 1`, or +`FLOW_GPUS` — because detecting them would mean depending on a vendor's +tooling. Until it is, a node asking for one is quietly given zero and runs +alongside every other; the log says so the first time it happens. + +Workers are kept warm on purpose, so a library that takes most of the card at +import would hold it after the run finished. The workers that ran on a card +are retired when the run ends, which gives the memory back; what it costs is +the next GPU run paying for its imports again. + +**The declaration is what buys that**, not touching the card. A node that +imports jax without `resources={"gpus": 1}` runs on the shared pool, is never +given `CUDA_VISIBLE_DEVICES`, and leaves a warm worker holding whatever it +preallocated. Declaring the card is what makes it a worker the engine knows +to retire — and what stops two such nodes running at once in the first place. + Declaring nothing is the default and is right for most nodes — a poll, a threshold, a message on its way somewhere. Those share the engine's worker pool and are given a fair share of `FLOW_CPUS` as a thread cap, which is what stops diff --git a/docs/getting-started/facility-automation.md b/docs/getting-started/facility-automation.md index 2f83b35..b3a8faf 100644 --- a/docs/getting-started/facility-automation.md +++ b/docs/getting-started/facility-automation.md @@ -134,7 +134,7 @@ safe, fan-in is free, and two flows can share a value by naming it. ### Read a sensor -Press **Add node** (or ⌘K / Ctrl-K, which opens the command palette) and pick +Press **Add node** (or ⌘P / Ctrl-P, which opens the command palette) and pick **MQTT**. In its panel on the right: - **Broker host** — your broker's hostname, `mosquitto` if you are using the diff --git a/docs/interface/dashboards.md b/docs/interface/dashboards.md index ca48fd1..cb36509 100644 --- a/docs/interface/dashboards.md +++ b/docs/interface/dashboards.md @@ -92,6 +92,15 @@ it, so swapping the store is a change to one flow and nothing else. The answer also states what it was computed for, so an answer to a different question is ignored rather than two charts overwriting each other's picture. +!!! note "Drop the bucket that is still filling" + + The request carries a window and an interval, and the binning is the flow's + own work — so the newest bucket only ever holds the part of an interval + that has elapsed. Drawn, it reads as a fall that never happened. The + engine's own [Activity charts](index.md#activity) end on the last closed + bin for that reason; a flow answering a chart has to drop or hold back its + newest bucket the same way. + ## Media tiles A media widget draws what its message points at: a picture, a clip with diff --git a/docs/interface/flow-editor.md b/docs/interface/flow-editor.md index 675a091..b484b59 100644 --- a/docs/interface/flow-editor.md +++ b/docs/interface/flow-editor.md @@ -29,7 +29,7 @@ banner tells you when it is not. ## Adding a node -**Add node** on the dock, or ⌘K / Ctrl-K for the command palette, which also +**Add node** on the dock, or ⌘P / Ctrl-P for the command palette, which also jumps between flows and offers your shared nodes. Pick a type and it appears on the canvas with its panel open. @@ -122,7 +122,7 @@ the edges, then publish. | Chord | Action | |---|---| -| ⌘K / Ctrl-K | command palette | +| ⌘P / Ctrl-P | command palette | | ⌘S / Ctrl-S | publish the flow — or, with focus in the code editor, apply the code | | ⌘Z / ⌘⇧Z | undo / redo (the flow; the code editor has its own) | | ⌘C / ⌘V | copy and paste nodes, including between flows | @@ -146,9 +146,12 @@ The canvas validates as you edit and marks the node each issue belongs to: - a node reading a message it also writes, with nothing to start it from - code that did not load - a webhook with no shared secret (advisory — it does not stop the flow) +- a node that loaded but reports itself down, such as a connector that cannot + reach its device -A flow with any of these except the last does not run, and the health summary -on Home counts it. +A flow with any of these except the last two does not run, and the health +summary on Home counts it. The last one clears on its own once the node reports +itself well again. ## See also diff --git a/docs/interface/index.md b/docs/interface/index.md index 8c25987..d9b932d 100644 --- a/docs/interface/index.md +++ b/docs/interface/index.md @@ -22,8 +22,20 @@ phone the sidebar collapses to a sheet. | **Modules** | the Python packages your node code may import | | **Alerts** | where failures get sent | | **Admin** | users (superusers only) | +| **Search** | anything in this installation, by name | | **Settings** | your account, appearance, and remote access | +### Search + +**Search** at the foot of the sidebar, or ⌘K / Ctrl-K from anywhere, opens a +panel that finds things by name as you type: flows and the nodes inside them, +dashboards and the widgets on them, panels, secrets, modules, workers and alert +channels. Picking a node opens its flow with that node in focus; picking a +widget opens its dashboard. + +It searches this installation. Reached through a portal, other installations +are behind **All installations** at the top of the sidebar. + ## Home The one screen you leave open. Four things share it. @@ -57,6 +69,8 @@ Always answers, degraded or not. The tiles cover: - **Flows** — total, running, paused, quarantined, and how many cannot run because their graph does not validate - **Nodes** — how many failed to load +- **Runs running** — batch runs in flight right now, and how many are + waiting. Only on an installation that has run something - **Queue** — depth, and how old the oldest pending item is - **Loop lag** — whether the engine's event loop is keeping up diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 914efce..323426e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -134,8 +134,11 @@ warning into a refusal to start. | `ARTIFACT_GC_GRACE_S` | `3600` | how long a freshly written artifact is spared, whatever refers to it | The three concurrency limits are also flags on `fluksio serve` — `--max-workers`, -`--max-cascades`, `--max-runs` — which outrank the file, and the engine says -which numbers it started with in its first lines. +`--max-cascades`, `--max-runs` — as is the card count, `--gpus`. The flags +outrank the file, and the engine says which numbers it started with in its +first lines. Each pool size must be at least 1 and the card count at least 0: +a number below that is refused as a flag error naming it, rather than read as +the default. Leave one empty (or unset) to get the default. An artifact is referred to by a run that recorded it or by a message currently holding it; anything else is what a camera published four hours ago, and the diff --git a/docs/reference/connector-contract.md b/docs/reference/connector-contract.md index 93cfec4..288062a 100644 --- a/docs/reference/connector-contract.md +++ b/docs/reference/connector-contract.md @@ -131,7 +131,9 @@ async def poll(self) -> dict[str, Any] | None: - Return `None` when there is nothing new. - **Only changed values are published.** A device polled every few seconds usually says the same thing, and every publication wakes everything - downstream, so the loop compares against what it last published. + downstream, so the loop compares against what it last published — what it + actually published, so a publication that failed is retried next tick rather + than counting as said. - Raising is not fatal: it is reported as a health problem and retried on the next tick. - The loop calls `inject`, which runs the graph, on a worker thread. `poll()` @@ -213,9 +215,10 @@ self.report_health("degraded", "3 of 5 registers timed out") self.report_health("down", str(exc)) ``` -Three values, `ok`, `degraded` and `down`, plus an optional detail string. The -engine forwards changes to the editor, which shows them on the node. Reporting -the same status twice is free — only changes are published. The polling loop +Three values, `ok`, `degraded` and `down`, plus an optional detail string. +Reporting the same status twice is free — only changes are published. A node +reporting `down` is named among its flow's issues and counted on the health +summary on Home; `degraded` means still working, and is not. The polling loop already reports around `poll()`; a connector managing its own connection should report when it connects and when it loses the connection. diff --git a/docs/reference/node-types.md b/docs/reference/node-types.md index 8753fb3..bb691a6 100644 --- a/docs/reference/node-types.md +++ b/docs/reference/node-types.md @@ -126,7 +126,7 @@ thing configured elsewhere. ### Inject -**`inject`** — emit a value on request, on a timer, or when the flow starts. +**`inject`** — emit a value on request, on a schedule, or when the flow starts. | Setting | Default | Notes | |---|---|---| @@ -137,8 +137,9 @@ thing configured elsewhere. | `at_start` | `false` | emit once when the flow starts | | `start_delay` | `1.0` | how long to wait before that first emission | -The most-placed trigger in a real installation — mostly as a button someone -presses. +The scheduler: a `cron` expression here is what makes a flow run by the clock. +It is also the most-placed node in a real installation — mostly as a button +someone presses. ### Delay & schedule @@ -169,7 +170,9 @@ needs a busy cascade slot waits for one. ### Trigger -**`trigger`** — send one value now and another once things go quiet. +**`trigger`** — send one value now and another once things go quiet. Despite +the name, a debounce and hold rather than a scheduler: everything it sends +starts from a value arriving. For a cron tick, see [`inject`](#inject). | Setting | Default | Notes | |---|---|---| diff --git a/frontend/scripts/capture-screenshots.mjs b/frontend/scripts/capture-screenshots.mjs index 2e0a4d5..d8d55de 100644 --- a/frontend/scripts/capture-screenshots.mjs +++ b/frontend/scripts/capture-screenshots.mjs @@ -81,7 +81,7 @@ for (const theme of ["light", "dark"]) { // Home's sections fetch independently, so networkidle can fall between them // and photograph the skeletons. The flow table is the last of them to land. await page - .getByText(/Flow activity/i) + .getByText(/activity over the last/i) .first() .waitFor({ timeout: 15000 }) await page.waitForTimeout(1500) diff --git a/frontend/src/client/core/OpenAPI.ts b/frontend/src/client/core/OpenAPI.ts index 106f4d8..327a9ad 100644 --- a/frontend/src/client/core/OpenAPI.ts +++ b/frontend/src/client/core/OpenAPI.ts @@ -48,7 +48,7 @@ export const OpenAPI: OpenAPIConfig = { PASSWORD: undefined, TOKEN: undefined, USERNAME: undefined, - VERSION: '0.1.4', + VERSION: '0.1.4+dev', WITH_CREDENTIALS: false, interceptors: { request: new Interceptors(), diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 93974ba..3ed0d50 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -2960,6 +2960,43 @@ export const RunRequestSchema = { title: 'RunRequest' } as const; +export const SearchEntrySchema = { + properties: { + category: { + type: 'string', + enum: ['flow', 'node', 'dashboard', 'widget', 'panel', 'secret', 'module', 'worker', 'alert'], + title: 'Category' + }, + name: { + type: 'string', + title: 'Name' + }, + title: { + type: 'string', + title: 'Title', + default: '' + }, + parent: { + type: 'string', + title: 'Parent', + default: '' + }, + kind: { + type: 'string', + title: 'Kind', + default: '' + } + }, + type: 'object', + required: ['category', 'name'], + title: 'SearchEntry', + description: `One thing somebody might be looking for. + +Deliberately not a route: where a category lands is the frontend's business, +and it already owns the router. This says what the thing is and what it is +called, which is all the matching needs.` +} as const; + export const SecretNamesSchema = { properties: { data: { @@ -3561,7 +3598,7 @@ export const ValidationIssueSchema = { properties: { code: { type: 'string', - enum: ['cycle', 'unconnected_input', 'missing_initial_value', 'node_error', 'unauthenticated_hook', 'self_loop_needs_initial', 'missing_source'], + enum: ['cycle', 'unconnected_input', 'missing_initial_value', 'node_error', 'unauthenticated_hook', 'self_loop_needs_initial', 'missing_source', 'node_unhealthy'], title: 'Code' }, message: { diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index 89bf192..c461744 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudDisconnectResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsReadRunsData, RunsReadRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadRunData, RunsReadRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; +import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudDisconnectResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsReadRunsData, RunsReadRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadRunData, RunsReadRunResponse, RunsDeleteRunData, RunsDeleteRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SearchReadSearchIndexResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; export class AlertsService { /** @@ -2036,6 +2036,44 @@ export class RunsService { }); } + /** + * Delete Run + * Forget a run and everything hanging off it. + * + * The same four statements ``_forget_runs`` uses when a flow goes: the run + * tables carry a plain string ``run_id`` and no foreign key, so nothing + * cascades on its own. ``flow_run``, ``metric_minute`` and ``engine_event`` + * stay — they are the observability record and are pruned on their own window. + * + * A live run is refused rather than raced: the driver writes its nodes back + * when it finishes, and those rows would arrive for a run that no longer + * exists. Cancel it first. + * + * Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage + * cache, so a later run loses hits this one would have served. And a node + * restored from this run points here through ``cached_from`` — ``_series`` + * already reads a missing source as an empty curve, which is what ``NO_CURVE`` + * explains on the screen. The artifact bytes need no help: ``sweep_artifacts`` + * keeps whatever a ``run_artifact`` row or a live message still names, so + * dropping the rows is enough and the hourly sweep reclaims the blobs. + * @param data The data for the request. + * @param data.runId + * @returns void Successful Response + * @throws ApiError + */ + public static deleteRun(data: RunsDeleteRunData): CancelablePromise { + return __request(OpenAPI, { + method: 'DELETE', + url: '/api/v1/runs/{run_id}', + path: { + run_id: data.runId + }, + errors: { + 422: 'Validation Error' + } + }); + } + /** * Cancel Run * Stop a run. One already past its last node is left as it finished. @@ -2120,6 +2158,27 @@ export class RunsService { } } +export class SearchService { + /** + * Read Search Index + * Everything searchable, for the client to match against as it is typed. + * + * The whole index rather than a query: it is a few hundred short rows for an + * installation of any ordinary size, so one fetch when the panel opens beats a + * round trip per keystroke — and the client already has a matcher. + * + * Secrets are named only to a superuser, which is who ``/secrets`` answers to. + * @returns SearchEntry Successful Response + * @throws ApiError + */ + public static readSearchIndex(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/search/' + }); + } +} + export class SecretsService { /** * Read Secrets diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 2069fc4..f09cedb 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -1052,6 +1052,23 @@ export type RunRequest = { }; }; +/** + * One thing somebody might be looking for. + * + * Deliberately not a route: where a category lands is the frontend's business, + * and it already owns the router. This says what the thing is and what it is + * called, which is all the matching needs. + */ +export type SearchEntry = { + category: 'flow' | 'node' | 'dashboard' | 'widget' | 'panel' | 'secret' | 'module' | 'worker' | 'alert'; + name: string; + title?: string; + parent?: string; + kind?: string; +}; + +export type category = 'flow' | 'node' | 'dashboard' | 'widget' | 'panel' | 'secret' | 'module' | 'worker' | 'alert'; + export type SecretNames = { data: Array<(string)>; count: number; @@ -1207,7 +1224,7 @@ export type ValidationError = { * Something wrong with a flow — a fault, or merely advisory. */ export type ValidationIssue = { - code: 'cycle' | 'unconnected_input' | 'missing_initial_value' | 'node_error' | 'unauthenticated_hook' | 'self_loop_needs_initial' | 'missing_source'; + code: 'cycle' | 'unconnected_input' | 'missing_initial_value' | 'node_error' | 'unauthenticated_hook' | 'self_loop_needs_initial' | 'missing_source' | 'node_unhealthy'; message: string; flow?: string; nodes?: Array<(string)>; @@ -1220,7 +1237,7 @@ export type ValidationIssue = { readonly advisory: boolean; }; -export type code = 'cycle' | 'unconnected_input' | 'missing_initial_value' | 'node_error' | 'unauthenticated_hook' | 'self_loop_needs_initial' | 'missing_source'; +export type code = 'cycle' | 'unconnected_input' | 'missing_initial_value' | 'node_error' | 'unauthenticated_hook' | 'self_loop_needs_initial' | 'missing_source' | 'node_unhealthy'; export type ValidationResult = { issues?: Array; @@ -1808,6 +1825,12 @@ export type RunsReadRunData = { export type RunsReadRunResponse = (RunDetail); +export type RunsDeleteRunData = { + runId: string; +}; + +export type RunsDeleteRunResponse = (void); + export type RunsCancelRunData = { runId: string; }; @@ -1830,6 +1853,8 @@ export type RunsCompareMetricData = { export type RunsCompareMetricResponse = (SeriesAnswer); +export type SearchReadSearchIndexResponse = (Array); + export type SecretsReadSecretsResponse = (SecretNames); export type SecretsSaveSecretData = { diff --git a/frontend/src/components/Common/DashboardMosaic.tsx b/frontend/src/components/Common/DashboardMosaic.tsx index 5db9d06..0c0e67c 100644 --- a/frontend/src/components/Common/DashboardMosaic.tsx +++ b/frontend/src/components/Common/DashboardMosaic.tsx @@ -65,10 +65,7 @@ type Block = { h: number } -/** - * The first page's widgets as one grid. - * - */ +/** The dashboard's widgets as one grid. */ function blocksOf(dashboard: DashboardDef_Output): { blocks: Block[] rows: number @@ -156,10 +153,13 @@ function Footprint({ dashboard }: { dashboard: DashboardDef_Output }) { function Tile({ dashboard, preview, + className, }: { dashboard: DashboardSummary /** Read the document for a footprint, or settle for the name alone. */ preview: boolean + /** What the layout needs of it — a width, in the scrolling strip. */ + className?: string }) { // The working copy, which is what the list itself is a summary of, so the // preview shows what an editor would open rather than the last publish. @@ -173,7 +173,10 @@ function Tile({ to="/dashboards/$name" params={{ name: dashboard.name }} data-testid="home-dashboard-tile" - className="grid content-start gap-2 rounded-lg border border-border p-2 transition-colors hover:bg-accent/50" + className={cn( + "grid content-start gap-2 rounded-lg border border-border p-2 transition-colors hover:bg-accent/50", + className, + )} > {preview && isPending ? ( @@ -216,19 +219,37 @@ export function byRecency< ) } -/** The dashboards, as the shapes they are, beside the flows on the home view. */ +/** + * The dashboards, as the shapes they are. + * + * Two layouts, because it is read two ways: a column of pairs where it sits + * beside something else, and one wide strip where it has the page to itself. + * A tile is `content-start` around an `aspect-video` footprint and so has no + * width of its own — the strip has to give it one. + */ export function DashboardMosaic({ dashboards, isPending, + row = false, }: { dashboards: DashboardSummary[] isPending: boolean + /** One scrolling strip instead of a two-column grid. */ + row?: boolean }) { + const container = row + ? "flex snap-x snap-mandatory gap-3 overflow-x-auto p-3" + : "grid gap-3 p-3 sm:grid-cols-2" + const tile = row ? "w-56 shrink-0 snap-start" : "" + if (isPending) { return ( -
+
{Array.from({ length: 2 }).map((_, index) => ( - + ))}
) @@ -251,12 +272,13 @@ export function DashboardMosaic({ } return ( -
+
{dashboards.map((dashboard, index) => ( ))}
diff --git a/frontend/src/components/Common/GlobalSearch.tsx b/frontend/src/components/Common/GlobalSearch.tsx new file mode 100644 index 0000000..4dbdba1 --- /dev/null +++ b/frontend/src/components/Common/GlobalSearch.tsx @@ -0,0 +1,185 @@ +import { useQuery } from "@tanstack/react-query" +import { useNavigate } from "@tanstack/react-router" +import { + Bell, + Box, + KeyRound, + LayoutDashboard, + LayoutGrid, + type LucideIcon, + MonitorSmartphone, + Package, + Server, + Workflow, +} from "lucide-react" +import { useState } from "react" + +import { type SearchEntry, SearchService } from "@/client" +import { + CommandDialog, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList, +} from "@/components/ui/command" + +export const searchQueryOptions = () => ({ + queryKey: ["search"] as const, + queryFn: () => SearchService.readSearchIndex(), + staleTime: 30_000, +}) + +/** The categories, in the order they are offered, with what to draw each as. */ +const GROUPS: { + category: SearchEntry["category"] + label: string + icon: LucideIcon +}[] = [ + { category: "flow", label: "Flows", icon: Workflow }, + { category: "node", label: "Nodes", icon: Box }, + { category: "dashboard", label: "Dashboards", icon: LayoutDashboard }, + { category: "widget", label: "Widgets", icon: LayoutGrid }, + { category: "panel", label: "Panels", icon: MonitorSmartphone }, + { category: "secret", label: "Secrets", icon: KeyRound }, + { category: "module", label: "Modules", icon: Package }, + { category: "worker", label: "Workers", icon: Server }, + { category: "alert", label: "Alerts", icon: Bell }, +] + +/** The second line: where the thing lives, and what kind it is. */ +function hint(entry: SearchEntry): string { + return [entry.parent, entry.kind].filter(Boolean).join(" · ") +} + +/** + * Everything in this installation, by name, from anywhere. + * + * The whole index arrives in one fetch and `cmdk` does the matching, so results + * narrow as they are typed without a round trip per keystroke. + * + * ponytail: every entry is rendered and cmdk hides the ones that do not match. + * Cap the groups if an installation ever grows big enough to feel it. + */ +export function GlobalSearch({ + open, + onOpenChange, +}: { + open: boolean + onOpenChange: (open: boolean) => void +}) { + const navigate = useNavigate() + const [query, setQuery] = useState("") + const { data } = useQuery({ ...searchQueryOptions(), enabled: open }) + + // Picking an item navigates, which can interrupt the dialog's exit animation + // and leave its overlay swallowing clicks — the same reason the flow canvas + // palette unmounts outright rather than fading out. + if (!open) return null + + const entries = data ?? [] + const typing = query.trim().length > 0 + + const go = (entry: SearchEntry) => { + onOpenChange(false) + setQuery("") + switch (entry.category) { + case "flow": + return navigate({ + to: "/flows/$flowName", + params: { flowName: entry.name }, + }) + case "node": + return navigate({ + to: "/flows/$flowName", + params: { flowName: entry.parent ?? "" }, + search: { node: entry.name }, + }) + case "dashboard": + return navigate({ + to: "/dashboards/$name", + params: { name: entry.name }, + }) + case "widget": + return navigate({ + to: "/dashboards/$name", + params: { name: entry.parent ?? "" }, + }) + // Panels are managed in a dialog on the dashboards screen, which opens + // itself when the address says so. + case "panel": + return navigate({ to: "/dashboards", search: { panels: true } }) + case "secret": + return navigate({ to: "/secrets" }) + case "module": + return navigate({ to: "/modules" }) + case "worker": + return navigate({ to: "/workers" }) + case "alert": + return navigate({ to: "/alerts" }) + } + } + + return ( + // Frosted chrome, a little above centre. `top-[40%]` against the dialog's + // own `-translate-y-1/2` puts the panel's middle at two fifths of the + // viewport; the inner Command paints its own surface, which has to give way + // to this one. + + + + {typing ? ( + <> + Nothing matches that. + {GROUPS.map(({ category, label, icon: Icon }) => { + const found = entries.filter( + (entry) => entry.category === category, + ) + if (found.length === 0) return null + return ( + + {found.map((entry) => ( + go(entry)} + className="min-h-11 md:min-h-8" + > + + + + {entry.title || entry.name} + + {hint(entry) ? ( + + {hint(entry)} + + ) : null} + + + ))} + + ) + })} + + ) : ( +

+ Start typing to search this installation. +

+ )} + + + ) +} diff --git a/frontend/src/components/Common/Loading.tsx b/frontend/src/components/Common/Loading.tsx index 004333e..f3294be 100644 --- a/frontend/src/components/Common/Loading.tsx +++ b/frontend/src/components/Common/Loading.tsx @@ -1,4 +1,4 @@ -import { Loader2 } from "lucide-react" +import { FluksioLoader } from "@/components/ui/fluksio-loader" /** * The router's pending screen, shown while a page's code chunk is on its way. @@ -10,12 +10,11 @@ import { Loader2 } from "lucide-react" */ export function PageLoading() { return ( - - - + +
) } diff --git a/frontend/src/components/Common/OverviewToolbar.tsx b/frontend/src/components/Common/OverviewToolbar.tsx index 593b5ff..4554bd8 100644 --- a/frontend/src/components/Common/OverviewToolbar.tsx +++ b/frontend/src/components/Common/OverviewToolbar.tsx @@ -1,5 +1,5 @@ import { useMutation } from "@tanstack/react-query" -import { Check, Loader2, Plus, Search, Trash2 } from "lucide-react" +import { Check, Plus, Search, Trash2 } from "lucide-react" import { motion } from "motion/react" import { type ReactNode, useRef, useState } from "react" @@ -13,6 +13,7 @@ import { DialogTitle, DialogTrigger, } from "@/components/ui/dialog" +import { FluksioLoader } from "@/components/ui/fluksio-loader" import { Input } from "@/components/ui/input" import { Tooltip, @@ -170,7 +171,7 @@ export function OverviewToolbar({ aria-label="Publish all changes" data-testid="publish-all" > - {publishing ? : } + {publishing ? : } @@ -278,6 +279,7 @@ export function ConfirmDelete({ names, noun, pending, + description, onConfirm, }: { open: boolean @@ -285,6 +287,8 @@ export function ConfirmDelete({ names: string[] noun: string pending: boolean + /** What is actually lost, when the git-backed answer below is not it. */ + description?: string onConfirm: () => void }) { return ( @@ -297,9 +301,14 @@ export function ConfirmDelete({ : `Delete these ${names.length} ${noun}s?`} - {names.length === 1 ? "It goes" : "They go"} from the installation - at once. The store's git history keeps what was there, but nothing - in the app brings {names.length === 1 ? "it" : "them"} back. + {description ?? ( + <> + {names.length === 1 ? "It goes" : "They go"} from the + installation at once. The store's git history keeps what was + there, but nothing in the app brings{" "} + {names.length === 1 ? "it" : "them"} back. + + )} diff --git a/frontend/src/components/Common/RangePicker.tsx b/frontend/src/components/Common/RangePicker.tsx index 8b90a7a..86fce44 100644 --- a/frontend/src/components/Common/RangePicker.tsx +++ b/frontend/src/components/Common/RangePicker.tsx @@ -1,8 +1,4 @@ -// The segmented shape's thumb transition lives beside the dashboard's own -// widgets, and CSS is chunked per entry — so the rule is pulled in wherever -// this picker is used, or the two copies of one shape would move differently. -import "@/components/Dashboard/dashboard.css" -import { cn } from "@/lib/utils" +import { Segmented } from "@/components/ui/segmented" /** * A window of history, and everything a query needs to ask for it. @@ -39,12 +35,7 @@ export const DEFAULT_RANGE = RANGES[2] export const rangeStart = (range: Range) => new Date(Date.now() - range.hours * 3600_000).toISOString() -/** - * The window a screen is showing, as presets. - * - * The one segmented shape: a single border pill, transparent segments, - * bg-accent on the selected one (root DESIGN-GUIDELINES.md). - */ +/** The window a screen is showing, as presets. */ export function RangePicker({ value, onChange, @@ -52,47 +43,20 @@ export function RangePicker({ value: Range onChange: (range: Range) => void }) { - const chosen = RANGES.findIndex((range) => range.hours === value.hours) return ( - // A `fieldset` carries `min-inline-size: min-content` from the UA sheet, - // which no width utility overrides. Equal tracks and no gap put the - // sliding thumb at its share of the padded box without measuring — a grid - // rather than a flex row because `flex-1` under `w-fit` sizes the segments - // to a share of the widest label instead of to the label itself. -
- Time range - {chosen >= 0 ? ( - - ) : null} - {RANGES.map((range, index) => ( - - ))} -
+ [String(range.hours), range.label] as const, + )} + onChange={(hours) => + onChange( + RANGES.find((range) => String(range.hours) === hours) ?? + DEFAULT_RANGE, + ) + } + /> ) } diff --git a/frontend/src/components/Common/UplotChart.tsx b/frontend/src/components/Common/UplotChart.tsx index 5943ffe..85528de 100644 --- a/frontend/src/components/Common/UplotChart.tsx +++ b/frontend/src/components/Common/UplotChart.tsx @@ -1,4 +1,4 @@ -import { useEffect, useLayoutEffect, useRef } from "react" +import { useEffect, useLayoutEffect, useRef, useState } from "react" import uPlot from "uplot" import "uplot/dist/uPlot.min.css" @@ -146,15 +146,23 @@ export const CURSOR: uPlot.Cursor = { mousemove: binder(false), } as unknown as uPlot.Cursor.Bind, drag: { - // No drag-to-zoom. `setData` re-ranges the scales from the data and runs - // on every render, so a dragged range was erased by the next reading — all - // it ever did here was flash a selection box over a live chart. - x: false, + // Drag across the plot to read a stretch of it closer. `setScale` stays + // off because the chart owns its x range itself: `setData` runs on every + // render and would re-range the scales from the data, so a held window is + // what tells it to leave them alone. Without that this only ever flashed a + // selection box over a live chart, which is why it used to be off. + x: true, y: false, setScale: false, }, } +/** Below this a drag is a click that moved, not a window. In pixels. */ +const DRAG_FLOOR = 4 + +/** How close two taps have to be to count as one gesture. */ +const DOUBLE_TAP_MS = 300 + /** Room for the axis ticks; uPlot measures the rest of the box itself. */ const PADDING: uPlot.Padding = [10, 12, 0, 0] @@ -278,6 +286,15 @@ export function UplotChart({ const host = useRef(null) const legend = useRef(null) const chart = useRef(null) + // A dragged x window, held so the next reading does not wash it away. The + // ref is what the data effect reads; the state is only what draws the way + // back out, and the two are set together. + const zoomed = useRef(false) + const [showReset, setShowReset] = useState(false) + const clearZoom = () => { + zoomed.current = false + setShowReset(false) + } // The chart outlives a render, so its handlers are read through a ref // rather than baked into the config it was built with. const report = useRef({ onCursor, onSelect }) @@ -302,6 +319,9 @@ export function UplotChart({ useLayoutEffect(() => { const element = host.current if (!element || labels.length === 0 || !ready) return + // A different set of series is a different picture; the window that was + // held over the old one means nothing on it. + clearZoom() const axis = { stroke: () => token("--muted-foreground", element), @@ -312,6 +332,8 @@ export function UplotChart({ /** The x value the page was last told about, so a move within one bucket * does not re-render it. */ let told: number | null = null + /** Whether the click about to arrive is the end of a drag. */ + let dragging = false // Resolved once for the whole chart: how many lines there are is part of // which slots they take, when nothing named them. const slots = slotsFor(labels.length, palette) @@ -343,11 +365,47 @@ export function UplotChart({ report.current.onCursor?.(ts) }, ], + setSelect: [ + (self) => { + // uPlot fires this for a plain click too. A few pixels is a + // slip of the hand, not a window anybody meant to ask for. + if (self.select.width <= DRAG_FLOOR) return + const from = self.posToVal(self.select.left, "x") + const to = self.posToVal( + self.select.left + self.select.width, + "x", + ) + // The box has done its job; the scale is what holds the window + // from here. `false` so this hook does not fire on itself. + self.setSelect({ left: 0, width: 0, top: 0, height: 0 }, false) + self.setScale("x", { min: from, max: to }) + dragging = true + zoomed.current = true + setShowReset(true) + }, + ], ready: [ (self) => { - self.over.addEventListener("click", () => - report.current.onSelect?.(under(self)), - ) + self.over.addEventListener("click", () => { + // The mouseup that ended a drag arrives here as a click as + // well; pinning a moment is not what it was asking for. + if (dragging) { + dragging = false + return + } + report.current.onSelect?.(under(self)) + }) + self.over.addEventListener("dblclick", clearZoom) + // ponytail: a touch screen gets no dblclick from every browser, + // and uPlot has no dbltap of its own. Two taps in a moment is + // the whole of the gesture. + let lastTap = 0 + self.over.addEventListener("pointerup", (event) => { + if (event.pointerType !== "touch") return + const now = event.timeStamp + if (now - lastTap < DOUBLE_TAP_MS) clearZoom() + lastTap = now + }) }, ], }, @@ -426,10 +484,14 @@ export function UplotChart({ // the blind spot a point count has: once a rolling window is full, a refetch // carrying different readings leaves the count where it was and never fires. // Safe to run this often because `setData` is idempotent and re-ranges the - // scales *from the data* — the opposite of the `redraw(false)` below. + // scales *from the data* — the opposite of the `redraw(false)` below. That + // re-ranging is exactly what a dragged window has to be spared, so while one + // is held the data goes in and the scales stay where they were put. Clearing + // the window renders, which brings the next pass through here with the reset + // back on: that is what puts the whole range back. useEffect(() => { if (!chart.current || plots.length === 0) return - chart.current.setData(table(plots)) + chart.current.setData(table(plots), !zoomed.current) }) // The canvas cannot follow a CSS variable, so a theme swap is a redraw. The @@ -448,6 +510,17 @@ export function UplotChart({
+ {/* Double-clicking does the same thing, but nothing says so. */} + {showReset ? ( + + ) : null} {points === 0 ? ( pending ? ( diff --git a/frontend/src/components/Dashboard/DashboardEditor.tsx b/frontend/src/components/Dashboard/DashboardEditor.tsx index f66bf97..72904c7 100644 --- a/frontend/src/components/Dashboard/DashboardEditor.tsx +++ b/frontend/src/components/Dashboard/DashboardEditor.tsx @@ -1,14 +1,6 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" import { useNavigate } from "@tanstack/react-router" -import { - Check, - ExternalLink, - Loader2, - Pencil, - Plus, - Settings2, - X, -} from "lucide-react" +import { Check, ExternalLink, Pencil, Plus, Settings2, X } from "lucide-react" import { motion } from "motion/react" import { type CSSProperties, useEffect, useRef, useState } from "react" import { @@ -36,6 +28,7 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog" +import { FluksioLoader } from "@/components/ui/fluksio-loader" import { Popover, PopoverContent, @@ -144,6 +137,14 @@ const AUTOSAVE_MS = 800 const INTERACTIVE = "button, a, input, select, textarea, [role='switch'], [role='combobox'], [role='slider'], .react-resizable-handle, .widget-grip" +/** + * What a click may land on without counting as a click off the widgets: the + * widget itself, and the handles the grid draws around it — resizing a tile is + * still working on it. The grid item covers both; the frame is what a stacked + * dashboard has instead. + */ +const ON_WIDGET = "[data-testid=widget-frame], .react-grid-item" + function nextId(dashboard: DashboardDef_Output, type: string): string { const taken = new Set(widgetsOf(dashboard).map((widget) => widget.id)) let candidate = type @@ -460,6 +461,7 @@ export function DashboardEditor({
@@ -524,12 +526,26 @@ export function DashboardEditor({ return ( + {/* Clicking off the widgets is how you put a panel away, as it is on the + flow pane — the dashboard is what you went back to look at. A menu a + widget portals to `body` lands outside this element but still bubbles + through React's tree, so only a press that landed in the subtree + counts. */} + {/* biome-ignore lint/a11y/useKeyWithClickEvents: Escape closes the panel; this is a pointer shortcut, not a control. */} + {/* biome-ignore lint/a11y/noStaticElementInteractions: see above. */}
{ + const target = event.target as Element + if (!event.currentTarget.contains(target)) return + if (target.closest(ON_WIDGET)) return + setSelected(null) + setSettingsOpen(false) + }} > {body}
@@ -698,7 +714,7 @@ export function DashboardEditor({ data-testid="publish-dashboard" > {save.isPending || publish.isPending ? ( - + ) : ( )} diff --git a/frontend/src/components/Dashboard/DashboardView.tsx b/frontend/src/components/Dashboard/DashboardView.tsx index c147230..7d2a72d 100644 --- a/frontend/src/components/Dashboard/DashboardView.tsx +++ b/frontend/src/components/Dashboard/DashboardView.tsx @@ -294,18 +294,19 @@ export const isPlaced = (widgets: WidgetDef[]) => }) /** - * One page, drawn as the single grid the panel shows. + * A dashboard, drawn as the single grid the panel shows. * * A dashboard is one canvas — several dashboards on a panel is what the rail - * is for — so the page's sections are one arrangement rather than a stack of - * headed grids. `SectionDef` stays in the schema, and the editor writes what - * it arranged back into the first section. + * is for — so its widgets are one arrangement rather than a stack of headed + * grids. Pages and sections left the schema in 7ff29ca; a document written + * the old way is flattened on read. */ export function DashboardView({ dashboard, renderWidget, stacked, rail, + editing, }: { dashboard: Dashboard renderWidget?: (widget: WidgetDef) => React.ReactNode @@ -313,6 +314,9 @@ export function DashboardView({ stacked?: boolean /** Whether the panel carries a rail, which takes a column of the canvas. */ rail?: boolean + /** Being arranged rather than used, so the lock is not mounted — the + * stacked editor draws through here, where the canvas draws its own grid. */ + editing?: boolean }) { const all = widgetsOf(dashboard) if (all.length === 0) { @@ -337,20 +341,26 @@ export function DashboardView({ : all const columns = columnsOf(dashboard) + const grid = ( + + + + ) + return ( - - - - - + {editing ? ( + grid + ) : ( + {grid} + )} ) } diff --git a/frontend/src/components/Dashboard/ui/core/controls.check.ts b/frontend/src/components/Dashboard/ui/core/controls.check.ts new file mode 100644 index 0000000..0668b06 --- /dev/null +++ b/frontend/src/components/Dashboard/ui/core/controls.check.ts @@ -0,0 +1,60 @@ +/** + * The tick-label thinning, checked. + * + * ponytail: a script rather than a suite, matching `color.check.ts` — pure + * arithmetic needs no browser: + * + * cd frontend && bun run src/components/Dashboard/ui/core/controls.check.ts + */ +import assert from "node:assert/strict" +import { fitMarks } from "./controls" + +const marks = (...labels: string[]) => labels.map((label) => ({ label })) +const labels = (kept: { label: string }[]) => kept.map((m) => m.label) + +const five = marks("20", "20.5", "21", "21.5", "22") + +// Unmeasured, so nothing is thinned yet. +assert.deepEqual(fitMarks(five, 0), five) + +// Two labels are the ends themselves and can never crowd. +assert.deepEqual(fitMarks(marks("0", "1"), 1), marks("0", "1")) + +// The default four columns keeps all five. +assert.deepEqual(labels(fitMarks(five, 261)), [ + "20", + "20.5", + "21", + "21.5", + "22", +]) + +// Three columns thins to the whole numbers, ends included. +assert.deepEqual(labels(fitMarks(five, 177)), ["20", "21", "22"]) + +// Across every width: the ends survive, and the stride divides the intervals +// so the last label lands on the end rather than short of it. Four intervals +// may thin by 1, 2 or 4 — never 3. +for (let width = 1; width <= 600; width++) { + const kept = labels(fitMarks(five, width)) + assert.equal(kept[0], "20", `first lost at ${width}px`) + assert.equal(kept[kept.length - 1], "22", `last lost at ${width}px`) + assert.ok( + [2, 3, 5].includes(kept.length), + `${kept.length} labels at ${width}px`, + ) +} + +// Six marks reach the divisor guard that five cannot: at this width the +// narrowest stride that fits is 3, which does not divide five intervals and +// would drop the last label. It must fall through to 5 and keep both ends. +const six = marks("0", "1", "2", "3", "4", "5") +assert.deepEqual(labels(fitMarks(six, 60)), ["0", "5"]) + +// Wider labels thin sooner than narrow ones at the same width. +assert.ok( + fitMarks(marks("1000.5", "1001", "1001.5"), 120).length <= + fitMarks(marks("1", "2", "3"), 120).length, +) + +console.log("fitMarks: ok") diff --git a/frontend/src/components/Dashboard/ui/core/controls.ts b/frontend/src/components/Dashboard/ui/core/controls.ts index f7efcb7..2cfa0cd 100644 --- a/frontend/src/components/Dashboard/ui/core/controls.ts +++ b/frontend/src/components/Dashboard/ui/core/controls.ts @@ -5,7 +5,7 @@ * either look: the state, the keyboard and every `aria-` live here, and a * renderer only decides what it looks like while doing it. */ -import { useCallback, useId, useRef, useState } from "react" +import { useCallback, useEffect, useId, useRef, useState } from "react" import { fractionOf } from "./config" @@ -141,6 +141,32 @@ export function tickIntervals(steps: number): number { return [4, 3, 2].find((count) => Number.isInteger(steps / count)) ?? 4 } +/** + * The labels that fit the width the scale was measured in. + * + * The marks are placed in percent, which says nothing about how wide a label + * is: five four-character labels crowd on a tile narrower than the four + * columns a slider is given by default. Keep every Nth instead, with N a + * divisor of the interval count so the first and last — the two that anchor + * the range — are always among those kept. + */ +export function fitMarks( + marks: T[], + width: number, +): T[] { + const intervals = marks.length - 1 + // Not measured yet, or nothing to thin. + if (width === 0 || intervals < 2) return marks + const widest = Math.max(...marks.map((mark) => mark.label.length)) + // A 0.75rem tabular digit runs about 7px, and neighbours need a gap. + const fits = Math.floor(width / (widest * 7 + 12)) + const stride = + [1, 2, 3, 4, 5].find( + (n) => intervals % n === 0 && intervals / n + 1 <= fits, + ) ?? intervals + return marks.filter((_, index) => index % stride === 0) +} + /** * A value set by dragging, published when the handle is let go. * @@ -175,6 +201,20 @@ export function useSliderDrag({ const [draft, setDraft] = useState(null) const current = draft ?? value + // How much room the scale under the track actually has, which is what says + // how many of its labels can be drawn without them running into each other. + const trackRef = useRef(null) + const [width, setWidth] = useState(0) + useEffect(() => { + const el = trackRef.current + if (!el) return + const observer = new ResizeObserver(([entry]) => + setWidth(entry.contentRect.width), + ) + observer.observe(el) + return () => observer.disconnect() + }, []) + const release = () => { if (draft === null) return onCommit(draft) @@ -187,8 +227,23 @@ export function useSliderDrag({ // without a precision setting of its own. const digits = (String(step).split(".")[1] ?? "").length + /** The scale under the track, drawn rather than declared: no browser + * renders `