diff --git a/.gitea/workflows/docs.yml b/.gitea/workflows/docs.yml new file mode 100644 index 0000000..cd7560e --- /dev/null +++ b/.gitea/workflows/docs.yml @@ -0,0 +1,27 @@ +name: Docs + +on: + push: + branches: + - main + pull_request: + types: + - opened + - synchronize + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@v7 + # Pinned to the version docker/Dockerfile.docs ships, so the gate matches + # what docs.fluksio.com actually builds with. --strict fails on warnings + # (dead internal links, unknown config keys), which is the point of + # running it here at all — the build itself takes about a second. + - name: Build the docs site + run: uvx zensical==0.0.46 build --strict --clean diff --git a/.gitignore b/.gitignore index d93df3f..3d98533 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ __pycache__/ # Output of `make verify` (root Makefile). /frontend/screenshots/ *.tsbuildinfo + +# Output of `make docs` (the zensical build of docs/). +/site/ diff --git a/Makefile b/Makefile index d122565..2e7cd8b 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ .PHONY: dev-utils dev dev-local up down update install dev-backend dev-frontend \ generate-client seed-example seed-demo seed-house seed-aircon seed-hosted-demo test test-backend test-frontend soak bench-startup lint lint-backend \ - lint-frontend format-frontend umami build clean help + lint-frontend format-frontend umami build docs docs-serve clean help COMPOSE_ROOT := $(CURDIR) # Explicit project name keeps this stack isolated from the sibling website @@ -178,9 +178,23 @@ lint-frontend: ## Lint frontend with biome format-frontend: ## Apply biome's fixes to the frontend (what `lint-frontend` only reports) cd frontend && bun run format +# ── Documentation ───────────────────────────────────────────────── +# The public site on docs.${DOMAIN}, served by the `docs` service in +# docker/compose.yml. These targets are for local authoring: zensical runs on +# demand via uvx, so it never touches the backend environment. Pinned to the +# version docker/Dockerfile.docs and .gitea/workflows/docs.yml ship, so local +# authoring builds with what docs.fluksio.com actually gets. +ZENSICAL := zensical==0.0.46 + +docs: ## Build the documentation site into ./site + uvx $(ZENSICAL) build --clean + +docs-serve: ## Serve the documentation site locally with live reload + uvx $(ZENSICAL) serve + # ── Cleanup ─────────────────────────────────────────────────────── clean: ## Remove build artifacts and caches rm -rf frontend/dist frontend/blob-report frontend/test-results - rm -rf backend/.pytest_cache backend/htmlcov + rm -rf backend/.pytest_cache backend/htmlcov site find backend -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true diff --git a/README.md b/README.md index c85e4f7..9d518de 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ backend/ FastAPI + SQLModel + Alembic + SQLite — the `fluksio` distribu fluksio/cli.py `fluksio serve` / `enroll` / `worker` worker/ the `fluksio-worker` distribution: the agent and the node runner frontend/ React 19 + TanStack Router + Tailwind 4 + shadcn/ui +docs/ the public documentation site (zensical), served on docs.${DOMAIN} docker/ compose.yml → compose.dev.yml → compose.local.yml (+ compose.traefik.yml) scripts/ generate-client.sh, test.sh ``` @@ -62,6 +63,20 @@ make generate-client # regenerate the frontend SDK from the OpenAPI schema ## Documentation +The public site lives in [`docs/`](docs/) and is built with zensical: + +```sh +make docs-serve # live preview on :8000 +make docs # static build into ./site +``` + +It is served at `docs.${DOMAIN}` by the `docs` service in `docker/compose.yml`, +and `.gitea/workflows/docs.yml` builds it with `--strict` on every push. Style +follows the root `DESIGN-GUIDELINES.md`; the tokens are mirrored in +`docs/stylesheets/extra.css`. + +Repo-only material: + - [`ROADMAP.md`](ROADMAP.md) — strategy and feature record - [`NOTEPAD.md`](NOTEPAD.md) — deferred work and findings - [`DESIGN.md`](DESIGN.md) — points at the workspace root's `DESIGN-GUIDELINES.md` diff --git a/docker/Dockerfile.docs b/docker/Dockerfile.docs new file mode 100644 index 0000000..3e59873 --- /dev/null +++ b/docker/Dockerfile.docs @@ -0,0 +1,19 @@ +# ─── Fluksio docs site (zensical) ─── +# Multi-stage: build the static documentation with zensical, serve it with Nginx. +# Routed to docs.${DOMAIN} by the `docs` service in compose.yml. The runtime +# image is just Nginx + static files (no Python), mirroring the SPA approach. + +# ── Stage 1: Build the static site ─────────────────────────────── +FROM python:3.13-slim AS docs-build +# Pin zensical so a rebuild is reproducible (it is only a build-time tool). +RUN pip install --no-cache-dir zensical==0.0.46 +WORKDIR /docs +COPY zensical.toml ./ +COPY docs/ ./docs/ +RUN zensical build --clean + +# ── Stage 2: Serve static files ────────────────────────────────── +FROM nginx:alpine +COPY docker/nginx.docs.conf /etc/nginx/conf.d/default.conf +COPY --from=docs-build /docs/site /usr/share/nginx/html +EXPOSE 80 diff --git a/docker/compose.dev.yml b/docker/compose.dev.yml index b493208..69a4fc3 100644 --- a/docker/compose.dev.yml +++ b/docker/compose.dev.yml @@ -137,6 +137,11 @@ services: - VITE_API_URL=http://localhost:8000 - NODE_ENV=development + # Reachable at http://docs.${DOMAIN}, but a container rebuild per edit is a + # poor authoring loop — use `make docs-serve` for that. + docs: + restart: "no" + playwright: build: context: .. diff --git a/docker/compose.yml b/docker/compose.yml index 9aeaf35..fb78247 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -188,6 +188,36 @@ services: - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.middlewares=https-redirect + # Static documentation site (zensical), served at docs.${DOMAIN}. Content and + # config live in ../docs/ and ../zensical.toml; `make docs-serve` previews it. + docs: + container_name: fluksio-docs + restart: always + security_opt: + - no-new-privileges:true + build: + context: .. + dockerfile: docker/Dockerfile.docs + expose: + - "80" + networks: + - proxy + labels: + - traefik.enable=true + - traefik.docker.network=proxy + - traefik.constraint-label=proxy + + - traefik.http.services.${STACK_NAME?Variable not set}-docs.loadbalancer.server.port=80 + + - traefik.http.routers.${STACK_NAME?Variable not set}-docs-http.rule=Host(`docs.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-docs-http.entrypoints=http + - traefik.http.routers.${STACK_NAME?Variable not set}-docs-http.middlewares=https-redirect + + - traefik.http.routers.${STACK_NAME?Variable not set}-docs-https.rule=Host(`docs.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-docs-https.entrypoints=https + - traefik.http.routers.${STACK_NAME?Variable not set}-docs-https.tls=true + - traefik.http.routers.${STACK_NAME?Variable not set}-docs-https.tls.certresolver=le + # Docker never restarts a merely *unhealthy* container on its own; autoheal # closes that gap for the services labeled autoheal=true. # diff --git a/docker/nginx.docs.conf b/docker/nginx.docs.conf new file mode 100644 index 0000000..03ba066 --- /dev/null +++ b/docker/nginx.docs.conf @@ -0,0 +1,27 @@ +# Static documentation site (zensical output). Clean-URL directories served from +# per-directory index.html, with a styled 404, gzip, and long-cached assets. +server { + listen 80; + server_name _; + root /usr/share/nginx/html; + index index.html; + + gzip on; + gzip_types text/css application/javascript application/json image/svg+xml text/plain; + gzip_min_length 256; + + location / { + try_files $uri $uri/ =404; + } + + # Fingerprinted CSS/JS/images emitted under assets/ can be cached hard. + location /assets/ { + expires 30d; + add_header Cache-Control "public, immutable"; + } + + error_page 404 /404.html; + location = /404.html { + internal; + } +} diff --git a/docs/assets/favicon-dark.svg b/docs/assets/favicon-dark.svg new file mode 100644 index 0000000..cccdc8a --- /dev/null +++ b/docs/assets/favicon-dark.svg @@ -0,0 +1,183 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/favicon.svg b/docs/assets/favicon.svg new file mode 100644 index 0000000..55fa8ac --- /dev/null +++ b/docs/assets/favicon.svg @@ -0,0 +1,186 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/code/agents.md b/docs/code/agents.md new file mode 100644 index 0000000..29527b3 --- /dev/null +++ b/docs/code/agents.md @@ -0,0 +1,111 @@ +# Agents over MCP + +The engine exposes the flow API to agents over the +[Model Context Protocol](https://modelcontextprotocol.io). Point Claude Code, +Claude Desktop, Codex or anything else that speaks MCP at it, and it can read, +build, publish and run flows. + +## Switching it on + +```sh +MCP_ENABLED=true +``` + +It is off by default, because switching it on opens client registration to +whoever can reach the host. Turn it on deliberately. + +The endpoint is `https://api.${DOMAIN}/mcp` — or `http://api.localhost/mcp` on +a local stack. + +## Connecting a client + +```sh +claude mcp add --transport http fluksio https://api.example.com/mcp +``` + +The OAuth flow does the rest: the client registers itself, you approve it in +the browser, and it exchanges the grant for a token. Nothing to paste. + +## What an agent may do + +An agent acts as **the person who approved it**, and only through the same REST +API the dashboard uses. It cannot do anything you could not do in the browser, +and it cannot do anything *you* cannot do — an agent approved by a non-superuser +is not one either. + +Its tokens are signed with a keypair of their own, kept beside the flow store. +Deleting that file revokes every agent's access without logging anyone out of +the dashboard. + +Registered clients are managed under **Admin**. + +## The tools + +Read-only: + +| Tool | Returns | +|---|---| +| `list_flows` | every flow with its node count and state | +| `get_flow` | one flow's definition, node status and issues | +| `get_node_source` | a node's Python | +| `list_node_types` | every type and its parameter schema | +| `get_flow_state` | the current value of each of a flow's messages | +| `get_message_history` | one message's recent numeric values | +| `get_graph` | every flow as one graph | +| `list_shared_nodes` | the shared library and its usages | +| `list_secrets` | secret **names** — never values | +| `get_modules` | the packages node code can import | +| `get_health` | flows, nodes, queue and loop lag | +| `get_metrics` | executions, errors and timings per minute | +| `list_failures` | what went wrong recently | +| `list_runs` | recent cascades | + +Editing: + +| Tool | Does | +|---|---| +| `save_flow` | write a draft | +| `save_node_source` | write a node's code into the draft | +| `validate_flow` | the issues, without saving | +| `publish_flow` | put the draft live (takes the version it saw) | +| `discard_draft` | throw the draft away | +| `delete_flow` | delete it | +| `apply_modules` | replace the package manifest | + +Operating: + +| Tool | Does | +|---|---| +| `run_flow` | run every node once from the values given | +| `trigger_node` | feed one node and run what is downstream | +| `start_flow` / `stop_flow` | activate or tear down | +| `pause_flow` / `resume_flow` | hold messages, or let them through | +| `cancel_node` | kill a node's code while it runs | + +## Why it is shaped this way + +The tools do not reach into the engine. Each one calls the same REST endpoint +the dashboard calls, over an in-process transport. + +That keeps exactly one description of what a flow is and how it may be changed +— the validation, the draft/publish split, the version check that stops two +clients overwriting each other. An agent that saved a flow through a private +back door would be an agent that could write one the canvas cannot open. + +The caller's token rides along on every hop, so the API sees the agent's own +identity rather than a service account. The audit trail on Home names it. + +## A note on working this way + +The pattern that works is the same one that works with a person: let the agent +read the flow and its issues, let it write a **draft**, and publish yourself +once you have looked. `validate_flow` before `publish_flow` is cheap and tells +you what the canvas would have told you. + +## Limits + +- MCP is not currently reachable through a [portal](../interface/portal.md) + tunnel — the proxy forwards `/api/v1/` only. Connect an agent on the same + network as the installation. +- Secrets are never readable, by an agent or by anyone else. `list_secrets` + returns names. diff --git a/docs/code/api.md b/docs/code/api.md new file mode 100644 index 0000000..254befe --- /dev/null +++ b/docs/code/api.md @@ -0,0 +1,223 @@ +# The HTTP API + +Everything the browser does, the API does first. The dashboard is a generated +client of this schema, not a privileged path into the engine — so anything you +can click, you can script. + +Base URL: `https://api.${DOMAIN}/api/v1`, or `http://127.0.0.1:8000/api/v1` for +a `fluksio serve` installation. + +```sh +export FLUKSIO=http://127.0.0.1:8000/api/v1 +``` + +## Authenticating + +```sh +export TOKEN=$(curl -s -X POST $FLUKSIO/login/access-token \ + -d "username=admin@example.com&password=..." | jq -r .access_token) + +curl -s $FLUKSIO/users/me -H "Authorization: Bearer $TOKEN" +``` + +A bearer token, valid for eight days. `POST /login/test-token` checks one. +Password recovery and reset are `POST /password-recovery/{email}` and +`POST /reset-password/`. + +Agents authenticate differently — see [Agents over MCP](agents.md). + +!!! tip "The interactive schema" + + When `ENVIRONMENT` is not `production`, the full OpenAPI schema is at + `/docs` (Swagger) and `/redoc`, and the raw document at + `/api/v1/openapi.json`. That is the authoritative reference; this page is + the tour. + + It is closed in production on purpose: the schema enumerates every endpoint + the installation serves, including the paths webhook nodes mounted at + runtime. + +## Flows + +| Method | Path | What | +|---|---|---| +| `GET` | `/flows/` | every flow, with node counts, draft and running state | +| `GET` | `/flows/{name}` | one flow — the draft if there is one — with node status and issues | +| `PUT` | `/flows/{name}` | save a draft (`version` must match, or 409) | +| `POST` | `/flows/{name}/publish` | put the draft live | +| `POST` | `/flows/{name}/discard-draft` | throw the draft away | +| `POST` | `/flows/{name}/rename` | rename it | +| `DELETE` | `/flows/{name}` | delete it | +| `GET` | `/flows/node-types` | every node type and its parameter schema | +| `GET` | `/flows/graph` | every flow as one graph — what Home draws | + +Reading a flow returns the *draft* when one exists, along with a preview of how +it would run: node statuses and validation issues for the version you are +editing, not the one running underneath it. + +### Node source + +```sh +curl -s $FLUKSIO/flows/house/nodes/decide/source -H "Authorization: Bearer $TOKEN" +curl -X PUT $FLUKSIO/flows/house/nodes/decide/source -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' -d '{"code": "def process(x):\n return {\"y\": x * 2}\n"}' +``` + +`POST .../share` promotes a node's source to the shared library; +`POST .../unshare` gives it a private copy back. `GET /flows/library` lists the +shared sources and which flows use each. + +### Running and controlling + +| Method | Path | What | +|---|---|---| +| `POST` | `/flows/{name}/start` · `/stop` | activate or tear down its subscriptions and schedules | +| `POST` | `/flows/{name}/pause` · `/resume` | hold messages instead of running them | +| `POST` | `/flows/{name}/step` | release exactly one held message | +| `POST` | `/flows/{name}/validate` | the issues, without saving | +| `POST` | `/flows/{name}/run` | run every node once, from the values you pass | +| `POST` | `/flows/{name}/nodes/{id}/trigger` | feed values into one node | +| `POST` | `/flows/{name}/nodes/{id}/cancel` | kill the worker running it right now | +| `POST` | `/flows/{name}/nodes/{id}/acknowledge` | clear the failure the canvas is marking | + +`POST /flows/{name}/run` on a **batch** flow submits a run instead, because +that is what running one means — the parameters, the series and the result are +the point, and a call that quietly did something else would be a trap. + +## Messages + +```sh +curl -s $FLUKSIO/messages/ -H "Authorization: Bearer $TOKEN" +curl -X POST $FLUKSIO/messages/house.setpoint -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' -d '{"value": 22.5}' +curl -s $FLUKSIO/messages/house.temperature/history -H "Authorization: Bearer $TOKEN" +``` + +`GET /messages/` is the catalogue: every message any published flow declares, +with its last value. Publishing puts a value into the graph exactly as a +dashboard control does — which means only a message some flow *declares* can be +published to. Flows own the namespace; everything else is a client of it. + +## Runs + +| Method | Path | What | +|---|---|---| +| `POST` | `/runs/flows/{name}` | queue one run — `{"params": {...}, "seed": 7, "draft": false}` | +| `POST` | `/runs/flows/{name}/sweep` | queue many, sharing a `group_id` | +| `GET` | `/runs` | the queryable history: `?flow=`, `?status=`, `?group=`, `?digest=`, `?limit=` | +| `GET` | `/runs/{id}` | one run in full: params, result, per-node record, artifacts | +| `POST` | `/runs/{id}/cancel` | stop it | +| `GET` | `/runs/{id}/metrics?name=&stride=` | one metric's series, in step order | +| `GET` | `/runs/series/compare?ids=a,b,c&metric=` | that metric across several runs | + +Submitting answers immediately with a `queued` run. Wrong parameters — an +undeclared name, a value of the wrong type — come back as a 422 naming the +problem, before anything executes. + +`?digest=` filters by the hash of the parameters, which is how you find "every +run that used exactly this configuration". + +`compare` answers in the same shape a chart widget draws, so three training +curves side by side is a widget binding rather than a screen of its own. + +See [Runs: pipelines that finish](../concepts/runs.md). + +## Artifacts + +```sh +curl -X PUT $FLUKSIO/artifacts -H "Authorization: Bearer $TOKEN" \ + --data-binary @model.pt +curl -s $FLUKSIO/artifacts/sha256:abc... -H "Authorization: Bearer $TOKEN" -o model.pt +``` + +Content-addressed, so uploading the same bytes twice stores them once. Node +code normally reaches these through `fluksio.save_artifact` / +`fluksio.load_artifact` rather than here. + +## Dashboards and panels + +| Method | Path | What | +|---|---|---| +| `GET` | `/dashboards/` | every dashboard, without its contents | +| `GET` | `/dashboards/{name}?draft=true` | the published document, or the editor's copy | +| `POST` | `/dashboards/{name}` | create one | +| `PUT` | `/dashboards/{name}` | save a draft | +| `POST` | `/dashboards/{name}/publish` · `/discard` · `/rename` | | +| `GET` `PUT` | `/panels/` | which device shows which dashboards | +| `POST` | `/panels/pair` | start a pairing | +| `GET` | `/panels/pair/{code}` | what is holding a code | + +## Secrets, modules, alerts + +```sh +curl -s $FLUKSIO/secrets/ -H "Authorization: Bearer $TOKEN" # names only +curl -X PUT $FLUKSIO/secrets/influx-token -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' -d '{"value": "..."}' + +curl -s $FLUKSIO/modules/ -H "Authorization: Bearer $TOKEN" +curl -X POST $FLUKSIO/modules/apply -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' -d '{"requirements": "numpy>=2\n"}' + +curl -s $FLUKSIO/alerts/config -H "Authorization: Bearer $TOKEN" +curl -X POST $FLUKSIO/alerts/test/my-phone -H "Authorization: Bearer $TOKEN" +``` + +Secrets are write-only over the API: you can list names and set values, never +read one back. + +## Observability + +| Path | What | +|---|---| +| `/observability/summary` | engine health — always 200, degraded or not | +| `/observability/timeseries` | executions and failures over a window | +| `/observability/flows` | per-flow rollups with a 60-slice trend | +| `/observability/runs` | recent cascades, with `?flow=`, `?since=`, `?until=` | +| `/observability/events?kind=failure\|audit` | what went wrong, or who changed what | +| `/observability/dead-letter` | work the engine gave up on | + +`GET /utils/health/` is the deep health check the container probe uses: it +fails when the event loop is wedged or the state backend is gone, not just when +the process is dead. + +## Workers + +| Method | Path | What | +|---|---|---| +| `GET` | `/workers` | what is attached, its labels and how busy it is | +| `POST` | `/workers/tokens` | mint a worker credential (superuser; shown once) | +| `GET` | `/workers/runtime` | the node runner's source, for a host without pip | +| `WS` | `/workers/attach` | where a worker dials in | + +See [Remote workers](workers.md). + +## Live events + +```text +ws://127.0.0.1:8000/api/v1/flows/ws?token= +``` + +The websocket authenticates from its query string. It sends a snapshot on +connect and then every engine event: node executions, values published, health +changes, run started and finished. This is what the canvas and the dashboards +draw from. + +## Errors + +| Code | Means | +|---|---| +| `400` | the request was malformed, or the node refused it with a message | +| `401` / `403` | not signed in, or not allowed | +| `404` | no such flow, dashboard, run or message | +| `409` | someone else saved first — the body carries `current_version` | +| `422` | a parameter, port or binding did not typecheck | +| `503` | that subsystem is not available on this installation | + +A 409 on a save or a publish is not an error to retry blindly: it means the +stored version moved past the one you were editing. Re-read, merge, save again. + +## Generating a client + +The frontend's TypeScript client is generated from the OpenAPI schema +(`make generate-client`). Any OpenAPI generator will do the same for your +language — point it at `/api/v1/openapi.json` on a non-production installation. diff --git a/docs/code/cli.md b/docs/code/cli.md new file mode 100644 index 0000000..9926781 --- /dev/null +++ b/docs/code/cli.md @@ -0,0 +1,153 @@ +# The `fluksio` command + +```sh +pip install fluksio +``` + +Installs the engine and the `fluksio` command. Python 3.10 or newer, Linux or +macOS. + +There is a second, smaller distribution — `fluksio-worker` — for a machine that +should only *run nodes* for an engine elsewhere. It has none of the engine in +it. See [Remote workers](workers.md). + +## `fluksio serve` + +Runs the engine. + +```sh +fluksio serve +``` + +On the first start it creates an admin account and prints its password **once**. +Nothing else has to be running: no database server, no message broker, no +Docker. + +| Option | Default | What it does | +|---|---|---| +| `--data-dir PATH` | `~/.fluksio` (or `$FLUKSIO_HOME`) | where this installation keeps everything | +| `--host HOST` | `127.0.0.1` | what to bind | +| `--port PORT` | `8000` | what to listen on | +| `--log-level LEVEL` | `info` | uvicorn's log level | +| `--admin-email ADDR` | `admin@example.com` | the account created on first run | +| `--admin-password PW` | generated | set it instead of having one generated | +| `--enroll CODE` | — | pair with a portal as part of coming up | +| `--portal URL` | — | the portal `--enroll` redeems at | + +`--enroll` with `--portal` is the one-command setup: it pairs before the engine +starts, so the connection is dialled as part of coming up rather than needing a +restart. It is skipped if the installation is already enrolled. + +!!! warning "One process" + + `fluksio serve` holds the flow engine. A second one is a *second engine* — + duplicated subscriptions, duplicated cron ticks, two webhooks answering the + same path. Run one, and distribute work with + [workers](workers.md) instead. + +!!! note "`$HOME` on a cluster" + + A login node's home directory is often NFS, where SQLite's write-ahead log + does not work — the database would be locked or corrupt. `fluksio serve` + warns when it notices; point `--data-dir` at local disk. + +### What it prints + +```text +Created the admin account admin@example.com + password: k3Qm-8vTpLdX + Shown once. Change it from the dashboard. +Fluksio 0.1.0 — data in /home/you/.fluksio + API http://127.0.0.1:8000/api/v1 + No portal. Pair this installation with: + fluksio enroll --portal https://hub.example.com +``` + +An enrolled installation says which portal it is on instead, and notes that the +dashboard is served from there rather than here. + +## `fluksio enroll` + +Pairs an existing installation with a portal. + +```sh +fluksio enroll ABCD-1234 --portal https://hub.fluksio.com +``` + +| Option | What it does | +|---|---| +| `--portal URL` | **required** — the portal the code was minted on | +| `--as EMAIL` | the local account a portal session arrives as | +| `--data-dir PATH` | which installation, if not the default | + +Get the code from the portal under **Installations → Add installation**. It is +single-use and expires in fifteen minutes. `--as` matters when the installation +has several superusers — without it, enrolment refuses rather than guessing. + +Afterwards, `fluksio serve` dials the portal as it comes up. See +[Accounts and the portal](../interface/portal.md). + +## `fluksio worker` + +Runs nodes for an engine elsewhere. Everything after `worker` belongs to the +agent's own parser — it is the same program `fluksio-worker` installs, so the +two are interchangeable: + +```sh +fluksio worker --url wss://api.example.com/api/v1/workers/attach \ + --token "$FLUKSIO_WORKER_TOKEN" --labels gpu +``` + +See [Remote workers](workers.md). + +## What lives in the data directory + +```text +~/.fluksio/ +├── fluksio.db SQLite: users, runs, metrics, observability, agents +├── flows/ a git repository — one directory per flow +│ ├── house/ +│ │ ├── flow.json the published structure +│ │ ├── nodes/*.py the published node code +│ │ ├── flow.draft.json unpublished edits, if any +│ │ └── nodes.draft/*.py +│ ├── _lib/ shared node sources +│ ├── _dashboards/ dashboards, drafts and all +│ └── requirements.txt what the Modules screen installs +├── artifacts/ content-addressed bytes, two levels deep +├── user-venv/ the interpreter your node code runs on +├── secrets.enc encrypted credentials, deliberately outside flows/ +├── alerts.json alert channels and rules +├── panels.json wall-panel pairings +├── oauth-key.pem signs agent tokens +├── cloud.json the portal enrolment, if there is one +├── secret_key signs sessions and derives the secrets key +└── env optional settings file +``` + +Two things follow from this layout and are worth internalising: + +**`flows/` is a real git repository.** `git log` is the history of every change +anyone made to any flow. A run records the commit it ran at, so `git show` on +that hash is literally the code that produced the number. + +**Backing up the data directory backs up the installation.** Everything else is +rebuildable. Copy it while the engine is stopped, or use SQLite's online backup +for the database if it is not. + +## Settings + +Settings come from the environment, or from an `env` file in the data +directory. The ones you are most likely to touch: + +| Variable | Default | What it does | +|---|---|---| +| `DATA_DIR` | `~/.fluksio` via the CLI | everything below it derives from this | +| `DATABASE_URL` | SQLite in the data dir | any SQLAlchemy URL | +| `REDIS_HOST` | unset | flow state in Redis instead of memory; survives a restart | +| `FRONTEND_HOST` | — | the address used in mails, OAuth metadata and panel links | +| `ENVIRONMENT` | `local` | `production` closes the interactive API schema | +| `MCP_ENABLED` | `false` | opens the [agent endpoint](agents.md) | +| `SECRET_KEY` | generated once | signs sessions, derives the secrets key | + +The full list is in [Configuration](../reference/configuration.md). diff --git a/docs/code/nodes.md b/docs/code/nodes.md new file mode 100644 index 0000000..6f6ad33 --- /dev/null +++ b/docs/code/nodes.md @@ -0,0 +1,202 @@ +# Writing node code + +A Function node is a Python file. That is all it is — no base class, no +decorator, no framework import unless you want one. + +```python +def process(temperature, setpoint=21.0): + """Ask for heat when the room is below the comfort point.""" + return {"heat": temperature < setpoint} +``` + +## The rules + +**One function called `process`.** If the file defines exactly one public +function under another name, that one is used instead. Two, and the node +refuses to load rather than guessing. + +**Arguments come from ports and settings, by name.** `temperature` above is an +input port; `setpoint` is a setting typed into the node's panel. Both arrive as +keyword arguments, which is why a setting may not share a name with a port. +See [Where a node's values come from](../concepts/values.md). + +**The return value is a dict keyed by output ports.** Every value is checked +against the port's declared type before it is published. A key that is not a +declared port is an error, not a silent drop — nothing leaves a node except +through a port it declared. + +**Nothing else is importable from the engine.** Node code runs in a separate +process, on a separate interpreter, with none of Fluksio's own modules on its +path. What it can import is what the [Modules](../interface/operations.md) +screen installed. + +**A node is a pure function of its inputs.** No context object, no global +store, no handle to reach for. A running total or a debounce timer has a +specific shape — see [Keeping state in a flow](../concepts/state.md). + +## Producing values over time + +A node that produces values *during* its execution is a generator. Every +`yield` is a dict keyed by output port, published the instant it happens: + +```python +def process(lr, steps): + loss = 1.0 + for _ in range(steps): + loss = train_one_step(lr) + yield {"loss": loss} # published now + return {"final_loss": loss} +``` + +Whatever the generator `return`s at the end is the node's result — what +downstream nodes read. If you never `return`, the last thing you yield is the +result instead. + +Mark the port so the flow says what it does: + +```json +{"name": "loss", "dtype": "float", "stream": true} +``` + +Two consequences. In a [run](../concepts/runs.md), the whole series is kept as +that run's metrics — this is why there is no `log_metric()` anywhere in the +API. And **the node's timeout starts measuring silence rather than duration**: +each emission resets the deadline, so a node yielding every few seconds can run +for hours under a timeout of 300. + +### `fluksio.emit` + +Where a `yield` cannot reach — the value comes from inside somebody else's +callback, and they call you rather than the other way round: + +```python +import fluksio + + +def process(): + model.fit(callbacks=[LambdaCallback( + on_epoch_end=lambda epoch, logs: fluksio.emit(loss=logs["loss"]) + )]) + return {"weights": ...} +``` + +Same ports, same type checking, same publication. Prefer `yield` where you can +reach it; `emit` where you cannot. + +## Bytes: artifacts + +Messages are JSON, which is what lets the same value pass through Redis, the +work queue and the worker protocol unchanged. A checkpoint is not that. + +```python +import fluksio + + +def process(dataset): + path = fluksio.load_artifact(dataset) # → a local path to read + ... + return { + "weights": fluksio.save_artifact("model.pt", media_type="application/octet-stream"), + "score": 0.94, + } +``` + +`save_artifact` takes bytes or a path, stores them by their SHA-256 digest, and +returns a small reference — digest, size, media type, name — which is what an +`artifact`-typed port carries. + +Because the address is the content's hash, a sweep whose fifty configs share +one preprocessed input stores it once, and a reference stays valid wherever the +store is reachable from — including on another machine. + +## Printing + +`print` works and is captured. The first 16 KB per call is kept and shown in +the flow editor's log panel and on the run's per-node record; the rest is +dropped, so a node printing in a loop cannot fill anything up. + +Use it to debug. Do not use it to record results — a number worth keeping is an +output port, not a line of text. + +## Errors + +An exception fails that node's execution, not the flow. The message you see is +one line from the frame in *your* code, not a stack through the engine — that +is a deliberate choice about what is actionable. + +The node keeps its last error visible after it recovers, so a failure that +fired an alert at 03:00 still says what it was at 09:00. It can also be +acknowledged from the canvas. + +## Timeouts + +`timeout` on a node is how many seconds its code may run before it is stopped. +The default is 30, and it covers the *first* call's imports, which can be much +slower than the body — a node importing torch is not being slow, it is loading. + +Above 60 seconds, a live flow may deliver the same work again while the node is +still running. In a batch run, which never redelivers, it is an idle timeout +instead: silence this long is a kill. + +## Running a node somewhere else + +A node declares the label of the machine it needs: + +```json +{"id": "train", "device": "gpu", "device_policy": "require", "timeout": 7200} +``` + +`require` (the default) waits for a worker carrying that label; `prefer` runs +locally when none is attached. A node bound to a device is compiled *on that +machine* — a node importing `torch` is correct on the GPU box and a missing +module on the engine, so checking it here would fail something that is fine. + +See [Remote workers](workers.md). + +## Sharing code between flows + +A node's source can be promoted to the shared library from its panel, and other +flows can then use it by reference. One copy, one place to edit — and every +flow using it runs the edit, which is the point and also the caution. + +Shared sources live in `_lib/` in the flow repository, so they are versioned +with everything else. + +## Packages + +Node code runs in a virtual environment of its own, on the installation's data +volume — deliberately separate from the one Fluksio itself runs on. + +Declare what you import in [Modules](../interface/operations.md), or over the +API: + +```sh +curl -X POST $FLUKSIO/modules/apply -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{"requirements": "numpy>=2\npandas\n"}' +``` + +It is a pip manifest installed with `uv pip sync`, versioned alongside your +flows. An install takes effect immediately; nothing restarts. + +## A worked example + +The repository ships a small supervised fit as a seedable demo — three nodes, +a batch flow, streaming metrics, artifacts between stages, and a GPU-labelled +node that falls back to the engine when no worker is attached. It is the +shortest complete thing to read: + +```text +prepare ──dataset(artifact)──▶ train ──weights(artifact)──▶ evaluate + │ + └── loss (streaming float) ──▶ chart +``` + +`make seed-demo` builds it against a running stack. + +## See also + +- [Where a node's values come from](../concepts/values.md) +- [Keeping state in a flow](../concepts/state.md) +- [Runs: pipelines that finish](../concepts/runs.md) +- [Node types](../reference/node-types.md) — the ones you do not have to write diff --git a/docs/code/workers.md b/docs/code/workers.md new file mode 100644 index 0000000..3e6f133 --- /dev/null +++ b/docs/code/workers.md @@ -0,0 +1,137 @@ +# Remote workers + +The engine runs where the automations are. The GPU is somewhere else, the +Raspberry Pi with the relays is in a shed, and neither of them is on the same +network as the other. + +A **worker** is a process that runs the code of nodes marked for it. It dials +*out* to the engine over one authenticated websocket, so nothing on that +machine has to be reachable — and nothing has to expose the engine's state +backend across hosts, which it never should. + +## Install and attach + +```sh +pip install fluksio-worker + +fluksio-worker \ + --url wss://api.example.com/api/v1/workers/attach \ + --token "$FLUKSIO_WORKER_TOKEN" \ + --labels gpu,cuda12 \ + --python /opt/torch-venv/bin/python +``` + +`fluksio-worker` is its own distribution — the agent, the node runner, and +`websockets`. Nothing of the engine, so a GPU box does not install a database +driver in order to run a training step. An engine host already has it, and +`fluksio worker …` is the same program. + +| Option | Default | What it does | +|---|---|---| +| `--url` | **required** | `wss://…/api/v1/workers/attach` | +| `--token` | `$FLUKSIO_WORKER_TOKEN` | the credential, minted on the engine | +| `--name` | this host's name | how it shows up in the worker list | +| `--labels` | none | comma-separated; what a node's `device` matches | +| `--python` | this interpreter | the interpreter node code runs on | +| `--parallel` | `1` | how many node calls it will take at once | +| `--artifact-url` | derived from `--url` | where the artifact store is, if not beside the socket | + +`--python` is the important one. It is how this machine keeps its own wheels — +the CUDA build, the vendor SDK, the thing that will not install anywhere else — +without the engine ever installing them or knowing about them. + +## Mint the token + +On the engine, as a superuser: + +```sh +curl -X POST $FLUKSIO/workers/tokens -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' -d '{"name": "gpu-dev"}' +``` + +Shown once, valid for a year — a worker is a machine somebody sets up and +leaves running. It is signed with the same keypair agent tokens use, so +rotating that key revokes every worker along with them. + +??? note "A host where pip is not an option" + + The two files work copied into one directory and run with `python agent.py + …`. The engine serves the runner itself at `GET /api/v1/workers/runtime` — + it is the same module its own local workers run, deliberately standard + library only. + +## Send a node to it + +A node declares the label of the machine it needs: + +```json +{ + "id": "train", + "device": "gpu", + "device_policy": "require", + "timeout": 7200 +} +``` + +| `device_policy` | Behaviour when nothing carrying the label is attached | +|---|---| +| `require` (default) | the run stays `queued` and says what it is waiting for | +| `prefer` | it runs on the engine instead | + +`prefer` is what makes a flow work before the GPU box exists. `require` is what +you want once it does. + +!!! note "Set from the API" + + `device` and `device_policy` are not yet fields in the node panel. Set them + with `PUT /flows/{name}`. + +## What follows from this + +- **The node's source travels with every call.** Nothing has to be deployed to + the worker, and changing a node's code takes effect on the next execution. +- **A node bound to a device is compiled on that machine.** A node importing + `torch` is correct on the GPU box and a missing module on the engine, so + checking it here would fail something that is fine. +- **`import fluksio` inside a node is the worker's own reporter.** `emit`, + `save_artifact`, `load_artifact` — installed before your code runs, so an + installed `fluksio` package on that box never shadows it. +- **Cancelling a run kills what it is executing**, there or here, and leaves + other runs of the same node alone. +- **If the worker disappears mid-call**, the run fails in seconds with + `worker went away mid-call` rather than waiting out its timeout. +- **A worker sends a heartbeat while it executes**, so a long node is + distinguishable from a dead socket. Ninety seconds of silence is gone. + +## Artifacts across machines + +An artifact reference names content by its hash, not a location, so it stays +valid wherever the store is reachable from. A worker that shares the engine's +filesystem writes to it directly; one that does not fetches and uploads over +HTTP, using the artifact endpoint beside the socket it already has. Either way +your node code is the same two calls. + +## Seeing what is attached + +```sh +curl -s $FLUKSIO/workers -H "Authorization: Bearer $TOKEN" | jq +``` + +Name, labels, how many calls it will take at once, how many are in flight, when +it attached, when it was last seen, its Python version, and a digest of its +environment. + +## What a worker is not + +It is not a second engine. Subscriptions, schedules, webhooks, the dashboards +and the run queue all stay in one process — that is what keeps a message having +one definition and a cron tick happening once. A worker executes node bodies. + +Running two engines against one data directory is not supported. Distribute +work with workers. + +## See also + +- [Runs: pipelines that finish](../concepts/runs.md#running-a-node-somewhere-else) +- [Writing node code](nodes.md) +- [Getting started: data science](../getting-started/data-science.md) diff --git a/docs/concepts/flows.md b/docs/concepts/flows.md new file mode 100644 index 0000000..518654d --- /dev/null +++ b/docs/concepts/flows.md @@ -0,0 +1,183 @@ +# Flows, nodes and messages + +Three ideas hold the whole system up. They are worth twenty minutes, because +almost everything else follows from them. + +## A flow is a graph you did not draw + +A **flow** is a set of **nodes**. Each node declares the messages it needs +(`requires`) and the messages it produces (`provides`). The graph is whatever +those declarations imply: + +```python +# node "read" +def process(): + return {"temperature": read_sensor()} + +# node "decide" +def process(temperature, setpoint=21.0): + return {"heat": temperature < setpoint} +``` + +`decide` is downstream of `read` because it needs `temperature` and `read` +produces it. Nobody drew a wire. + +This is the one structural decision everything else rests on, so it is worth +being explicit about the consequences: + +- **Fan-in is free.** Two nodes providing `temperature` are two producers of + one message. The consumer does not change. +- **A wire cannot be wrong.** There is no wire. There is a name that either + matches or does not, and the canvas tells you at edit time which it is. +- **Layout is not a document.** The canvas computes the arrangement, so a flow + has no stored positions to maintain, merge or fight over. +- **Flows stay small.** A graph nobody can hand-arrange is one worth keeping + small — which is the intent. Several atomic flows that name each other beat + one flow with sixty nodes in it. + +### Message names are namespaced + +Inside flow `house`, a message named `temperature` is really +`house.temperature`. A bare name is qualified with its own flow; a dotted name +is used as written. That is how two flows share a value: + +```python +# in flow `dashboard`, reading a message that flow `house` produces +def process(house_temperature): # port bound to "house.temperature" + ... +``` + +The canvas draws messages arriving from another flow as labelled endpoints, so +you can see where they come from without opening the other flow. + +## A node is a function with declared ports + +Most nodes are **Function** nodes: a Python file defining `process(...)`. Its +arguments are its input ports by name; its return value is a dict keyed by +output ports. + +```python +def process(reading, unit="C"): + return {"shown": reading if unit == "C" else reading * 1.8 + 32} +``` + +`reading` is a port. `unit` is a **setting** — a constant of this node's code, +typed into its panel and stored with the flow. Both arrive as arguments, which +is why a setting may not share a name with a port. See +[Where a node's values come from](values.md). + +The rest of the node types are the ones that would be tedious or unsafe to +write yourself: MQTT, HTTP, InfluxDB, schedules, switches, notifications. Each +one is configured by filling in a form the editor generates from its parameter +schema, so they all behave the same way. The full list is in +[Node types](../reference/node-types.md). + +### Ports are typed + +A port declares a `dtype`: `float`, `int`, `str`, `bool`, `json`, `record`, +`list`, `series` or `artifact`. Every value that passes through is checked +against it. + +Types are not decoration. They are what lets the dashboard editor offer you +only the messages a gauge can actually draw, and what lets the canvas refuse a +binding before anything runs. See [Payload types](../reference/payload-types.md). + +Everything on the wire is JSON. Bytes — a checkpoint, an image, a model — +travel as an `artifact`: the bytes go to a content-addressed store and the +message carries a small reference to them. + +### Nodes are pure + +A node is called with the values of the messages it declares and returns the +values of the messages it provides. There is no context object, no global +store, no handle to reach for. + +That is deliberate: a node with hidden state cannot run twice in parallel, +cannot be replayed, and cannot be moved to another machine. Plenty of real +automations do need to remember something, and there is a specific way to say +so — see [Keeping state in a flow](state.md). + +## Two shapes of flow + +Set `mode` on the flow: + +| | `live` (default) | `batch` | +|---|---|---| +| Runs | continuously | once per run, on request | +| Started by | subscriptions, schedules, webhooks | `POST /runs/flows/{name}` | +| Ends | never | when the graph drains | +| Keeps | the last value of each message | a run record: params, result, metrics, artifacts | +| Is | a thermostat, an ETL job on a cron | an experiment, a CI-style job | + +A batch flow is built and validated like any other, appears on the same canvas +and is type-checked the same way. It is simply never *activated*: no +subscriptions, no schedules, no webhooks. See +[Runs: pipelines that finish](runs.md). + +## Editing is separate from running + +Every flow has a published version and, while you are working, a draft. + +- **Saving** writes the draft. The engine keeps running the published version. +- **Publishing** promotes the draft. The engine reloads and picks it up. +- **Discarding** throws the draft away. + +The store is a git repository — `flow.json` for the structure, `nodes/*.py` for +the code — and each save is a commit. So a flow's history is readable with +ordinary git tooling, and copying a flow between installations is copying a +directory. + +Saving carries the version you last saw. If someone else saved in between, you +get a 409 instead of quietly overwriting their work. + +## What can be wrong, and when you find out + +The canvas validates continuously and names problems on the nodes they belong +to: + +| Issue | What it means | +|---|---| +| `unconnected_input` | a port needs a message nothing in reach provides | +| `missing_initial_value` | the message exists but has never held a value, and nothing will give it one | +| `cycle` | A waits for B and B waits for A — nothing could ever start | +| `self_loop_needs_initial` | a node reads a message it also writes, with no starting value | +| `node_error` | the node's code did not load: a syntax error, a missing import | +| `unauthenticated_hook` | advisory — a webhook with no shared secret is open to anyone | + +A flow with any of these except the advisory one does not run. The health +summary on Home counts them, so "why is nothing happening?" has an answer that +does not involve reading logs. + +## What happens at runtime + +- **A flow can be started and stopped.** Stopped means its subscriptions and + schedules are torn down. +- **A flow can be paused and stepped.** Paused holds messages instead of + running them; step releases exactly one. This is how you test something + before it moves a relay. +- **A failing node does not take the flow down.** It reports an error, keeps + its last error visible after it recovers, and can fire an alert. +- **A flow whose background tasks keep crashing is quarantined.** The engine + stops restarting them and says so, rather than spinning. Publishing a change + gives it another chance. + +## Values that arrive from outside + +Some messages are not computed by any node: a dashboard control writes them, +the API publishes them, a batch run passes them in. Declare those as the flow's +**inputs**, with the value they start from: + +```json +{"inputs": [{"spec": {"name": "setpoint", "dtype": "float"}, "initial": 21.0}]} +``` + +Without that, the node reading `setpoint` waits for something nothing provides, +and the canvas says so. With it, the flow starts at 21.0 and whatever writes +the message afterwards takes over. + +## Where to next + +- [Where a node's values come from](values.md) — ports, settings, flow inputs +- [Keeping state in a flow](state.md) — the sanctioned way to remember something +- [Runs: pipelines that finish](runs.md) — the batch half of the engine +- [Writing node code](../code/nodes.md) — the practical side of `process()` diff --git a/docs/concepts/runs.md b/docs/concepts/runs.md new file mode 100644 index 0000000..d740048 --- /dev/null +++ b/docs/concepts/runs.md @@ -0,0 +1,271 @@ +# Runs: pipelines that finish + +A flow that automates a house never ends — a value arrives, nodes fire, and it +waits for the next one. A research pipeline is the other shape: parameters go +in, stages execute in order, and at some point it is *done* and has produced +something worth keeping. Fluksio calls the second one a **run**, and it is the +same engine either way. + +This is what makes Fluksio usable where Kedro, MLflow or ClearML would be: a +run has parameters that identify it, a result, per-step metrics, artifacts and +a place in a queryable history — without a second server, and without paying a +project bootstrap on every execution. + +## A batch flow + +Set `mode: "batch"` on the flow and name the messages its result should hold: + +```json +{ + "name": "train_polymer_gnn", + "mode": "batch", + "outputs": ["final_loss", "report"], + "inputs": [ + {"spec": {"name": "lr", "dtype": "float"}, "initial": 0.1}, + {"spec": {"name": "steps", "dtype": "int"}, "initial": 3000} + ], + "nodes": [{"id": "train", "timeout": 7200, "device": "gpu", "...": "..."}] +} +``` + +A batch flow is built and validated like any other — it appears on the canvas, +its ports are type-checked — but it is never *activated*: no subscriptions, no +schedules, no webhooks. It runs when a run asks it to, and not otherwise. + +Its `inputs` are its parameters. A run supplies values for them; anything it +does not supply keeps the declared initial value. + +One thing a batch flow may not do is rate-limit a port (`interval`). A rate +limit holds a value back for a timer to release, and a run has no timer — the +value would be dropped rather than delayed, so submitting is refused instead. + +## Submitting + +```bash +curl -X POST $FLUKSIO/runs/flows/train_polymer_gnn \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + -d '{"params": {"lr": 0.3, "steps": 4000}, "seed": 7}' +``` + +The answer is immediate and the run is `queued`; a training run is measured in +hours, so nothing waits for it. Poll `GET /api/v1/runs/{id}` for its status, +result, per-node record and artifacts. + +Wrong parameters are refused before anything executes — an undeclared name, or +a value of the wrong type, comes back as a 422 naming the problem. + +### Sweeps + +An ensemble is the same parameters at different seeds; a grid search is the +parameters spread out. Both are one call, and the caller builds the list: + +```bash +curl -X POST $FLUKSIO/runs/flows/train_polymer_gnn/sweep \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' -d '{ + "runs": [{"params": {"lr": 0.1}, "seed": 1}, {"params": {"lr": 0.3}, "seed": 1}] +}' +``` + +They share a `group_id`, so `GET /api/v1/runs?group=…` is the sweep, and they +execute in parallel. That is safe because **each run has a state backend of its +own**: message names are global keys, so two runs of one flow would otherwise +overwrite each other's values. They do not. + +## Producing values before you are finished + +A training loop has numbers worth keeping long before it has a result. Those +numbers are **outputs**, not logs: a node declares a port for them and produces +them over time, which in Python is a generator. + +```python +def process(lr, steps): + loss = 1.0 + for _ in range(steps): + loss = train_one_step(lr) + yield {"loss": loss} # published now, on the `loss` port + return { + "weights": fluksio.save_artifact(dump(model), "weights.npz"), + "final_loss": loss, + } +``` + +Mark the port it streams on, so the flow says what it does: + +```json +{"name": "loss", "dtype": "float", "stream": true} +``` + +Every yield is published the instant it happens — same port, same type check, +same place on the canvas as any other value. Whatever the generator `return`s +is the node's result, and is what downstream nodes read. If you never +`return`, the last thing you yield is the result instead. + +This is the whole reason the framework does not have a logging API. A metric +that escapes through `log_metric()` is undeclared: invisible to validation, +absent from the canvas, and stored somewhere the graph knows nothing about. +A metric that leaves through a port is a message — so a chart binds to it +directly, a downstream node can consume it, and the run keeps its series +without anyone asking. + +**Where a yield cannot reach** — the value comes from inside somebody else's +callback, and they call you rather than the other way round — `fluksio.emit` +writes the same ports the same way: + +```python +import fluksio + + +def process(): + model.fit(callbacks=[LambdaCallback( + on_epoch_end=lambda epoch, logs: fluksio.emit(loss=logs["loss"]) + )]) + return {"weights": ...} +``` + +### What a run does with them + +Every number a node emits is kept as the run's series, stepped by the count of +emissions on that message. Read one back with +`GET /api/v1/runs/{id}/metrics?name=.loss`, or compare runs: + +``` +GET /api/v1/runs/series/compare?ids=,,&metric=.loss +``` + +That answers in the `series` shape a chart widget already draws, so three +training curves side by side is a widget binding. During a run the values also +arrive live on the flow socket, so a chart bound to the port fills in as the +training goes. + +A streaming port may set `interval` to thin out what reaches the canvas — the +run's history still keeps every value, because the interval is asking for the +display not to be flooded, not for the curve to have holes in it. + +Emitting has a second effect: **a node's timeout measures silence, not +duration**. A node that yields every few seconds can run for hours under a +`timeout` of 300; one that says nothing for longer than its timeout is killed. +Set `timeout` on a long node to how long it may plausibly go quiet. + +In a *live* flow, an emission also wakes whatever is downstream of it, exactly +as a subscriber publishing does. In a run it does not: a run's graph is +scheduled once, and three thousand mid-node cascades would leave "the run has +finished" with nothing to mean. + +## Artifacts + +Bytes never travel as a message. `save_artifact` writes them to a +content-addressed store and returns a small reference — digest, size, media +type, name — which is what an `artifact`-typed port carries: + +```python +def process(weights): # requires: weights, dtype "artifact" + path = fluksio.load_artifact(weights) + ... +``` + +Because the address is the content's hash, a sweep whose fifty configs share +one preprocessed input stores it once, and a reference stays valid wherever +the store is reachable from. Artifacts a run produced are listed on it and +downloadable at `GET /api/v1/artifacts/{digest}`. + +## Objects that cannot be serialized + +A live model, a `DataLoader`, a JAX-compiled function — these do not cross a +node boundary, and no framework flag will make them. There are exactly two +patterns, and they are both deliberate: + +* **Keep them in one node.** Stages that must share live memory are one node. + Building the model and training it is one stage; the fact that Kedro would + make them two nodes is Kedro's problem, not a structure worth reproducing. +* **Cross at a checkpoint.** Save what matters as an artifact and rebuild from + it on the other side. That is the boundary that also survives the next node + running on a different machine. + +## Running a node somewhere else + +A node that needs a GPU declares the label of the machine that has one: + +```json +{"id": "train", "device": "gpu", "device_policy": "require", "timeout": 7200} +``` + +A worker on that machine dials **out** to the engine, because the engine +generally cannot reach it — different network, no inbound route — and because +nothing should expose Redis across hosts. Install it on the box, mint it a +token, and start it: + +```bash +curl -X POST $FLUKSIO/workers/tokens -d '{"name": "gpu-dev"}' # once, as an admin + +pip install fluksio-worker + +fluksio-worker \ + --url wss://api.example.com/api/v1/workers/attach \ + --token "$FLUKSIO_WORKER_TOKEN" \ + --labels gpu,cuda12 \ + --python /opt/torch-venv/bin/python +``` + +`fluksio-worker` is its own distribution — the agent, the node runner, and +`websockets`. Nothing of the engine, so a GPU box does not install a database +driver to run a training step. Where pip is not an option, the two files still +work copied into one directory and run with `python agent.py …`; the engine +serves the runner at `GET /api/v1/workers/runtime`. + +`--python` is the interpreter node code runs on, which is how the GPU box keeps +its CUDA wheels without the engine ever installing them. The node's source +travels with every call, so nothing has to be deployed there. + +A few consequences worth knowing: + +* `import fluksio` inside a node is the worker's own reporter — `emit`, + `save_artifact`, `load_artifact` — installed before the node's code runs, so + the installed `fluksio` package (if the box has one) never shadows it. +* A node bound to a device is **compiled on that machine**. A node importing + `torch` is correct on the GPU box and a missing module on the engine, so + checking it on the engine would fail a node that is fine. +* If nothing carrying the label is attached, the run stays `queued` and says + what it is waiting for. Submit first, switch the GPU box on later. +* Cancelling a run kills what it is executing, there or here, and leaves other + runs of the same node alone. +* If the worker disappears mid-call, the run fails in seconds with + `worker went away mid-call` rather than waiting out its timeout. +* `device_policy: "prefer"` runs locally when no such worker is attached; + `"require"` (the default) waits for one. + +## Durability + +Submitting journals the run to a Redis stream of its own, separate from the +one the automations use — a burst of five hundred sweep runs must not stand +between a house and its heating. An engine that is down when a run is +submitted picks it up when it starts. + +From the moment a run is claimed, its database row is the record and the queue +is finished with it. Redelivering two hours of training because an +acknowledgement was late is not recovery; instead a running run refreshes a +lease, and one whose lease goes stale is marked `abandoned` — which is what a +run whose engine was killed mid-training becomes. + +## What this costs, compared + +The repository ships a benchmark that measures submitting a run against a +Kedro project doing the same nothing: + +``` +fluksio — submit accepted median 15.3 ms + submit -> result median 60.8 ms +kedro — kedro run median 1109.5 ms +``` + +The difference is not the orchestration; it is that Fluksio does not boot a +project per run. The engine is already up, and the workers already have the +node's code compiled. On a 510-run sweep, that gap is about nine minutes of +pure startup that never happens. + +## See also + +- [Getting started: data science](../getting-started/data-science.md) — the same + material as a first setup +- [Writing node code](../code/nodes.md) — generators, `fluksio.emit`, artifacts +- [Remote workers](../code/workers.md) — attaching the machine with the GPU +- [The HTTP API](../code/api.md) — every endpoint used above diff --git a/docs/concepts/state.md b/docs/concepts/state.md new file mode 100644 index 0000000..c41df0d --- /dev/null +++ b/docs/concepts/state.md @@ -0,0 +1,75 @@ +# Keeping state in a flow + +Logic nodes are pure functions of their inputs. There is no `context`, no +`global`, no handle to a store — a node is called with the values of the +messages it declares and returns the values of the messages it provides. + +That is deliberate: a node with hidden state cannot be run twice in parallel, +cannot be replayed, and cannot be moved to another worker. But plenty of real +automations need to remember something. This is how. + +## State is a message the node both reads and writes + +A running total, a debounce timer, a last-seen reading — each is a value that +survives between runs. Give it a message name, declare it as both an input and +an output, and it is state: + +```python +def process(reading, total=0.0): + return {"total": total + reading} +``` + +with `reading` and `total` as inputs and `total` as the output. Each run reads +the total the previous run wrote. + +Two things make this work rather than loop forever: + +- **A node never depends on itself.** The graph is built from message names, but + a node providing a message it also requires is not placed downstream of + itself. Publishing `total` does not re-run the node that wrote it. +- **The value must start somewhere.** The first run has nothing to read. Declare + the message as a flow input with a starting value; otherwise the flow reports + `self_loop_needs_initial` at edit time and refuses to publish. + +The port needs a default in the function signature (`total=0.0` above) so the +first call is well-formed even if the value is missing. + +## Feeding a value back between two nodes + +Sometimes the value comes from a different node — a controller reading back what +an estimator computed from its own last output. Written plainly that is a cycle, +and the validator rejects it, because a graph where A waits for B and B waits +for A can never start. + +What is actually meant is that the fed-back value is *state*, not a trigger. +Say so, by marking the input non-triggering: + +```json +{"name": "estimate", "port": "estimate", "dtype": "float", "trigger": false} +``` + +A non-triggering input: + +- creates no dependency, so it cannot form a cycle; +- never makes the node wait — if the message has no value yet, the port is + simply left out of the call, and the function's default applies; +- is read fresh from state whenever the node does run, for whatever reason. + +Use it for the back edge only. An input that should wake the node is an ordinary +input, and marking it non-triggering would mean the node never runs at all. + +## What still holds engine-side state + +Built-in nodes that are *about* time or change — rate limiting, filter-on-change, +delay, cron — keep what they need in the engine's own state backend, under keys +that never appear as messages. That is the engine's business, not a flow's: the +line is that node *code* you write never reaches for state, while node *types* +the engine ships may. + +Storage and direct-I/O nodes are the other sanctioned exception, since talking +to a device or a database is stateful by nature. + +## See also + +- [Where a node's values come from](values.md) — ports, settings and flow inputs +- [Node types](../reference/node-types.md) — the built-ins that do keep engine-side state diff --git a/docs/concepts/values.md b/docs/concepts/values.md new file mode 100644 index 0000000..69f5325 --- /dev/null +++ b/docs/concepts/values.md @@ -0,0 +1,67 @@ +# Where a node's values come from + +A node function is called with one argument per thing it needs, and every one +of them is declared somewhere you can see. There are two kinds. + +## Ports: what the graph carries + +A port binds to a message name. Whatever last published that message is the +value the node is called with, and the wiring follows from the name rather than +from a line somebody drew. + +```python +def process(reading, setpoint): + return {"heat": reading < setpoint} +``` + +with `reading` and `setpoint` as inputs and `heat` as an output. + +## Settings: constants of this node + +A setting is a value that belongs to this node's code rather than to the graph: +how many retries, which unit, how long to wait. It is typed into the node's +**Settings** section, stored with the flow, and arrives as an argument like a +port: + +```python +def process(reading, unit="C"): + return {"shown": reading if unit == "C" else reading * 1.8 + 32} +``` + +Because both arrive by name, a setting cannot share a name with a port — the +node reports it rather than picking one. + +Settings are part of the flow document, so changing one is an edit that gets +published, not something that happens at runtime. **A value that should change +while the flow runs is a message, not a setting.** + +There is no `global` and no `flow` context. A constant several nodes need is a +node that provides it: one place it is set, an ordinary message out, and every +consumer visibly downstream of it. + +## Flow inputs: what arrives from outside + +Some messages are not computed by any node in the flow — a dashboard control +writes them, the API publishes them, a batch run passes them in. Declare those +as the flow's **inputs**, with the value they start from: + +```json +{"inputs": [{"spec": {"name": "setpoint", "dtype": "float"}, "initial": 21.0}]} +``` + +Without that, the node reading `setpoint` waits for something nothing provides, +and the canvas says so. With it, the flow starts from 21.0 and whatever writes +the message afterwards takes over. + +The canvas draws each one as a labelled endpoint feeding the nodes that read +it, the same way it draws a dashboard tile or another flow — so a value never +appears from nowhere. They are edited in the flow's own panel. + +A batch flow's inputs are also its run parameters, and its **outputs** name what +a run reports as its result; see [Runs](runs.md). + +## See also + +- [Flows, nodes and messages](flows.md) — how the graph is built from the names +- [Keeping state in a flow](state.md) — a value a node reads *and* writes +- [Writing node code](../code/nodes.md) — the practical side diff --git a/docs/getting-started/data-science.md b/docs/getting-started/data-science.md new file mode 100644 index 0000000..155ee0e --- /dev/null +++ b/docs/getting-started/data-science.md @@ -0,0 +1,323 @@ +# Getting started: data science + +You have a training script. It works. What you do not have is any reliable +answer to "what was the learning rate on the run that got 94%?", and the +`results_final_v3_ACTUAL.csv` in your home directory is not helping. + +This page adds Fluksio to what you already have. It takes about five minutes, +installs one Python package, and does not ask you to restructure anything. + +## Install + +```sh +pip install fluksio +fluksio serve +``` + +That is the whole installation. No Docker, no database server, no ports to +open. The first run prints something like: + +```text +Created the admin account admin@example.com + password: k3Qm-8vTpLdX + Shown once. Change it from the dashboard. +Fluksio 0.1.0 — data in /home/you/.fluksio + API http://127.0.0.1:8000/api/v1 + No portal. Pair this installation with: + fluksio enroll --portal https://hub.example.com +``` + +**Write that password down.** It is shown once and it is how you authenticate +from here on. + +Everything the installation owns lives in `~/.fluksio`: a SQLite database, a +git repository holding your flows, the artifact store, and a virtual +environment your node code runs in. Move it with `--data-dir`, which is worth +doing on a cluster where `$HOME` is a network filesystem — SQLite's +write-ahead log does not work on NFS, and `fluksio serve` warns you when it +notices. + +!!! tip "Keep it running" + + The engine is meant to be resident. That is the whole reason submitting a + run costs about 15 ms instead of the second a project-bootstrapping + orchestrator spends before it does anything. Leave it in a `tmux` window, + or write a small `systemd --user` unit for it. + +## Get a token + +Everything below is the HTTP API. Grab a token once: + +```sh +export FLUKSIO=http://127.0.0.1:8000/api/v1 +export TOKEN=$(curl -s -X POST $FLUKSIO/login/access-token \ + -d "username=admin@example.com&password=k3Qm-8vTpLdX" | jq -r .access_token) +``` + +While you are experimenting, the interactive schema at + is the fastest way to see what is available. + +## Tell it about your packages + +Node code runs in `~/.fluksio/user-venv`, deliberately separate from the +environment Fluksio itself is installed in — so a pin of yours can never +collide with one of ours. That venv starts empty, so the first thing to do is +say what your script imports: + +```sh +curl -X POST $FLUKSIO/modules/apply -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d "{\"requirements\": $(jq -Rs . < requirements.txt)}" +``` + +It is a pip manifest, installed with `uv pip sync`, and it is versioned +alongside your flows — so what a run imported is recorded with what it ran. +Adding a package takes effect immediately; nothing restarts. + +??? note "Already have a venv you would rather not duplicate?" + + Attach it as a worker instead of reinstalling into it. Mint a token, then + point the agent at your existing interpreter: + + ```sh + curl -X POST $FLUKSIO/workers/tokens -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' -d '{"name": "laptop"}' + + fluksio worker --url ws://127.0.0.1:8000/api/v1/workers/attach \ + --token "$WORKER_TOKEN" --labels local --python "$(which python)" + ``` + + Then mark the node `"device": "local"` and it runs on that interpreter. It + is the same mechanism that sends a node to a GPU box, and it is worth + knowing about early — see [Remote workers](../code/workers.md). + +## Wrap your training script + +A **flow** is a graph of nodes. A **batch flow** is one that runs on demand +from parameters to a result, which is what an experiment is. Your existing +script becomes the body of a node. + +Say your script looks roughly like this: + +```python +def train(lr, epochs): + model = build_model() + for epoch in range(epochs): + loss = step(model, lr) + print(f"epoch {epoch}: {loss}") # ← this is what you are losing + torch.save(model.state_dict(), "weights.pt") + return loss +``` + +Two changes turn it into a node: + +```python +"""Fit the model. A generator, so numbers escape while it is still running.""" + +import fluksio + + +def process(lr, epochs): + model = build_model() + for epoch in range(int(epochs)): + loss = step(model, lr) + yield {"loss": loss} # ← published now, on the loss port + torch.save(model.state_dict(), "weights.pt") + return { + "weights": fluksio.save_artifact("weights.pt"), + "final_loss": loss, + } +``` + +`yield` replaces `print`. Each one publishes on the node's `loss` port the +instant it happens, and the run keeps every value as a series — which is why +Fluksio has no `log_metric()` call. A metric that leaves through a port is a +message like any other: a chart can bind to it, a downstream node can consume +it, and it shows up on the canvas. A metric that escapes through a logging +function is invisible to all three. + +`fluksio.save_artifact` handles the things too big to be messages — a +checkpoint, a dataset, a plot. It stores the bytes by their hash and returns a +small reference. Nothing changes about how you write the file. + +!!! note "Where a `yield` cannot reach" + + If the number comes from inside somebody else's callback — Keras, Lightning, + HuggingFace `Trainer` — `fluksio.emit(loss=...)` writes the same port the + same way: + + ```python + model.fit(callbacks=[LambdaCallback( + on_epoch_end=lambda epoch, logs: fluksio.emit(loss=logs["loss"]) + )]) + ``` + +## Create the flow + +There is no scaffolding command yet, so a flow is created by PUTting its +definition. That is a fifteen-line script you run once: + +```python +"""Create the `train` flow. Run once; edit it in the canvas afterwards.""" + +import httpx + +API = "http://127.0.0.1:8000/api/v1" +api = httpx.Client(base_url=API, timeout=60) +token = api.post( + "/login/access-token", + data={"username": "admin@example.com", "password": "k3Qm-8vTpLdX"}, +).json()["access_token"] +api.headers["Authorization"] = f"Bearer {token}" + +api.put("/flows/train", json={ + "name": "train", + "title": "Model training", + # Batch: nothing is activated, nothing fires until a run asks. + "mode": "batch", + # Its inputs are the run's parameters, with the values a run gets when it + # names none. + "inputs": [ + {"spec": {"name": "lr", "dtype": "float"}, "initial": 0.01}, + {"spec": {"name": "epochs", "dtype": "int"}, "initial": 50}, + ], + # What a run reports as its result. + "outputs": ["final_loss", "weights"], + "nodes": [{ + "id": "train", + "type": "python", + "title": "Fit the model", + # An *idle* timeout once the node streams: this is how long it may go + # quiet, not how long it may run. + "timeout": 600, + "requires": [ + {"name": "lr", "dtype": "float"}, + {"name": "epochs", "dtype": "int"}, + ], + "provides": [ + # `stream` says this port publishes repeatedly during one execution. + {"name": "loss", "dtype": "float", "stream": True}, + {"name": "final_loss", "dtype": "float"}, + {"name": "weights", "dtype": "artifact"}, + ], + }], +}).raise_for_status() + +api.put("/flows/train/nodes/train/source", + json={"code": open("train_node.py").read()}).raise_for_status() + +version = api.get("/flows/train", params={"draft": True}).json()["definition"]["version"] +api.post("/flows/train/publish", json={"version": version}).raise_for_status() +print("published") +``` + +Two things worth noticing. Ports are declared, not inferred — `process(lr, +epochs)` gets its arguments from the ports of the same name, and the types are +checked on every value. And saving writes a *draft*; `publish` is what the +engine picks up. That separation is what lets you edit a flow that is running. + +## Run it + +```sh +curl -X POST $FLUKSIO/runs/flows/train -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{"params": {"lr": 0.003, "epochs": 200}, "seed": 7}' +``` + +It answers immediately with a queued run — training is measured in hours, so +nothing waits for it. A parameter you did not declare, or one of the wrong +type, is refused with a 422 before anything executes. + +Then, whenever you like: + +```sh +curl -s $FLUKSIO/runs/ -H "Authorization: Bearer $TOKEN" | jq +curl -s "$FLUKSIO/runs//metrics?name=train.loss" -H "Authorization: Bearer $TOKEN" | jq +``` + +The run carries its parameters, a digest of them, the seed, its result, how +long each node took, what it logged, and every artifact it produced. That is +the answer to "what was the learning rate on the run that got 94%?". + +## Sweep it + +A grid search and an ensemble are the same call — you build the list, Fluksio +runs them in parallel: + +```sh +curl -X POST $FLUKSIO/runs/flows/train/sweep -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' -d '{"runs": [ + {"params": {"lr": 0.001}, "seed": 1}, + {"params": {"lr": 0.003}, "seed": 1}, + {"params": {"lr": 0.010}, "seed": 1} + ]}' +``` + +They share a `group_id`, so the sweep is `GET /runs?group=…`. Running them +concurrently is safe because each run gets a state backend of its own — two +runs of one flow cannot overwrite each other's values. + +Compare the curves in one call: + +```sh +curl -s "$FLUKSIO/runs/series/compare?ids=$A,$B,$C&metric=train.loss" \ + -H "Authorization: Bearer $TOKEN" +``` + +which answers in exactly the shape a chart widget draws. + +## Small scripts you are just playing with + +The same machinery, minus the ceremony. If what you want is "keep a record of +every version of this thing I keep tweaking", note that **your flows are +already a git repository**: + +```sh +cd ~/.fluksio/flows +git log --oneline +``` + +Every save is a commit — the node source, the parameters, the graph. A run +records the commit it ran at, so `git show` on that hash is literally the code +that produced the number. You get the versioning without adopting anything. + +For quick iteration, keep the flow small (one node is fine), keep the engine +running, and submit from wherever you are working: + +```python +import httpx +run = httpx.post(f"{API}/runs/flows/train", json={"params": {"lr": lr}}, + headers=auth).json() +``` + +A submit is around 15 ms, so calling that in a loop is a reasonable thing to do. + +## Get a browser onto it + +The pip install gives you the engine and the API, not a web interface — a +machine with no inbound route cannot serve one usefully anyway. To see the +canvas, the run history and live loss curves, pair the installation with a +portal, which serves the dashboard from its side: + +```sh +fluksio enroll --portal https://hub.fluksio.com +``` + +Get the claim code from the portal under **Installations → Add installation**. +Nothing needs to be exposed: your machine dials out and holds the connection +open. See [Accounts and the portal](../interface/portal.md). + +If you would rather stay entirely offline, run the dashboard SPA yourself from +the app's Docker image — see the [facility path](facility-automation.md), which +is the same stack. + +## Where to go next + +- [Runs: pipelines that finish](../concepts/runs.md) — the full picture: + artifacts, sweeps, durability, what happens when your engine dies mid-training +- [Writing node code](../code/nodes.md) — generators, settings, what a node may + and may not do +- [Remote workers](../code/workers.md) — send the training node to the GPU box + and keep the rest on your laptop +- [The flow editor](../interface/flow-editor.md) — once you have a portal, this + is where the graph gets easier to change than the script did diff --git a/docs/getting-started/facility-automation.md b/docs/getting-started/facility-automation.md new file mode 100644 index 0000000..7801fef --- /dev/null +++ b/docs/getting-started/facility-automation.md @@ -0,0 +1,272 @@ +# Getting started: facility automation + +You have a box in a cupboard, a handful of sensors that already publish +somewhere, and an ambition to make the house do something about them. This +page brings up a Fluksio instance you can leave running for years, then wires +the first sensor through to a dashboard. + +Budget an afternoon. Most of it is the browser, which is the point: you will be +in this interface a lot, so it is worth learning it properly. + +## What you are standing up + +```text + sensors ──MQTT──▶ ┌──────────────┐ ──▶ InfluxDB (history) + │ Fluksio │ + HTTP / webhooks ──▶│ flow engine │ ──▶ dashboards (what you look at) + │ │ + schedules ────────▶└──────────────┘ ──▶ MQTT / HTTP (what you control) + │ + └── workers on other boxes (optional) +``` + +One engine holds every flow. Nothing here is a plugin you install separately — +the broker client, the time-series writer, the dashboards and the alerting are +all part of the same process, editing the same graph. + +## Prerequisites + +- Docker and Compose v2 on the host +- A hostname you can point at it. `fluksio.local`, a subdomain, or just + `localhost` if you only ever reach it from that machine +- Optionally: an MQTT broker and an InfluxDB you already run. If not, the stack + can start both for you + +## Bring up the stack + +Clone the app repository and start it: + +```sh +git clone https://git.stroblme.de/Fluksio/app.git ~/fluksio +cd ~/fluksio +cp .env.example .env +$EDITOR .env # DOMAIN, FIRST_SUPERUSER, ENVIRONMENT=production +make up +``` + +`.env` is the whole configuration. The four settings that matter on day one: + +| Setting | What it does | +|---|---| +| `DOMAIN` | the hostname everything is served under; the SPA lands on `app.${DOMAIN}` and the API on `api.${DOMAIN}` | +| `FIRST_SUPERUSER` | the account you sign in with | +| `FIRST_SUPERUSER_PASSWORD` | leave it as `changethis` and one is generated for you | +| `ENVIRONMENT` | `production` closes the interactive API schema; `local` leaves it open | + +Everything the installation owns — the database, your flows, secrets, +artifacts, the packages your node code imports — is on one Docker volume. +Backing that volume up is backing up the installation. + +!!! tip "Reverse proxy" + + The stack emits Traefik labels and ships a Traefik you can bring up + alongside it (`docker/compose.traefik.yml`). If you already run Nginx + Proxy Manager or Caddy, attach it to the `proxy` network instead and + forward `app.${DOMAIN}` → `fluksio-app:80` and `api.${DOMAIN}` → + `fluksio-api:8000`. + +??? note "Even smaller: no Docker at all" + + `pip install fluksio && fluksio serve` gives you the same engine with no + containers, keeping its data in `~/.fluksio`. What it does not give you is + the web interface, which the SPA container serves — so you would drive it + from the API, or [pair it with a portal](../interface/portal.md) that + serves the dashboard for you. Good for a Raspberry Pi that only runs flows; + less good as your main instance. + +Open `http://app.${DOMAIN}` and sign in. You should be looking at Home: an +empty brain graph, a health summary, and a flow list with nothing in it. + +## Your first flow + +Go to **Flows → New flow** and call it `house`. You land on the canvas. + +A flow is a set of nodes that talk to each other through **named messages**. +You do not draw wires: a node says which messages it needs and which it +produces, and the canvas draws the graph that follows from those names. That +sounds like a small difference and turns out to be a large one — renaming is +safe, fan-in is free, and two flows can share a value by naming it. + +### Read a sensor + +Press **Add node** (or ⌘K / Ctrl-K, which opens the command palette) and pick +**MQTT**. In its panel on the right: + +- **Broker host** — your broker's hostname, `mosquitto` if you are using the + one the stack can start +- **Topic** — map each output to a topic: + `{"living_temperature": "zigbee2mqtt/living/temperature"}` +- **Provides** — add one output port named `living_temperature`, type `float` + +That is a working node. Press **Publish** (⌘S) and the engine picks it up. + +The canvas now draws your node with a live value on its output as soon as the +broker sends one. Click the wire to see the last payload and its history. + +### Do something with it + +Add a **Function** node. This is a Python node — the code editor opens in its +panel: + +```python +def process(living_temperature, comfortable=21.0): + """Ask for heat when the room is below the comfort point.""" + return {"heat_wanted": living_temperature < comfortable} +``` + +Declare `living_temperature` as an input (type `float`) and `heat_wanted` as an +output (type `bool`). `comfortable` is not a port — it is a **setting**, +because it is a constant of this node rather than something the graph carries. +It shows up as a field in the node's Settings section. + +The canvas now draws MQTT → your function, because the message names line up. +Nothing else was needed. + +### Act on it + +Add a second **MQTT** node, this time with `heat_wanted` as an *input*, and a +topic mapping to whatever your relay listens on. A node with inputs publishes; +a node with outputs subscribes. + +Publish the flow. You have a thermostat. + +!!! warning "Test before it touches a relay" + + A flow can be **paused** (it holds messages instead of running them) and + **stepped** (release exactly one). Together with the run button — which + injects a value by hand — that is how you convince yourself the logic is + right before the contactor finds out. Both live on the dock at the bottom + of the canvas. + +## Store the history + +A live value is enough to control something and useless for answering "was last +February colder?". That is what the **InfluxDB** node is for. + +Add one, and configure it to write the message you already have: + +```json +{ + "url": "http://influxdb:8086", + "token": {"$secret": "influx-token"}, + "org": "home", + "bucket": "sensors", + "writes": { + "living_temperature": { + "measurement": "environment", + "field": "temp_c", + "tags": {"room": "living"} + } + } +} +``` + +Give it `living_temperature` as an input and every value that passes gets a +point. + +Note the token. Credentials never sit in a flow: `{"$secret": "influx-token"}` +is a reference into an encrypted store, and the editor renders those fields as +a secret picker. Add the actual value once under **Secrets**. Flows are a git +repository you might well push somewhere — this is what keeps a password out of +it. + +Reading back is the same node with `queries` instead of `writes`, or — for +anything a chart asks for — a pair of small Python nodes on either side that +build a Flux query and shape its rows. That indirection is deliberate: the +database node holds the connection and nothing else, so a dashboard widget +never learns which database answered it. + +## Put it on a screen + +**Dashboards → New dashboard**, then drag widgets onto the grid and bind each +one to a message. A gauge on `house.living_temperature`, a switch on +`house.heat_wanted`, a chart on the history. + +Widgets are typed the same way ports are: a switch binds to a `bool`, a gauge +to a number, an agenda to a `list`. Bind it wrong and the editor says so rather +than drawing nothing. + +Controls work in the other direction — a switch on a dashboard publishes the +message it is bound to, exactly as a node would. The canvas draws it as a +labelled endpoint feeding the nodes that read it, so a value never appears from +nowhere. + +For a tablet on the wall, see [Dashboards and panels](../interface/dashboards.md): +a panel is a named device, it pairs with a six-character code instead of a +login, and it can only reach the dashboards you gave it. + +## Spread it across machines + +You now have one box doing everything. Two reasons to change that: something +lives on a different network, or something needs hardware the engine's host +does not have. + +The unit of distribution is the **worker**. It runs the code of nodes you mark +for it, and it dials *out* to the engine — so the Pi in the shed does not need +an inbound route, and the engine does not need to reach it. + +On the engine, mint a token: + +```sh +curl -X POST https://api.${DOMAIN}/api/v1/workers/tokens \ + -H "Authorization: Bearer $TOKEN" -d '{"name": "shed-pi"}' +``` + +On the other machine: + +```sh +pip install fluksio-worker +fluksio-worker \ + --url wss://api.${DOMAIN}/api/v1/workers/attach \ + --token "$WORKER_TOKEN" \ + --labels shed,gpio \ + --parallel 2 +``` + +Then mark the node that talks to the shed's GPIO with `device: shed`, and it +runs there. Everything else stays where it is. A node bound to a label no +attached worker carries simply waits rather than failing, so you can write the +flow before the hardware arrives. + +!!! note "What a worker is not" + + It is not a second engine. Subscriptions, schedules, webhooks and the + dashboards all stay in one process — that is what keeps a value having one + definition. A worker executes node bodies, nothing else. Scaling *the + engine* to several processes is not supported: run one. + +## Make it tell you when something breaks + +Under **Alerts**, add a channel and a rule. Channels are ntfy (a push +notification on your phone), SMTP, a webhook, or a message a dashboard +notification widget reads. + +What you probably want on day one is everything, to ntfy: + +- add an ntfy channel with your server and topic +- add a rule with no events ticked, which means *all of them* + +The engine deduplicates aggressively — the same node failing every second is +one alert, not thirty-six thousand — and caps the total at ten an hour however +bad it gets. See [Secrets, modules and alerts](../interface/operations.md). + +## Keep it alive + +- **Back up the data volume.** That is the database, the flows, the secrets and + the artifacts. Everything else is rebuildable. +- **Watch Home.** The health summary names what is wrong — a quarantined flow, a + node that will not load, a stalled queue, a flow that cannot run because its + graph does not validate. +- **Flows are git.** `git log` inside the flow store is the history of every + change anyone made, and reverting one is a revert. + +## Where to go next + +- [Flows, nodes and messages](../concepts/flows.md) — the model, properly +- [Node types](../reference/node-types.md) — everything you can put on a canvas + without writing Python +- [The flow editor](../interface/flow-editor.md) — the canvas, in detail +- [Keeping state in a flow](../concepts/state.md) — running totals, debounces, + and the one rule that makes them safe +- [Accounts and the portal](../interface/portal.md) — reach the installation + from outside the house without opening a port diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md new file mode 100644 index 0000000..118a1d3 --- /dev/null +++ b/docs/getting-started/index.md @@ -0,0 +1,72 @@ +# Pick your starting point + +People arrive at Fluksio from two directions, and the honest answer to "how do +I set this up?" is different for each — not just in the commands, but in how +much of an afternoon it is reasonable to spend. + +Pick the one that sounds like you. Everything past this section is the same for +both. + +
+ +
+ +### Data science + +*"I have a training script. I want to stop losing track of what I ran."* + +One `pip install`, one command, and you are writing Python again. No Docker, no +database, no ports to open. Flows are files, runs are rows, and the metrics are +just the numbers your loop already produces. + +[Set up for experiments →](data-science.md) + +
+ +
+ +### Facility automation + +*"I have a homelab and a pile of sensors. I want them to do something."* + +A stack you bring up once and leave running: the engine, a broker, a +time-series database, dashboards, alerting. Most of the work happens in the +browser, and it is worth doing properly because you will live in it. + +[Set up a homelab instance →](facility-automation.md) + +
+ +
+ +## Not sure? + +Some rough tells: + +| | Data science | Facility automation | +|---|---|---| +| **The flow** | starts, finishes, has a result | never ends | +| **You mostly** | write Python | wire nodes in the browser | +| **Time to first result** | a few minutes | an afternoon | +| **Runs on** | your laptop, or a login node | a box in a cupboard | +| **Data lives in** | SQLite beside the flows | InfluxDB, usually | +| **The thing you look at** | run history and loss curves | a dashboard, maybe on a wall | + +If both describe you — a lab with instruments to drive *and* models to +fit — start with the data-science path. It is the smaller installation, and it +grows into the other one without being reinstalled: the same engine, the same +flows, just more of them running all the time. + +## What is the same either way + +Whichever door you came in: + +- **Flows are files in a git repository.** Every save is a commit. You can read + the history with ordinary git, and you can copy a flow between installations + by copying a directory. +- **Editing is separate from running.** You edit a draft; the engine keeps + running what was published until you publish. +- **Nodes are typed.** A port declares what it carries, and a mismatch is + caught at edit time rather than at three in the morning. +- **Everything the browser does is an API call.** The dashboard is a client of + the same REST API you can script against. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..063ba9d --- /dev/null +++ b/docs/index.md @@ -0,0 +1,57 @@ +# Fluksio + +Fluksio is a node-based automation engine. You describe what should happen as a +graph of small pieces of logic, and it keeps that graph running — reacting to +what arrives, or executing once from parameters to a result. + +Two very different jobs turn out to be the same shape, which is why the same +engine does both: + +- **A house, a lab or a plant** produces values forever. A sensor publishes, a + rule fires, a relay closes, a dashboard on the wall shows what happened. The + flow never ends. +- **An experiment** produces a value once. Parameters go in, stages execute, + and at some point it is *done* and has left behind metrics, artifacts and a + record you can compare against last month's. + +The first is a **live flow**, the second is a **run**. Both are the same nodes, +the same type checking, the same editor and the same API. + +## What you actually get + +- A **flow engine** that owns your graph, checks the types on every edge, and + keeps running when a node fails rather than taking the rest down with it. +- A **canvas** that lays flows out for you and an editor for the Python inside + each node — with the running values drawn on the wires while you work. +- **Dashboards** built next to the logic that feeds them, including ones you + can hang on a wall tablet that has no keyboard. +- **Runs**: parameters, metrics, artifacts, sweeps, and a queryable history — + without a second server and without paying a project bootstrap per execution. +- **Distributed workers**: a node marked `device: gpu` runs on the machine that + has one, which dials out to the engine rather than needing to be reachable. +- An **HTTP API** that came first — everything the browser does, you can do from + a script — plus an MCP endpoint for agents. + +## Start here + +The setup is genuinely different depending on what you are here for, so +[Getting started](getting-started/index.md) splits in two: one path installs a +Python package and gets out of your way, the other stands up a server you will +be running for years. Everything after that is shared. + +If you would rather look before installing, the hosted demo at +[fluksio.com](https://fluksio.com) runs a real instance with a small-house panel +and a training pipeline on it. + +## Where things are + +| If you want to… | Read | +|---|---| +| Understand what a flow, a node and a message are | [Concepts](concepts/flows.md) | +| Drive Fluksio from the browser | [The interface](interface/index.md) | +| Drive it from Python, a shell or CI | [Code and the CLI](code/cli.md) | +| Look up a node type or a payload type | [Reference](reference/node-types.md) | + +Fluksio is self-hosted by default. An installation runs offline, keeps its data +on its own disk, and never contacts anything unless you +[connect it to a portal](interface/portal.md) yourself. diff --git a/docs/interface/dashboards.md b/docs/interface/dashboards.md new file mode 100644 index 0000000..945edba --- /dev/null +++ b/docs/interface/dashboards.md @@ -0,0 +1,137 @@ +# Dashboards and panels + +A dashboard is a grid of widgets bound to message names — the same names that +wire the graph. It is its own document rather than a set of nodes, so it reads +across flows without belonging to any of them, and a flow stays the logic it +was. + +There is no second tool and no separate deployment. You build the panel next to +the thing feeding it. + +## Building one + +**Dashboards → New dashboard**, then drag widgets onto the grid from the widget +picker and bind each to a message. + +Like flows, dashboards are drafts until you publish. A wall panel reads only +the published version, so a half-arranged page never reaches the wall. + +On a phone the dashboard is read, not arranged: the grid stacks to one column +and dragging is off. Picking a widget and editing its settings still works. + +## The widgets + +### Display + +| Widget | Binds to | Notes | +|---|---|---| +| **Value** | anything | a formatted reading with a unit and a precision | +| **Gauge** | `float`, `int` | min, max, unit | +| **Chart** | `float`, `int` | up to five series; see *Two kinds of chart* below | +| **Bar** | `float`, `int` | a reading, optionally nesting up to three inside it | +| **Icon** | numbers, booleans, weather strings | maps a value onto a glyph | +| **Text** | — | markdown you write; a label, a note, an instruction | +| **Agenda** | `list` | upcoming items, e.g. from a calendar connector | +| **Forecast** | `list` | a short outlook strip | +| **Notification** | `record` | title, body and severity — what an alert channel writes | +| **Clock** | — | the time, in a size a wall can read | + +### Input + +| Widget | Publishes | Notes | +|---|---|---| +| **Button** | a fixed value | one-shot: run it, open it, reset it | +| **Switch** | `bool` | on/off | +| **Slider** | `float`, `int` | min, max, step | +| **Input** | text or a number | free entry | +| **Dropdown** | one of a list | a mode, a scene, a preset | + +A control publishes the message it is bound to, exactly as a node would. On the +flow canvas it is drawn as a labelled endpoint feeding the nodes that read it, +so nobody has to guess where the value came from. + +### Typed bindings + +Widgets are type-checked against the message the same way ports are: a switch +takes a `bool`, a gauge takes a number, an agenda takes a `list`, a +notification takes a `record`. Bind one wrong and the editor says so rather +than drawing nothing. The same table is enforced on the server. + +Charts are the exception worth knowing: because identity across five series is +carried by lightness alone, a chart with more than one series always draws a +legend. + +## Two kinds of chart + +**Live.** Bind up to five messages and the chart draws the engine's own ring +buffer of recent values, extended over the socket as new ones land. Nothing +else is involved. This is what you want for "the last few hours of the living +room". + +**Querying.** The chart publishes a *request* — the window and resolution it +wants — exactly as a slider publishes a value, and draws the `series` some flow +answers with. What serves that request is the flow's business: typically a +small Python node that builds a Flux query, an InfluxDB node that runs it, and +another Python node that shapes the rows. + +That indirection is the point. The widget never learns which database answered +it, so swapping the store is a change to one flow and nothing else. The answer +also states what it was computed for, so an answer to a different question is +ignored rather than two charts overwriting each other's picture. + +## Showing one + +- `/view/{name}` — a browser tab pointed at one dashboard. Needs an ordinary + session. +- **Panels** — a named device that pairs instead of logging in. Below. + +## Panels: hanging a screen on a wall + +A wall tablet has no keyboard, so it pairs. + +1. **Dashboards → Panels**, add a panel named after where it hangs, and tick + the dashboards it shows. More than one and the screen draws a rail to switch + between them. +2. Point the device's browser at the link the dialog shows. The device then + displays a six-character code. +3. Type that code into the same panel's **Pair device** field. The line under + it names what is holding the code — check it is the screen you just hung, + because approving adopts whatever answered. The screen picks the credential + up within a few seconds and never asks again. + +What the screen holds is not a login. It reaches that panel's published +dashboards and the message endpoints its widgets speak, and nothing else. + +It cannot be made strictly read-only, and that is honest rather than an +oversight: a querying chart publishes its request, and a control on a panel is +the reason you put one there. Deleting the panel revokes the credential, which +is also how you retire a device — the screen falls back to asking for a new +code. + +!!! note "If the link is wrong" + + The pairing link is built from the installation's `FRONTEND_HOST`. If that + is not the address devices on your network actually reach, fix the setting + rather than the link: it is the same one password-reset mails and the OAuth + metadata are built from. + +### A screen somewhere you cannot reach + +Another building, someone else's network, no route in. An installation +[enrolled with a portal](portal.md) shows a second link, +`https://hub.${DOMAIN}/i/{installation-id}/panel`, and the same three steps +work through it: the portal serves that one page without a session, forwards +the pairing calls down the tunnel, and mints the credential when you approve +the code. The pairing line then reads *via portal*. + +The portal names the panel and nothing else. What the panel may read is decided +on the installation, on every call, by the same check a locally paired screen +passes. Two differences: it acts as the account the installation was enrolled +with rather than as whoever approved it, and deleting the panel stops it here +immediately while the portal's copy of the token expires on its own. + +## See also + +- [Payload types](../reference/payload-types.md) — what a widget can bind to +- [Flows, nodes and messages](../concepts/flows.md) — where the names come from +- [Accounts and the portal](portal.md) — reaching all of this from outside diff --git a/docs/interface/flow-editor.md b/docs/interface/flow-editor.md new file mode 100644 index 0000000..dc52dfb --- /dev/null +++ b/docs/interface/flow-editor.md @@ -0,0 +1,156 @@ +# The flow editor + +The canvas is where most of the work happens. It floats its chrome over a +full-bleed graph, so what you are looking at is always the flow. + +## The canvas + +Nodes are laid out for you — left to right on a desktop, top to bottom on a +phone — and the arrangement is recomputed as the graph changes. There is +nothing to drag and nothing to tidy. + +That is on purpose. The edges come from message names, so a stored layout would +be a second, weaker description of the same thing, and something to merge when +two people edit the same flow. It also keeps flows honest: a graph nobody can +hand-arrange is one worth splitting. + +What is drawn: + +- **Your nodes**, with a status dot — running, succeeded, failed, or nothing at + all when idle. Status is always named in words too, never colour alone. +- **Edges**, which pulse when a value goes down them. Click one to inspect it. +- **Endpoints** — anything wired into this flow that is not a node in it: a + dashboard control writing a message, a tile reading one, a node in another + flow on the far side of a dotted name. Drawn as labelled boxes at the + boundary, so a value never appears to come from nowhere. + +Values are live. The editor holds a websocket to the engine, and the connection +banner tells you when it is not. + +## Adding a node + +**Add node** on the dock, or ⌘K / Ctrl-K for the command palette, which also +jumps between flows and offers your shared nodes. Pick a type and it appears on +the canvas with its panel open. + +## The node panel + +Floats on the right; a full-screen sheet on a phone. + +**Consumes** and **Provides** are the ports. Each row is a message name, a +payload type, and — on an output — a *streaming* toggle for ports that publish +repeatedly during one execution. The name field suggests messages already in +the flow, which is usually how you wire something: type the name that already +exists. + +Renaming an output renames the message everywhere it is read. Inputs are not +renamed, because an input is more often re-pointed at a different message than +renamed. + +**Settings** is generated from the node type's parameter schema — so an MQTT +node shows broker, topic and QoS, and an ntfy node shows server, topic and +priority. A field marked as a credential renders as a picker over your stored +[secrets](operations.md) instead of a text box. + +A Function node gets a second settings section where you add your own: a +setting is a constant of this node's code and arrives as an argument beside its +ports. See [Where a node's values come from](../concepts/values.md). + +**Timeout** (Function nodes) is how many seconds the code may run before it is +stopped. Once a node streams, this becomes an *idle* timeout: it measures +silence, not duration, so a node that yields every few seconds can run for +hours under a timeout of 300. + +**Code** is the editor. It saves as you type and applies on ⌘S; the node +reloads without the flow stopping. A node file defines `process(...)`; if it +defines exactly one public function under another name, that one is used +instead. + +**Shared** turns a node's code into a library entry other flows can reuse. +Editing a shared node edits the copy every flow using it runs, which is the +point and also the caution. + +!!! note "Not in the panel (yet)" + + `device` and `device_policy` — which machine a node's code runs on — are set + through the API rather than the panel. See + [Remote workers](../code/workers.md). + +## The flow panel + +Click the canvas title to open the flow's own settings. + +- **Running** — the switch that starts and stops the flow. Stopped means none + of its subscriptions, schedules or webhooks exist. +- **Mode** — `live` or `batch`. Live reacts to what arrives; batch only runs + when a run asks it to. See [Runs](../concepts/runs.md). +- **Inputs** — messages the flow takes from outside, with the value each starts + from. For a batch flow these are also the run's parameters. +- **Outputs** (batch only) — which messages a run reports as its result. +- **Contents** — a reminder that this flow's name namespaces every message + inside it. + +## Inspecting an edge + +Click a wire. You get the last payload that went down it, when, and a sparkline +of its recent numeric history. You can also republish that value from the node +that produced it, which re-runs everything downstream — the fastest way to test +a change without waiting for the real sensor. + +## The dock + +Along the bottom, floating over the canvas. + +| Control | What it does | +|---|---| +| **Add node** | opens the palette | +| **Fit** | returns to the view the flow opened with | +| **Run** | injects a value by hand; on a batch flow, opens the parameters dialog | +| **Pause / Resume** | holds messages instead of running them | +| **Step** | releases exactly one held message | +| **Publish** | puts the draft live | +| **Discard** | throws the draft away | +| **Issues** | what is stopping this flow from running, per node | +| **Logs** | what the nodes printed, filterable | + +Pause and step together are how you test a flow before it moves anything +physical: pause it, inject a value, step it through, watch the values appear on +the edges, then publish. + +## Shortcuts + +| Chord | Action | +|---|---| +| ⌘K / Ctrl-K | command palette | +| ⌘S / Ctrl-S | publish the flow — or, with focus in the code editor, apply the code | +| ⌘Z / ⌘⇧Z | undo / redo (the flow; the code editor has its own) | +| ⌘C / ⌘V | copy and paste nodes, including between flows | + +## Drafts and publishing + +Saving writes a draft; the engine keeps running what was published. Publishing +promotes it and reloads. + +If someone else saved the same flow while you were working, publishing answers +with a conflict rather than overwriting them. The Flows list offers **Publish +all changes** when several flows are sitting on drafts. + +## Issues + +The canvas validates as you edit and marks the node each issue belongs to: + +- a port that needs a message nothing in reach provides +- a message that exists but has never held a value +- a dependency cycle +- a node reading a message it also writes, with nothing to start it from +- code that did not load +- a webhook with no shared secret (advisory — it does not stop the flow) + +A flow with any of these except the last does not run, and the health summary +on Home counts it. + +## See also + +- [Flows, nodes and messages](../concepts/flows.md) — the model behind the canvas +- [Node types](../reference/node-types.md) — everything the palette offers +- [Writing node code](../code/nodes.md) — what goes in the editor diff --git a/docs/interface/index.md b/docs/interface/index.md new file mode 100644 index 0000000..df99f09 --- /dev/null +++ b/docs/interface/index.md @@ -0,0 +1,88 @@ +# The dashboard app + +The web interface is a single-page app served at `app.${DOMAIN}` — or, for an +installation reached through a portal, at `${DOMAIN}/i/{installation-id}`. +Either way it is the same application, and it is a client of the same REST API +you can script against. + +Sign in with the account the installation was created with. On a fresh +installation that account was printed once, on the first start. + +## The shell + +A floating sidebar on the left, the selected screen filling the rest. On a +phone the sidebar collapses to a sheet. + +| Entry | What lives there | +|---|---| +| **Home** | the brain graph, health, and everything that recently happened | +| **Flows** | the list of flows, and the canvas for each | +| **Dashboards** | the widget canvases, and the panels that display them | +| **Secrets** | credentials your nodes reference without holding | +| **Modules** | the Python packages your node code may import | +| **Alerts** | where failures get sent | +| **Admin** | users (superusers only) | +| **Settings** | your account, appearance, and remote access | + +## Home + +The one screen you leave open. Three things share it. + +### The brain graph + +Every flow drawn as a neuron, wired to the flows it exchanges messages with. +This is the brand mark made live, and it is also the fastest read on the +installation: a neuron pulses when its flow is running work, and its ring turns +terracotta when the flow cannot run as written. A neuron with a problem keeps +its label showing so you can see which one it is without hovering. + +Each flow also has a switch beside it in the list, which starts and stops it. + +### Health + +Always answers, degraded or not. The tiles cover: + +- **Flows** — total, running, paused, quarantined, and how many cannot run + because their graph does not validate +- **Nodes** — how many failed to load +- **Queue** — depth, and how old the oldest pending item is +- **Loop lag** — whether the engine's event loop is keeping up + +`status: degraded` comes with a list of named problems, in words. "3 flow(s) +cannot run: house, pv, hallway" is more useful than a red dot, so that is what +it says. + +### Activity + +Charts of executions and failures over the selected range (1h / 6h / 24h / 7d), +with the recent runs, recent failures, dead-lettered work and the audit trail +underneath. + +The charts are scrubbable: hover a moment and the lists below filter to it, +click to hold it while you read. That turns "something went wrong around two +o'clock" into the actual rows. + +## Flows + +The list shows each flow's title, node count, whether it has unpublished +changes, and whether it is enabled, paused or quarantined. The toolbar searches, +creates, and offers **Publish all changes** when several flows have drafts. + +Opening one takes you to [the flow editor](flow-editor.md). + +## Everything else + +- [The flow editor](flow-editor.md) — the canvas, the code editor, running and + testing +- [Dashboards and panels](dashboards.md) — widgets, bindings, and hanging a + screen on a wall +- [Secrets, modules and alerts](operations.md) — the three screens that keep an + installation running +- [Accounts and the portal](portal.md) — reaching an installation from outside + its network + +## Appearance + +Light and dark follow your system by default; **Settings → Appearance** +overrides it. Both themes are first-class — the wall-panel view in particular is +designed to be legible in dark from across a room. diff --git a/docs/interface/operations.md b/docs/interface/operations.md new file mode 100644 index 0000000..3dc6a6f --- /dev/null +++ b/docs/interface/operations.md @@ -0,0 +1,131 @@ +# Secrets, modules and alerts + +Three screens that have nothing to do with each other except that an +installation you actually depend on needs all of them. + +## Secrets + +A node never holds a credential. A parameter written as + +```json +{"$secret": "influx-token"} +``` + +is replaced with the stored value when the node is built, and the editor +renders any field a node type marks as a credential as a picker over what you +have stored. + +**Secrets** is where the values live. Add a name and a value; the value is +never shown again, and the list only ever shows names. + +They are encrypted at rest with a key derived from the installation's +`SECRET_KEY`, and kept **outside** the flow repository. That matters because +flows are a git repository you may well push somewhere: what gets committed and +shared never contains a password. + +A name that no longer resolves is a clear error on the node that wanted it — +"No secret named 'x' — add it under Secrets" — rather than a mysterious +authentication failure. + +!!! warning "Rotating `SECRET_KEY`" + + The secrets store is encrypted with a key derived from it. Change + `SECRET_KEY` and the store no longer decrypts. Re-enter the secrets, or + plan the rotation properly. + +## Modules + +Node code runs in a virtual environment of its own, on the installation's data +volume — separate from the packages Fluksio itself runs on. A pin of yours can +never shadow one of ours, and vice versa. + +**Modules** is a pip manifest and an **Apply** button. It is installed with +`uv pip sync`, so a line you delete is uninstalled rather than left behind. +The manifest is versioned alongside your flows, so what a deployment installed +is recorded with what uses it. + +The page shows what is currently installed, the interpreter's Python version, +and whether the manifest and the environment agree. A failed resolve comes back +as `uv`'s own output, in the pane, because that is the only thing anyone can +act on. + +An install takes effect immediately — the worker processes pick up the new +interpreter state without the engine restarting. + +!!! note "Big scientific stacks" + + Installing torch into this venv works and is often the simplest thing. The + alternative, when you already have an environment you would rather not + duplicate, is to attach it as a worker: `fluksio-worker --python + /path/to/venv/bin/python`. See [Remote workers](../code/workers.md). + +## Alerts + +Everything that goes wrong already travels the engine's event bus. Alerts is +where you say who hears about it. + +### Channels + +| Kind | Settings | +|---|---| +| **ntfy** | server, topic, and a token for a protected topic | +| **SMTP** | an address to send to (the installation's mail settings do the rest) | +| **Webhook** | a URL to POST to | +| **Dashboard** | a message name a notification widget reads | + +Any channel setting may hold a `{"$secret": "name"}` reference instead of a +literal, so a webhook URL with a token in it does not sit in the config. + +### Rules + +A rule is a set of events and the channels they go to. **A rule with no events +ticked covers all of them**, including ones added in later versions — which is +usually what you want for the first rule you write. + +| Event | Fires when | +|---|---| +| `node_error` | a node failed | +| `node_health` | a connection dropped | +| `flow_quarantined` | a flow crashed often enough that the engine stopped restarting it | +| `task_crashed` | a background task crashed | +| `engine_degraded` | the engine is struggling | +| `cascade_dropped` | work was given up on | +| `queue_unavailable` | the work queue is unreachable | + +**Test** sends one message through a channel so you find out it is misconfigured +now rather than at three in the morning. + +### What it mostly does is not send + +This is the part worth knowing before you decide it is broken: + +- the same fault repeating is **one** alert, with a cooldown (15 minutes by + default per rule) +- a connection flapping is one story, not one alert per transition +- however bad it gets, **ten alerts an hour** is the ceiling + +An engine that is genuinely on fire produces a handful of messages, not +thirty-six thousand. + +## Admin: users + +Superusers get an **Admin** screen: add, edit and remove accounts, and see who +is a superuser. + +Accounts that arrived through a portal are badged *Portal*. They have no +password — they sign in at the portal, not here — and they are never +superusers. Deleting one cuts their access immediately and independently of the +portal, so it works even if the portal cannot be reached. See +[Accounts and the portal](portal.md). + +The same screen manages OAuth clients, which is how [agents](../code/agents.md) +are registered. + +## Settings + +Your own account: name, email, password, and appearance (light, dark, or +follow the system). + +**Remote access** is where an installation is paired with a portal, remote +users are admitted, and the link is cut again. That has [its own +page](portal.md). diff --git a/docs/interface/portal.md b/docs/interface/portal.md new file mode 100644 index 0000000..1703869 --- /dev/null +++ b/docs/interface/portal.md @@ -0,0 +1,117 @@ +# Accounts and the portal + +Fluksio is self-hosted by default. An installation runs offline, keeps its data +on its own disk, and never contacts anything unless you tell it to. + +The **portal** is optional, and it exists to solve two specific problems: + +1. **Your machine has no inbound route.** A homelab behind CGNAT, a cluster + node with no open ports, a laptop. Opening one is work, and often not + allowed. +2. **You want a browser on it anyway.** A `pip install fluksio` installation + has no web server for the dashboard at all. + +An enrolled installation dials *out* to the portal and holds one websocket +open. The portal serves the dashboard from its own side, and only the API calls +travel down the tunnel — so the interface loads at portal speed and your +machine stays unreachable from the internet. + +## Enrolling + +Two halves, deliberately: whoever performs the second step decides what the +installation's owner gets. + +**On the portal** (`hub.${DOMAIN}`, or [fluksio.com](https://fluksio.com) for +the hosted one): **Installations → Add installation**, give it a name, and copy +the code. + +**On the installation**, either from the dashboard: + +> **Settings → Remote access**, enter the portal URL and the code, press +> **Connect**. + +or from the command line, which is the path for an installation with no web +interface of its own: + +```sh +fluksio enroll ABCD-1234 --portal https://hub.fluksio.com +``` + +The code is single-use and expires in fifteen minutes. + +A portal session then arrives as *that local account* — the settings screen +states this plainly, because it is the whole security model in one sentence. +Use `--as someone@example.com` to enrol as a specific local account when the +installation has several superusers. + +Once enrolled, the installation appears under **Installations** with its +status, when it was last seen and its version. **Open** takes you to its +dashboard at `${DOMAIN}/i/{installation-id}`. + +## What the portal can and cannot do + +The portal holds one credential for your installation and proxies requests down +the tunnel. What those requests may do is decided **on the installation**, by +the same checks a local session passes. + +The trust anchor is a signing keypair on the portal. Every installation pins +its public half at enrolment and rejects anything else — which is what stops a +hijacked DNS entry or a mis-issued certificate from impersonating the portal. + +## Letting someone else in + +Anyone else on the portal reaches your installation only if a superuser there +admits them, and they arrive as a local user of their own rather than as you. + +1. **They**: **Installations → Join an installation**, and copy the code. It is + bound to their portal account and expires in fifteen minutes. +2. **You**, on the installation: **Settings → Remote access → Add remote + user**, and enter the code. +3. They now see the installation under **Installations**, marked *Shared by*, + with **Open** and nothing else. Renaming, re-keying and removing stay with + you. + +The installation redeems that code against the portal using its own credential. +A portal session cannot do this — which is what stops somebody you let in from +letting others in. + +On the installation they appear under **Admin → Users**, badged *Portal*, never +a superuser and with no password. + +## Cutting it off + +| From | Action | Effect | +|---|---|---| +| The portal | **New code** | rotates the credential and drops the current link | +| The portal | **Remove** | deletes the registration and cuts the connection | +| The installation | **Disconnect** | unilateral and immediate — the portal's tokens stop verifying here whatever the portal still has on file | +| The installation | delete a user under **Admin → Users** | that one person, immediately, independent of the portal | + +**New code** also cuts every credential the portal minted for this installation, +including [wall panels paired through it](dashboards.md#a-screen-somewhere-you-cannot-reach). + +The installation's own **Disconnect** is the one to reach for if you are ever +unsure: it does not need the portal's cooperation. + +## Running your own portal + +The portal is the `index` stack's `hub` service — accounts, the registry of +connected installations, and the websocket each one dials in on. Two things +about it are load-bearing: + +- **It runs a single process.** It keeps its attached installations in the + memory of the process holding their sockets, so a second worker would answer + for links it does not hold. Scaling out needs a routing layer first. +- **Back up the signing keypair with the database.** Replacing it forces every + installation to be enrolled again. + +Websocket upgrades must be enabled on the `hub` hostname in whatever proxy +fronts it. Without them every installation sits in a reconnect loop and the +portal shows them all offline. + +## See also + +- [The `fluksio` command](../code/cli.md) — `fluksio enroll`, and what it writes +- [Dashboards and panels](dashboards.md) — pairing a wall screen through the portal +- [Getting started: data science](../getting-started/data-science.md) — the + shortest path from a pip install to a browser diff --git a/docs/llms.txt b/docs/llms.txt new file mode 100644 index 0000000..e36af6a --- /dev/null +++ b/docs/llms.txt @@ -0,0 +1,38 @@ +# Fluksio + +> Fluksio is a node-based automation engine. A flow is a graph of small +> functions wired by message name, kept running by one resident engine. The +> same engine serves two shapes of work: live flows that never end (buildings, +> labs, homelabs) and batch runs that finish and leave a record (experiments, +> ML pipelines, CI-style jobs). Self-hosted by default; an optional portal +> exists only to reach an installation that has no inbound route. + +## Getting started +- [Pick your starting point](/getting-started/): the two setup paths and how to choose +- [Data science](/getting-started/data-science/): pip install, wrap a training script, runs and sweeps +- [Facility automation](/getting-started/facility-automation/): Docker stack, MQTT to dashboard, InfluxDB, workers + +## Concepts +- [Flows, nodes and messages](/concepts/flows/): the graph comes from message names, not drawn wires +- [Where a node's values come from](/concepts/values/): ports, settings, flow inputs +- [Keeping state in a flow](/concepts/state/): a message a node both reads and writes +- [Runs: pipelines that finish](/concepts/runs/): batch mode, metrics, artifacts, sweeps, durability + +## The interface +- [The dashboard app](/interface/): the shell, Home, health and activity +- [The flow editor](/interface/flow-editor/): canvas, node panel, dock, shortcuts, drafts +- [Dashboards and panels](/interface/dashboards/): widgets, typed bindings, wall screens +- [Secrets, modules and alerts](/interface/operations/): credentials, packages, failure routing +- [Accounts and the portal](/interface/portal/): enrolment, remote users, revoking + +## Code and the CLI +- [The fluksio command](/code/cli/): serve, enroll, worker, and the data directory +- [Writing node code](/code/nodes/): process(), generators, fluksio.emit, artifacts +- [The HTTP API](/code/api/): auth, flows, messages, runs, observability, websocket +- [Remote workers](/code/workers/): fluksio-worker, device labels, artifacts across hosts +- [Agents over MCP](/code/agents/): the tool list and the security model + +## Reference +- [Node types](/reference/node-types/): every built-in type and its settings +- [Payload types](/reference/payload-types/): float, int, str, bool, json, record, list, series, artifact +- [Configuration](/reference/configuration/): every environment variable diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md new file mode 100644 index 0000000..63ec324 --- /dev/null +++ b/docs/reference/configuration.md @@ -0,0 +1,127 @@ +# Configuration + +Every setting comes from the environment, or from an env file. Which file +depends on how the installation was started: + +| Started with | Reads | +|---|---| +| `fluksio serve` | `env` inside the data directory (`$FLUKSIO_ENV_FILE`) | +| the Docker stack | `.env` beside `docker/` | + +Anything already exported wins over the file. + +## Storage + +| Variable | Default | Notes | +|---|---|---| +| `DATA_DIR` | `flow-data` (`~/.fluksio` via the CLI) | everything below derives from this | +| `DATABASE_URL` | SQLite in `DATA_DIR` | any SQLAlchemy URL | +| `FLOWS_DIR` | `$DATA_DIR/flows` | the git repository holding flows | +| `SECRETS_FILE` | `$DATA_DIR/secrets.enc` | encrypted credentials, deliberately outside the repo | +| `ALERTS_FILE` | `$DATA_DIR/alerts.json` | channels and rules | +| `PANELS_FILE` | `$DATA_DIR/panels.json` | wall-panel pairings | +| `OAUTH_PRIVATE_KEY_FILE` | `$DATA_DIR/oauth-key.pem` | signs agent and worker tokens | +| `CLOUD_CONFIG_FILE` | `$DATA_DIR/cloud.json` | the portal enrolment, if any | + +Set `DATA_DIR` and the rest follow. Set one explicitly and it wins — which is +what the container images do to pin everything onto `/data`. + +!!! warning "The four files that must be on persistent storage" + + `secrets.enc`, `alerts.json`, `panels.json` and `oauth-key.pem` are written + at runtime. In a container, anything not on a volume lands in the writable + layer and is lost on the next rebuild — un-pairing every screen and + revoking every agent. + +## State + +| Variable | Default | Notes | +|---|---|---| +| `REDIS_HOST` | unset | without it, flow state lives in memory and does not survive a restart | +| `REDIS_PORT` | `6379` | | + +Flow state is the last value of every message, node memory, and the run queue. +Redis here is persistence, not a cache — run it with append-only persistence +on. + +## Identity and access + +| Variable | Default | Notes | +|---|---|---| +| `SECRET_KEY` | generated | signs sessions and derives the secrets-store key | +| `ACCESS_TOKEN_EXPIRE_MINUTES` | `11520` (8 days) | | +| `FIRST_SUPERUSER` | — | absent means the CLI creates one on first run | +| `FIRST_SUPERUSER_PASSWORD` | — | absent means one is generated and printed once | +| `DOMAIN` | `localhost` | what the API and OAuth issuer are built from | +| `FRONTEND_HOST` | `http://localhost:5173` | used in mails, OAuth metadata and panel pairing links | +| `BACKEND_CORS_ORIGINS` | `[]` | comma-separated; `FRONTEND_HOST` is always allowed | + +!!! danger "Rotating `SECRET_KEY`" + + The secrets store is encrypted with a key derived from it. Change it and + the store stops decrypting, and every session is signed out. Re-enter your + secrets, or plan the rotation properly. + +## Environment + +| Variable | Default | Notes | +|---|---|---| +| `ENVIRONMENT` | `local` | `local`, `staging` or `production` | +| `PRIVATE_API_ENABLED` | `false` | unauthenticated test-only endpoints; needs `ENVIRONMENT=local` too | + +`production` closes `/docs`, `/redoc` and the OpenAPI document, because the +schema enumerates every endpoint the installation serves — including the paths +webhook nodes mounted at runtime. It also turns a `changethis` secret from a +warning into a refusal to start. + +## The engine + +| Variable | Default | Notes | +|---|---|---| +| `FLOW_MAX_WORKERS` | `4` | node-code subprocesses run in parallel | +| `FLOW_NODE_TIMEOUT` | `30.0` | seconds a node may run, unless it sets its own | +| `OBS_RETENTION_DAYS` | `30` | how long metrics, events and run records are kept | + +## Agents + +| Variable | Default | Notes | +|---|---|---| +| `MCP_ENABLED` | `false` | opens the `/mcp` endpoint **and** OAuth client registration | +| `MCP_TOKEN_EXPIRE_MINUTES` | `60` | an agent's token is a bearer secret held by a program | +| `MCP_REFRESH_EXPIRE_DAYS` | `30` | | +| `OAUTH_CODE_EXPIRE_SECONDS` | `60` | | + +See [Agents over MCP](../code/agents.md). + +## Mail + +Needed for password-reset mails. Without `SMTP_HOST` and `EMAILS_FROM_EMAIL`, +mail is simply off. + +| Variable | Default | +|---|---| +| `SMTP_HOST` | — | +| `SMTP_PORT` | `587` | +| `SMTP_USER` / `SMTP_PASSWORD` | — | +| `SMTP_TLS` / `SMTP_SSL` | `true` / `false` | +| `EMAILS_FROM_EMAIL` | — | +| `EMAILS_FROM_NAME` | `Fluksio` | +| `EMAIL_RESET_TOKEN_EXPIRE_HOURS` | `48` | + +## Monitoring + +| Variable | Default | Notes | +|---|---|---| +| `SENTRY_DSN` | — | error reporting, if you want it | + +## Health check + +`GET /api/v1/utils/health/` is a *deep* check: it fails when the event loop is +wedged or the state backend is gone, not just when the process is up. That is +what the container healthcheck probes, and what an autoheal sidecar restarts +on. + +## See also + +- [The `fluksio` command](../code/cli.md) — what the data directory holds +- [Getting started: facility automation](../getting-started/facility-automation.md) diff --git a/docs/reference/node-types.md b/docs/reference/node-types.md new file mode 100644 index 0000000..d0ce49c --- /dev/null +++ b/docs/reference/node-types.md @@ -0,0 +1,286 @@ +# Node types + +Every type the canvas can place. Each is configured by filling in a form the +editor generates from its parameter schema, so they all behave the same way. + +Anything here could be written as a **Function** node — that is what the +function node is for. These exist because the same handful of shapes account +for most of a real installation, and a rule you fill in is easier to read on a +canvas, and to change, than five lines of code repeated eighty times. + +`GET /flows/node-types` returns this list with each type's full parameter +schema, including any [connectors](#connectors) installed on your instance. + +## Function + +**`python`** — your own code, run on every incoming message. + +The one type with a source file. Its arguments are its input ports and its own +settings; its return value is a dict keyed by output ports. See +[Writing node code](../code/nodes.md). + +Settings on a Function node are free-form: you add them by name, and they +arrive as keyword arguments. + +## Integrations + +### MQTT + +**`mqtt`** — subscribe to topics, or publish what arrives on its inputs. + +A node with *outputs only* subscribes; a node with *inputs* publishes. + +| Setting | Default | Notes | +|---|---|---| +| `topic` | `*` | one topic for every port, or `{"message": "some/topic"}` per port | +| `broker_host` | `localhost` | | +| `broker_port` | `1883` | | +| `username` / `password` | — | `password` takes a secret reference | +| `client_id` | — | | +| `qos` | `0` | 0, 1 or 2 | +| `retain` | `false` | on published messages | +| `keepalive` | `60` | seconds | + +Nodes sharing a broker share one connection. + +### HTTP + +**`http`** — receive data on a webhook, or send it to a URL. + +Outputs only makes it a **webhook**: the engine mounts a route at +`/hooks/{flow}/{url}/{secret}` while the flow runs. Inputs make it a **sender**. + +| Setting | Default | Notes | +|---|---|---| +| `url` | — | the route path in webhook mode, the full URL in sender mode | +| `method` | `POST` | `GET` or `POST` | +| `timeout` | `30` | seconds, sender mode | +| `headers` | `{}` | | +| `secret` | — | shared secret appended to the webhook URL; takes a secret reference | + +!!! warning "A webhook with no secret is open to anyone who can reach the host." + + The canvas flags this as an advisory issue rather than stopping the flow, + because a webhook on a private network is a legitimate thing to want. + +### InfluxDB + +**`influxdb`** — write measurements to a bucket, or read them back. + +| Setting | Notes | +|---|---| +| `url`, `token`, `org`, `bucket` | connection; `token` takes a secret reference | +| `write_precision` | `ns`, `us`, `ms` (default) or `s` | +| `query_range` | default window for queries, e.g. `-1h` | +| `writes` | per-input: `measurement`, `field`, `tags` | +| `queries` | per-output: `measurement`, `field`, `tags`, `range`, `aggregation` | + +```json +{ + "writes": { + "living_temperature": { + "measurement": "environment", + "field": "temp_c", + "tags": {"room": "living"} + } + } +} +``` + +**Query passthrough.** An incoming message holding a `flux` key is run as +written, and the rows come back on the first output port as +`{"rows": [{ts, value, field, measurement, tags}], ...}`. + +That is what keeps a database node a database node: it holds the credentials +and the connection and nothing else. Building a query and shaping its rows are +ordinary Function nodes on either side, so a dashboard widget never learns which +database answered it. + +### Notification + +**`ntfy`** — push an incoming value to a phone through [ntfy](https://ntfy.sh). + +| Setting | Default | Notes | +|---|---|---| +| `server` | `https://ntfy.sh` | | +| `topic` | — | required | +| `title` | — | | +| `priority` | `default` | `min`, `low`, `default`, `high`, `urgent` | +| `tags` | — | comma-separated ntfy tags | +| `token` | — | for a protected topic; takes a secret reference | + +This is a *flow* deciding something is worth saying. The engine reporting that +it broke is [Alerts](../interface/operations.md#alerts), which is a different +thing configured elsewhere. + +## Timing + +### Inject + +**`inject`** — emit a value on request, on a timer, or when the flow starts. + +| Setting | Default | Notes | +|---|---|---| +| `payload` | current time | what to emit | +| `payloads` | `{}` | per-output-port payloads, keyed by port | +| `interval` | `0` | emit every n seconds; 0 means never on its own | +| `cron` | — | a five-field cron expression | +| `at_start` | `false` | emit once when the flow starts | +| `start_delay` | `1.0` | how long to wait before that first emission | + +The most-placed trigger in a real installation — mostly as a button someone +presses. + +### Delay & schedule + +**`delay`** — hold messages back, limit their rate, or emit on a schedule. + +| Setting | Default | Notes | +|---|---|---| +| `delay` | `0` | seconds to hold each message | +| `interval` | `0` | minimum seconds between forwards | +| `mapping` | `{}` | input port → output port; paired in order when empty | +| `cron` | — | five-field expression | + +Order of operations: rate check → delay → forward. With a `cron` and no inputs +it emits the current time on each tick; with inputs it emits the last value it +received. + +!!! note "Not in a batch flow" + + A rate limit holds a value back for a timer to release, and a run has no + timer — the value would be dropped rather than delayed. Submitting a batch + flow with a rate-limited port is refused instead. + +### Trigger + +**`trigger`** — send one value now and another once things go quiet. + +| Setting | Default | Notes | +|---|---|---| +| `first` | `true` | sent as soon as a value arrives | +| `then` | `false` | sent when the wait expires; empty sends nothing | +| `wait` | `60` | seconds of quiet before the second value | +| `extend` | `true` | a value arriving during the wait starts it over | + +The shape this exists for: *the door opened — turn the light on, and off again +in two minutes unless it opens again.* + +## Logic + +### Switch + +**`switch`** — send a value down one branch or another, by rule. + +Each rule names an output port; a matching value leaves through that port. +Comparisons: `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `contains`, `between`. + +| Setting | Default | Notes | +|---|---|---| +| `rules` | `[]` | checked in order; each names the output it routes to | +| `stop_at_first` | `true` | leave through the first matching rule only | +| `otherwise` | — | output for a value that matched nothing | + +Comparing a string to a number is a mistake in the rule, not a reason to take +the flow down: the rule simply does not match. + +### Change + +**`change`** — scale, offset, round or map a value on its way past. + +| Setting | Default | Notes | +|---|---|---| +| `scale` | `1.0` | multiply numbers by this | +| `offset` | `0.0` | then add this | +| `round_to` | — | decimal places | +| `mapping` | `{}` | replace a value with another, looked up as text | +| `default` | — | value when the lookup misses; empty passes it through | + +### Filter unchanged + +**`rbe`** — pass a value on only when it has actually changed. + +| Setting | Default | Notes | +|---|---|---| +| `deadband` | `0.0` | ignore numeric changes smaller than this | +| `deadband_percent` | `false` | read the deadband as a percentage | + +The cheapest fix for a sensor that reports every second and changes every hour. + +### Join + +**`join`** — gather several inputs into one object or list. + +| Setting | Default | Notes | +|---|---|---| +| `mode` | `object` | `object` or `array` | + +## System + +### Command + +**`exec`** — run a command in the engine's container and read its output. + +| Setting | Default | Notes | +|---|---|---| +| `command` | — | the command to run | +| `append_payload` | `false` | add the incoming value as one argument | +| `timeout` | `30` | seconds | +| `fail_on_error` | `false` | treat a non-zero exit as a node failure rather than output | + +Outputs the command's stdout, stderr and exit code. + +!!! warning "Inside the container, not on the host" + + A flow ported from something that read the host's journal, or poked a host + script, needs either a mount or a small listener on the host side. This + node cannot see the host. + +### File + +**`file`** — read a file into the flow, or write one out of it. + +| Setting | Default | Notes | +|---|---|---| +| `path` | — | relative to the engine's files directory | +| `mode` | `read` | `read`, `write` or `append` | +| `format` | `text` | `text` or `json` | +| `newline` | `true` | end each written record with a newline | + +Confined to a directory the engine owns. A flow that could name any path would +be a way to read the secrets store or overwrite a node's source. + +## Numeric + +### Perceptron + +**`mlp`** — a small neural layer over its numeric inputs. + +`output = weights @ inputs + biases`, with weights drawn from `seed` so a node +reloads identically. + +| Setting | Default | Notes | +|---|---|---| +| `seed` | `0` | | + +Kept as a worked example of numeric logic rather than as a modelling tool. If +you are training something, that is a [batch flow](../concepts/runs.md) and a +Function node. + +## Connectors + +Anything else in the palette came from an installed **connector** package — a +node type written against a published contract and discovered through the +`fluksio.node_types` entry point group. The editor shows which package supplied +it. + +A connector declares its contract version, and one written for a version this +engine does not speak is ignored rather than half-loaded. Installing or +upgrading one takes effect on the next engine restart, because Python does not +re-import a changed module and a rescan would promise more than it delivers. + +## See also + +- [Payload types](payload-types.md) — what a port may carry +- [Writing node code](../code/nodes.md) — the Function node +- [The flow editor](../interface/flow-editor.md) — placing and configuring them diff --git a/docs/reference/payload-types.md b/docs/reference/payload-types.md new file mode 100644 index 0000000..65f37c2 --- /dev/null +++ b/docs/reference/payload-types.md @@ -0,0 +1,129 @@ +# Payload types + +Every port declares a `dtype`, and every value that passes through it is +checked against that declaration. + +This is not decoration. It is what lets the dashboard editor offer you only the +messages a gauge can actually draw, what lets the canvas refuse a binding before +anything runs, and what lets a downstream node know the shape of what it is +getting before the flow starts. + +Everything on the wire is JSON. That is what lets the same value pass through +the state backend, the work queue and the worker protocol unchanged. + +## The scalars + +| `dtype` | Accepts | +|---|---| +| `float` | any number — `int` or `float`, but not `bool` | +| `int` | a whole number, not `bool` | +| `str` | a string | +| `bool` | exactly `true` or `false` | + +`bool` is an `int` subclass in Python and deliberately not a number here: a flag +is not a measurement, and a switch bound to a temperature is a mistake worth +catching. + +## The structured ones + +These are *declared shapes* rather than "some JSON", which is what makes a +widget binding checkable. + +### `record` + +Flat named scalars. + +```json +{"title": "Boiler", "body": "Pressure low", "severity": "warning"} +``` + +Nesting is deliberately out: a record that can contain a record is a schema +language, and the shape stops being readable from the declaration alone. + +Read by the **Notification** widget. It is also what an alert channel of kind +*dashboard* writes. + +### `list` + +Ordered items of one declared shape. The port also declares `item`: + +| `item` | Meaning | +|---|---| +| unset | `record` — what the agenda and forecast widgets read | +| `float`, `int`, `str`, `bool` | a list of scalars | +| `json` | anything | + +A list of lists, or a list of series, is refused. One declared level is the +point. + +### `series` + +Labelled lines of `(timestamp, value)` pairs — what a chart draws. + +```json +{ + "lines": [ + {"label": "living", "points": [[1717000000, 21.4], [1717000060, 21.5]]} + ], + "range": "-24h" +} +``` + +Keys beside `lines` are carried through untouched, which is how a querying +chart puts the window and resolution it asked for on the request and reads them +back off the answer. That is what stops an answer to a *different* question +from overwriting the picture. + +`GET /runs/series/compare` answers in this shape, which is why comparing three +training curves is a widget binding rather than a screen of its own. + +### `artifact` + +A reference to stored bytes. + +```json +{"digest": "sha256:…", "size": 4194304, "media_type": "application/octet-stream", "name": "weights.pt"} +``` + +Binary payloads — tensors, checkpoints, images — never travel as a message. The +bytes go to a content-addressed store and the message carries this. A +thirty-megabyte checkpoint never sits in the state backend, and the reference +stays valid wherever the store is reachable from, including on another machine. + +Node code produces one with `fluksio.save_artifact` and opens one with +`fluksio.load_artifact`. See [Writing node code](../code/nodes.md#bytes-artifacts). + +### `json` + +Anything JSON-serializable. The escape hatch, and the right answer when a +payload genuinely has no fixed shape. + +Reach for it last. A `json` port tells the canvas, the widget picker and the +next author nothing. + +## What a widget will bind to + +| Widget | Accepts | +|---|---| +| Gauge, Chart, Slider, Bar | `float`, `int` | +| Switch | `bool` | +| Agenda, Forecast | `list` | +| Notification | `record` | +| Value | anything | +| Icon | weather strings, booleans and numbers alike | +| Clock, Text | nothing — they bind to no message | + +Enforced on the server as well as in the editor. + +## Type failures + +A value that does not match its port's declaration raises on the node that +published it, naming the port and what arrived. It does not get published, and +it does not reach anything downstream — a wrong value stopping at its source is +much easier to diagnose than one propagating. + +## See also + +- [Flows, nodes and messages](../concepts/flows.md) +- [Node types](node-types.md) +- [Dashboards and panels](../interface/dashboards.md) diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 0000000..9e53374 --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,177 @@ +/* Fluksio brand theming for the documentation site. + * + * Follows the zensical custom color scheme approach + * (https://zensical.org/docs/setup/colors/#custom-color-schemes): override the + * Material `--md-*` CSS variables per [data-md-color-scheme]. Colors mirror the + * design tokens in ../DESIGN.md → ../../DESIGN-GUIDELINES.md: neutral white / + * near-black surfaces carrying one slate-blue accent, with terracotta reserved + * for the single secondary affordance per view. The built-in `default` (light) + * and `slate` (dark) schemes stay as the base so all of Material's derived + * styling keeps working; only the brand-relevant variables are remapped. + * + * NOTE: the modern theme paints a frosted, translucent header via + * `.md-header { backdrop-filter: blur(...); background: var(--md-default-bg-color--light) }`, + * which is the same `bg-card/80 backdrop-blur` shell both frontends float their + * chrome on. `--md-default-bg-color--light` therefore has to stay a + * *translucent* alpha of the background — an opaque value stops the header + * being see-through. + */ + +/* Slate-blue brand primary, shared by both schemes. + * + * `#59849b` is the identity blue in the wordmark, and deliberately not used + * here: it measures 4.04:1 against white, below WCAG AA for normal text. The + * values below are the same ones the apps ship as `--primary`. */ +[data-md-color-scheme="default"], +[data-md-color-scheme="slate"] { + --md-primary-fg-color: #4a7189; + --md-primary-fg-color--light: #7ba3b8; + --md-primary-fg-color--dark: #395767; + --md-primary-bg-color: #ffffff; + --md-primary-bg-color--light: rgba(255, 255, 255, 0.7); + + /* The one secondary accent. Used for the facility lane on Getting started + * and nowhere else, mirroring "one --brand-secondary affordance per view". */ + --fluksio-brand-secondary: #de8f6e; + + /* Footer stays dark in both schemes (Material convention), neutral tokens. */ + --md-footer-bg-color: #1a1a1a; + --md-footer-bg-color--dark: #0a0a0a; +} + +/* Light: white page, near-black text (--foreground is #333232, not pure black). */ +[data-md-color-scheme="default"] { + --md-default-bg-color: #ffffff; + --md-default-bg-color--light: rgba(255, 255, 255, 0.7); + --md-default-bg-color--lighter: rgba(255, 255, 255, 0.3); + --md-default-bg-color--lightest: rgba(255, 255, 255, 0.12); + + --md-default-fg-color: #333232; + --md-default-fg-color--light: #6b6b6b; + --md-default-fg-color--lighter: #9e9e9e; + --md-default-fg-color--lightest: #e4e4e4; + + --md-accent-fg-color: #395767; + --md-typeset-a-color: #4a7189; + + --md-code-bg-color: #f2f2f2; + --md-code-fg-color: #333232; +} + +/* Dark: near-black page, off-white text. Surfaces lift to #1a1a1a (--card) so + * code and quoted blocks read against the #0a0a0a backdrop. */ +[data-md-color-scheme="slate"] { + --md-default-bg-color: #0a0a0a; + --md-default-bg-color--light: rgba(10, 10, 10, 0.7); + --md-default-bg-color--lighter: rgba(10, 10, 10, 0.3); + --md-default-bg-color--lightest: rgba(10, 10, 10, 0.12); + + --md-default-fg-color: #f5f5f5; + --md-default-fg-color--light: #a3a3a3; + --md-default-fg-color--lighter: #6b6b6b; + --md-default-fg-color--lightest: #2a2a2a; + + --md-accent-fg-color: #7ba3b8; + --md-typeset-a-color: #7ba3b8; + + --md-code-bg-color: #1a1a1a; + --md-code-fg-color: #f5f5f5; + + --fluksio-brand-secondary: #e5a184; +} + +/* The theme highlights the active sidebar entry with a + * `--md-accent-fg-color--transparent` fill, which is still the built-in indigo + * because only the opaque accent is remapped above. Drop the fill in both + * schemes; the brand accent colour and the heavier weight the theme also + * applies keep the selection visible. */ +.md-nav--primary .md-nav__item .md-nav__link--active { + background: none; +} + +/* Radius follows the app's map: 16px for panels and cards, 12px for the + * smaller inline surfaces. */ +.md-typeset .admonition, +.md-typeset details, +.md-typeset pre > code, +.md-typeset .tabbed-content pre > code { + border-radius: 16px; +} + +.md-typeset code { + border-radius: 12px; +} + +/* Tables are the one place these docs go wide. Nothing may scroll the page + * sideways, so the box scrolls instead — DESIGN-GUIDELINES.md → Responsive. */ +.md-typeset .md-typeset__table { + overflow-x: auto; +} + +/* ── Audience cards ──────────────────────────────────────────────── + * + * Used on Getting started, where the split mirrors the landing page's audience + * switch: data science takes --primary, facility automation takes the one + * terracotta accent. Built out of Markdown lists via attr_list + md_in_html, + * so there is no HTML soup in the page source. + */ +.fluksio-lanes { + display: grid; + gap: 1rem; + margin: 1.5rem 0; +} + +@media screen and (min-width: 768px) { + .fluksio-lanes { + grid-template-columns: 1fr 1fr; + } +} + +.fluksio-lane { + border: 1px solid var(--md-default-fg-color--lightest); + border-radius: 16px; + padding: 1.25rem 1.5rem; + /* min-width:0 keeps a long line from stretching the grid track and, with it, + * the page. */ + min-width: 0; +} + +.fluksio-lane > :first-child { + margin-top: 0; +} + +.fluksio-lane > :last-child { + margin-bottom: 0; +} + +.fluksio-lane--science { + border-color: var(--md-primary-fg-color); +} + +.fluksio-lane--science h3 { + color: var(--md-primary-fg-color); +} + +.fluksio-lane--facility { + border-color: var(--fluksio-brand-secondary); +} + +.fluksio-lane--facility h3 { + color: var(--fluksio-brand-secondary); +} + +/* ── Brand mark ──────────────────────────────────────────────────── + * + * The mark's disc is painted `--foreground`, so the light file is a dark disc + * and disappears on the dark header. The theme renders the logo as an , + * whose src CSS cannot swap — so the image is hidden under `slate` and the + * anchor paints the dark mark instead. Both files are the same 24×24 viewBox, + * so the two look identical apart from that inversion. + */ +[data-md-color-scheme="slate"] .md-logo img { + visibility: hidden; +} + +[data-md-color-scheme="slate"] .md-logo { + background: url("../assets/favicon-dark.svg") center / 1.2rem 1.2rem no-repeat; +} diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 0000000..650bdea --- /dev/null +++ b/zensical.toml @@ -0,0 +1,124 @@ +# Fluksio documentation site, built with zensical (https://zensical.org). +# Content lives in docs/, the static build goes to site/ (gitignored). +# Build: `make docs` | Live preview: `make docs-serve` +# This config is zensical-native (TOML). It is portable to Material for MkDocs +# by translating to mkdocs.yml if the engine ever needs to change. + +[project] +site_name = "Fluksio" +site_description = "Documentation for Fluksio, node-based automation for experiments and for buildings." +site_url = "https://docs.fluksio.com/" +# Footer line. HTML renders as-is, so the legal links (mirroring the index +# landing footer) and a link back to the main site live here. +copyright = """Copyright © 2026 Fluksio ·
fluksio.com · Impressum · Privacy Policy""" + +# Fluksio brand color schemes (slate blue on neutral white/near-black, with +# terracotta as the one secondary accent), layered over the built-in +# default/slate schemes. See docs/stylesheets/extra.css. +extra_css = ["stylesheets/extra.css"] + +# Site navigation. Must stay above the first [project.*] table header, or TOML +# would nest it inside that table and zensical would silently fall back to +# auto-generated (alphabetical) navigation. +nav = [ + { "Home" = "index.md" }, + { "Getting started" = [ + { "Pick your starting point" = "getting-started/index.md" }, + { "Data science" = "getting-started/data-science.md" }, + { "Facility automation" = "getting-started/facility-automation.md" }, + ] }, + { "Concepts" = [ + { "Flows, nodes and messages" = "concepts/flows.md" }, + { "Where a node's values come from" = "concepts/values.md" }, + { "Keeping state in a flow" = "concepts/state.md" }, + { "Runs: pipelines that finish" = "concepts/runs.md" }, + ] }, + { "The interface" = [ + { "The dashboard app" = "interface/index.md" }, + { "The flow editor" = "interface/flow-editor.md" }, + { "Dashboards and panels" = "interface/dashboards.md" }, + { "Secrets, modules and alerts" = "interface/operations.md" }, + { "Accounts and the portal" = "interface/portal.md" }, + ] }, + { "Code and the CLI" = [ + { "The fluksio command" = "code/cli.md" }, + { "Writing node code" = "code/nodes.md" }, + { "The HTTP API" = "code/api.md" }, + { "Remote workers" = "code/workers.md" }, + { "Agents over MCP" = "code/agents.md" }, + ] }, + { "Reference" = [ + { "Node types" = "reference/node-types.md" }, + { "Payload types" = "reference/payload-types.md" }, + { "Configuration" = "reference/configuration.md" }, + ] }, + # The roadmap, the design guidelines and the architecture material stay in + # the repositories — repo-only, intentionally not published. +] + +# Drop the "Made with Zensical" generator notice from the footer. +[project.extra] +generator = false + +[project.theme] +language = "en" +# The brand mark the dashboard SPA uses (frontend/public/favicon.svg), as both +# the browser favicon and the header logo, so the docs match the app. +favicon = "assets/favicon.svg" +logo = "assets/favicon.svg" +features = [ + "content.code.copy", + "navigation.footer", + "navigation.indexes", + "navigation.instant", + "navigation.sections", + "navigation.top", + "navigation.tracking", + "search.highlight", + "toc.follow", +] + +# Body font mirrors both frontends (Inter); code font stays the theme default, +# which matches the app's system monospace stack. +[project.theme.font] +text = "Inter" + +# Three states, the same as both frontends: follow the system, then an explicit +# light and an explicit dark the toggle cycles through. +[[project.theme.palette]] +media = "(prefers-color-scheme)" +toggle.icon = "lucide/monitor-cog" +toggle.name = "Follow the system theme" + +[[project.theme.palette]] +media = "(prefers-color-scheme: light)" +scheme = "default" +toggle.icon = "lucide/sun" +toggle.name = "Switch to dark mode" + +[[project.theme.palette]] +media = "(prefers-color-scheme: dark)" +scheme = "slate" +toggle.icon = "lucide/moon" +toggle.name = "Switch to light mode" + +# Markdown extensions: admonitions, grids (attr_list + md_in_html), code blocks +# with highlighting and copy, content tabs, collapsible details, mermaid. +[project.markdown_extensions.abbr] +[project.markdown_extensions.admonition] +[project.markdown_extensions.attr_list] +[project.markdown_extensions.def_list] +[project.markdown_extensions.footnotes] +[project.markdown_extensions.md_in_html] +[project.markdown_extensions.toc] +permalink = true +[project.markdown_extensions.pymdownx.details] +[project.markdown_extensions.pymdownx.highlight] +anchor_linenums = true +[project.markdown_extensions.pymdownx.inlinehilite] +[project.markdown_extensions.pymdownx.superfences] +custom_fences = [ + { name = "mermaid", class = "mermaid", format = "pymdownx.superfences.fence_code_format" } +] +[project.markdown_extensions.pymdownx.tabbed] +alternate_style = true