Files
app/Makefile
T
stroblmeandClaude Fable 5 19bc2810cf Add a Python SDK: flows declared in your own repository
A data scientist keeps their code where it is and decorates it: `@node`
declares a function's ports beside the function, `Flow(name, nodes=[...])`
says which of them make a flow, and `use(fn, wire=..., **settings)` rebinds
one for a single flow. `fluksio sync` uploads the document plus a generated
import shim per node, so the store still holds a complete, runnable,
git-versioned definition while the code it imports stays theirs.

`fluksio login|run|runs` and `flow.submit().wait()` are the client half, over
the run endpoints that already existed. Runs record the user repository's
commit beside the store's, so "what code produced this number" is answerable
on the side that now holds the code.

- `fluksio/sdk/`: ports, decorators, the flow builder and its checks, the shim
  generator, an HTTP client and sync. Standard library only at import, so
  `from fluksio import node` in a training script pulls in no engine.
- `FlowDef.origin` marks a flow code-defined; `Run.origin_commit` carries the
  repository's commit; `POST /modules/refresh` retires the workers without an
  install, which every sync calls — a worker holds the imported package in
  memory, so an edit to it is invisible until the process goes.
- The canvas shows a generated body read-only and names the repository to edit
  instead; a body edited there stops the next sync rather than being discarded.
- The worker's reporter carries inert `Port`, `node`, `use` and `Flow`, since
  the shim imports a module whose first line declares them.
- `examples/myresearch` is the worked example, `make sync-example` uploads it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ue1tkFWB1bcGy3aWhCKpU
2026-08-23 20:16:08 +02:00

227 lines
12 KiB
Makefile

# ─── Fluksio app Makefile ───
# Convenience targets for development, testing, linting, and deployment.
# The workspace root delegates to these (see ../Makefile).
.PHONY: dev-utils dev dev-local dev-lan up down update install dev-backend dev-frontend \
generate-client seed-example seed-demo sync-example seed-house seed-aircon seed-tinyhouse seed-hosted-demo test test-backend test-frontend soak bench-startup lint lint-backend \
lint-frontend format-frontend umami build docs docs-serve clean help
COMPOSE_ROOT := $(CURDIR)
# Explicit project name keeps this stack isolated from the sibling website
# stack (otherwise both default to "docker", the directory their compose
# files live in).
COMPOSE_PROJECT := fluksio-app
# Compose interpolation needs the project-local .env before reading compose.yml.
COMPOSE := docker compose -p $(COMPOSE_PROJECT) --env-file $(COMPOSE_ROOT)/.env
COMPOSE_PROD := $(COMPOSE) -f docker/compose.yml
# Production also runs autoheal, which restarts the backend when its deep
# health check fails. It is profile-gated because it mounts the Docker socket.
COMPOSE_PROD_RUN := $(COMPOSE_PROD) --profile autoheal
# Host port `make dev-lan` publishes the frontend on.
APP_PORT ?= 8080
COMPOSE_DEV := $(COMPOSE_PROD) -f docker/compose.dev.yml
# Integrated local stack: dev stack wired onto the shared `proxy` network.
COMPOSE_LOCAL := $(COMPOSE_DEV) -f docker/compose.local.yml
# Same stack, with the frontend also published on APP_PORT and proxying the API
# there, for clients that cannot resolve app.$(DOMAIN).
COMPOSE_LAN := $(COMPOSE_LOCAL) -f docker/compose.lan.yml
help: ## Show available targets
@awk 'BEGIN{FS=":.*?## "} /^[a-zA-Z_-]+:.*?##/ {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
# ── Development (Docker) ──────────────────────────────────────────
dev-utils: ## Start only the utility containers (proxy, mailcatcher)
$(COMPOSE_DEV) up --build proxy mailcatcher
dev: ## Start the full dev stack (includes a local Traefik proxy)
$(COMPOSE_DEV) up --build
dev-local: ## Start the integrated local stack (called by the root `make dev`)
DOMAIN=$${DOMAIN:-localhost} ENVIRONMENT=$${ENVIRONMENT:-local} \
$(COMPOSE_LOCAL) up --build -d proxy backend frontend mailcatcher
dev-lan: ## Same, plus the app on http://<host-ip>:$(APP_PORT) (no DNS needed)
DOMAIN=$${DOMAIN:-localhost} ENVIRONMENT=$${ENVIRONMENT:-local} APP_PORT=$(APP_PORT) \
$(COMPOSE_LAN) up --build -d proxy backend frontend mailcatcher
up: ## Start the production stack
$(COMPOSE_PROD_RUN) up --build -d
umami: ## Provision + start the optional Umami site-analytics service (idempotent)
# Guard: the dashboard is internet-facing on analytics.$(DOMAIN); refuse to
# start when its session-signing secret is unset or still the placeholder.
@secret=$$(grep -E '^UMAMI_APP_SECRET=' $(COMPOSE_ROOT)/.env 2>/dev/null | head -1 | cut -d= -f2-); \
if [ -z "$$secret" ] || [ "$$secret" = "changethis" ]; then \
echo "ERROR: UMAMI_APP_SECRET is empty or 'changethis' in .env — run the workspace root's scripts/setup.sh (or set it: openssl rand -hex 32) before 'make umami'." >&2; \
exit 1; \
fi
# --no-recreate: reuse an already-running db instead of recreating it under
# the prod compose set, which would blip every service that depends on it.
$(COMPOSE_PROD) up -d --wait --no-recreate db
# Least-privilege role owning only the umami database, so the third-party
# image never holds the shared superuser credentials.
@pguser=$$(grep -E '^POSTGRES_USER=' $(COMPOSE_ROOT)/.env 2>/dev/null | head -1 | cut -d= -f2-); pguser=$${pguser:-postgres}; \
role=$$(grep -E '^UMAMI_DB_USER=' $(COMPOSE_ROOT)/.env 2>/dev/null | head -1 | cut -d= -f2-); role=$${role:-umami}; \
pw=$$(grep -E '^UMAMI_DB_PASSWORD=' $(COMPOSE_ROOT)/.env 2>/dev/null | head -1 | cut -d= -f2-); \
db=$$(grep -E '^UMAMI_DB=' $(COMPOSE_ROOT)/.env 2>/dev/null | head -1 | cut -d= -f2-); db=$${db:-umami}; \
if [ -z "$$pw" ] || [ "$$pw" = "changethis" ]; then \
echo "ERROR: UMAMI_DB_PASSWORD is empty or 'changethis' in .env — run the workspace root's scripts/setup.sh before 'make umami'." >&2; \
exit 1; \
fi; \
$(COMPOSE_PROD) exec -T db psql -v ON_ERROR_STOP=1 -U "$$pguser" -d postgres \
-v role="$$role" -v pw="$$pw" -v db="$$db" < docker/provision-umami-db.sql
# --no-deps: only (re)create umami, never restart the shared db underneath it.
$(COMPOSE_PROD) --profile analytics up -d --no-deps umami
update: ## Pull, rebuild using the layer cache, and recreate changed containers
git pull
$(COMPOSE_PROD_RUN) build
$(COMPOSE_PROD_RUN) up -d --remove-orphans
docker image prune -f
down: ## Stop all running containers
-$(COMPOSE_LOCAL) down
-$(COMPOSE_PROD_RUN) down
# ── Development (local, no Docker) ───────────────────────────────
# Run `make dev-backend` and `make dev-frontend` in two separate terminals.
install: ## Install all dependencies (backend + frontend)
cd backend && uv sync
cd frontend && bun install
dev-backend: ## Start the FastAPI backend with hot-reload (local)
cd backend && uv run fastapi dev fluksio/main.py
dev-frontend: ## Start the Vite dev server (local)
cd frontend && bun dev
generate-client: ## Regenerate the frontend SDK from the backend's OpenAPI schema
bash scripts/generate-client.sh
seed-example: ## Seed the querying-chart example (needs a running stack + InfluxDB)
cd backend && uv run python ../scripts/seed_example_chart.py
seed-demo: ## Seed the training-run example: a batch flow and its dashboard
cd backend && uv run python ../scripts/seed_demo_training.py
sync-example: ## Upload `examples/myresearch` the way a data scientist would
# The same command the docs give, against whichever engine `fluksio login`
# last talked to. The nodes import the package from this checkout, so the
# engine has to be one that can see this path.
cd backend && uv run fluksio sync ../examples/myresearch
seed-house: ## Seed the house write-path rig (needs the real broker reachable)
cd backend && uv run python ../scripts/seed_house_control.py
seed-aircon: ## Seed the aircon write-path rig (needs the unit reachable)
cd backend && uv run python ../scripts/seed_aircon_control.py
# The house itself. Reads ../house.json for this installation's addresses and
# DMX wiring; everything it creates is published and stopped. ARGS=--dry checks
# the whole set without an installation and pushes nothing.
seed-tinyhouse: ## Seed the TinyHouse flows (ARGS=--dry to check only)
@user=$$(grep -E '^FIRST_SUPERUSER=' $(COMPOSE_ROOT)/.env 2>/dev/null | head -1 | cut -d= -f2-); \
pass=$$(grep -E '^FIRST_SUPERUSER_PASSWORD=' $(COMPOSE_ROOT)/.env 2>/dev/null | head -1 | cut -d= -f2-); \
cd backend && \
FIRST_SUPERUSER="$${FIRST_SUPERUSER:-$$user}" \
FIRST_SUPERUSER_PASSWORD="$${FIRST_SUPERUSER_PASSWORD:-$$pass}" \
uv run python ../scripts/seed_tinyhouse.py $(ARGS)
# Operators of the hosted demo only — NOT part of any deployment, and nothing a
# self-hosted instance needs. It wipes and recreates its three flows and its
# dashboard, so re-running it is how the public demo is reset.
# `uv run` does not read .env, so the credentials are lifted out of it here;
# anything already exported wins, which is how a remote instance is targeted:
# API_URL=https://api.example.com make seed-hosted-demo
seed-hosted-demo: ## Seed the hosted demo panel (operators only; API_URL selects the instance)
@user=$$(grep -E '^FIRST_SUPERUSER=' $(COMPOSE_ROOT)/.env 2>/dev/null | head -1 | cut -d= -f2-); \
pass=$$(grep -E '^FIRST_SUPERUSER_PASSWORD=' $(COMPOSE_ROOT)/.env 2>/dev/null | head -1 | cut -d= -f2-); \
cd backend && \
FIRST_SUPERUSER="$${FIRST_SUPERUSER:-$$user}" \
FIRST_SUPERUSER_PASSWORD="$${FIRST_SUPERUSER_PASSWORD:-$$pass}" \
uv run python ../scripts/seed_demo.py
# ── Testing ───────────────────────────────────────────────────────
test: test-backend test-frontend ## Run all tests (backend + frontend)
test-backend: ## Run backend tests (pytest + coverage)
# Its own SQLite file in a temp directory (tests/__init__.py), so this needs
# nothing running and touches no development data.
cd backend && uv run bash scripts/test.sh
# The hostname the stack is served under. A checkout configured for a
# deployment carries the deployment's domain, and *.fluksio.com resolves to the
# live instance from here — which is why the run below maps both names onto the
# local Traefik by address and never lets DNS decide.
export DOMAIN ?= $(shell sed -n 's/^DOMAIN=//p' $(COMPOSE_ROOT)/.env | head -1)
PW_VERSION = $(shell sed -n 's/.*"@playwright\/test": "[^0-9]*\([0-9.]*\)".*/\1/p' frontend/package.json | head -1)
test-frontend: ## Run frontend tests (Playwright e2e) against the local stack
@ip=$$(docker network inspect proxy \
--format '{{range .Containers}}{{if eq .Name "fluksio-app-proxy-1"}}{{.IPv4Address}}{{end}}{{end}}' \
2>/dev/null | cut -d/ -f1); \
[ -n "$$ip" ] || { echo " ✗ proxy network or Traefik container not found — is the stack up?"; exit 1; }; \
docker run --rm --network proxy --ipc=host \
--user $$(id -u):$$(id -g) -e HOME=/tmp \
--add-host app.$(DOMAIN):$$ip --add-host api.$(DOMAIN):$$ip \
-v $(COMPOSE_ROOT):/app -w /app/frontend \
-e PLAYWRIGHT_BASE_URL=http://app.$(DOMAIN) \
-e PLAYWRIGHT_API_URL=http://api.$(DOMAIN) \
-e HOST_RESOLVER_RULES="MAP app.$(DOMAIN) $$ip, MAP api.$(DOMAIN) $$ip" \
-e CI=$${CI:-1} \
-e FIRST_SUPERUSER="$$(sed -n 's/^FIRST_SUPERUSER=//p' $(COMPOSE_ROOT)/.env | head -1)" \
-e FIRST_SUPERUSER_PASSWORD="$$(sed -n 's/^FIRST_SUPERUSER_PASSWORD=//p' $(COMPOSE_ROOT)/.env | head -1)" \
mcr.microsoft.com/playwright:v$(PW_VERSION)-noble \
npx playwright test $(PLAYWRIGHT_ARGS)
# Load and chaos against a *running* stack, never part of `make test`: it
# restarts this stack's containers. `SOAK_ARGS="--dry-run"` only looks.
soak: ## Run the soak/chaos harness (SOAK_ARGS="--minutes 30 --scenario redis")
cd backend && uv run python scripts/soak.py $(SOAK_ARGS)
bench-startup: ## Time submitting a run (BENCH_ARGS="--kedro ../some/kedro/project")
cd backend && uv run python scripts/bench_startup.py $(BENCH_ARGS)
# ── Linting ───────────────────────────────────────────────────────
build: ## Build the fluksio and fluksio-worker wheels into dist/
uv build --all-packages --out-dir dist
lint: lint-backend lint-frontend ## Run all linters
lint-backend: ## Lint backend with ruff + mypy
cd backend && uv run ruff check .
cd backend && uv run ruff format --check .
cd backend && uv run mypy fluksio
cd worker && uv run --no-project --with mypy mypy fluksio_worker
lint-frontend: ## Lint frontend with biome
cd frontend && bun run lint
format-frontend: ## Apply biome's fixes to the frontend (what `lint-frontend` only reports)
cd frontend && bun run format
# ── Documentation ─────────────────────────────────────────────────
# The public site on docs.${DOMAIN}, served by the `docs` service in
# docker/compose.yml. These targets are for local authoring: zensical runs on
# demand via uvx, so it never touches the backend environment. Pinned to the
# version docker/Dockerfile.docs and .gitea/workflows/docs.yml ship, so local
# authoring builds with what docs.fluksio.com actually gets.
ZENSICAL := zensical==0.0.46
docs: ## Build the documentation site into ./site
uvx $(ZENSICAL) build --clean
docs-serve: ## Serve the documentation site locally with live reload
uvx $(ZENSICAL) serve
# ── Cleanup ───────────────────────────────────────────────────────
clean: ## Remove build artifacts and caches
rm -rf frontend/dist frontend/blob-report frontend/test-results
rm -rf backend/.pytest_cache backend/htmlcov site
find backend -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true