diff --git a/backend/app/core/db.py b/backend/app/core/db.py index ba991fb..3692575 100644 --- a/backend/app/core/db.py +++ b/backend/app/core/db.py @@ -4,7 +4,9 @@ from app import crud from app.core.config import settings from app.models import User, UserCreate -engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) +# A connection idle across a Postgres restart is dead but still pooled; the +# pre-ping spends a round trip to find out instead of failing the request. +engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI), pool_pre_ping=True) # make sure all SQLModel models are imported (app.models) before initializing DB diff --git a/backend/app/flow/queue.py b/backend/app/flow/queue.py index ed72f75..c02a96c 100644 --- a/backend/app/flow/queue.py +++ b/backend/app/flow/queue.py @@ -156,7 +156,8 @@ class WorkQueue(ABC): @abstractmethod def stats(self) -> dict[str, Any]: - """Queue depth and age, for the health endpoint.""" + """In-flight, delayed and parked counts plus the oldest pending age, + for the health endpoint.""" @abstractmethod def dead_letters(self, count: int = 50) -> list[dict[str, Any]]: @@ -266,7 +267,6 @@ class MemoryWorkQueue(WorkQueue): def stats(self) -> dict[str, Any]: with self._lock: return { - "depth": len(self._items), "pending": self._in_flight, "delayed": len(self._delayed), "parked": sum(len(v) for v in self._parked.values()), @@ -302,7 +302,9 @@ class RedisWorkQueue(WorkQueue): self, host: str, port: int = 6379, - namespace: str = "pipeline", + # A root of its own: the state backend owns `pipeline:`, and sharing the + # prefix meant `RedisState.clear()` could DEL the queue stream. + namespace: str = "queue", consumer: str | None = None, ) -> None: self._redis = redis.Redis(host=host, port=port, decode_responses=True) @@ -448,7 +450,6 @@ class RedisWorkQueue(WorkQueue): for key in self._redis.scan_iter(f"{self._ns}:__parked__:*") ) return { - "depth": cast(int, self._redis.xlen(self._stream)), "pending": count, "delayed": cast(int, self._redis.zcard(self._delayed_key)), "parked": parked, diff --git a/backend/app/main.py b/backend/app/main.py index df039e9..9d0134e 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -51,7 +51,7 @@ def _state_backend() -> StateBackend: return MemoryState() -def _work_queue(namespace: str = "pipeline") -> WorkQueue: +def _work_queue(namespace: str = "queue") -> WorkQueue: """Redis makes queued work survive the process; memory does not pretend to.""" if settings.REDIS_HOST: return RedisWorkQueue( diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py index af0bd68..d3d75c7 100644 --- a/backend/tests/__init__.py +++ b/backend/tests/__init__.py @@ -16,3 +16,8 @@ os.environ["MCP_ENABLED"] = "false" # The private seeding endpoints are opt-in; the suite is one of the two places # (with the dev stack) where they are meant to work. os.environ["PRIVATE_API_ENABLED"] = "true" +# The suite is the development configuration by definition: `/private` is +# gated on it (app/api/routes/private.py) and the MCP host allow-list is +# built from DOMAIN (app/mcp/http.py), while the tests speak to api.localhost. +os.environ["ENVIRONMENT"] = "local" +os.environ["DOMAIN"] = "localhost" diff --git a/backend/tests/flow/test_queue.py b/backend/tests/flow/test_queue.py index 07c8225..4f61d56 100644 --- a/backend/tests/flow/test_queue.py +++ b/backend/tests/flow/test_queue.py @@ -66,6 +66,8 @@ def test_claimed_work_counts_as_in_flight_until_it_is_acknowledged(): queue.add(WorkItem(kind="cascade", node="f.n", flow="f")) assert queue.stats()["pending"] == 0 + # The stream length was never a backlog, so the key is gone from both queues. + assert "depth" not in queue.stats() (item,) = queue.claim(1, 10) assert queue.stats()["pending"] == 1 diff --git a/deployment.md b/deployment.md deleted file mode 100644 index 4b8ebc1..0000000 --- a/deployment.md +++ /dev/null @@ -1,344 +0,0 @@ -# FastAPI Project - Deployment - -You can deploy the project using Docker Compose to a remote server. - -This project expects you to have a Traefik proxy handling communication to the outside world and HTTPS certificates. - -You can use CI/CD (continuous integration and continuous deployment) systems to deploy automatically, there are already configurations to do it with GitHub Actions. - -But you have to configure a couple things first. 🤓 - -## Preparation - -* Have a remote server ready and available. -* Configure the DNS records of your domain to point to the IP of the server you just created. -* Configure a wildcard subdomain for your domain, so that you can have multiple subdomains for different services, e.g. `*.fastapi-project.example.com`. This will be useful for accessing different components, like `dashboard.fastapi-project.example.com`, `api.fastapi-project.example.com`, `traefik.fastapi-project.example.com`, `adminer.fastapi-project.example.com`, etc. And also for `staging`, like `dashboard.staging.fastapi-project.example.com`, `adminer.staging.fastapi-project.example.com`, etc. -* Install and configure [Docker](https://docs.docker.com/engine/install/) on the remote server (Docker Engine, not Docker Desktop). - -## Public Traefik - -We need a Traefik proxy to handle incoming connections and HTTPS certificates. - -You need to do these next steps only once. - -### Traefik Docker Compose - -* Create a remote directory to store your Traefik Docker Compose file: - -```bash -mkdir -p /root/code/traefik-public/ -``` - -Copy the Traefik Docker Compose file to your server. You could do it by running the command `rsync` in your local terminal: - -```bash -rsync -a compose.traefik.yml root@your-server.example.com:/root/code/traefik-public/ -``` - -### Traefik Public Network - -This Traefik will expect a Docker "public network" named `traefik-public` to communicate with your stack(s). - -This way, there will be a single public Traefik proxy that handles the communication (HTTP and HTTPS) with the outside world, and then behind that, you could have one or more stacks with different domains, even if they are on the same single server. - -To create a Docker "public network" named `traefik-public` run the following command in your remote server: - -```bash -docker network create traefik-public -``` - -### Traefik Environment Variables - -The Traefik Docker Compose file expects some environment variables to be set in your terminal before starting it. You can do it by running the following commands in your remote server. - -* Create the username for HTTP Basic Auth, e.g.: - -```bash -export USERNAME=admin -``` - -* Create an environment variable with the password for HTTP Basic Auth, e.g.: - -```bash -export PASSWORD=changethis -``` - -* Use openssl to generate the "hashed" version of the password for HTTP Basic Auth and store it in an environment variable: - -```bash -export HASHED_PASSWORD=$(openssl passwd -apr1 $PASSWORD) -``` - -To verify that the hashed password is correct, you can print it: - -```bash -echo $HASHED_PASSWORD -``` - -* Create an environment variable with the domain name for your server, e.g.: - -```bash -export DOMAIN=fastapi-project.example.com -``` - -* Create an environment variable with the email for Let's Encrypt, e.g.: - -```bash -export EMAIL=admin@example.com -``` - -**Note**: you need to set a different email, an email `@example.com` won't work. - -### Start the Traefik Docker Compose - -Go to the directory where you copied the Traefik Docker Compose file in your remote server: - -```bash -cd /root/code/traefik-public/ -``` - -Now with the environment variables set and the `compose.traefik.yml` in place, you can start the Traefik Docker Compose running the following command: - -```bash -docker compose -f compose.traefik.yml up -d -``` - -## Deploy the FastAPI Project - -Now that you have Traefik in place you can deploy your FastAPI project with Docker Compose. - -**Note**: You might want to jump ahead to the section about Continuous Deployment with GitHub Actions. - -## Copy the Code - -```bash -rsync -av --filter=":- .gitignore" ./ root@your-server.example.com:/root/code/app/ -``` - -Note: `--filter=":- .gitignore"` tells `rsync` to use the same rules as git, ignore files ignored by git, like the Python virtual environment. - -## Environment Variables - -You need to set some environment variables first. - -### Generate secret keys - -Some environment variables in the `.env` file have a default value of `changethis`. - -You have to change them with a secret key, to generate secret keys you can run the following command: - -```bash -python -c "import secrets; print(secrets.token_urlsafe(32))" -``` - -Copy the content and use that as password / secret key. And run that again to generate another secure key. - -### Required Environment Variables - -Set the `ENVIRONMENT`, by default `local` (for development), but when deploying to a server you would put something like `staging` or `production`: - -```bash -export ENVIRONMENT=production -``` - -Set the `DOMAIN`, by default `localhost` (for development), but when deploying you would use your own domain, for example: - -```bash -export DOMAIN=fastapi-project.example.com -``` - -Set the `POSTGRES_PASSWORD` to something different than `changethis`: - -```bash -export POSTGRES_PASSWORD="changethis" -``` - -Set the `SECRET_KEY`, used to sign tokens: - -```bash -export SECRET_KEY="changethis" -``` - -Note: you can use the Python command above to generate a secure secret key. - -Set the `FIRST_SUPER_USER_PASSWORD` to something different than `changethis`: - -```bash -export FIRST_SUPERUSER_PASSWORD="changethis" -``` - -Set the `BACKEND_CORS_ORIGINS` to include your domain: - -```bash -export BACKEND_CORS_ORIGINS="https://dashboard.${DOMAIN?Variable not set},https://api.${DOMAIN?Variable not set}" -``` - -You can set several other environment variables: - -* `PROJECT_NAME`: The name of the project, used in the API for the docs and emails. -* `STACK_NAME`: The name of the stack used for Docker Compose labels and project name, this should be different for `staging`, `production`, etc. You could use the same domain replacing dots with dashes, e.g. `fastapi-project-example-com` and `staging-fastapi-project-example-com`. -* `BACKEND_CORS_ORIGINS`: A list of allowed CORS origins separated by commas. -* `FIRST_SUPERUSER`: The email of the first superuser, this superuser will be the one that can create new users. -* `SMTP_HOST`: The SMTP server host to send emails, this would come from your email provider (E.g. Mailgun, Sparkpost, Sendgrid, etc). -* `SMTP_USER`: The SMTP server user to send emails. -* `SMTP_PASSWORD`: The SMTP server password to send emails. -* `EMAILS_FROM_EMAIL`: The email account to send emails from. -* `POSTGRES_SERVER`: The hostname of the PostgreSQL server. You can leave the default of `db`, provided by the same Docker Compose. You normally wouldn't need to change this unless you are using a third-party provider. -* `POSTGRES_PORT`: The port of the PostgreSQL server. You can leave the default. You normally wouldn't need to change this unless you are using a third-party provider. -* `POSTGRES_USER`: The Postgres user, you can leave the default. -* `POSTGRES_DB`: The database name to use for this application. You can leave the default of `app`. -* `SENTRY_DSN`: The DSN for Sentry, if you are using it. - -## GitHub Actions Environment Variables - -There are some environment variables only used by GitHub Actions that you can configure: - -* `LATEST_CHANGES`: Used by the GitHub Action [latest-changes](https://github.com/tiangolo/latest-changes) to automatically add release notes based on the PRs merged. It's a personal access token, read the docs for details. -* `SMOKESHOW_AUTH_KEY`: Used to handle and publish the code coverage using [Smokeshow](https://github.com/samuelcolvin/smokeshow), follow their instructions to create a (free) Smokeshow key. - -### Deploy with Docker Compose - -With the environment variables in place, you can deploy with Docker Compose: - -```bash -cd /root/code/app/ -docker compose -f compose.yml build -docker compose -f compose.yml up -d -``` - -For production you wouldn't want to have the overrides in `compose.override.yml`, that's why we explicitly specify `compose.yml` as the file to use. - -## Continuous Deployment (CD) - -You can use GitHub Actions to deploy your project automatically. 😎 - -You can have multiple environment deployments. - -There are already two environments configured, `staging` and `production`. 🚀 - -### Install GitHub Actions Runner - -* On your remote server, create a user for your GitHub Actions: - -```bash -sudo adduser github -``` - -* Add Docker permissions to the `github` user: - -```bash -sudo usermod -aG docker github -``` - -* Temporarily switch to the `github` user: - -```bash -sudo su - github -``` - -* Go to the `github` user's home directory: - -```bash -cd -``` - -* [Install a GitHub Action self-hosted runner following the official guide](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/adding-self-hosted-runners#adding-a-self-hosted-runner-to-a-repository). - -* When asked about labels, add a label for the environment, e.g. `production`. You can also add labels later. - -After installing, the guide would tell you to run a command to start the runner. Nevertheless, it would stop once you terminate that process or if your local connection to your server is lost. - -To make sure it runs on startup and continues running, you can install it as a service. To do that, exit the `github` user and go back to the `root` user: - -```bash -exit -``` - -After you do it, you will be on the previous user again. And you will be on the previous directory, belonging to that user. - -Before being able to go the `github` user directory, you need to become the `root` user (you might already be): - -```bash -sudo su -``` - -* As the `root` user, go to the `actions-runner` directory inside of the `github` user's home directory: - -```bash -cd /home/github/actions-runner -``` - -* Install the self-hosted runner as a service with the user `github`: - -```bash -./svc.sh install github -``` - -* Start the service: - -```bash -./svc.sh start -``` - -* Check the status of the service: - -```bash -./svc.sh status -``` - -You can read more about it in the official guide: [Configuring the self-hosted runner application as a service](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/configuring-the-self-hosted-runner-application-as-a-service). - -### Set Secrets - -On your repository, configure secrets for the environment variables you need, the same ones described above, including `SECRET_KEY`, etc. Follow the [official GitHub guide for setting repository secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions#creating-secrets-for-a-repository). - -The current Github Actions workflows expect these secrets: - -* `DOMAIN_PRODUCTION` -* `DOMAIN_STAGING` -* `STACK_NAME_PRODUCTION` -* `STACK_NAME_STAGING` -* `EMAILS_FROM_EMAIL` -* `FIRST_SUPERUSER` -* `FIRST_SUPERUSER_PASSWORD` -* `POSTGRES_PASSWORD` -* `SECRET_KEY` -* `LATEST_CHANGES` -* `SMOKESHOW_AUTH_KEY` - -## GitHub Action Deployment Workflows - -There are GitHub Action workflows in the `.github/workflows` directory already configured for deploying to the environments (GitHub Actions runners with the labels): - -* `staging`: after pushing (or merging) to the branch `master`. -* `production`: after publishing a release. - -If you need to add extra environments you could use those as a starting point. - -## URLs - -Replace `fastapi-project.example.com` with your domain. - -### Main Traefik Dashboard - -Traefik UI: `https://traefik.fastapi-project.example.com` - -### Production - -Frontend: `https://dashboard.fastapi-project.example.com` - -Backend API docs: `https://api.fastapi-project.example.com/docs` - -Backend API base URL: `https://api.fastapi-project.example.com` - -Adminer: `https://adminer.fastapi-project.example.com` - -### Staging - -Frontend: `https://dashboard.staging.fastapi-project.example.com` - -Backend API docs: `https://api.staging.fastapi-project.example.com/docs` - -Backend API base URL: `https://api.staging.fastapi-project.example.com` - -Adminer: `https://adminer.staging.fastapi-project.example.com` diff --git a/development.md b/development.md index 7879ffc..07fd33a 100644 --- a/development.md +++ b/development.md @@ -1,221 +1,171 @@ -# FastAPI Project - Development - -## Docker Compose - -* Start the local stack with Docker Compose: - -```bash -docker compose watch -``` - -* Now you can open your browser and interact with these URLs: - -Frontend, built with Docker, with routes handled based on the path: - -Backend, JSON based web API based on OpenAPI: - -Automatic interactive documentation with Swagger UI (from the OpenAPI backend): - -Adminer, database web administration: - -Traefik UI, to see how the routes are being handled by the proxy: - -**Note**: The first time you start your stack, it might take a minute for it to be ready. While the backend waits for the database to be ready and configures everything. You can check the logs to monitor it. - -To check the logs, run (in another terminal): - -```bash -docker compose logs -``` - -To check the logs of a specific service, add the name of the service, e.g.: - -```bash -docker compose logs backend -``` - -## Mailcatcher - -Mailcatcher is a simple SMTP server that catches all emails sent by the backend during local development. Instead of sending real emails, they are captured and displayed in a web interface. - -This is useful for: - -* Testing email functionality during development -* Verifying email content and formatting -* Debugging email-related functionality without sending real emails - -The backend is automatically configured to use Mailcatcher when running with Docker Compose locally (SMTP on port 1025). All captured emails can be viewed at . - -## Local Development - -The Docker Compose files are configured so that each of the services is available in a different port in `localhost`. - -For the backend and frontend, they use the same port that would be used by their local development server, so, the backend is at `http://localhost:8000` and the frontend at `http://localhost:5173`. - -This way, you could turn off a Docker Compose service and start its local development service, and everything would keep working, because it all uses the same ports. - -For example, you can stop that `frontend` service in the Docker Compose, in another terminal, run: - -```bash -docker compose stop frontend -``` - -And then start the local frontend development server: - -```bash -bun run dev -``` - -Or you could stop the `backend` Docker Compose service: - -```bash -docker compose stop backend -``` - -And then you can run the local development server for the backend: - -```bash -cd backend -fastapi dev app/main.py -``` - -## Docker Compose in `localhost.tiangolo.com` - -When you start the Docker Compose stack, it uses `localhost` by default, with different ports for each service (backend, frontend, adminer, etc). - -When you deploy it to production (or staging), it will deploy each service in a different subdomain, like `api.example.com` for the backend and `dashboard.example.com` for the frontend. - -In the guide about [deployment](deployment.md) you can read about Traefik, the configured proxy. That's the component in charge of transmitting traffic to each service based on the subdomain. - -If you want to test that it's all working locally, you can edit the local `.env` file, and change: - -```dotenv -DOMAIN=localhost.tiangolo.com -``` - -That will be used by the Docker Compose files to configure the base domain for the services. - -Traefik will use this to transmit traffic at `api.localhost.tiangolo.com` to the backend, and traffic at `dashboard.localhost.tiangolo.com` to the frontend. - -The domain `localhost.tiangolo.com` is a special domain that is configured (with all its subdomains) to point to `127.0.0.1`. This way you can use that for your local development. - -After you update it, run again: - -```bash -docker compose watch -``` - -When deploying, for example in production, the main Traefik is configured outside of the Docker Compose files. For local development, there's an included Traefik in `compose.override.yml`, just to let you test that the domains work as expected, for example with `api.localhost.tiangolo.com` and `dashboard.localhost.tiangolo.com`. - -## Docker Compose files and env vars - -There is a main `compose.yml` file with all the configurations that apply to the whole stack, it is used automatically by `docker compose`. - -And there's also a `compose.override.yml` with overrides for development, for example to mount the source code as a volume. It is used automatically by `docker compose` to apply overrides on top of `compose.yml`. - -These Docker Compose files use the `.env` file containing configurations to be injected as environment variables in the containers. - -They also use some additional configurations taken from environment variables set in the scripts before calling the `docker compose` command. - -After changing variables, make sure you restart the stack: - -```bash -docker compose watch -``` - -## The .env file - -The `.env` file is the one that contains all your configurations, generated keys and passwords, etc. - -Depending on your workflow, you could want to exclude it from Git, for example if your project is public. In that case, you would have to make sure to set up a way for your CI tools to obtain it while building or deploying your project. - -One way to do it could be to add each environment variable to your CI/CD system, and updating the `compose.yml` file to read that specific env var instead of reading the `.env` file. - -## Pre-commits and code linting - -we are using a tool called [prek](https://prek.j178.dev/) (modern alternative to [Pre-commit](https://pre-commit.com/)) for code linting and formatting. - -When you install it, it runs right before making a commit in git. This way it ensures that the code is consistent and formatted even before it is committed. - -You can find a file `.pre-commit-config.yaml` with configurations at the root of the project. - -#### Install prek to run automatically - -`prek` is already part of the dependencies of the project. - -After having the `prek` tool installed and available, you need to "install" it in the local repository, so that it runs automatically before each commit. - -Using `uv`, you could do it with (make sure you are inside `backend` folder): - -```bash -❯ uv run prek install -f -prek installed at `../.git/hooks/pre-commit` -``` - -The `-f` flag forces the installation, in case there was already a `pre-commit` hook previously installed. - -Now whenever you try to commit, e.g. with: - -```bash -git commit -``` - -...prek will run and check and format the code you are about to commit, and will ask you to add that code (stage it) with git again before committing. - -Then you can `git add` the modified/fixed files again and now you can commit. - -#### Running prek hooks manually - -you can also run `prek` manually on all the files, you can do it using `uv` with: - -```bash -❯ uv run prek run --all-files -check for added large files..............................................Passed -check toml...............................................................Passed -check yaml...............................................................Passed -fix end of files.........................................................Passed -trim trailing whitespace.................................................Passed -ruff.....................................................................Passed -ruff-format..............................................................Passed -biome check..............................................................Passed -``` +# Development + +How to run, test and lint the `app` stack. The workspace root owns the rest: +[`README.md`](../README.md) for the quick start, [`DEPLOY.md`](../DEPLOY.md) for +production, [`DESIGN-GUIDELINES.md`](../DESIGN-GUIDELINES.md) before any UI work. +[`ROADMAP.md`](ROADMAP.md) is what is planned, [`NOTEPAD.md`](NOTEPAD.md) what is deferred. + +## Running it + +**Integrated — the normal way.** From the workspace root, once: `make init` (submodules, +secrets, the per-stack `.env` files, the shared external `proxy` docker network). Then +`make dev` brings both stacks up detached behind a single Traefik — which is what lets the +SPA, the API and the marketing site answer on one port. `make status` lists the containers, +`make down` stops everything. + +The hostname comes from `DOMAIN` in `app/.env`, which `scripts/setup.sh` copies out of the +root `.env` — change it there and re-run `make init`. `make dev DOMAIN=…` at the root does +*not* reach this stack: `app/Makefile` assigns `DOMAIN` from `app/.env`, and a makefile +assignment overrides an inherited environment variable. + +**App only.** `cd app && make dev` starts this stack on its own Traefik with all host ports +published, in the foreground. `make dev-utils` starts just db, adminer, proxy, mailcatcher +and prestart — the useful half when the backend runs on the host. + +**No Docker.** `make install` once, then `make dev-backend` (FastAPI on :8000) and +`make dev-frontend` (Vite on :5173) in two terminals. A local stack's CORS list already +contains `http://localhost:5173`, so a host Vite server can talk to a containerised API. + +Nothing invokes `docker compose` bare, and neither should you: both stacks keep their +compose files in a `docker/` directory, so without `-p fluksio-app` they collide in a +project named after that directory, and `--env-file` is required because compose +interpolates `.env` before it reads `compose.yml`. The `COMPOSE` variable in `Makefile` has +both. The files layer as `compose.yml` (production) → `compose.dev.yml` (local Traefik, +published ports, hot reload, test helpers) → `compose.local.yml` (integrated). CI layers +`compose.ci.yml` instead of the last one; `compose.traefik.yml` is the production edge +proxy, deployed on its own. ## URLs -The production or staging URLs would use these same paths, but with your own domain. +Integrated stack (root `make dev`). Traefik's port 80 is the **only** published port: +`compose.local.yml` drops the rest, since behind the proxy they add nothing and one of them +already being taken would stop the whole stack from starting. -### Development URLs +| URL | Service | +|---|---| +| http://app.localhost | dashboard SPA | +| http://api.localhost | backend API | +| http://api.localhost/docs | OpenAPI / Swagger UI | +| http://localhost | marketing site (the `index` stack) | -Development URLs, for local development. +Adminer and mailcatcher have no route here, so use the containers directly: +`docker exec -it fluksio-db psql -U postgres -d app` for the database, +`docker logs -f fluksio-app-mailcatcher-1` for captured mail. -Frontend: +Standalone (`cd app && make dev`) publishes the full set: -Backend: +| URL | Service | +|---|---| +| http://app.localhost, http://api.localhost | this stack's own Traefik on :80 | +| http://localhost:5173 | SPA (the nginx image, not the Vite dev server) | +| http://localhost:8000 | backend API | +| http://localhost:8090 | Traefik dashboard | +| http://127.0.0.1:8080 | adminer | +| http://localhost:1080 | mailcatcher web UI (SMTP on 1025) | +| localhost:5432 | Postgres | -Automatic Interactive Docs (Swagger UI): +## Hot reload, and what it misses -Automatic Alternative Docs (ReDoc): +`compose.dev.yml` bind-mounts `backend/app` into the api container and runs uvicorn with +`--reload`, so a backend edit is live. The `develop.watch` block next to it syncs the same +directory, but only under `docker compose watch`, which no target runs — the detached +`up -d` flow never triggers it. The mount is what does the work, so an api container created +before the mount existed keeps serving the source baked into its image and has to be +recreated once. -Adminer: +The frontend is built into an nginx image, so a UI change needs an explicit rebuild, from +`app/`: -Traefik UI: +```sh +docker compose -p fluksio-app --env-file "$PWD/.env" \ + -f docker/compose.yml -f docker/compose.dev.yml -f docker/compose.local.yml \ + up --build -d frontend +``` -MailCatcher: +## Services the dev stack adds -### Development URLs with `localhost.tiangolo.com` Configured +`compose.dev.yml` brings up a broker and a time-series database of the stack's own, so the +mqtt and influx node types are testable without external hardware. Neither publishes a host +port, on purpose — nothing outside the stack needs them. -Development URLs, for local development. +- **mosquitto** — anonymous MQTT. Point a node at `broker_host: mosquitto`, port 1883. +- **influxdb 2.7** — `http://influxdb:8086`, org and bucket `fluksio`, token + `fluksio-dev-token`. No volume, so `down -v` starts it over. +- **mailcatcher** — SMTP on 1025, web UI on 1080; the backend is pointed at it, so no + development mail leaves the machine. -Frontend: +`adminer` and `redis` come from `compose.yml` and exist in production too; dev only differs +in that adminer gets a host port. Without `REDIS_HOST` the flow engine keeps state in +memory instead of Redis. -Backend: +## Configuration and secrets -Automatic Interactive Docs (Swagger UI): +The root `.env` is the source of truth. `scripts/setup.sh` creates `app/.env` from +`app/.env.example` and keeps the shared keys — domain, secret key, database password, +superuser — in step with it. Only the `.env.example` files are tracked; `make secrets` +reprints the generated values. A release that adds a key needs `scripts/setup.sh --secrets` +before `make update`, so the key exists before the stack is rebuilt. -Automatic Alternative Docs (ReDoc): +The backend settings load `../.env` relative to `backend/` through pydantic-settings, which +is why an exported environment variable always outranks the file — how CI points the suite +at a Docker-assigned Postgres port. -Adminer: +## Tests -Traefik UI: +`make test` is `test-backend` plus `test-frontend`. -MailCatcher: +**Backend.** `cd backend && uv run bash scripts/tests-start.sh` waits for Postgres, then +runs pytest under coverage; the HTML report lands in `backend/htmlcov`. Postgres is the only +service needed — mail is patched out in `tests/`, and the flow engine falls back to +`MemoryState` while `REDIS_HOST` is empty. `tests/__init__.py` pins the suite's own +environment before anything imports the settings: database `app_test` (created and dropped +per session, so a run never touches development data), plus `ENVIRONMENT=local` and +`DOMAIN=localhost`, so a checkout configured for a deployment cannot drag that deployment's +configuration into the run. `make test-backend` resolves `POSTGRES_SERVER` from the running +`fluksio-db` container's address, because the integrated stack publishes no 5432. + +**Frontend (e2e).** `make test-frontend` runs the suite inside the pinned +`mcr.microsoft.com/playwright:v-noble` image (version read from +`frontend/package.json`) on the `proxy` network, against a stack that must already be up. It +maps `app.$(DOMAIN)` and `api.$(DOMAIN)` onto the Traefik container's address twice, with +`--add-host` and with `HOST_RESOLVER_RULES`, because Chromium pins `*.localhost` to loopback +whatever `/etc/hosts` says. Narrow a run with +`make test-frontend PLAYWRIGHT_ARGS="--grep flows"`. + +The specs are `frontend/tests/*.spec.ts`; `playwright.config.ts` defines `setup` → +`chromium` and `mobile`. `auth.setup.ts` logs in once into `playwright/.auth/user.json`, +which both browser projects reuse; `mobile` runs `mobile.spec.ts` only, on a Pixel 5 at +393px. + +Both origins come from `PLAYWRIGHT_BASE_URL` and `PLAYWRIGHT_API_URL`, deliberately never +from `VITE_API_URL` — that one belongs to the app build and in a deployment checkout it +names the deployment, which would mean driving a browser at the local stack while sending +teardown `DELETE`s to the live instance. `tests/guard.ts` runs first and refuses the whole +run when either origin resolves outside loopback or the private ranges, because the suite +creates and deletes flows, dashboards and users. `PLAYWRIGHT_ALLOW_PUBLIC=1` overrides it. + +**Visual check.** From the root, `make verify` logs in and screenshots both themes into +`app/frontend/screenshots/{light,dark}/`; `make verify-docker` does the same from the +Playwright container when the host lacks the browser libraries. Both default to `localhost` +rather than reading `.env`, so on a checkout configured for a deployment name the domain the +stack actually serves: `make verify DOMAIN=example.com`. + +## Lint, hooks and the generated client + +`make lint` runs ruff, mypy and biome. `make hooks` at the root installs the pre-commit +hooks in both stacks; `.pre-commit-config.yaml` holds them. One is worth knowing about: +`generate-frontend-sdk` regenerates the SPA's API client from the backend's OpenAPI schema +whenever anything under `backend/` changes, so a route change reaches the frontend without +being asked. `make generate-client` forces it. + +Run the root's `make design-check` before pushing anything that touches the design tokens, +`lib/motion.ts`, `components.json` or the font asset — the design system is a duplication +contract between the two frontends, and that target is the only thing checking it. + +## CI + +`.gitea/workflows/` runs four jobs, each writing `.env` from `.env.example` first: +`test-backend.yml` (pytest against a compose Postgres), `playwright.yml` (the e2e suite in +two shards, one compose project each, the browser loading the SPA from the nginx `frontend` +service), `pre-commit.yml` (the hooks above, plus strict mypy, which they do not cover) and +`test-compose.yml` (a smoke test that the production images actually come up). diff --git a/docker/compose.yml b/docker/compose.yml index a80656f..05d71b6 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -154,6 +154,11 @@ services: # credential and the key it pinned, so losing it on a rebuild would mean # enrolling again by hand. Absent until someone connects a portal. - CLOUD_CONFIG_FILE=/data/cloud.json + # Alert routing and the wall-panel pairings. Both are written at runtime, + # so off the volume they would land in the container's writable layer and + # be lost on the next rebuild — un-pairing every screen. + - ALERTS_FILE=/data/alerts.json + - PANELS_FILE=/data/panels.json volumes: - app-flow-data:/data