Keep the engine's state in SQLite, not Postgres

One process owns this database — the image has run a single uvicorn
worker for that reason since the four-engines bug — so a file beside the
flows is the honest shape for it, and it is what lets `fluksio serve`
need no infrastructure at all. Live values, node execution and the work
queue never came here anyway; what does is a rollup a minute at a time,
a row per cascade and the run history, and WAL keeps the readers going
while that one writer works.

DATA_DIR is now the one setting that moves everything an installation
keeps; the rest derive from it and the images still spell theirs out.
The schema is prepared in-process at startup, so the prestart service is
gone, and the ten Postgres-only revisions collapse into one portable
baseline.

Three things only worked because psycopg was casting for us: a token's
subject arriving as a string where the column is a UUID, `greatest`, and
`date_bin`. The timestamps needed a column type of their own — SQLite
stores no offset, and a naive datetime read back either raises against an
aware `now` or serialises as local time.

Postgres stays in the stack only for Umami, behind the analytics profile.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/fluksio/core/config.py

@@ -1,3 +1,4 @@
+import os
 import secrets
 import warnings
 from pathlib import Path
@@ -8,13 +9,24 @@ from pydantic import (
     BeforeValidator,
     EmailStr,
     HttpUrl,
-    PostgresDsn,
     computed_field,
     model_validator,
 )
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from typing_extensions import Self
 
+#: Everything the engine keeps on disk, relative to :attr:`Settings.DATA_DIR`.
+#: One setting to move the lot; each still overridable on its own, which is
+#: what the container images do.
+DERIVED_PATHS = {
+    "FLOWS_DIR": "flows",
+    "SECRETS_FILE": "secrets.enc",
+    "ALERTS_FILE": "alerts.json",
+    "PANELS_FILE": "panels.json",
+    "OAUTH_PRIVATE_KEY_FILE": "oauth-key.pem",
+    "CLOUD_CONFIG_FILE": "cloud.json",
+}
+
 
 def parse_cors(v: Any) -> list[str] | str:
     if isinstance(v, str) and not v.startswith("["):
@@ -26,8 +38,10 @@ def parse_cors(v: Any) -> list[str] | str:
 
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(
-        # Use top level .env file (one level above ./backend/)
-        env_file="../.env",
+        # The stack's own file, one level above ./backend/. An installed
+        # `fluksio` has no such tree, so its CLI points this at the data
+        # directory instead — and at nothing it might find in the cwd.
+        env_file=os.environ.get("FLUKSIO_ENV_FILE", "../.env"),
         env_ignore_empty=True,
         extra="ignore",
     )
@@ -38,6 +52,14 @@ class Settings(BaseSettings):
     FRONTEND_HOST: str = "http://localhost:5173"
     ENVIRONMENT: Literal["local", "staging", "production"] = "local"
 
+    #: Everything this installation keeps: the database, the flow repository,
+    #: secrets, artifacts and the user venv. The paths below derive from it
+    #: unless they are set explicitly.
+    DATA_DIR: Path = Path("flow-data")
+    #: Any SQLAlchemy URL. The default puts SQLite in the data directory, which
+    #: is what makes `fluksio serve` need no infrastructure at all.
+    DATABASE_URL: str | None = None
+
     # Flows live on disk as a git repository; secrets stay outside it.
     FLOWS_DIR: Path = Path("flow-data/flows")
     SECRETS_FILE: Path = Path("flow-data/secrets.enc")
@@ -80,6 +102,22 @@ class Settings(BaseSettings):
         list[AnyUrl] | str, BeforeValidator(parse_cors)
     ] = []
 
+    @model_validator(mode="before")
+    @classmethod
+    def _derive_data_paths(cls, data: Any) -> Any:
+        """Put every stored thing under ``DATA_DIR`` unless it was named.
+
+        ``setdefault``, so the container images keep their explicit ``/data``
+        paths and a checkout keeps ``flow-data/``.
+        """
+        if not isinstance(data, dict):
+            return data
+        base = Path(str(data.get("DATA_DIR", "flow-data"))).expanduser()
+        data["DATA_DIR"] = base
+        for key, name in DERIVED_PATHS.items():
+            data.setdefault(key, base / name)
+        return data
+
     @computed_field  # type: ignore[prop-decorator]
     @property
     def oauth_issuer(self) -> str:
@@ -105,25 +143,21 @@ class Settings(BaseSettings):
             self.FRONTEND_HOST
         ]
 
-    PROJECT_NAME: str
+    PROJECT_NAME: str = "Fluksio"
     SENTRY_DSN: HttpUrl | None = None
-    POSTGRES_SERVER: str
-    POSTGRES_PORT: int = 5432
-    POSTGRES_USER: str
-    POSTGRES_PASSWORD: str = ""
-    POSTGRES_DB: str = ""
 
     @computed_field  # type: ignore[prop-decorator]
     @property
-    def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn:
-        return PostgresDsn.build(
-            scheme="postgresql+psycopg",
-            username=self.POSTGRES_USER,
-            password=self.POSTGRES_PASSWORD,
-            host=self.POSTGRES_SERVER,
-            port=self.POSTGRES_PORT,
-            path=self.POSTGRES_DB,
-        )
+    def SQLALCHEMY_DATABASE_URI(self) -> str:
+        """SQLite in the data directory, unless a URL says otherwise.
+
+        One engine process owns this database — the same reason the image runs
+        a single uvicorn worker — so a file beside the flows is the honest
+        shape for it, and needs nothing running to be one.
+        """
+        if self.DATABASE_URL:
+            return self.DATABASE_URL
+        return f"sqlite:///{(self.DATA_DIR / 'fluksio.db').expanduser().resolve()}"
 
     SMTP_TLS: bool = True
     SMTP_SSL: bool = False
@@ -148,8 +182,10 @@ class Settings(BaseSettings):
         return bool(self.SMTP_HOST and self.EMAILS_FROM_EMAIL)
 
     EMAIL_TEST_USER: EmailStr = "test@example.com"
-    FIRST_SUPERUSER: EmailStr
-    FIRST_SUPERUSER_PASSWORD: str
+    # Absent means "the CLI will make one on first run" — a pip install is not
+    # asked for two environment variables before it can start.
+    FIRST_SUPERUSER: EmailStr | None = None
+    FIRST_SUPERUSER_PASSWORD: str | None = None
 
     def _check_default_secret(self, var_name: str, value: str | None) -> None:
         if value == "changethis":
@@ -165,7 +201,6 @@ class Settings(BaseSettings):
     @model_validator(mode="after")
     def _enforce_non_default_secrets(self) -> Self:
         self._check_default_secret("SECRET_KEY", self.SECRET_KEY)
-        self._check_default_secret("POSTGRES_PASSWORD", self.POSTGRES_PASSWORD)
         self._check_default_secret(
             "FIRST_SUPERUSER_PASSWORD", self.FIRST_SUPERUSER_PASSWORD
         )
@@ -173,4 +208,6 @@ class Settings(BaseSettings):
         return self
 
 
-settings = Settings()  # type: ignore
+# No arguments and no required environment: a fresh install boots on the
+# defaults above, into a data directory of its own.
+settings = Settings()

backend/fluksio/core/db.py

@@ -1,28 +1,103 @@
-from sqlmodel import Session, create_engine, select
+"""The database: one file beside the flows, or whatever ``DATABASE_URL`` says.
+
+SQLite is the default because one engine process owns this database. Node
+execution, live values and the work queue never come here — they are the state
+backend's and the bus's — so what lands in a transaction is the rollups a
+minute at a time, a row per cascade, and the run history. WAL lets the readers
+carry on while that single writer works.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import Engine, event, inspect, make_url
+from sqlmodel import Session, SQLModel, create_engine, select
 
 from fluksio import crud
 from fluksio.core.config import settings
 from fluksio.models import User, UserCreate
 
-# A connection idle across a Postgres restart is dead but still pooled; the
-# pre-ping spends a round trip to find out instead of failing the request.
-engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI), pool_pre_ping=True)
+logger = logging.getLogger(__name__)
 
 
-# make sure all SQLModel models are imported (app.models) before initializing DB
-# otherwise, SQLModel might fail to initialize relationships properly
-# for more details: https://github.com/fastapi/full-stack-fastapi-template/issues/28
+def make_engine(url: str) -> Engine:
+    """One engine for the process, configured for whichever dialect it names."""
+    if make_url(url).get_backend_name() != "sqlite":
+        # A connection idle across a server restart is dead but still pooled;
+        # the pre-ping spends a round trip to find out instead of failing the
+        # request.
+        return create_engine(url, pool_pre_ping=True)
+
+    database = make_url(url).database
+    if database and database != ":memory:":
+        Path(database).parent.mkdir(parents=True, exist_ok=True)
+    # The engine's threads — the run service, the metrics collector, the
+    # request pool — share this engine, so connections cross threads. Each one
+    # is still used by a single thread at a time; the pool sees to that.
+    return create_engine(url, connect_args={"check_same_thread": False})
+
+
+engine = make_engine(settings.SQLALCHEMY_DATABASE_URI)
+
+
+@event.listens_for(engine, "connect")
+def _sqlite_pragmas(dbapi_connection: Any, _record: Any) -> None:
+    """What makes concurrent readers, cascades and waiting-out a writer work."""
+    if engine.dialect.name != "sqlite":
+        return
+    cursor = dbapi_connection.cursor()
+    # Readers do not block on the writer, which is the whole reason this is
+    # usable while a run is streaming metrics into it.
+    cursor.execute("PRAGMA journal_mode=WAL")
+    # Commit without waiting for the platter. A power cut can lose the last
+    # transactions; it cannot corrupt the file.
+    cursor.execute("PRAGMA synchronous=NORMAL")
+    # Off by default in SQLite, and `ondelete="CASCADE"` on the OAuth tables is
+    # the whole of revoking a client.
+    cursor.execute("PRAGMA foreign_keys=ON")
+    # A second writer waits rather than raising "database is locked".
+    cursor.execute("PRAGMA busy_timeout=30000")
+    cursor.close()
+
+
+def _alembic_config(connection: Any) -> Config:
+    """Alembic driven from here, so an installed wheel needs no alembic.ini."""
+    config = Config()
+    config.set_main_option(
+        "script_location", str(Path(__file__).parents[1] / "alembic")
+    )
+    config.attributes["connection"] = connection
+    return config
+
+
+def migrate(engine: Engine) -> None:
+    """Bring the schema up to date, creating it if there is nothing there."""
+    with engine.begin() as connection:
+        config = _alembic_config(connection)
+        tables = inspect(connection).get_table_names()
+        if not tables:
+            # Nothing to upgrade from. Building the schema from the models is
+            # both faster and exactly what the revisions would have produced.
+            SQLModel.metadata.create_all(connection)
+            command.stamp(config, "head")
+            logger.info("created the database schema")
+            return
+        command.upgrade(config, "head")
 
 
 def init_db(session: Session) -> None:
-    # Tables should be created with Alembic migrations
-    # But if you don't want to use migrations, create
-    # the tables un-commenting the next lines
-    # from sqlmodel import SQLModel
-
-    # This works because the models are already imported and registered from app.models
-    # SQLModel.metadata.create_all(engine)
+    """The first superuser, when the deployment names one.
 
+    A `fluksio serve` with no configuration makes its own and prints it once;
+    that path calls into `fluksio.core.bootstrap` instead of this.
+    """
+    if not (settings.FIRST_SUPERUSER and settings.FIRST_SUPERUSER_PASSWORD):
+        return
     user = session.exec(
         select(User).where(User.email == settings.FIRST_SUPERUSER)
     ).first()
@@ -33,3 +108,10 @@ def init_db(session: Session) -> None:
             is_superuser=True,
         )
         user = crud.create_user(session=session, user_create=user_in)
+
+
+def prepare(engine: Engine) -> None:
+    """Everything that has to be true before the app serves a request."""
+    migrate(engine)
+    with Session(engine) as session:
+        init_db(session)

backend/fluksio/core/types.py

@@ -0,0 +1,44 @@
+"""Column types the models share.
+
+Its own module because :mod:`fluksio.core.db` imports the models, so the
+models cannot import from there.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from sqlalchemy import DateTime, TypeDecorator
+
+
+class UTCDateTime(TypeDecorator[datetime]):
+    """A timestamp that is still UTC-aware when it comes back.
+
+    SQLite has no timezone-aware type: it stores what it is handed and returns
+    it naive, so a column read back would compare against ``now(timezone.utc)``
+    with a TypeError, or — worse — serialise without an offset and be read as
+    local time by whoever gets the JSON. Every stored instant is converted to
+    UTC on the way in and labelled UTC on the way out, on every dialect.
+    """
+
+    impl = DateTime(timezone=True)
+    cache_ok = True
+
+    def process_bind_param(
+        self, value: datetime | None, dialect: Any
+    ) -> datetime | None:
+        if value is None:
+            return None
+        if value.tzinfo is None:
+            return value.replace(tzinfo=timezone.utc)
+        return value.astimezone(timezone.utc)
+
+    def process_result_value(
+        self, value: datetime | None, dialect: Any
+    ) -> datetime | None:
+        if value is None:
+            return None
+        if value.tzinfo is None:
+            return value.replace(tzinfo=timezone.utc)
+        return value.astimezone(timezone.utc)