Melvin Strobl
72c1d73e96
C4: a device's `viewers` (superusers by default, or members) covers the image and video messages its ports publish. Anyone else gets none of their values on the socket, in the catalogue or a flow's state, 403 for their history, for the frames they hold (the last 8 included) and for TURN credentials, no bytes when asking for them, and `bye: forbidden` for a live view. A device credential stays bounded by its dashboards. The device page sets it in its Camera section, and a tile the viewer may not see says "No access to this camera" and asks the camera for nothing. C3: - A volatile upload is held to MAX_FRAME_BYTES (4 MiB, 413 above it, read without extra copies), and so is a frame on a device's socket. - A frame the engine holds in memory is written to the ring directory before a local worker is handed it, so load_artifact finds it. - A socket the bus dropped events for catches up with a snapshot, demand included, and a device's credential is checked again per frame, cached by the flow store revision and its nonce, so a lost event never keeps an unpaired device's frames coming. - A report is at most 64 KB once its named config_schemas are put in place. - A device away for 60 s has its true bool messages set to false. - Refused frames are logged by fixed keys, a frame is named after its message, `fluksio serve` waits 5 s for open connections, a tile with a stream URL creates no demand, the 8-name cap keeps the newest names. - Every container's log rotates (json-file, 3 x 10 MB).
358 lines
17 KiB
Python
358 lines
17 KiB
Python
import os
|
|
import secrets
|
|
import warnings
|
|
from pathlib import Path
|
|
from typing import Annotated, Any, Literal, Self
|
|
|
|
from pydantic import (
|
|
AnyUrl,
|
|
BeforeValidator,
|
|
EmailStr,
|
|
Field,
|
|
HttpUrl,
|
|
PositiveInt,
|
|
computed_field,
|
|
model_validator,
|
|
)
|
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
|
|
#: Everything the engine keeps on disk, relative to :attr:`Settings.DATA_DIR`.
|
|
#: One setting to move the lot; each still overridable on its own, which is
|
|
#: what the container images do.
|
|
DERIVED_PATHS = {
|
|
"FLOWS_DIR": "flows",
|
|
"SECRETS_FILE": "secrets.enc",
|
|
"ALERTS_FILE": "alerts.json",
|
|
"WEBPUSH_FILE": "webpush.json",
|
|
"PANELS_FILE": "panels.json",
|
|
"DEVICES_FILE": "devices.json",
|
|
"OAUTH_PRIVATE_KEY_FILE": "oauth-key.pem",
|
|
"CLOUD_CONFIG_FILE": "cloud.json",
|
|
"PROVISIONERS_FILE": "provisioners.json",
|
|
"AI_CONFIG_FILE": "ai.json",
|
|
"CHATS_DIR": "chats",
|
|
}
|
|
|
|
|
|
def parse_cors(v: Any) -> list[str] | str:
|
|
if isinstance(v, str) and not v.startswith("["):
|
|
return [i.strip() for i in v.split(",") if i.strip()]
|
|
elif isinstance(v, list | str):
|
|
return v
|
|
raise ValueError(v)
|
|
|
|
|
|
class Settings(BaseSettings):
|
|
model_config = SettingsConfigDict(
|
|
# The stack's own file, one level above ./backend/. An installed
|
|
# `fluksio` has no such tree, so its CLI points this at the data
|
|
# directory instead — and at nothing it might find in the cwd.
|
|
env_file=os.environ.get("FLUKSIO_ENV_FILE", "../.env"),
|
|
env_ignore_empty=True,
|
|
extra="ignore",
|
|
)
|
|
API_V1_STR: str = "/api/v1"
|
|
SECRET_KEY: str = secrets.token_urlsafe(32)
|
|
# 60 minutes * 24 hours * 8 days = 8 days
|
|
ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8
|
|
#: The address people and devices reach this instance at: panel links,
|
|
#: mails and the MCP issuer are built from it.
|
|
FRONTEND_HOST: str = "http://localhost:5173"
|
|
#: The built dashboard, served on the API's own origin when set. The image
|
|
#: sets it; a checkout runs Vite instead and leaves it empty.
|
|
FRONTEND_DIST: Path | None = None
|
|
ENVIRONMENT: Literal["local", "staging", "production"] = "local"
|
|
#: The proxies whose X-Forwarded-For is believed: uvicorn's own setting,
|
|
#: read here too so a call replayed off the portal tunnel resolves its
|
|
#: caller the same way. IPs or networks, comma-separated; `*` trusts all.
|
|
FORWARDED_ALLOW_IPS: str = "127.0.0.1"
|
|
|
|
#: Everything this instance keeps: the database, the flow repository,
|
|
#: secrets, artifacts and the user venv. The paths below derive from it
|
|
#: unless they are set explicitly.
|
|
DATA_DIR: Path = Path("flow-data")
|
|
#: Where the database file goes, as a SQLite URL. The default puts it in
|
|
#: the data directory, which is what makes `fluksio serve` need no
|
|
#: infrastructure at all. **SQLite only**: the upserts the metrics
|
|
#: collector and the run recorder are built on are written against that
|
|
#: dialect, so another backend would parse, migrate, serve — and then
|
|
#: silently drop every rollup and fail every run.
|
|
DATABASE_URL: str | None = None
|
|
|
|
# Flows live on disk as a git repository; secrets stay outside it.
|
|
FLOWS_DIR: Path = Path("flow-data/flows")
|
|
SECRETS_FILE: Path = Path("flow-data/secrets.enc")
|
|
# Which failures reach which channel. Beside the flows, not in them:
|
|
# alerting is the deployment's concern, not any one flow's.
|
|
ALERTS_FILE: Path = Path("flow-data/alerts.json")
|
|
# This instance's web push keypair and the browsers subscribed to it.
|
|
# Beside the alerts it serves; deleting it makes every device subscribe
|
|
# again.
|
|
WEBPUSH_FILE: Path = Path("flow-data/webpush.json")
|
|
# Where machines can be started from when a node needs one and nothing that
|
|
# could take it is attached. Operator-authored, like the alerts beside it,
|
|
# and absent on an instance that has nowhere to start one.
|
|
PROVISIONERS_FILE: Path = Path("flow-data/provisioners.json")
|
|
# Where panels were kept before they became devices. Read once at startup,
|
|
# moved into the flow repository and `DEVICES_FILE`, and renamed
|
|
# `.migrated`.
|
|
PANELS_FILE: Path = Path("flow-data/panels.json")
|
|
# Each device's credential nonce and capability report. Outside the flow
|
|
# repository, so reverting it can never undo an unpair.
|
|
DEVICES_FILE: Path = Path("flow-data/devices.json")
|
|
# Which interpreter node code runs on. "auto" adopts the venv the engine
|
|
# was installed into, when it was installed into one and there is no venv
|
|
# of its own to lose — which is the `pip install fluksio` beside your own
|
|
# packages case. "managed" always builds a separate one, which is what a
|
|
# container wants. A path names an interpreter outright.
|
|
NODE_VENV: str = "auto"
|
|
# The MCP endpoint, and the OAuth server agents authenticate against. Off
|
|
# until someone asks for it: it opens client registration to the network.
|
|
MCP_ENABLED: bool = False
|
|
# Unauthenticated test-only endpoints (user seeding). Requires an explicit
|
|
# opt-in on top of ENVIRONMENT=local, so a deployment that merely kept the
|
|
# default environment never exposes them.
|
|
PRIVATE_API_ENABLED: bool = False
|
|
# `/docs`, `/redoc` and the OpenAPI document for any caller, rather than
|
|
# only one on this machine. Never in production.
|
|
API_DOCS_ENABLED: bool = False
|
|
# Whether anyone who reaches the login page may create an account. Off
|
|
# unless asked for: an account can change what flows do, so who gets one
|
|
# is a superuser's call (Admin → Users).
|
|
SIGNUP_ENABLED: bool = False
|
|
OAUTH_PRIVATE_KEY_FILE: Path = Path("flow-data/oauth-key.pem")
|
|
# Written only when someone enrols this instance with a portal.
|
|
# Its absence is what keeps remote access off.
|
|
CLOUD_CONFIG_FILE: Path = Path("flow-data/cloud.json")
|
|
# The assistant's provider: written from Settings, key encrypted with the
|
|
# app key like the secrets store. Absent until someone configures it, and
|
|
# its absence is what keeps every AI surface off.
|
|
AI_CONFIG_FILE: Path = Path("flow-data/ai.json")
|
|
# One transcript per user, beside the config it talks through.
|
|
CHATS_DIR: Path = Path("flow-data/chats")
|
|
OAUTH_CODE_EXPIRE_SECONDS: int = 60
|
|
# Short, because an agent's token is a bearer secret held by a program
|
|
# rather than a person, and it can refresh unattended.
|
|
MCP_TOKEN_EXPIRE_MINUTES: int = 60
|
|
MCP_REFRESH_EXPIRE_DAYS: int = 30
|
|
# The three below are all pool sizes, so 0 says neither "none" nor
|
|
# "unlimited" — it is a pool that cannot be built and an engine that would
|
|
# accept no work. Rejected here rather than quietly read as the default,
|
|
# because a limit somebody set and did not get is the worse surprise.
|
|
FLOW_MAX_WORKERS: PositiveInt = 4
|
|
# How many cascades may be in flight at once. Sustained throughput is this
|
|
# over the mean cascade time, so an instance whose nodes wait on the
|
|
# network rather than on a CPU wants it higher than the core count.
|
|
FLOW_MAX_CASCADES: PositiveInt = 4
|
|
# How many batch runs are driven at once. A different limit from the one
|
|
# above: a run drives a whole graph, and its nodes are bounded by the worker
|
|
# pool rather than by cascade slots. A sweep is what this governs.
|
|
FLOW_MAX_RUNS: PositiveInt = 4
|
|
# How long a python node may be silent before its worker is killed, unless
|
|
# the node sets its own. 0, the default, disables it: a dead worker still
|
|
# fails fast, and a slow one is left to finish. Set it where silence means
|
|
# stuck rather than working.
|
|
FLOW_NODE_TIMEOUT: float = 0.0
|
|
# Cores nodes may be given, for the ones that declare `resources`. 0 works
|
|
# it out: every core but two, which are what keeps the engine's own event
|
|
# loop answering while the machine is busy. Nodes that declare nothing are
|
|
# not accounted against it — they only get its fair share as a thread cap.
|
|
FLOW_CPUS: int = 0
|
|
# GPUs on this machine, each held by one node at a time. 0 counts the
|
|
# NVIDIA device nodes, the way 0 cores works the core count out; anything
|
|
# else — another vendor, or keeping a card back — is said outright.
|
|
FLOW_GPUS: int = 0
|
|
# Megabytes nodes that declare `ram` may be given. 0 works it out: the
|
|
# machine's memory, or the container's limit where that is lower, less a
|
|
# gigabyte for the engine. Like cores, a node declaring nothing is not
|
|
# counted against it.
|
|
FLOW_RAM: int = 0
|
|
# A warm worker is retired rather than kept when its node returns with the
|
|
# process holding more than this many megabytes resident. A library that
|
|
# grew it then gives the memory back, and the next call pays its imports
|
|
# again. 0, the default, keeps every worker warm.
|
|
FLOW_WORKER_MAX_RSS_MB: int = 0
|
|
# How long a node's `stop` may take before a rebuild moves on without it.
|
|
# A broker that has gone away can leave a node waiting on an
|
|
# acknowledgement that never comes.
|
|
FLOW_NODE_STOP_TIMEOUT: float = 5.0
|
|
# How long a request waits for a rebuild that is already running before it
|
|
# is answered as busy: one for the whole pipeline, one for a single flow's.
|
|
# Past either, the engine is wedged rather than busy.
|
|
FLOW_PIPELINE_REBUILD_WAIT: float = 15.0
|
|
FLOW_REBUILD_WAIT: float = 5.0
|
|
# Seconds between restarts of a node's crashed loop, one step per attempt
|
|
# with the last repeating. A JSON list in the environment.
|
|
FLOW_RESTART_BACKOFF: Annotated[tuple[float, ...], Field(min_length=1)] = (
|
|
1.0,
|
|
5.0,
|
|
30.0,
|
|
60.0,
|
|
)
|
|
# How long a cancelled node loop gets to stop before it is abandoned.
|
|
FLOW_CANCEL_GRACE: float = 5.0
|
|
# How long the engine's own metrics, events and run records are kept.
|
|
#: The largest body `PUT /artifacts` will take, in bytes. A checkpoint or a
|
|
#: video segment is a legitimate artifact, so this is generous rather than
|
|
#: small; 0 removes the limit. Nothing capped it at all before, so any
|
|
#: account or worker credential could fill the data volume.
|
|
MAX_ARTIFACT_BYTES: int = 2 * 1024 * 1024 * 1024
|
|
|
|
OBS_RETENTION_DAYS: int = 30
|
|
# How often artifact bytes nothing refers to any more are swept away; 0
|
|
# never sweeps. A flow streaming media writes one artifact per frame, so
|
|
# without this the store only grows.
|
|
ARTIFACT_GC_INTERVAL_S: int = 3600
|
|
# How long a freshly written artifact is spared, whatever refers to it.
|
|
# Storing bytes and recording the reference are two steps; this is the
|
|
# window between them.
|
|
ARTIFACT_GC_GRACE_S: int = 3600
|
|
#: Where a worker in this container writes the frames a flow only shows
|
|
#: live: memory rather than the data volume, so a camera at ten frames a
|
|
#: second is not writing to an SD card. The engine holds its own frames in
|
|
#: its memory. Empty works it out — a directory under `/dev/shm` named for
|
|
#: this data directory, so two instances on one host do not trim each
|
|
#: other — and falls back to the temporary directory where there is no
|
|
#: `/dev/shm`.
|
|
ARTIFACT_VOLATILE_DIR: Path | None = None
|
|
#: How many bytes of live frames the engine holds before the oldest fall
|
|
#: out, and the workers' directory too; 0 turns the ring off, and a
|
|
#: volatile save then lands in the durable store like any other. Under
|
|
#: Docker's default 64 MB `/dev/shm`: raise `shm_size` with it when
|
|
#: workers publish frames.
|
|
ARTIFACT_VOLATILE_BYTES: int = 48 * 1024 * 1024
|
|
#: The most one live frame may weigh: a volatile `PUT /artifacts` is
|
|
#: refused with 413 above it, and a device's frame on its socket is
|
|
#: dropped. Held in memory whole, which is why it is far below the
|
|
#: durable store's cap. Keep it under uvicorn's 16 MiB websocket limit.
|
|
MAX_FRAME_BYTES: int = 4 * 1024 * 1024
|
|
# Without a Redis host the engine keeps its state in memory.
|
|
REDIS_HOST: str | None = None
|
|
REDIS_PORT: int = 6379
|
|
|
|
BACKEND_CORS_ORIGINS: Annotated[
|
|
list[AnyUrl] | str, BeforeValidator(parse_cors)
|
|
] = []
|
|
|
|
#: Where a live view's two ends look for a way to each other, comma
|
|
#: separated. None by default: on one network the peers find each other
|
|
#: without, and a third-party server would learn who watches what. coturn
|
|
#: serves STUN as well, so its `stun:` address goes here.
|
|
STUN_URLS: Annotated[list[str] | str, BeforeValidator(parse_cors)] = []
|
|
#: The relay a live view falls back to when the two ends cannot reach each
|
|
#: other, as `turn:` and `turns:` URLs. Used only with `TURN_SECRET`, the
|
|
#: secret the TURN server shares (coturn's `static-auth-secret`), from
|
|
#: which each viewer and device gets credentials of its own.
|
|
TURN_URLS: Annotated[list[str] | str, BeforeValidator(parse_cors)] = []
|
|
TURN_SECRET: str = ""
|
|
|
|
@model_validator(mode="before")
|
|
@classmethod
|
|
def _derive_data_paths(cls, data: Any) -> Any:
|
|
"""Put every stored thing under ``DATA_DIR`` unless it was named.
|
|
|
|
``setdefault``, so the container images keep their explicit ``/data``
|
|
paths and a checkout keeps ``flow-data/``.
|
|
"""
|
|
if not isinstance(data, dict):
|
|
return data
|
|
base = Path(str(data.get("DATA_DIR", "flow-data"))).expanduser()
|
|
data["DATA_DIR"] = base
|
|
for key, name in DERIVED_PATHS.items():
|
|
data.setdefault(key, base / name)
|
|
return data
|
|
|
|
@computed_field # type: ignore[prop-decorator]
|
|
@property
|
|
def oauth_issuer(self) -> str:
|
|
"""Who issues MCP tokens — this app, at the address it is reached on.
|
|
|
|
The API is a path on the dashboard's origin, so that one address is
|
|
both. Kept a property of its own because a hosted deployment can later
|
|
point agents at a different issuer without the resource server
|
|
changing: it validates whatever issuer it is configured to trust.
|
|
"""
|
|
return self.FRONTEND_HOST.rstrip("/")
|
|
|
|
@computed_field # type: ignore[prop-decorator]
|
|
@property
|
|
def mcp_resource(self) -> str:
|
|
"""The resource an MCP token is issued for (RFC 8707)."""
|
|
return f"{self.oauth_issuer}/mcp"
|
|
|
|
@computed_field # type: ignore[prop-decorator]
|
|
@property
|
|
def all_cors_origins(self) -> list[str]:
|
|
return [str(origin).rstrip("/") for origin in self.BACKEND_CORS_ORIGINS] + [
|
|
self.FRONTEND_HOST
|
|
]
|
|
|
|
PROJECT_NAME: str = "Fluksio"
|
|
SENTRY_DSN: HttpUrl | None = None
|
|
|
|
@computed_field # type: ignore[prop-decorator]
|
|
@property
|
|
def SQLALCHEMY_DATABASE_URI(self) -> str:
|
|
"""SQLite in the data directory, unless a URL names another file.
|
|
|
|
One engine process owns this database — the same reason the image runs
|
|
a single uvicorn worker — so a file beside the flows is the honest
|
|
shape for it, and needs nothing running to be one.
|
|
"""
|
|
if self.DATABASE_URL:
|
|
return self.DATABASE_URL
|
|
return f"sqlite:///{(self.DATA_DIR / 'fluksio.db').expanduser().resolve()}"
|
|
|
|
SMTP_TLS: bool = True
|
|
SMTP_SSL: bool = False
|
|
SMTP_PORT: int = 587
|
|
SMTP_HOST: str | None = None
|
|
SMTP_USER: str | None = None
|
|
SMTP_PASSWORD: str | None = None
|
|
EMAILS_FROM_EMAIL: EmailStr | None = None
|
|
EMAILS_FROM_NAME: str | None = None
|
|
|
|
@model_validator(mode="after")
|
|
def _set_default_emails_from(self) -> Self:
|
|
if not self.EMAILS_FROM_NAME:
|
|
self.EMAILS_FROM_NAME = self.PROJECT_NAME
|
|
return self
|
|
|
|
EMAIL_RESET_TOKEN_EXPIRE_HOURS: int = 48
|
|
|
|
@computed_field # type: ignore[prop-decorator]
|
|
@property
|
|
def emails_enabled(self) -> bool:
|
|
return bool(self.SMTP_HOST and self.EMAILS_FROM_EMAIL)
|
|
|
|
EMAIL_TEST_USER: EmailStr = "test@example.com"
|
|
# Absent means "the CLI will make one on first run" — a pip install is not
|
|
# asked for two environment variables before it can start.
|
|
FIRST_SUPERUSER: EmailStr | None = None
|
|
FIRST_SUPERUSER_PASSWORD: str | None = None
|
|
|
|
def _check_default_secret(self, var_name: str, value: str | None) -> None:
|
|
if value == "changethis":
|
|
message = (
|
|
f'The value of {var_name} is "changethis", '
|
|
"for security, please change it, at least for deployments."
|
|
)
|
|
if self.ENVIRONMENT == "local":
|
|
warnings.warn(message, stacklevel=1)
|
|
else:
|
|
raise ValueError(message)
|
|
|
|
@model_validator(mode="after")
|
|
def _enforce_non_default_secrets(self) -> Self:
|
|
self._check_default_secret("SECRET_KEY", self.SECRET_KEY)
|
|
self._check_default_secret(
|
|
"FIRST_SUPERUSER_PASSWORD", self.FIRST_SUPERUSER_PASSWORD
|
|
)
|
|
|
|
return self
|
|
|
|
|
|
# No arguments and no required environment: a fresh install boots on the
|
|
# defaults above, into a data directory of its own.
|
|
settings = Settings()
|