From 22c682e505b40e89d50d646b734ffacac42d99c2 Mon Sep 17 00:00:00 2001 From: stroblme Date: Sat, 29 Aug 2026 14:06:50 +0200 Subject: [PATCH] Put the deployment-only dependencies behind a server extra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A data-science environment installing fluksio waited for lxml, aiohttp and the rest of a connector stack it has nothing to talk to. Outbound mail, error reporting and the MQTT and InfluxDB clients moved to `fluksio[server]`, which the image installs; each import is guarded and names the extra. `tenacity` had no import site at all and is gone. 23 fewer packages and the compiled ones among them — a bare `pip install fluksio` still serves, runs every python node, and registers the mqtt and influxdb node types, which only need the library when one is actually built. sentry-sdk arrives anyway underneath `fastapi[standard]`; what changed there is that nothing of ours requires it. The dev environment keeps every extra: the suite exercises the connectors and strict mypy checks their call sites. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019Hra4ndWMCLU5F3KjUuVAc --- Makefile | 14 ++++++------ backend/Dockerfile | 4 ++-- backend/fluksio/flow/nodes/influx.py | 26 ++++++++++++++++++---- backend/fluksio/flow/nodes/mqtt.py | 22 ++++++++++++++++--- backend/fluksio/main.py | 5 ++++- backend/fluksio/sdk/cli.py | 10 ++++----- backend/fluksio/utils.py | 7 +++++- backend/pyproject.toml | 16 +++++++++----- backend/tests/api/routes/test_runs.py | 4 +--- uv.lock | 31 ++++++++++----------------- worker/fluksio_worker/worker_main.py | 4 ++++ 11 files changed, 93 insertions(+), 50 deletions(-) diff --git a/Makefile b/Makefile index e3df5de..291035a 100644 --- a/Makefile +++ b/Makefile @@ -105,11 +105,13 @@ down: ## Stop all running containers # Run `make dev-backend` and `make dev-frontend` in two separate terminals. install: ## Install all dependencies (backend + frontend) - cd backend && uv sync + # Every extra: the suite exercises the connectors that moved into + # `fluksio[server]`, and strict mypy checks their call sites. + cd backend && uv sync --all-extras cd frontend && bun install dev-backend: ## Start the FastAPI backend with hot-reload (local) - cd backend && uv run fastapi dev fluksio/main.py + cd backend && uv run --all-extras fastapi dev fluksio/main.py dev-frontend: ## Start the Vite dev server (local) cd frontend && bun dev @@ -130,7 +132,7 @@ test: test-backend test-frontend ## Run all tests (backend + frontend) test-backend: ## Run backend tests (pytest + coverage) # Its own SQLite file in a temp directory (tests/__init__.py), so this needs # nothing running and touches no development data. - cd backend && uv run bash scripts/test.sh + cd backend && uv run --all-extras bash scripts/test.sh PW_VERSION = $(shell sed -n 's/.*"@playwright\/test": "[^0-9]*\([0-9.]*\)".*/\1/p' frontend/package.json | head -1) @@ -181,9 +183,9 @@ build: ## Build the fluksio and fluksio-worker wheels into dist/ lint: lint-backend lint-frontend ## Run all linters lint-backend: ## Lint backend with ruff + mypy - cd backend && uv run ruff check . - cd backend && uv run ruff format --check . - cd backend && uv run mypy fluksio + cd backend && uv run --all-extras ruff check . + cd backend && uv run --all-extras ruff format --check . + cd backend && uv run --all-extras mypy fluksio cd worker && uv run --no-project --with mypy mypy fluksio_worker lint-frontend: ## Lint frontend with biome diff --git a/backend/Dockerfile b/backend/Dockerfile index 27041f8..93f978e 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -31,7 +31,7 @@ ENV NODE_VENV=managed RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=uv.lock,target=uv.lock \ --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - uv sync --frozen --no-install-workspace --package fluksio + uv sync --frozen --no-install-workspace --package fluksio --extra server COPY ./backend/scripts /app/backend/scripts @@ -52,7 +52,7 @@ COPY ./worker /app/worker RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=uv.lock,target=uv.lock \ --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - uv sync --frozen --package fluksio + uv sync --frozen --package fluksio --extra server # Connectors are ordinary installed packages found through the # `fluksio.node_types` entry point. `make connectors` builds them into here; diff --git a/backend/fluksio/flow/nodes/influx.py b/backend/fluksio/flow/nodes/influx.py index c0018f8..08cb4b8 100644 --- a/backend/fluksio/flow/nodes/influx.py +++ b/backend/fluksio/flow/nodes/influx.py @@ -14,6 +14,22 @@ from fluksio.flow.nodes.base import Node, NodeResult logger = logging.getLogger(__name__) +def _influxdb() -> Any: + """The client library, which is a `fluksio[server]` extra. + + Imported per use rather than at module level, because the node type is + registered at boot and an installation with no InfluxDB behind it should + not have to carry the library to start. + """ + try: + import influxdb_client + except ImportError: + raise RuntimeError( + "the influxdb node needs the server extra: pip install 'fluksio[server]'" + ) from None + return influxdb_client + + class InfluxDbNode(Node): """ InfluxDB node for writing to and reading from InfluxDB. @@ -290,7 +306,7 @@ class InfluxDbNode(Node): :returns: ``{"rows": [...], **echo}``. :rtype: dict """ - from influxdb_client import InfluxDBClient + InfluxDBClient = _influxdb().InfluxDBClient flux = str(request["flux"]) echo = {key: value for key, value in request.items() if key != "flux"} @@ -338,8 +354,10 @@ class InfluxDbNode(Node): - Dict with "value" and "tags" keys: Value written with merged tags :type data: dict """ - from influxdb_client import InfluxDBClient, Point, WritePrecision - from influxdb_client.client.write_api import SYNCHRONOUS + influxdb_client = _influxdb() + InfluxDBClient = influxdb_client.InfluxDBClient + Point, WritePrecision = influxdb_client.Point, influxdb_client.WritePrecision + SYNCHRONOUS = influxdb_client.client.write_api.SYNCHRONOUS try: with InfluxDBClient( @@ -438,7 +456,7 @@ class InfluxDbNode(Node): :returns: Dict of port name to queried value. :rtype: dict """ - from influxdb_client import InfluxDBClient + InfluxDBClient = _influxdb().InfluxDBClient results = {} diff --git a/backend/fluksio/flow/nodes/mqtt.py b/backend/fluksio/flow/nodes/mqtt.py index f6b8e35..80ffaef 100644 --- a/backend/fluksio/flow/nodes/mqtt.py +++ b/backend/fluksio/flow/nodes/mqtt.py @@ -20,6 +20,22 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +def _aiomqtt() -> Any: + """The client library, which is a `fluksio[server]` extra. + + Imported per use rather than at module level, because the node type is + registered at boot and an installation that talks to no broker should not + have to carry the library to start. + """ + try: + import aiomqtt + except ImportError: + raise RuntimeError( + "the mqtt node needs the server extra: pip install 'fluksio[server]'" + ) from None + return aiomqtt + + def topic_matches(filter_: str, topic: str) -> bool: """Does an MQTT topic filter cover this topic? @@ -382,7 +398,7 @@ class MqttNode(Node): A dropped connection raises, and the supervisor decides when to reconnect — the same arrangement the subscriber uses. """ - import aiomqtt + aiomqtt = _aiomqtt() queue = self._publish_queue if queue is None: @@ -408,7 +424,7 @@ class MqttNode(Node): async def _publish_once(self, data: dict[str, Any]) -> None: """Connect, publish, disconnect — the unstarted node's path.""" - import aiomqtt + aiomqtt = _aiomqtt() async with aiomqtt.Client( hostname=self.broker_host, @@ -570,7 +586,7 @@ class MqttNode(Node): """ import json - import aiomqtt + aiomqtt = _aiomqtt() if not (self._stop_event and self._stop_event.is_set()): try: diff --git a/backend/fluksio/main.py b/backend/fluksio/main.py index 2edba0c..8b271c7 100644 --- a/backend/fluksio/main.py +++ b/backend/fluksio/main.py @@ -4,7 +4,6 @@ import logging from collections.abc import AsyncIterator from contextlib import AbstractAsyncContextManager, asynccontextmanager -import sentry_sdk from fastapi import FastAPI, Request from fastapi.concurrency import run_in_threadpool from fastapi.responses import JSONResponse @@ -51,6 +50,10 @@ def custom_generate_unique_id(route: APIRoute) -> str: if settings.SENTRY_DSN and settings.ENVIRONMENT != "local": + # Imported here rather than at the top: it is a `fluksio[server]` extra, so + # a pip install without one has no sentry to import — and no DSN either. + import sentry_sdk + # `enable_tracing` was removed in sentry-sdk 2.x; this is what it meant. sentry_sdk.init(dsn=str(settings.SENTRY_DSN), traces_sample_rate=1.0) diff --git a/backend/fluksio/sdk/cli.py b/backend/fluksio/sdk/cli.py index 2866798..bc18d3b 100644 --- a/backend/fluksio/sdk/cli.py +++ b/backend/fluksio/sdk/cli.py @@ -485,14 +485,14 @@ def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]: f"{', '.join(sorted(types)) or 'none'}){hint}" ) params: dict[str, Any] = {} - for name, value in raw.items(): - if value is True: - params[name] = True + for key, written in raw.items(): + if isinstance(written, bool): + params[key] = written continue try: - params[name] = _coerce(value, types[name]) + params[key] = _coerce(written, types[key]) except ValueError as exc: - raise SyncError(f"'{name}' takes {types[name]}: {exc}") from exc + raise SyncError(f"'{key}' takes {types[key]}: {exc}") from exc return params diff --git a/backend/fluksio/utils.py b/backend/fluksio/utils.py index d49532b..702a744 100644 --- a/backend/fluksio/utils.py +++ b/backend/fluksio/utils.py @@ -4,7 +4,6 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any -import emails # type: ignore import jwt from jinja2 import Template from jwt.exceptions import InvalidTokenError @@ -37,6 +36,12 @@ def send_email( html_content: str = "", ) -> None: assert settings.emails_enabled, "no provided configuration for email variables" + try: + import emails # type: ignore + except ImportError: + raise RuntimeError( + "sending mail needs the server extra: pip install 'fluksio[server]'" + ) from None message = emails.Message( subject=subject, html=html_content, diff --git a/backend/pyproject.toml b/backend/pyproject.toml index a5de93d..2c8b82a 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -25,15 +25,12 @@ dependencies = [ "fastapi[standard]<1.0.0,>=0.114.2", "python-multipart<1.0.0,>=0.0.7", "email-validator<3.0.0.0,>=2.1.0.post1", - "tenacity<9.0.0,>=8.2.3", "pydantic>2.0", - "emails<1.0,>=0.6", "jinja2<4.0.0,>=3.1.4", "alembic<2.0.0,>=1.12.1", "httpx<1.0.0,>=0.25.1", "sqlmodel<1.0.0,>=0.0.21", "pydantic-settings<3.0.0,>=2.2.1", - "sentry-sdk[fastapi]>=2.20.0", "pyjwt<3.0.0,>=2.8.0", "pwdlib[argon2,bcrypt]>=0.3.0", "numpy>=2.2.6", @@ -43,8 +40,6 @@ dependencies = [ # copied onto other people's machines and stays dependency-free. "orjson>=3.10", "cryptography>=44.0.0", - "aiomqtt>=2.0.0", - "influxdb-client[async]>=1.40.0", "croniter>=1.3.0", "mcp>=1.29,<2", "fluksio-worker>=0.1,<0.2", @@ -62,6 +57,17 @@ dependencies = [ # extra rather than a dependency: csv and jsonl need nothing, and pyarrow is # tens of megabytes for whoever wants dtypes kept. parquet = ["pyarrow>=17"] +# What a deployment has and a laptop does not: the device connectors, outbound +# mail and error reporting. The engine, the CLI and every python node work +# without them — `pip install fluksio` in a data-science environment is the +# case this exists for, and lxml and the aiohttp stack are most of its wait. +# Each import is guarded and names this extra. The image installs it. +server = [ + "emails<1.0,>=0.6", + "sentry-sdk[fastapi]>=2.20.0", + "aiomqtt>=2.0.0", + "influxdb-client[async]>=1.40.0", +] [project.urls] Homepage = "https://fluksio.com" diff --git a/backend/tests/api/routes/test_runs.py b/backend/tests/api/routes/test_runs.py index 6aff598..6dff91a 100644 --- a/backend/tests/api/routes/test_runs.py +++ b/backend/tests/api/routes/test_runs.py @@ -433,9 +433,7 @@ def test_a_run_records_the_inputs_it_actually_starts_from(): name="study", mode="batch", inputs=[ - FlowInput( - spec=MessageSpec(name="lr", dtype=DType.FLOAT), initial=0.01 - ), + FlowInput(spec=MessageSpec(name="lr", dtype=DType.FLOAT), initial=0.01), FlowInput(spec=MessageSpec(name="epochs", dtype=DType.INT)), ], ) diff --git a/uv.lock b/uv.lock index 8d79503..849c897 100644 --- a/uv.lock +++ b/uv.lock @@ -872,16 +872,13 @@ name = "fluksio" version = "0.1.4+dev" source = { editable = "backend" } dependencies = [ - { name = "aiomqtt" }, { name = "alembic" }, { name = "croniter" }, { name = "cryptography" }, { name = "email-validator" }, - { name = "emails" }, { name = "fastapi", extra = ["standard"] }, { name = "fluksio-worker" }, { name = "httpx" }, - { name = "influxdb-client", extra = ["async"] }, { name = "jinja2" }, { name = "mcp" }, { name = "numpy" }, @@ -893,9 +890,7 @@ dependencies = [ { name = "python-multipart" }, { name = "redis" }, { name = "rich" }, - { name = "sentry-sdk", extra = ["fastapi"] }, { name = "sqlmodel" }, - { name = "tenacity" }, { name = "uv" }, ] @@ -903,6 +898,12 @@ dependencies = [ parquet = [ { name = "pyarrow" }, ] +server = [ + { name = "aiomqtt" }, + { name = "emails" }, + { name = "influxdb-client", extra = ["async"] }, + { name = "sentry-sdk", extra = ["fastapi"] }, +] [package.dev-dependencies] dev = [ @@ -915,16 +916,16 @@ dev = [ [package.metadata] requires-dist = [ - { name = "aiomqtt", specifier = ">=2.0.0" }, + { name = "aiomqtt", marker = "extra == 'server'", specifier = ">=2.0.0" }, { name = "alembic", specifier = ">=1.12.1,<2.0.0" }, { name = "croniter", specifier = ">=1.3.0" }, { name = "cryptography", specifier = ">=44.0.0" }, { name = "email-validator", specifier = ">=2.1.0.post1,<3.0.0.0" }, - { name = "emails", specifier = ">=0.6,<1.0" }, + { name = "emails", marker = "extra == 'server'", specifier = ">=0.6,<1.0" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.114.2,<1.0.0" }, { name = "fluksio-worker", editable = "worker" }, { name = "httpx", specifier = ">=0.25.1,<1.0.0" }, - { name = "influxdb-client", extras = ["async"], specifier = ">=1.40.0" }, + { name = "influxdb-client", extras = ["async"], marker = "extra == 'server'", specifier = ">=1.40.0" }, { name = "jinja2", specifier = ">=3.1.4,<4.0.0" }, { name = "mcp", specifier = ">=1.29,<2" }, { name = "numpy", specifier = ">=2.2.6" }, @@ -937,12 +938,11 @@ requires-dist = [ { name = "python-multipart", specifier = ">=0.0.7,<1.0.0" }, { name = "redis", specifier = ">=7.1.0" }, { name = "rich", specifier = ">=13" }, - { name = "sentry-sdk", extras = ["fastapi"], specifier = ">=2.20.0" }, + { name = "sentry-sdk", extras = ["fastapi"], marker = "extra == 'server'", specifier = ">=2.20.0" }, { name = "sqlmodel", specifier = ">=0.0.21,<1.0.0" }, - { name = "tenacity", specifier = ">=8.2.3,<9.0.0" }, { name = "uv", specifier = ">=0.5" }, ] -provides-extras = ["parquet"] +provides-extras = ["parquet", "server"] [package.metadata.requires-dev] dev = [ @@ -2673,15 +2673,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/cb/6a6a47d5b464bd08695d254f3da6e7986cc70c9fa5d778eda57538edfe56/starlette-1.6.0-py3-none-any.whl", hash = "sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c", size = 75969, upload-time = "2026-08-08T18:27:56.196Z" }, ] -[[package]] -name = "tenacity" -version = "8.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/4d/6a19536c50b849338fcbe9290d562b52cbdcf30d8963d3588a68a4107df1/tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78", size = 47309, upload-time = "2024-07-05T07:25:31.836Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/3f/8ba87d9e287b9d385a02a7114ddcef61b26f86411e121c9003eb509a1773/tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687", size = 28165, upload-time = "2024-07-05T07:25:29.591Z" }, -] - [[package]] name = "typer" version = "0.21.1" diff --git a/worker/fluksio_worker/worker_main.py b/worker/fluksio_worker/worker_main.py index 5ba0880..50142f2 100644 --- a/worker/fluksio_worker/worker_main.py +++ b/worker/fluksio_worker/worker_main.py @@ -89,6 +89,10 @@ def _emit(event: dict[str, Any]) -> None: class _Reporter(ModuleType): """``import fluksio`` — the parts of a node's job that need the engine.""" + #: The same name the SDK exports at top level, so a node written against + #: one surface finds it on the other. Set by ``_install_reporter``. + logger: logging.Logger + def emit(self, **ports: Any) -> None: """Publish on this node's output ports without returning yet.