stroblmeandClaude Opus 5
180da3d640
The engine was I/O-bound on its own state backend. `RedisState.lock()` is one key — `pipeline:_lock` — for the whole process, taken five times a message at two round trips each, and every cascade and every node read queued behind it. Inside it, reading a node's inputs was three round trips per input (an EXISTS for `in`, then EXISTS and GET for the value), writing was two updates that a single transaction already gives, and the version counters went one INCR at a time. Replaced with the atomic command that was always available: `get_present` is one MGET and tells a missing key from one holding null, so the lock it used to be read under bought nothing; value and timestamp land in one `update`, which is a MULTI/EXEC; `increment_multi` pipelines the counters. `values()` — what every websocket snapshot calls — is two reads whatever the message count instead of two per message. Beside that: every webhook did its blocking XADD on the asyncio event loop (MQTT already used `to_thread`); the per-execution `NodeOutcome` was built and validated even with no run watching; `_minute` built a tz-aware datetime per event on the loop thread to key a dict, and now keys on an int; `move_due` promoted delayed items one round trip each, every second; `FLOW_MAX_CASCADES` makes the in-flight ceiling a setting rather than a constant. `orjson` replaces stdlib json where a message pays for it — state, the journal, the engine side of the worker pipe. `fluksio-worker` stays dependency-free, and the run-cache digest stays on stdlib so no stored key is invalidated. A non-finite number now stores as `null` rather than the bare `NaN` that was never JSON. Measured with `scripts/bench_engine.py` against a real Redis, 200 messages: a five-node chain went from 43.9 to 103.1 msg/s with p50 latency 2110ms → 782ms and p95 3913ms → 1439ms; one source into twenty consumers went from 5.4 to 33.7 msg/s. In memory, twenty consumers went from 187 to 448 msg/s. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BpfSinyCBfjuieikyfMPbf
161 lines
5.1 KiB
TOML
161 lines
5.1 KiB
TOML
[project]
|
|
name = "fluksio"
|
|
version = "0.1.2"
|
|
description = "Node-based automation engine: flows, dashboards, batch runs"
|
|
readme = "README.md"
|
|
license = "AGPL-3.0-or-later"
|
|
license-files = ["LICENSE"]
|
|
requires-python = ">=3.12"
|
|
authors = [{ name = "Fluksio", email = "stroblme@posteo.de" }]
|
|
keywords = ["automation", "workflow", "dataflow", "experiment-tracking", "mlops"]
|
|
classifiers = [
|
|
"Development Status :: 3 - Alpha",
|
|
"Intended Audience :: Developers",
|
|
"Intended Audience :: Science/Research",
|
|
"Operating System :: MacOS",
|
|
"Operating System :: POSIX :: Linux",
|
|
"Programming Language :: Python :: 3",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Programming Language :: Python :: 3.13",
|
|
"Topic :: Home Automation",
|
|
"Topic :: Scientific/Engineering",
|
|
"Topic :: System :: Distributed Computing",
|
|
]
|
|
dependencies = [
|
|
"fastapi[standard]<1.0.0,>=0.114.2",
|
|
"python-multipart<1.0.0,>=0.0.7",
|
|
"email-validator<3.0.0.0,>=2.1.0.post1",
|
|
"tenacity<9.0.0,>=8.2.3",
|
|
"pydantic>2.0",
|
|
"emails<1.0,>=0.6",
|
|
"jinja2<4.0.0,>=3.1.4",
|
|
"alembic<2.0.0,>=1.12.1",
|
|
"httpx<1.0.0,>=0.25.1",
|
|
"sqlmodel<1.0.0,>=0.0.21",
|
|
"pydantic-settings<3.0.0,>=2.2.1",
|
|
"sentry-sdk[fastapi]>=2.20.0",
|
|
"pyjwt<3.0.0,>=2.8.0",
|
|
"pwdlib[argon2,bcrypt]>=0.3.0",
|
|
"numpy>=2.2.6",
|
|
"redis>=7.1.0",
|
|
# Every message costs a round trip's worth of encode and decode — state,
|
|
# the journal, the worker pipe. Engine-side only: `fluksio-worker` is
|
|
# copied onto other people's machines and stays dependency-free.
|
|
"orjson>=3.10",
|
|
"cryptography>=44.0.0",
|
|
"aiomqtt>=2.0.0",
|
|
"influxdb-client[async]>=1.40.0",
|
|
"croniter>=1.3.0",
|
|
"mcp>=1.29,<2",
|
|
"fluksio-worker>=0.1,<0.2",
|
|
# The Modules screen installs node code's packages with it, into a venv of
|
|
# the user's own. Present in the image; a pip install would otherwise have
|
|
# to find one on PATH, and quietly fall back to the engine's interpreter.
|
|
"uv>=0.5",
|
|
# `fluksio status` draws with it. Already here underneath fastapi's CLI,
|
|
# named because a command that depends on it should say so.
|
|
"rich>=13",
|
|
]
|
|
|
|
[project.urls]
|
|
Homepage = "https://fluksio.com"
|
|
Documentation = "https://docs.fluksio.com"
|
|
"Getting started" = "https://docs.fluksio.com/getting-started/data-science/"
|
|
|
|
[project.scripts]
|
|
fluksio = "fluksio.cli:main"
|
|
|
|
[tool.uv.sources]
|
|
fluksio-worker = { workspace = true }
|
|
|
|
[dependency-groups]
|
|
dev = [
|
|
"pytest<8.0.0,>=7.4.3",
|
|
"mypy<2.0.0,>=1.8.0",
|
|
"ruff<1.0.0,>=0.2.2",
|
|
"prek>=0.2.24,<1.0.0",
|
|
"coverage<8.0.0,>=7.4.3",
|
|
]
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["fluksio"]
|
|
|
|
# Named rather than excluded: this directory also holds a working tree's
|
|
# runtime data — `flow-data/` with the user venv in it — which is not source.
|
|
[tool.hatch.build.targets.sdist]
|
|
include = ["fluksio", "tests", "scripts", "alembic.ini", "pyproject.toml", "README.md"]
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[tool.mypy]
|
|
strict = true
|
|
exclude = ["venv", ".venv", "alembic"]
|
|
# influxdb_client's Point builder carries no annotations; calling it is not a
|
|
# reason to stop checking the caller (fluksio/flow/nodes/influx.py).
|
|
untyped_calls_exclude = ["influxdb_client"]
|
|
|
|
# Every module is checked strictly. influxdb_client is typed but re-exports its
|
|
# names implicitly, which strict mode refuses to follow. (croniter, the other
|
|
# untyped dependency, is ignored at its two import sites.)
|
|
[[tool.mypy.overrides]]
|
|
module = ["influxdb_client"]
|
|
implicit_reexport = true
|
|
|
|
|
|
[tool.ruff]
|
|
target-version = "py312"
|
|
exclude = ["alembic"]
|
|
|
|
[tool.ruff.lint]
|
|
select = [
|
|
"E", # pycodestyle errors
|
|
"W", # pycodestyle warnings
|
|
"F", # pyflakes
|
|
"I", # isort
|
|
"B", # flake8-bugbear
|
|
"C4", # flake8-comprehensions
|
|
"UP", # pyupgrade
|
|
"ARG001", # unused arguments in functions
|
|
"T201", # print statements are not allowed
|
|
]
|
|
ignore = [
|
|
"E501", # line too long, handled by black
|
|
"B008", # do not perform function calls in argument defaults
|
|
"W191", # indentation contains tabs
|
|
"B904", # Allow raising exceptions without from e, for HTTPException
|
|
]
|
|
|
|
[tool.ruff.lint.per-file-ignores]
|
|
# The node API's stubs carry the real signatures and raise; unused arguments
|
|
# are what a stub is.
|
|
"fluksio/__init__.py" = ["ARG001"]
|
|
# It talks to whoever ran it; that is what a command line is.
|
|
"fluksio/cli.py" = ["T201"]
|
|
"fluksio/sdk/cli.py" = ["T201"]
|
|
# Node functions take `params` whether or not they use it — that is the
|
|
# contract the engine calls them with.
|
|
"fluksio/flow/nodes.py" = ["ARG001", "ARG002"]
|
|
"tests/flow/*" = ["ARG001"]
|
|
# Node functions declared for a test take the ports they declare, used or not.
|
|
"tests/sdk/*" = ["ARG001"]
|
|
# Printing is what this one is about: node code is user code, and `print` is
|
|
# how it says things.
|
|
"tests/flow/test_logs.py" = ["ARG001", "T201"]
|
|
|
|
[tool.ruff.lint.pyupgrade]
|
|
# Preserve types, even if a file imports `from __future__ import annotations`.
|
|
keep-runtime-typing = true
|
|
|
|
[tool.coverage.run]
|
|
source = ["fluksio"]
|
|
dynamic_context = "test_function"
|
|
|
|
[tool.coverage.report]
|
|
show_missing = true
|
|
sort = "-Cover"
|
|
|
|
[tool.coverage.html]
|
|
show_contexts = true
|