Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Node settings arrive as keyword arguments, not a params dict

Browse Source 
A python node's settings are constants of its own function, so they are passed
the way its ports are: by name. The controller binds them to the compiled
function, the `params` field is gone from the worker and remote protocols, and
a setting sharing a port's name is reported as a node error rather than
shadowing it. The panel's scaffold follows suit and keeps the header in step
with both ports and settings.

The demo's `pace` moves from a flow input to a setting of the training node,
which is what it always was.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NUb8YpL2s3gmN9WTACTt4q
This commit is contained in:
stroblme
2026-08-20 17:47:45 +02:00
co-authored by Claude Opus 5
parent 2552c92a45
commit 3508713e85
21 changed files with 202 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/api/routes/test_flows.py
+4 -4
View File
@@ -5,12 +5,12 @@ from app.core.config import settings
PREFIX = f"{settings.API_V1_STR}/flows"
WORKING_NODE = """
def process(params):
def process():
return {"reading": 21.5}
"""
BROKEN_NODE = """
def process(params):
def process():
raise RuntimeError("boom")
"""
@@ -75,7 +75,7 @@ def test_broken_node_is_reported_and_siblings_stay_active(
response = client.put(
f"{PREFIX}/demo/nodes/logger/source",
headers=superuser_token_headers,
json={"code": "def process(reading, params:\n"},
json={"code": "def process(reading:\n"},
)
assert response.status_code == 200
assert response.json()["status"] == "error"
@@ -98,7 +98,7 @@ def test_running_a_flow_produces_values(
client.put(
f"{PREFIX}/demo/nodes/logger/source",
headers=superuser_token_headers,
json={"code": "def process(reading, params):\n return {}\n"},
json={"code": "def process(reading):\n return {}\n"},
)
response = client.post(
backend/tests/flow/test_drafts.py
+2 -2
View File
@@ -8,8 +8,8 @@ from app.flow.messages import MessageSpec
from app.flow.schemas import FlowDef, NodeDef
from app.flow.store import FlowStore, StaleVersion
SOURCE = "def process(params):\n return {}\n"
EDITED = "def process(params):\n return {'temp': 1}\n"
SOURCE = "def process():\n return {}\n"
EDITED = "def process():\n return {'temp': 1}\n"
@pytest.fixture
backend/tests/flow/test_library.py
+1 -1
View File
@@ -8,7 +8,7 @@ from app.flow.messages import MessageSpec
from app.flow.schemas import FlowDef, NodeDef
from app.flow.store import FlowStore, LibExists, LibNotFound
SOURCE = "def process(params):\n return {'temp': 1}\n"
SOURCE = "def process():\n return {'temp': 1}\n"
@pytest.fixture
backend/tests/flow/test_logs.py
+7 -2
View File
@@ -4,6 +4,7 @@ import sys
from typing import Any
from app.flow import logs
from app.flow.controller import with_settings
from app.flow.messages import DType, MessageSpec
from app.flow.nodes import Node
from app.flow.pipeline import Pipeline
@@ -73,7 +74,7 @@ def test_a_failing_node_reports_its_traceback():
namespace: dict[str, Any] = {}
exec(
compile(
'def process(params):\n print("about to fail")\n'
'def process():\n print("about to fail")\n'
' raise RuntimeError("boom")\n',
"<node demo.broken>",
"exec",
@@ -82,7 +83,11 @@ def test_a_failing_node_reports_its_traceback():
)
bus = RecordingBus()
run_with_capture([make_node("broken", namespace["process"])], bus)
# Wrapped the way the controller wraps it, so the settings a node declares
# arrive as keyword arguments and the frames match the real call.
run_with_capture(
[make_node("broken", with_settings(namespace["process"], {}))], bus
)
captured = logs_of(bus)
assert len(captured) == 1
backend/tests/flow/test_node_settings.py
+63
View File
@@ -0,0 +1,63 @@
"""A node's settings are arguments of its function, like its ports.
What distinguishes them is where the value comes from: a port carries whatever
the graph last published, a setting is a constant stored with the flow. Both
arrive by name, so one name cannot mean both.
"""
from pathlib import Path
import pytest
from app.flow.controller import FlowController
from app.flow.messages import DType, MessageSpec
from app.flow.pipeline import Pipeline
from app.flow.schemas import FlowDef, NodeDef
from app.flow.state import MemoryState
from app.flow.store import FlowStore
SOURCE = "def process(reading, factor):\n return {'scaled': reading * factor}\n"
def a_flow(**params: object) -> FlowDef:
return FlowDef(
name="house",
nodes=[
NodeDef(
id="scale",
params=dict(params),
requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
provides=[MessageSpec(name="scaled", dtype=DType.FLOAT)],
)
],
)
@pytest.fixture
def store(tmp_path: Path) -> FlowStore:
return FlowStore(tmp_path / "flows")
def test_a_setting_reaches_the_function_as_a_keyword_argument(store: FlowStore):
store.write_flow(a_flow(factor=3))
store.write_node_source("house", "scale", SOURCE)
controller = FlowController(store)
nodes, _loaded, _initial, _inputs = controller._build_flows(
[(store.read_flow("house"), False)]
)
pipeline = Pipeline(nodes=nodes, state=MemoryState())
pipeline.run({"house.reading": 2.0})
assert pipeline.values()["house.scaled"]["value"] == 6.0
def test_a_setting_named_after_a_port_is_refused(store: FlowStore):
store.write_draft(a_flow(reading=3), 0)
store.write_node_source("house", "scale", SOURCE, draft=True)
controller = FlowController(store)
preview = controller.preview("house")
assert [node.status for node in preview.nodes] == ["error"]
assert "both an input and a setting" in (preview.nodes[0].error or "")
backend/tests/flow/test_remote.py
+6 -9
View File
@@ -64,7 +64,7 @@ def test_a_call_crosses_to_the_thread_and_the_answer_comes_back(loop):
thread = call_in_thread(
lambda: result.update(
value=hub.run(
"gpu", "flow", "node", "src", {"x": 1}, {}, "flow.node", timeout=5
"gpu", "flow", "node", "src", {"x": 1}, "flow.node", timeout=5
)
)
)
@@ -93,7 +93,6 @@ def test_reports_arrive_before_the_answer_and_a_heartbeat_is_not_one(loop):
"node",
"src",
{},
{},
"flow.node",
timeout=5,
run_id="r1",
@@ -125,7 +124,7 @@ def test_a_failure_keeps_its_class_across_the_socket(loop):
def call() -> None:
try:
hub.run("gpu", "flow", "node", "src", {}, {}, "flow.node", timeout=5)
hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=5)
except Exception as exc:
caught.append(exc)
@@ -150,7 +149,7 @@ def test_a_worker_that_goes_away_fails_the_call_rather_than_hanging(loop):
def call() -> None:
try:
hub.run("gpu", "flow", "node", "src", {}, {}, "flow.node", timeout=30)
hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=30)
except Exception as exc:
caught.append(exc)
@@ -173,7 +172,7 @@ def test_silence_past_the_deadline_is_a_timeout(loop):
def call() -> None:
try:
hub.run("gpu", "flow", "node", "src", {}, {}, "flow.node", timeout=0.3)
hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=0.3)
except Exception as exc:
caught.append(exc)
@@ -186,7 +185,7 @@ def test_a_label_nothing_carries_is_named_rather_than_waited_on(loop):
attach(hub, loop)
with pytest.raises(NoWorker, match="tpu"):
hub.run("tpu", "flow", "node", "src", {}, {}, "flow.node", timeout=5)
hub.run("tpu", "flow", "node", "src", {}, "flow.node", timeout=5)
# Compiling against a machine that is not attached is not a broken node —
# a node importing torch is correct there and missing here.
assert hub.compile("tpu", "flow", "node", "src") is None
@@ -208,9 +207,7 @@ def test_cancelling_a_run_reaches_only_that_run(loop):
def call(run_id: str) -> None:
try:
hub.run(
"gpu", "flow", "node", "src", {}, {}, "flow.node", 30, run_id=run_id
)
hub.run("gpu", "flow", "node", "src", {}, "flow.node", 30, run_id=run_id)
except Exception:
pass
backend/tests/flow/test_store.py
+3 -9
View File
@@ -43,9 +43,7 @@ def test_every_change_is_committed(store: FlowStore):
store.write_flow(a_flow())
assert commit_count(store) == before + 1
store.write_node_source(
"heating", "sensor", "def process(params):\n return {}\n"
)
store.write_node_source("heating", "sensor", "def process():\n return {}\n")
assert commit_count(store) == before + 2
@@ -65,9 +63,7 @@ def test_missing_flow_is_reported(store: FlowStore):
def test_deleting_removes_flow_and_its_nodes(store: FlowStore):
store.write_flow(a_flow())
store.write_node_source(
"heating", "sensor", "def process(params):\n return {}\n"
)
store.write_node_source("heating", "sensor", "def process():\n return {}\n")
store.delete_flow("heating")
@@ -77,9 +73,7 @@ def test_deleting_removes_flow_and_its_nodes(store: FlowStore):
def test_renaming_a_flow_carries_its_nodes(store: FlowStore):
store.write_flow(a_flow())
store.write_node_source(
"heating", "sensor", "def process(params):\n return {}\n"
)
store.write_node_source("heating", "sensor", "def process():\n return {}\n")
renamed = store.rename_flow("heating", "warmth")
backend/tests/flow/test_workers.py
+26 -38
View File
@@ -23,18 +23,17 @@ def pool() -> Iterator[PythonWorkerPool]:
def run(pool: PythonWorkerPool, code: str, node: str = "demo", **kwargs):
return pool.run(
"demo", node, code, kwargs, {"factor": 2}, f"demo.{node}", timeout=5
)
return pool.run("demo", node, code, kwargs, f"demo.{node}", timeout=5)
def test_a_node_returns_its_value_and_what_it_printed(pool, capsys):
result = run(
pool,
"def process(value, params):\n"
"def process(value, factor):\n"
" print('seen', value)\n"
" return {'out': value * params['factor']}\n",
" return {'out': value * factor}\n",
value=21,
factor=2,
)
assert result == {"out": 42}
# The proxy writes them to stdout, which is where the engine's tee is.
@@ -43,7 +42,7 @@ def test_a_node_returns_its_value_and_what_it_printed(pool, capsys):
def test_a_failure_keeps_its_class_and_points_at_the_node(pool):
with pytest.raises(Exception) as caught:
run(pool, "def process(params):\n raise ValueError('bad input')\n")
run(pool, "def process():\n raise ValueError('bad input')\n")
# The engine renders a node error as "<class>: <message>", so both have to
# survive the trip.
@@ -55,9 +54,9 @@ def test_a_failure_keeps_its_class_and_points_at_the_node(pool):
def test_a_node_that_kills_its_worker_is_an_ordinary_error(pool):
with pytest.raises(Exception, match="worker died"):
run(pool, "import os\n\n\ndef process(params):\n os._exit(1)\n")
run(pool, "import os\n\n\ndef process():\n os._exit(1)\n")
assert run(pool, "def process(params):\n return {'out': 1}\n") == {"out": 1}
assert run(pool, "def process():\n return {'out': 1}\n") == {"out": 1}
def test_a_node_that_runs_too_long_is_killed_and_the_pool_recovers(pool):
@@ -66,8 +65,7 @@ def test_a_node_that_runs_too_long_is_killed_and_the_pool_recovers(pool):
pool.run(
"demo",
"slow",
"import time\n\n\ndef process(params):\n time.sleep(30)\n",
{},
"import time\n\n\ndef process():\n time.sleep(30)\n",
{},
"demo.slow",
timeout=1,
@@ -75,7 +73,7 @@ def test_a_node_that_runs_too_long_is_killed_and_the_pool_recovers(pool):
assert time.monotonic() - started < 10
# The killed worker's slot is refilled on the next call.
assert run(pool, "def process(params):\n return {'out': 2}\n") == {"out": 2}
assert run(pool, "def process():\n return {'out': 2}\n") == {"out": 2}
def test_a_running_node_can_be_cancelled(pool):
@@ -92,8 +90,7 @@ def test_a_running_node_can_be_cancelled(pool):
pool.run(
"demo",
"slow",
"import time\n\n\ndef process(params):\n time.sleep(30)\n",
{},
"import time\n\n\ndef process():\n time.sleep(30)\n",
{},
"demo.slow",
timeout=30,
@@ -104,12 +101,12 @@ def test_a_running_node_can_be_cancelled(pool):
def test_a_result_that_is_not_json_is_refused(pool):
with pytest.raises(Exception, match="cannot be sent back as JSON"):
run(pool, "def process(params):\n return {'out': {1, 2}}\n")
run(pool, "def process():\n return {'out': {1, 2}}\n")
def test_compiling_reports_where_the_source_is_wrong(pool):
assert pool.compile("demo", "broken", "def process(params)\n return {}\n")
assert pool.compile("demo", "fine", "def process(params):\n return {}\n") is None
assert pool.compile("demo", "broken", "def process()\n return {}\n")
assert pool.compile("demo", "fine", "def process():\n return {}\n") is None
def test_a_node_imports_the_standard_library_not_the_engines_own_modules(pool):
@@ -118,7 +115,7 @@ def test_a_node_imports_the_standard_library_not_the_engines_own_modules(pool):
result = run(
pool,
"import queue\nimport secrets\n\n\n"
"def process(params):\n"
"def process():\n"
" return {'out': [queue.Queue().qsize(), len(secrets.token_hex(4))]}\n",
)
assert result == {"out": [0, 8]}
@@ -134,7 +131,7 @@ def test_the_engines_secrets_are_not_in_a_workers_environment(pool, monkeypatch)
result = run(
pool,
"import os\n\n\n"
"def process(params):\n"
"def process():\n"
" return {'out': [k for k in ('SECRET_KEY', 'POSTGRES_PASSWORD',\n"
" 'FLUKSIO_HARMLESS') if k in os.environ]}\n",
)
@@ -152,8 +149,7 @@ def test_a_pool_can_stop_while_a_node_is_running(pool):
pool.run(
"demo",
node,
"import time\n\n\ndef process(params):\n time.sleep(60)\n",
{},
"import time\n\n\ndef process():\n time.sleep(60)\n",
{},
f"demo.{node}",
timeout=60,
@@ -177,7 +173,7 @@ def test_a_pool_can_stop_while_a_node_is_running(pool):
assert len(outcomes) == 2
with pytest.raises(Exception, match="shutting down"):
run(pool, "def process(params):\n return {'out': 1}\n")
run(pool, "def process():\n return {'out': 1}\n")
# -----------------------------------------------------------------------------
@@ -190,12 +186,11 @@ def test_a_generator_node_publishes_each_yield_and_returns_the_end(pool):
result = pool.run(
"demo",
"train",
"def process(params):\n"
"def process():\n"
" for step in range(3):\n"
" yield {'loss': 1.0 / (step + 1)}\n"
" return {'weights': 'w', 'final_loss': 0.25}\n",
{},
{},
"demo.train",
timeout=5,
run_id="r1",
@@ -219,12 +214,11 @@ def test_without_a_return_the_last_yield_is_the_result(pool):
result = pool.run(
"demo",
"count",
"def process(params):\n"
"def process():\n"
" yield {'out': 1}\n"
" yield {'out': 2}\n"
" yield {'out': 3}\n",
{},
{},
"demo.count",
timeout=5,
on_event=seen.append,
@@ -242,14 +236,13 @@ def test_emit_reaches_the_same_ports_from_inside_a_callback(pool):
"demo",
"fit",
"import fluksio\n"
"def process(params):\n"
"def process():\n"
" def on_epoch(n):\n"
" fluksio.emit(loss=1.0 / (n + 1))\n"
" for epoch in range(2):\n"
" on_epoch(epoch)\n"
" return {'done': True}\n",
{},
{},
"demo.fit",
timeout=5,
on_event=seen.append,
@@ -261,7 +254,7 @@ def test_emit_reaches_the_same_ports_from_inside_a_callback(pool):
def test_a_plain_function_still_just_returns(pool):
seen = []
assert run(pool, "def process(params):\n return {'out': 7}\n") == {"out": 7}
assert run(pool, "def process():\n return {'out': 7}\n") == {"out": 7}
assert seen == []
@@ -272,13 +265,12 @@ def test_events_hold_off_the_timeout_but_silence_does_not(pool):
"demo",
"slow",
"import time\n"
"def process(params):\n"
"def process():\n"
" for step in range(12):\n"
" time.sleep(0.05)\n"
" yield {'beat': step}\n"
" return {'done': True}\n",
{},
{},
"demo.slow",
timeout=0.3,
run_id="r2",
@@ -290,8 +282,7 @@ def test_events_hold_off_the_timeout_but_silence_does_not(pool):
pool.run(
"demo",
"quiet",
"import time\ndef process(params):\n time.sleep(2)\n return {}\n",
{},
"import time\ndef process():\n time.sleep(2)\n return {}\n",
{},
"demo.quiet",
timeout=0.3,
@@ -309,8 +300,7 @@ def test_cancelling_one_run_leaves_the_same_node_in_another_alone(pool):
pool.run(
"demo",
"hold",
"import time\ndef process(params):\n time.sleep(5)\n return {}\n",
{},
"import time\ndef process():\n time.sleep(5)\n return {}\n",
{},
"demo.hold",
timeout=10,
@@ -344,10 +334,9 @@ def test_a_node_saves_and_loads_an_artifact(tmp_path):
"demo",
"save",
"import fluksio\n"
"def process(params):\n"
"def process():\n"
" return {'weights': fluksio.save_artifact(b'x' * 2048, 'w.npz')}\n",
{},
{},
"demo.save",
timeout=10,
)["weights"]
@@ -360,11 +349,10 @@ def test_a_node_saves_and_loads_an_artifact(tmp_path):
"demo",
"load",
"import fluksio\n"
"def process(weights, params):\n"
"def process(weights):\n"
" with open(fluksio.load_artifact(weights), 'rb') as f:\n"
" return {'size': len(f.read())}\n",
{"weights": ref},
{},
"demo.load",
timeout=10,
)