Add the flow API: typed messages, git-backed store, REST and live events
Makes the flow engine reachable from the API, which is what M3 needs before
any of it can reach the browser.
- app/flow is a package now; the prototype's watch-dir scripts and the
matplotlib/networkx visualiser are gone with their dependencies.
- Messages carry a serializable dtype instead of a live Python type, and a
port name, so the graph can speak qualified names while node functions keep
local arguments. Redis state is JSON, not pickle.
- Message names are namespaced per flow ("heating.temp"); a bare name resolves
to its own flow, a dotted one crosses flows.
- Several nodes may provide the same message: producers are a list, so fan-in
is a real edge instead of a silently dropped one.
- Flows are stored as flow.json plus node sources in a git repository, one
commit per save, with identical saves skipped so autosave stays quiet.
- Node failures are isolated and reported per node; validate() returns cycles
and unconnected inputs instead of raising deep in a run.
- Credentials live in an encrypted store and are referenced as {"$secret": …}.
- Engine events reach websocket clients through a bus, so values, node status
and execution show up live.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016WzrvW7rjQbynnhF6pxh6i
This commit is contained in:
Melvin Strobl
co-authored by
Claude Fable 5
Claude Fable 5
parent
61be29827d
commit
06a4506767
@@ -0,0 +1,146 @@
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
PREFIX = f"{settings.API_V1_STR}/flows"
|
||||
|
||||
WORKING_NODE = """
|
||||
def process(params):
|
||||
return {"reading": 21.5}
|
||||
"""
|
||||
|
||||
BROKEN_NODE = """
|
||||
def process(params):
|
||||
raise RuntimeError("boom")
|
||||
"""
|
||||
|
||||
|
||||
def a_flow(name: str = "demo") -> dict:
|
||||
return {
|
||||
"name": name,
|
||||
"title": "Demo",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "sensor",
|
||||
"type": "python",
|
||||
"position": {"x": 0, "y": 0},
|
||||
"provides": [{"name": "reading", "dtype": "float"}],
|
||||
},
|
||||
{
|
||||
"id": "logger",
|
||||
"type": "python",
|
||||
"position": {"x": 240, "y": 0},
|
||||
"requires": [{"name": "reading", "dtype": "float"}],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_flows_require_authentication(client: TestClient) -> None:
|
||||
assert client.get(f"{PREFIX}/").status_code == 401
|
||||
|
||||
|
||||
def test_save_then_read_flow(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
response = client.put(
|
||||
f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow()
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
response = client.get(f"{PREFIX}/demo", headers=superuser_token_headers)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["definition"]["title"] == "Demo"
|
||||
assert {node["id"] for node in body["definition"]["nodes"]} == {"sensor", "logger"}
|
||||
|
||||
|
||||
def test_name_mismatch_is_rejected(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
response = client.put(
|
||||
f"{PREFIX}/other", headers=superuser_token_headers, json=a_flow("demo")
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def test_broken_node_is_reported_and_siblings_stay_active(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
|
||||
client.put(
|
||||
f"{PREFIX}/demo/nodes/sensor/source",
|
||||
headers=superuser_token_headers,
|
||||
json={"code": WORKING_NODE},
|
||||
)
|
||||
|
||||
response = client.put(
|
||||
f"{PREFIX}/demo/nodes/logger/source",
|
||||
headers=superuser_token_headers,
|
||||
json={"code": "def process(reading, params:\n"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "error"
|
||||
|
||||
statuses = client.get(f"{PREFIX}/demo", headers=superuser_token_headers).json()
|
||||
by_id = {node["id"]: node for node in statuses["nodes"]}
|
||||
assert by_id["demo.sensor"]["status"] == "active"
|
||||
assert by_id["demo.logger"]["status"] == "error"
|
||||
|
||||
|
||||
def test_running_a_flow_produces_values(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
|
||||
client.put(
|
||||
f"{PREFIX}/demo/nodes/sensor/source",
|
||||
headers=superuser_token_headers,
|
||||
json={"code": WORKING_NODE},
|
||||
)
|
||||
client.put(
|
||||
f"{PREFIX}/demo/nodes/logger/source",
|
||||
headers=superuser_token_headers,
|
||||
json={"code": "def process(reading, params):\n return {}\n"},
|
||||
)
|
||||
|
||||
response = client.post(
|
||||
f"{PREFIX}/demo/run", headers=superuser_token_headers, json={"inputs": {}}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["values"]["demo.reading"]["value"] == 21.5
|
||||
|
||||
|
||||
def test_unconnected_input_is_surfaced(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
flow = a_flow()
|
||||
flow["nodes"][0]["provides"] = [] # nothing produces "reading" any more
|
||||
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=flow)
|
||||
|
||||
issues = client.post(
|
||||
f"{PREFIX}/demo/validate", headers=superuser_token_headers
|
||||
).json()["issues"]
|
||||
assert any(issue["code"] == "unconnected_input" for issue in issues)
|
||||
|
||||
|
||||
def test_delete_flow(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
|
||||
|
||||
assert (
|
||||
client.delete(f"{PREFIX}/demo", headers=superuser_token_headers).status_code
|
||||
== 200
|
||||
)
|
||||
assert (
|
||||
client.get(f"{PREFIX}/demo", headers=superuser_token_headers).status_code == 404
|
||||
)
|
||||
|
||||
|
||||
def test_node_types_are_listed(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
types = client.get(f"{PREFIX}/node-types", headers=superuser_token_headers).json()
|
||||
by_type = {entry["type"]: entry for entry in types}
|
||||
assert by_type["python"]["has_source"] is True
|
||||
assert "properties" in by_type["mqtt"]["params_schema"]
|
||||
@@ -12,6 +12,15 @@ from tests.utils.user import authentication_token_from_email
|
||||
from tests.utils.utils import get_superuser_token_headers
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def flow_data(tmp_path_factory: pytest.TempPathFactory) -> Generator[None, None, None]:
|
||||
"""Keep flows and secrets written by tests out of the real store."""
|
||||
root = tmp_path_factory.mktemp("flow-data")
|
||||
settings.FLOWS_DIR = root / "flows"
|
||||
settings.SECRETS_FILE = root / "secrets.enc"
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def db() -> Generator[Session, None, None]:
|
||||
with Session(engine) as session:
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
from collections.abc import Generator
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def db() -> Generator[None, None, None]:
|
||||
"""The engine holds no database state, so these tests need no database."""
|
||||
yield
|
||||
@@ -0,0 +1,55 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.flow.messages import DType, MessageSpec, flow_of, qualify
|
||||
|
||||
|
||||
def test_port_defaults_to_last_name_segment():
|
||||
assert MessageSpec(name="temperature").port == "temperature"
|
||||
assert MessageSpec(name="heating.temperature").port == "temperature"
|
||||
assert MessageSpec(name="heating.temperature", port="t").port == "t"
|
||||
|
||||
|
||||
def test_int_rejects_bool():
|
||||
# bool is a subclass of int, but a flag is not a number here.
|
||||
spec = MessageSpec(name="count", dtype=DType.INT)
|
||||
spec.check(3)
|
||||
with pytest.raises(TypeError):
|
||||
spec.check(True)
|
||||
|
||||
|
||||
def test_float_accepts_int_but_not_bool():
|
||||
spec = MessageSpec(name="temp", dtype=DType.FLOAT)
|
||||
spec.check(21)
|
||||
spec.check(21.5)
|
||||
with pytest.raises(TypeError):
|
||||
spec.check(True)
|
||||
with pytest.raises(TypeError):
|
||||
spec.check("21")
|
||||
|
||||
|
||||
def test_json_dtype_round_trips():
|
||||
spec = MessageSpec(name="payload", dtype=DType.JSON)
|
||||
value = {"a": [1, 2], "b": None}
|
||||
spec.check(value)
|
||||
assert json.loads(json.dumps(value)) == value
|
||||
|
||||
|
||||
def test_coerce_from_text():
|
||||
assert MessageSpec(name="a", dtype=DType.FLOAT).coerce("2.5") == 2.5
|
||||
assert MessageSpec(name="a", dtype=DType.INT).coerce("7") == 7
|
||||
assert MessageSpec(name="a", dtype=DType.BOOL).coerce("yes") is True
|
||||
assert MessageSpec(name="a", dtype=DType.BOOL).coerce("0") is False
|
||||
|
||||
|
||||
def test_spec_serializes():
|
||||
spec = MessageSpec(name="heating.temp", dtype=DType.FLOAT)
|
||||
assert MessageSpec.model_validate_json(spec.model_dump_json()) == spec
|
||||
|
||||
|
||||
def test_qualify_scopes_bare_names_only():
|
||||
assert qualify("heating", "temp") == "heating.temp"
|
||||
assert qualify("heating", "solar.power") == "solar.power"
|
||||
assert qualify("heating", "") == ""
|
||||
assert flow_of("heating.temp") == "heating"
|
||||
@@ -0,0 +1,152 @@
|
||||
"""The wiring fundamentals: name binding, fan-in, namespaces, validation."""
|
||||
|
||||
from app.flow.messages import DType, MessageSpec
|
||||
from app.flow.nodes import Node
|
||||
from app.flow.pipeline import Pipeline
|
||||
|
||||
|
||||
def spec(name: str, dtype: DType = DType.FLOAT, port: str = "") -> MessageSpec:
|
||||
return MessageSpec(name=name, dtype=dtype, port=port)
|
||||
|
||||
|
||||
def make_node(node_id: str, flow: str, f, requires=(), provides=()) -> Node:
|
||||
node = Node(f=f, requires=list(requires), provides=list(provides), name=node_id)
|
||||
node.assign_flow(flow, node_id)
|
||||
return node
|
||||
|
||||
|
||||
def test_bare_names_are_scoped_to_their_flow():
|
||||
source = make_node(
|
||||
"source", "heating", lambda params: {"temp": 20.0}, provides=[spec("temp")]
|
||||
)
|
||||
assert "heating.temp" in source.provides
|
||||
|
||||
|
||||
def test_consumer_receives_from_any_producer():
|
||||
# Two producers of one message: each publication reaches the consumer.
|
||||
seen = []
|
||||
|
||||
def emit_a(params):
|
||||
return {"temp": 1.0}
|
||||
|
||||
def emit_b(params):
|
||||
return {"temp": 2.0}
|
||||
|
||||
def consume(temp, params):
|
||||
seen.append(temp)
|
||||
return None
|
||||
|
||||
a = make_node("a", "heating", emit_a, provides=[spec("temp")])
|
||||
b = make_node("b", "heating", emit_b, provides=[spec("temp")])
|
||||
c = make_node("c", "heating", consume, requires=[spec("temp")])
|
||||
|
||||
pipeline = Pipeline(nodes=[a, b, c])
|
||||
|
||||
assert pipeline.produces["heating.temp"] == [a, b]
|
||||
assert pipeline.dependencies[c] == frozenset({a, b})
|
||||
|
||||
a.inject()
|
||||
b.inject()
|
||||
|
||||
assert seen == [1.0, 2.0]
|
||||
# Latest value wins.
|
||||
assert pipeline.state["heating.temp"] == 2.0
|
||||
|
||||
|
||||
def test_flows_connect_through_qualified_names():
|
||||
def emit(params):
|
||||
return {"power": 500.0}
|
||||
|
||||
def consume(power, params):
|
||||
return {"used": power}
|
||||
|
||||
producer = make_node("meter", "solar", emit, provides=[spec("power")])
|
||||
# A bare name would be heating.power; the dotted one crosses the flow.
|
||||
consumer = make_node(
|
||||
"load",
|
||||
"heating",
|
||||
consume,
|
||||
requires=[spec("solar.power")],
|
||||
provides=[spec("used")],
|
||||
)
|
||||
|
||||
pipeline = Pipeline(nodes=[producer, consumer])
|
||||
assert pipeline.dependencies[consumer] == frozenset({producer})
|
||||
|
||||
producer.inject()
|
||||
assert pipeline.state["heating.used"] == 500.0
|
||||
|
||||
|
||||
def test_ports_keep_function_arguments_local():
|
||||
def convert(celsius, params):
|
||||
return {"fahrenheit": celsius * 9 / 5 + 32}
|
||||
|
||||
node = make_node(
|
||||
"convert",
|
||||
"heating",
|
||||
convert,
|
||||
requires=[spec("solar.celsius", port="celsius")],
|
||||
provides=[spec("fahrenheit")],
|
||||
)
|
||||
Pipeline(nodes=[node])
|
||||
|
||||
assert node.execute({"solar.celsius": 100.0}) == {"heating.fahrenheit": 212.0}
|
||||
|
||||
|
||||
def test_validate_reports_cycles_and_dangling_inputs():
|
||||
a = make_node(
|
||||
"a", "f", lambda b, params: {"a": b}, requires=[spec("b")], provides=[spec("a")]
|
||||
)
|
||||
b = make_node(
|
||||
"b", "f", lambda a, params: {"b": a}, requires=[spec("a")], provides=[spec("b")]
|
||||
)
|
||||
lonely = make_node(
|
||||
"lonely", "f", lambda missing, params: None, requires=[spec("missing")]
|
||||
)
|
||||
|
||||
issues = Pipeline(nodes=[a, b, lonely]).validate()
|
||||
codes = {issue.code for issue in issues}
|
||||
|
||||
assert "cycle" in codes
|
||||
assert "unconnected_input" in codes
|
||||
dangling = next(i for i in issues if i.code == "unconnected_input")
|
||||
assert dangling.message_name == "f.missing"
|
||||
assert dangling.node == "f.lonely"
|
||||
|
||||
|
||||
def test_declared_flow_inputs_are_not_dangling():
|
||||
node = make_node(
|
||||
"n", "f", lambda setpoint, params: None, requires=[spec("setpoint")]
|
||||
)
|
||||
issues = Pipeline(nodes=[node]).validate(known_inputs={"f.setpoint"})
|
||||
assert issues == []
|
||||
|
||||
|
||||
def test_a_failing_node_does_not_stop_its_siblings():
|
||||
ran = []
|
||||
|
||||
def boom(params):
|
||||
raise RuntimeError("nope")
|
||||
|
||||
def fine(params):
|
||||
ran.append("fine")
|
||||
return {"ok": 1.0}
|
||||
|
||||
bad = make_node("bad", "f", boom, provides=[spec("bad_out")])
|
||||
good = make_node("good", "f", fine, provides=[spec("ok")])
|
||||
|
||||
pipeline = Pipeline(nodes=[bad, good])
|
||||
pipeline.run()
|
||||
|
||||
assert ran == ["fine"]
|
||||
assert pipeline.state["f.ok"] == 1.0
|
||||
|
||||
|
||||
def test_values_carry_timestamps():
|
||||
node = make_node("n", "f", lambda params: {"out": 1.0}, provides=[spec("out")])
|
||||
pipeline = Pipeline(nodes=[node])
|
||||
node.inject()
|
||||
|
||||
values = pipeline.values("f")
|
||||
assert values["f.out"]["value"] == 1.0
|
||||
assert values["f.out"]["ts"] > 0
|
||||
@@ -0,0 +1,75 @@
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.flow.messages import MessageSpec
|
||||
from app.flow.schemas import FlowDef, NodeDef
|
||||
from app.flow.store import FlowNotFound, FlowStore
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> FlowStore:
|
||||
return FlowStore(tmp_path / "flows")
|
||||
|
||||
|
||||
def commit_count(store: FlowStore) -> int:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(store.root), "rev-list", "--count", "HEAD"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
return int(result.stdout.strip())
|
||||
|
||||
|
||||
def a_flow() -> FlowDef:
|
||||
return FlowDef(
|
||||
name="heating",
|
||||
nodes=[NodeDef(id="sensor", provides=[MessageSpec(name="temp")])],
|
||||
)
|
||||
|
||||
|
||||
def test_flow_round_trips(store: FlowStore):
|
||||
store.write_flow(a_flow())
|
||||
|
||||
assert store.list_flows() == ["heating"]
|
||||
assert store.read_flow("heating").nodes[0].provides[0].name == "temp"
|
||||
|
||||
|
||||
def test_every_change_is_committed(store: FlowStore):
|
||||
before = commit_count(store)
|
||||
|
||||
store.write_flow(a_flow())
|
||||
assert commit_count(store) == before + 1
|
||||
|
||||
store.write_node_source(
|
||||
"heating", "sensor", "def process(params):\n return {}\n"
|
||||
)
|
||||
assert commit_count(store) == before + 2
|
||||
|
||||
|
||||
def test_saving_unchanged_content_does_nothing(store: FlowStore):
|
||||
store.write_flow(a_flow())
|
||||
commits = commit_count(store)
|
||||
|
||||
# Autosave repeats the same document; history should not grow.
|
||||
assert store.write_flow(a_flow()) is False
|
||||
assert commit_count(store) == commits
|
||||
|
||||
|
||||
def test_missing_flow_is_reported(store: FlowStore):
|
||||
with pytest.raises(FlowNotFound):
|
||||
store.read_flow("nope")
|
||||
|
||||
|
||||
def test_deleting_removes_flow_and_its_nodes(store: FlowStore):
|
||||
store.write_flow(a_flow())
|
||||
store.write_node_source(
|
||||
"heating", "sensor", "def process(params):\n return {}\n"
|
||||
)
|
||||
|
||||
store.delete_flow("heating")
|
||||
|
||||
assert store.list_flows() == []
|
||||
assert not (store.root / "heating").exists()
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Synchronous nodes wait until every input is fresh.
|
||||
|
||||
The ordering guarantee at the stateful boundary rests on the state backend's
|
||||
atomic operations, so those are covered here too.
|
||||
"""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from app.flow.messages import DType, MessageSpec
|
||||
from app.flow.nodes import Node
|
||||
from app.flow.pipeline import Pipeline
|
||||
from app.flow.state import MemoryState
|
||||
|
||||
|
||||
def spec(name: str) -> MessageSpec:
|
||||
return MessageSpec(name=name, dtype=DType.FLOAT)
|
||||
|
||||
|
||||
def node(node_id: str, f, requires=(), provides=(), params=None) -> Node:
|
||||
n = Node(
|
||||
f=f,
|
||||
requires=list(requires),
|
||||
provides=list(provides),
|
||||
params=params or {},
|
||||
name=node_id,
|
||||
)
|
||||
n.assign_flow("f", node_id)
|
||||
return n
|
||||
|
||||
|
||||
def test_synchronous_node_waits_for_all_inputs_to_be_fresh():
|
||||
runs: list[str] = []
|
||||
|
||||
def sensor_a(params):
|
||||
return {"a": 1.0}
|
||||
|
||||
def sensor_b(params):
|
||||
return {"b": 2.0}
|
||||
|
||||
def sync(a, b, params):
|
||||
runs.append("sync")
|
||||
return None
|
||||
|
||||
def eager(a, b, params):
|
||||
runs.append("eager")
|
||||
return None
|
||||
|
||||
a = node("a", sensor_a, provides=[spec("a")])
|
||||
b = node("b", sensor_b, provides=[spec("b")])
|
||||
sync_node = node(
|
||||
"sync", sync, requires=[spec("a"), spec("b")], params={"synchronous": True}
|
||||
)
|
||||
eager_node = node("eager", eager, requires=[spec("a"), spec("b")])
|
||||
|
||||
Pipeline(nodes=[a, b, sync_node, eager_node], max_workers=1)
|
||||
|
||||
a.inject()
|
||||
assert runs == [] # b has never arrived
|
||||
|
||||
b.inject()
|
||||
assert runs.count("eager") == 1
|
||||
assert runs.count("sync") == 1
|
||||
|
||||
# Only a is new: the eager node runs again, the synchronous one waits.
|
||||
runs.clear()
|
||||
a.inject()
|
||||
assert runs == ["eager"]
|
||||
|
||||
# Now b is new as well, so both have moved on.
|
||||
runs.clear()
|
||||
b.inject()
|
||||
assert sorted(runs) == ["eager", "sync"]
|
||||
|
||||
|
||||
def test_increment_and_multi_get():
|
||||
state = MemoryState()
|
||||
|
||||
assert state.increment("counter") == 1
|
||||
assert state.increment("counter") == 2
|
||||
|
||||
state.update({"a": 1, "b": 2})
|
||||
assert state.get_multi(["a", "b", "missing"]) == {
|
||||
"a": 1,
|
||||
"b": 2,
|
||||
"missing": None,
|
||||
}
|
||||
|
||||
|
||||
def test_compare_and_swap_only_applies_on_match():
|
||||
state = MemoryState()
|
||||
state.update({"a": 1, "b": 2})
|
||||
|
||||
assert state.compare_and_swap_multi({"a": 1, "b": 2}, {"a": 10, "new": 100})
|
||||
assert state.get("a") == 10
|
||||
assert state.get("new") == 100
|
||||
|
||||
assert not state.compare_and_swap_multi({"a": 1}, {"a": 99})
|
||||
assert state.get("a") == 10
|
||||
|
||||
|
||||
def test_only_one_thread_wins_a_compare_and_swap():
|
||||
state = MemoryState()
|
||||
state.set("version", 1)
|
||||
|
||||
def claim() -> bool:
|
||||
return state.compare_and_swap_multi({"version": 1}, {"version": 2})
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(lambda _: claim(), range(8)))
|
||||
|
||||
assert sum(results) == 1
|
||||
Reference in New Issue
Block a user