Add the flow API: typed messages, git-backed store, REST and live events

Makes the flow engine reachable from the API, which is what M3 needs before
any of it can reach the browser.

- app/flow is a package now; the prototype's watch-dir scripts and the
  matplotlib/networkx visualiser are gone with their dependencies.
- Messages carry a serializable dtype instead of a live Python type, and a
  port name, so the graph can speak qualified names while node functions keep
  local arguments. Redis state is JSON, not pickle.
- Message names are namespaced per flow ("heating.temp"); a bare name resolves
  to its own flow, a dotted one crosses flows.
- Several nodes may provide the same message: producers are a list, so fan-in
  is a real edge instead of a silently dropped one.
- Flows are stored as flow.json plus node sources in a git repository, one
  commit per save, with identical saves skipped so autosave stays quiet.
- Node failures are isolated and reported per node; validate() returns cycles
  and unconnected inputs instead of raising deep in a run.
- Credentials live in an encrypted store and are referenced as {"$secret": …}.
- Engine events reach websocket clients through a bus, so values, node status
  and execution show up live.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016WzrvW7rjQbynnhF6pxh6i

backend/tests/flow/conftest.py

@@ -0,0 +1,9 @@
+from collections.abc import Generator
+
+import pytest
+
+
+@pytest.fixture(scope="session", autouse=True)
+def db() -> Generator[None, None, None]:
+    """The engine holds no database state, so these tests need no database."""
+    yield

backend/tests/flow/test_messages.py

@@ -0,0 +1,55 @@
+import json
+
+import pytest
+
+from app.flow.messages import DType, MessageSpec, flow_of, qualify
+
+
+def test_port_defaults_to_last_name_segment():
+    assert MessageSpec(name="temperature").port == "temperature"
+    assert MessageSpec(name="heating.temperature").port == "temperature"
+    assert MessageSpec(name="heating.temperature", port="t").port == "t"
+
+
+def test_int_rejects_bool():
+    # bool is a subclass of int, but a flag is not a number here.
+    spec = MessageSpec(name="count", dtype=DType.INT)
+    spec.check(3)
+    with pytest.raises(TypeError):
+        spec.check(True)
+
+
+def test_float_accepts_int_but_not_bool():
+    spec = MessageSpec(name="temp", dtype=DType.FLOAT)
+    spec.check(21)
+    spec.check(21.5)
+    with pytest.raises(TypeError):
+        spec.check(True)
+    with pytest.raises(TypeError):
+        spec.check("21")
+
+
+def test_json_dtype_round_trips():
+    spec = MessageSpec(name="payload", dtype=DType.JSON)
+    value = {"a": [1, 2], "b": None}
+    spec.check(value)
+    assert json.loads(json.dumps(value)) == value
+
+
+def test_coerce_from_text():
+    assert MessageSpec(name="a", dtype=DType.FLOAT).coerce("2.5") == 2.5
+    assert MessageSpec(name="a", dtype=DType.INT).coerce("7") == 7
+    assert MessageSpec(name="a", dtype=DType.BOOL).coerce("yes") is True
+    assert MessageSpec(name="a", dtype=DType.BOOL).coerce("0") is False
+
+
+def test_spec_serializes():
+    spec = MessageSpec(name="heating.temp", dtype=DType.FLOAT)
+    assert MessageSpec.model_validate_json(spec.model_dump_json()) == spec
+
+
+def test_qualify_scopes_bare_names_only():
+    assert qualify("heating", "temp") == "heating.temp"
+    assert qualify("heating", "solar.power") == "solar.power"
+    assert qualify("heating", "") == ""
+    assert flow_of("heating.temp") == "heating"

backend/tests/flow/test_pipeline.py

@@ -0,0 +1,152 @@
+"""The wiring fundamentals: name binding, fan-in, namespaces, validation."""
+
+from app.flow.messages import DType, MessageSpec
+from app.flow.nodes import Node
+from app.flow.pipeline import Pipeline
+
+
+def spec(name: str, dtype: DType = DType.FLOAT, port: str = "") -> MessageSpec:
+    return MessageSpec(name=name, dtype=dtype, port=port)
+
+
+def make_node(node_id: str, flow: str, f, requires=(), provides=()) -> Node:
+    node = Node(f=f, requires=list(requires), provides=list(provides), name=node_id)
+    node.assign_flow(flow, node_id)
+    return node
+
+
+def test_bare_names_are_scoped_to_their_flow():
+    source = make_node(
+        "source", "heating", lambda params: {"temp": 20.0}, provides=[spec("temp")]
+    )
+    assert "heating.temp" in source.provides
+
+
+def test_consumer_receives_from_any_producer():
+    # Two producers of one message: each publication reaches the consumer.
+    seen = []
+
+    def emit_a(params):
+        return {"temp": 1.0}
+
+    def emit_b(params):
+        return {"temp": 2.0}
+
+    def consume(temp, params):
+        seen.append(temp)
+        return None
+
+    a = make_node("a", "heating", emit_a, provides=[spec("temp")])
+    b = make_node("b", "heating", emit_b, provides=[spec("temp")])
+    c = make_node("c", "heating", consume, requires=[spec("temp")])
+
+    pipeline = Pipeline(nodes=[a, b, c])
+
+    assert pipeline.produces["heating.temp"] == [a, b]
+    assert pipeline.dependencies[c] == frozenset({a, b})
+
+    a.inject()
+    b.inject()
+
+    assert seen == [1.0, 2.0]
+    # Latest value wins.
+    assert pipeline.state["heating.temp"] == 2.0
+
+
+def test_flows_connect_through_qualified_names():
+    def emit(params):
+        return {"power": 500.0}
+
+    def consume(power, params):
+        return {"used": power}
+
+    producer = make_node("meter", "solar", emit, provides=[spec("power")])
+    # A bare name would be heating.power; the dotted one crosses the flow.
+    consumer = make_node(
+        "load",
+        "heating",
+        consume,
+        requires=[spec("solar.power")],
+        provides=[spec("used")],
+    )
+
+    pipeline = Pipeline(nodes=[producer, consumer])
+    assert pipeline.dependencies[consumer] == frozenset({producer})
+
+    producer.inject()
+    assert pipeline.state["heating.used"] == 500.0
+
+
+def test_ports_keep_function_arguments_local():
+    def convert(celsius, params):
+        return {"fahrenheit": celsius * 9 / 5 + 32}
+
+    node = make_node(
+        "convert",
+        "heating",
+        convert,
+        requires=[spec("solar.celsius", port="celsius")],
+        provides=[spec("fahrenheit")],
+    )
+    Pipeline(nodes=[node])
+
+    assert node.execute({"solar.celsius": 100.0}) == {"heating.fahrenheit": 212.0}
+
+
+def test_validate_reports_cycles_and_dangling_inputs():
+    a = make_node(
+        "a", "f", lambda b, params: {"a": b}, requires=[spec("b")], provides=[spec("a")]
+    )
+    b = make_node(
+        "b", "f", lambda a, params: {"b": a}, requires=[spec("a")], provides=[spec("b")]
+    )
+    lonely = make_node(
+        "lonely", "f", lambda missing, params: None, requires=[spec("missing")]
+    )
+
+    issues = Pipeline(nodes=[a, b, lonely]).validate()
+    codes = {issue.code for issue in issues}
+
+    assert "cycle" in codes
+    assert "unconnected_input" in codes
+    dangling = next(i for i in issues if i.code == "unconnected_input")
+    assert dangling.message_name == "f.missing"
+    assert dangling.node == "f.lonely"
+
+
+def test_declared_flow_inputs_are_not_dangling():
+    node = make_node(
+        "n", "f", lambda setpoint, params: None, requires=[spec("setpoint")]
+    )
+    issues = Pipeline(nodes=[node]).validate(known_inputs={"f.setpoint"})
+    assert issues == []
+
+
+def test_a_failing_node_does_not_stop_its_siblings():
+    ran = []
+
+    def boom(params):
+        raise RuntimeError("nope")
+
+    def fine(params):
+        ran.append("fine")
+        return {"ok": 1.0}
+
+    bad = make_node("bad", "f", boom, provides=[spec("bad_out")])
+    good = make_node("good", "f", fine, provides=[spec("ok")])
+
+    pipeline = Pipeline(nodes=[bad, good])
+    pipeline.run()
+
+    assert ran == ["fine"]
+    assert pipeline.state["f.ok"] == 1.0
+
+
+def test_values_carry_timestamps():
+    node = make_node("n", "f", lambda params: {"out": 1.0}, provides=[spec("out")])
+    pipeline = Pipeline(nodes=[node])
+    node.inject()
+
+    values = pipeline.values("f")
+    assert values["f.out"]["value"] == 1.0
+    assert values["f.out"]["ts"] > 0

backend/tests/flow/test_store.py

@@ -0,0 +1,75 @@
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from app.flow.messages import MessageSpec
+from app.flow.schemas import FlowDef, NodeDef
+from app.flow.store import FlowNotFound, FlowStore
+
+
+@pytest.fixture
+def store(tmp_path: Path) -> FlowStore:
+    return FlowStore(tmp_path / "flows")
+
+
+def commit_count(store: FlowStore) -> int:
+    result = subprocess.run(
+        ["git", "-C", str(store.root), "rev-list", "--count", "HEAD"],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return int(result.stdout.strip())
+
+
+def a_flow() -> FlowDef:
+    return FlowDef(
+        name="heating",
+        nodes=[NodeDef(id="sensor", provides=[MessageSpec(name="temp")])],
+    )
+
+
+def test_flow_round_trips(store: FlowStore):
+    store.write_flow(a_flow())
+
+    assert store.list_flows() == ["heating"]
+    assert store.read_flow("heating").nodes[0].provides[0].name == "temp"
+
+
+def test_every_change_is_committed(store: FlowStore):
+    before = commit_count(store)
+
+    store.write_flow(a_flow())
+    assert commit_count(store) == before + 1
+
+    store.write_node_source(
+        "heating", "sensor", "def process(params):\n    return {}\n"
+    )
+    assert commit_count(store) == before + 2
+
+
+def test_saving_unchanged_content_does_nothing(store: FlowStore):
+    store.write_flow(a_flow())
+    commits = commit_count(store)
+
+    # Autosave repeats the same document; history should not grow.
+    assert store.write_flow(a_flow()) is False
+    assert commit_count(store) == commits
+
+
+def test_missing_flow_is_reported(store: FlowStore):
+    with pytest.raises(FlowNotFound):
+        store.read_flow("nope")
+
+
+def test_deleting_removes_flow_and_its_nodes(store: FlowStore):
+    store.write_flow(a_flow())
+    store.write_node_source(
+        "heating", "sensor", "def process(params):\n    return {}\n"
+    )
+
+    store.delete_flow("heating")
+
+    assert store.list_flows() == []
+    assert not (store.root / "heating").exists()

backend/tests/flow/test_synchronous_nodes.py

@@ -0,0 +1,111 @@
+"""Synchronous nodes wait until every input is fresh.
+
+The ordering guarantee at the stateful boundary rests on the state backend's
+atomic operations, so those are covered here too.
+"""
+
+from concurrent.futures import ThreadPoolExecutor
+
+from app.flow.messages import DType, MessageSpec
+from app.flow.nodes import Node
+from app.flow.pipeline import Pipeline
+from app.flow.state import MemoryState
+
+
+def spec(name: str) -> MessageSpec:
+    return MessageSpec(name=name, dtype=DType.FLOAT)
+
+
+def node(node_id: str, f, requires=(), provides=(), params=None) -> Node:
+    n = Node(
+        f=f,
+        requires=list(requires),
+        provides=list(provides),
+        params=params or {},
+        name=node_id,
+    )
+    n.assign_flow("f", node_id)
+    return n
+
+
+def test_synchronous_node_waits_for_all_inputs_to_be_fresh():
+    runs: list[str] = []
+
+    def sensor_a(params):
+        return {"a": 1.0}
+
+    def sensor_b(params):
+        return {"b": 2.0}
+
+    def sync(a, b, params):
+        runs.append("sync")
+        return None
+
+    def eager(a, b, params):
+        runs.append("eager")
+        return None
+
+    a = node("a", sensor_a, provides=[spec("a")])
+    b = node("b", sensor_b, provides=[spec("b")])
+    sync_node = node(
+        "sync", sync, requires=[spec("a"), spec("b")], params={"synchronous": True}
+    )
+    eager_node = node("eager", eager, requires=[spec("a"), spec("b")])
+
+    Pipeline(nodes=[a, b, sync_node, eager_node], max_workers=1)
+
+    a.inject()
+    assert runs == []  # b has never arrived
+
+    b.inject()
+    assert runs.count("eager") == 1
+    assert runs.count("sync") == 1
+
+    # Only a is new: the eager node runs again, the synchronous one waits.
+    runs.clear()
+    a.inject()
+    assert runs == ["eager"]
+
+    # Now b is new as well, so both have moved on.
+    runs.clear()
+    b.inject()
+    assert sorted(runs) == ["eager", "sync"]
+
+
+def test_increment_and_multi_get():
+    state = MemoryState()
+
+    assert state.increment("counter") == 1
+    assert state.increment("counter") == 2
+
+    state.update({"a": 1, "b": 2})
+    assert state.get_multi(["a", "b", "missing"]) == {
+        "a": 1,
+        "b": 2,
+        "missing": None,
+    }
+
+
+def test_compare_and_swap_only_applies_on_match():
+    state = MemoryState()
+    state.update({"a": 1, "b": 2})
+
+    assert state.compare_and_swap_multi({"a": 1, "b": 2}, {"a": 10, "new": 100})
+    assert state.get("a") == 10
+    assert state.get("new") == 100
+
+    assert not state.compare_and_swap_multi({"a": 1}, {"a": 99})
+    assert state.get("a") == 10
+
+
+def test_only_one_thread_wins_a_compare_and_swap():
+    state = MemoryState()
+    state.set("version", 1)
+
+    def claim() -> bool:
+        return state.compare_and_swap_multi({"version": 1}, {"version": 2})
+
+    with ThreadPoolExecutor(max_workers=8) as pool:
+        results = list(pool.map(lambda _: claim(), range(8)))
+
+    assert sum(results) == 1