Add the flow API: typed messages, git-backed store, REST and live events

Makes the flow engine reachable from the API, which is what M3 needs before
any of it can reach the browser.

- app/flow is a package now; the prototype's watch-dir scripts and the
  matplotlib/networkx visualiser are gone with their dependencies.
- Messages carry a serializable dtype instead of a live Python type, and a
  port name, so the graph can speak qualified names while node functions keep
  local arguments. Redis state is JSON, not pickle.
- Message names are namespaced per flow ("heating.temp"); a bare name resolves
  to its own flow, a dotted one crosses flows.
- Several nodes may provide the same message: producers are a list, so fan-in
  is a real edge instead of a silently dropped one.
- Flows are stored as flow.json plus node sources in a git repository, one
  commit per save, with identical saves skipped so autosave stays quiet.
- Node failures are isolated and reported per node; validate() returns cycles
  and unconnected inputs instead of raising deep in a run.
- Credentials live in an encrypted store and are referenced as {"$secret": …}.
- Engine events reach websocket clients through a bus, so values, node status
  and execution show up live.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016WzrvW7rjQbynnhF6pxh6i

backend/app/api/deps.py

@@ -2,7 +2,7 @@ from collections.abc import Generator
 from typing import Annotated
 
 import jwt
-from fastapi import Depends, HTTPException, status
+from fastapi import Depends, HTTPException, Request, status
 from fastapi.security import OAuth2PasswordBearer
 from jwt.exceptions import InvalidTokenError
 from pydantic import ValidationError
@@ -11,6 +11,7 @@ from sqlmodel import Session
 from app.core import security
 from app.core.config import settings
 from app.core.db import engine
+from app.flow.controller import FlowController
 from app.models import TokenPayload, User
 
 reusable_oauth2 = OAuth2PasswordBearer(
@@ -27,6 +28,24 @@ SessionDep = Annotated[Session, Depends(get_db)]
 TokenDep = Annotated[str, Depends(reusable_oauth2)]
 
 
+def user_from_token(session: Session, token: str) -> User | None:
+    """Resolve a bearer token to its user, or None if it does not hold up.
+
+    Shared with the websocket, which cannot use the HTTP security scheme.
+    """
+    try:
+        payload = jwt.decode(
+            token, settings.SECRET_KEY, algorithms=[security.ALGORITHM]
+        )
+        token_data = TokenPayload(**payload)
+    except (InvalidTokenError, ValidationError):
+        return None
+    user = session.get(User, token_data.sub)
+    if user is None or not user.is_active:
+        return None
+    return user
+
+
 def get_current_user(session: SessionDep, token: TokenDep) -> User:
     try:
         payload = jwt.decode(
@@ -49,6 +68,18 @@ def get_current_user(session: SessionDep, token: TokenDep) -> User:
 CurrentUser = Annotated[User, Depends(get_current_user)]
 
 
+def get_flow_controller(request: Request) -> FlowController:
+    controller: FlowController | None = getattr(
+        request.app.state, "flow_controller", None
+    )
+    if controller is None:
+        raise HTTPException(status_code=503, detail="The flow engine is not running")
+    return controller
+
+
+FlowControllerDep = Annotated[FlowController, Depends(get_flow_controller)]
+
+
 def get_current_active_superuser(current_user: CurrentUser) -> User:
     if not current_user.is_superuser:
         raise HTTPException(

backend/app/api/main.py

@@ -1,6 +1,6 @@
 from fastapi import APIRouter
 
-from app.api.routes import items, login, private, users, utils
+from app.api.routes import flows, items, login, private, secrets, users, utils
 from app.core.config import settings
 
 api_router = APIRouter()
@@ -8,6 +8,9 @@ api_router.include_router(login.router)
 api_router.include_router(users.router)
 api_router.include_router(utils.router)
 api_router.include_router(items.router)
+api_router.include_router(flows.router)
+api_router.include_router(flows.ws_router)
+api_router.include_router(secrets.router)
 
 
 if settings.ENVIRONMENT == "local":

backend/app/api/routes/flows.py

@@ -0,0 +1,295 @@
+"""The flow API. Everything the editor can do is available here first."""
+
+import asyncio
+from typing import Any
+
+from fastapi import (
+    APIRouter,
+    Depends,
+    HTTPException,
+    WebSocket,
+    WebSocketDisconnect,
+)
+from fastapi.concurrency import run_in_threadpool
+from pydantic import BaseModel
+from sqlmodel import Session
+
+from app.api.deps import FlowControllerDep, get_current_user, user_from_token
+from app.core.db import engine
+from app.flow.controller import FlowController
+from app.flow.events import event_bus
+from app.flow.messages import qualify
+from app.flow.pipeline import ValidationIssue
+from app.flow.schemas import (
+    FlowDef,
+    FlowsPublic,
+    FlowStatePublic,
+    FlowSummary,
+    MessageValue,
+    NodeSource,
+    NodeStatusPublic,
+    NodeTypeInfo,
+)
+from app.flow.store import FlowNotFound
+from app.models import Message
+
+router = APIRouter(
+    prefix="/flows", tags=["flows"], dependencies=[Depends(get_current_user)]
+)
+# The websocket authenticates from its query string, so it stays off the
+# router that enforces the HTTP bearer scheme.
+ws_router = APIRouter(prefix="/flows", tags=["flows"])
+
+
+class FlowDetail(BaseModel):
+    """A flow plus how it is currently doing."""
+
+    definition: FlowDef
+    nodes: list[NodeStatusPublic] = []
+    issues: list[ValidationIssue] = []
+
+
+class ValidationResult(BaseModel):
+    issues: list[ValidationIssue] = []
+
+
+class RunRequest(BaseModel):
+    inputs: dict[str, Any] = {}
+
+
+class TriggerRequest(BaseModel):
+    values: dict[str, Any] = {}
+
+
+def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail:
+    return FlowDetail(
+        definition=definition,
+        nodes=controller.node_statuses(definition.name),
+        issues=controller.flow_issues(definition.name),
+    )
+
+
+def _read_flow(controller: FlowController, name: str) -> FlowDef:
+    try:
+        return controller.store.read_flow(name)
+    except FlowNotFound:
+        raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
+
+
+def _flow_state(controller: FlowController, name: str) -> FlowStatePublic:
+    return FlowStatePublic(
+        values={
+            key: MessageValue(**value) for key, value in controller.values(name).items()
+        },
+        nodes=controller.node_statuses(name),
+    )
+
+
+# -----------------------------------------------------------------------------
+# Flows
+# -----------------------------------------------------------------------------
+
+
+@router.get("/", response_model=FlowsPublic)
+def read_flows(controller: FlowControllerDep) -> Any:
+    """List every flow."""
+    summaries = []
+    for definition in controller.store.read_all():
+        statuses = controller.node_statuses(definition.name)
+        summaries.append(
+            FlowSummary(
+                name=definition.name,
+                title=definition.title,
+                node_count=len(definition.nodes),
+                error_count=sum(1 for s in statuses if s.status == "error"),
+            )
+        )
+    return FlowsPublic(data=summaries, count=len(summaries))
+
+
+@router.get("/node-types", response_model=list[NodeTypeInfo])
+def read_node_types() -> Any:
+    """The node types that can be placed on a canvas."""
+    from app.flow.controller import node_type_info
+
+    return node_type_info()
+
+
+@router.get("/{name}", response_model=FlowDetail)
+def read_flow(name: str, controller: FlowControllerDep) -> Any:
+    """Read one flow, with the state of its nodes."""
+    return _detail(controller, _read_flow(controller, name))
+
+
+@router.put("/{name}", response_model=FlowDetail)
+async def save_flow(
+    name: str,
+    definition: FlowDef,
+    controller: FlowControllerDep,
+) -> Any:
+    """Create or replace a flow. Saving the same content again changes nothing."""
+    if definition.name != name:
+        raise HTTPException(
+            status_code=400, detail="The flow name in the body must match the URL"
+        )
+
+    duplicates = {n.id for n in definition.nodes}
+    if len(duplicates) != len(definition.nodes):
+        raise HTTPException(status_code=400, detail="Node names must be unique")
+
+    changed = await run_in_threadpool(controller.store.write_flow, definition)
+    if changed:
+        await controller.reload()
+    return _detail(controller, definition)
+
+
+@router.delete("/{name}", response_model=Message)
+async def delete_flow(name: str, controller: FlowControllerDep) -> Any:
+    """Delete a flow and everything in it."""
+    try:
+        await run_in_threadpool(controller.store.delete_flow, name)
+    except FlowNotFound:
+        raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
+    await controller.reload()
+    return Message(message=f"Deleted flow '{name}'")
+
+
+# -----------------------------------------------------------------------------
+# Node source
+# -----------------------------------------------------------------------------
+
+
+@router.get("/{name}/nodes/{node_id}/source", response_model=NodeSource)
+def read_node_source(
+    name: str,
+    node_id: str,
+    controller: FlowControllerDep,
+) -> Any:
+    """Read a node's Python source."""
+    _read_flow(controller, name)
+    return NodeSource(code=controller.store.read_node_source(name, node_id))
+
+
+@router.put("/{name}/nodes/{node_id}/source", response_model=NodeStatusPublic)
+async def save_node_source(
+    name: str,
+    node_id: str,
+    source: NodeSource,
+    controller: FlowControllerDep,
+) -> Any:
+    """Save a node's source and report whether it loads."""
+    _read_flow(controller, name)
+    changed = await run_in_threadpool(
+        controller.store.write_node_source, name, node_id, source.code
+    )
+    if changed:
+        await controller.reload()
+
+    status = next(
+        (s for s in controller.node_statuses(name) if s.id == f"{name}.{node_id}"),
+        None,
+    )
+    return status or NodeStatusPublic(id=f"{name}.{node_id}")
+
+
+# -----------------------------------------------------------------------------
+# Validation and execution
+# -----------------------------------------------------------------------------
+
+
+@router.post("/{name}/validate", response_model=ValidationResult)
+def validate_flow(name: str, controller: FlowControllerDep) -> Any:
+    """Report what would keep this flow from running."""
+    _read_flow(controller, name)
+    return ValidationResult(issues=controller.flow_issues(name))
+
+
+@router.post("/{name}/run", response_model=FlowStatePublic)
+async def run_flow(
+    name: str,
+    body: RunRequest,
+    controller: FlowControllerDep,
+) -> Any:
+    """Run every node of a flow once."""
+    _read_flow(controller, name)
+    inputs = {qualify(name, key): value for key, value in body.inputs.items()}
+    await run_in_threadpool(controller.run_flow, name, inputs)
+    return _flow_state(controller, name)
+
+
+@router.post("/{name}/nodes/{node_id}/trigger", response_model=FlowStatePublic)
+async def trigger_node(
+    name: str,
+    node_id: str,
+    body: TriggerRequest,
+    controller: FlowControllerDep,
+) -> Any:
+    """Feed values into a single node."""
+    try:
+        await run_in_threadpool(
+            controller.trigger_node, f"{name}.{node_id}", body.values
+        )
+    except KeyError:
+        raise HTTPException(
+            status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
+        )
+    return _flow_state(controller, name)
+
+
+@router.get("/{name}/state", response_model=FlowStatePublic)
+def read_flow_state(name: str, controller: FlowControllerDep) -> Any:
+    """The last value seen on every message of this flow."""
+    _read_flow(controller, name)
+    return _flow_state(controller, name)
+
+
+# -----------------------------------------------------------------------------
+# Live updates
+# -----------------------------------------------------------------------------
+
+
+@ws_router.websocket("/ws")
+async def flow_events(websocket: WebSocket, token: str = "") -> None:
+    """Stream values, node status and execution events as they happen.
+
+    The token goes in the query string because browsers cannot set headers on
+    a websocket handshake.
+    """
+    with Session(engine) as session:
+        user = user_from_token(session, token)
+    if user is None:
+        await websocket.close(code=1008)
+        return
+
+    await websocket.accept()
+
+    controller: FlowController | None = getattr(
+        websocket.app.state, "flow_controller", None
+    )
+    if controller is not None:
+        await websocket.send_json(
+            {
+                "type": "snapshot",
+                "values": controller.values(),
+                "nodes": [s.model_dump() for s in controller.node_statuses()],
+                "issues": [i.model_dump() for i in controller.issues],
+            }
+        )
+
+    async with event_bus.subscribe() as queue:
+        receiver = asyncio.create_task(websocket.receive_text())
+        try:
+            while True:
+                sender = asyncio.create_task(queue.get())
+                done, _ = await asyncio.wait(
+                    {sender, receiver}, return_when=asyncio.FIRST_COMPLETED
+                )
+                if receiver in done:
+                    # The client went away.
+                    sender.cancel()
+                    break
+                await websocket.send_json(sender.result())
+        except WebSocketDisconnect:
+            pass
+        finally:
+            receiver.cancel()

backend/app/api/routes/secrets.py

@@ -0,0 +1,51 @@
+"""Credentials for node integrations.
+
+Values go in and are never handed back out — the API only ever lists names.
+"""
+
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.concurrency import run_in_threadpool
+from pydantic import BaseModel
+
+from app.api.deps import get_current_user
+from app.flow.secrets import SecretNotFound, get_secrets
+from app.models import Message
+
+router = APIRouter(
+    prefix="/secrets", tags=["secrets"], dependencies=[Depends(get_current_user)]
+)
+
+
+class SecretNames(BaseModel):
+    data: list[str]
+    count: int
+
+
+class SecretValue(BaseModel):
+    value: str
+
+
+@router.get("/", response_model=SecretNames)
+def read_secrets() -> Any:
+    """List the names of stored secrets."""
+    names = get_secrets().list()
+    return SecretNames(data=names, count=len(names))
+
+
+@router.put("/{name}", response_model=Message)
+async def save_secret(name: str, body: SecretValue) -> Any:
+    """Store a secret under a name that nodes can reference."""
+    await run_in_threadpool(get_secrets().set, name, body.value)
+    return Message(message=f"Saved secret '{name}'")
+
+
+@router.delete("/{name}", response_model=Message)
+async def delete_secret(name: str) -> Any:
+    """Delete a secret."""
+    try:
+        await run_in_threadpool(get_secrets().delete, name)
+    except SecretNotFound:
+        raise HTTPException(status_code=404, detail=f"No secret named '{name}'")
+    return Message(message=f"Deleted secret '{name}'")