Add the flow API: typed messages, git-backed store, REST and live events

Makes the flow engine reachable from the API, which is what M3 needs before
any of it can reach the browser.

- app/flow is a package now; the prototype's watch-dir scripts and the
  matplotlib/networkx visualiser are gone with their dependencies.
- Messages carry a serializable dtype instead of a live Python type, and a
  port name, so the graph can speak qualified names while node functions keep
  local arguments. Redis state is JSON, not pickle.
- Message names are namespaced per flow ("heating.temp"); a bare name resolves
  to its own flow, a dotted one crosses flows.
- Several nodes may provide the same message: producers are a list, so fan-in
  is a real edge instead of a silently dropped one.
- Flows are stored as flow.json plus node sources in a git repository, one
  commit per save, with identical saves skipped so autosave stays quiet.
- Node failures are isolated and reported per node; validate() returns cycles
  and unconnected inputs instead of raising deep in a run.
- Credentials live in an encrypted store and are referenced as {"$secret": …}.
- Engine events reach websocket clients through a bus, so values, node status
  and execution show up live.
This commit is contained in:
Melvin Strobl
2026-08-15 17:23:36 +02:00
parent ec19e2beda
commit e8895faa47
54 changed files with 2586 additions and 4978 deletions
+295
View File
@@ -0,0 +1,295 @@
"""The flow API. Everything the editor can do is available here first."""
import asyncio
from typing import Any
from fastapi import (
APIRouter,
Depends,
HTTPException,
WebSocket,
WebSocketDisconnect,
)
from fastapi.concurrency import run_in_threadpool
from pydantic import BaseModel
from sqlmodel import Session
from app.api.deps import FlowControllerDep, get_current_user, user_from_token
from app.core.db import engine
from app.flow.controller import FlowController
from app.flow.events import event_bus
from app.flow.messages import qualify
from app.flow.pipeline import ValidationIssue
from app.flow.schemas import (
FlowDef,
FlowsPublic,
FlowStatePublic,
FlowSummary,
MessageValue,
NodeSource,
NodeStatusPublic,
NodeTypeInfo,
)
from app.flow.store import FlowNotFound
from app.models import Message
router = APIRouter(
prefix="/flows", tags=["flows"], dependencies=[Depends(get_current_user)]
)
# The websocket authenticates from its query string, so it stays off the
# router that enforces the HTTP bearer scheme.
ws_router = APIRouter(prefix="/flows", tags=["flows"])
class FlowDetail(BaseModel):
"""A flow plus how it is currently doing."""
definition: FlowDef
nodes: list[NodeStatusPublic] = []
issues: list[ValidationIssue] = []
class ValidationResult(BaseModel):
issues: list[ValidationIssue] = []
class RunRequest(BaseModel):
inputs: dict[str, Any] = {}
class TriggerRequest(BaseModel):
values: dict[str, Any] = {}
def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail:
return FlowDetail(
definition=definition,
nodes=controller.node_statuses(definition.name),
issues=controller.flow_issues(definition.name),
)
def _read_flow(controller: FlowController, name: str) -> FlowDef:
try:
return controller.store.read_flow(name)
except FlowNotFound:
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
def _flow_state(controller: FlowController, name: str) -> FlowStatePublic:
return FlowStatePublic(
values={
key: MessageValue(**value) for key, value in controller.values(name).items()
},
nodes=controller.node_statuses(name),
)
# -----------------------------------------------------------------------------
# Flows
# -----------------------------------------------------------------------------
@router.get("/", response_model=FlowsPublic)
def read_flows(controller: FlowControllerDep) -> Any:
"""List every flow."""
summaries = []
for definition in controller.store.read_all():
statuses = controller.node_statuses(definition.name)
summaries.append(
FlowSummary(
name=definition.name,
title=definition.title,
node_count=len(definition.nodes),
error_count=sum(1 for s in statuses if s.status == "error"),
)
)
return FlowsPublic(data=summaries, count=len(summaries))
@router.get("/node-types", response_model=list[NodeTypeInfo])
def read_node_types() -> Any:
"""The node types that can be placed on a canvas."""
from app.flow.controller import node_type_info
return node_type_info()
@router.get("/{name}", response_model=FlowDetail)
def read_flow(name: str, controller: FlowControllerDep) -> Any:
"""Read one flow, with the state of its nodes."""
return _detail(controller, _read_flow(controller, name))
@router.put("/{name}", response_model=FlowDetail)
async def save_flow(
name: str,
definition: FlowDef,
controller: FlowControllerDep,
) -> Any:
"""Create or replace a flow. Saving the same content again changes nothing."""
if definition.name != name:
raise HTTPException(
status_code=400, detail="The flow name in the body must match the URL"
)
duplicates = {n.id for n in definition.nodes}
if len(duplicates) != len(definition.nodes):
raise HTTPException(status_code=400, detail="Node names must be unique")
changed = await run_in_threadpool(controller.store.write_flow, definition)
if changed:
await controller.reload()
return _detail(controller, definition)
@router.delete("/{name}", response_model=Message)
async def delete_flow(name: str, controller: FlowControllerDep) -> Any:
"""Delete a flow and everything in it."""
try:
await run_in_threadpool(controller.store.delete_flow, name)
except FlowNotFound:
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
await controller.reload()
return Message(message=f"Deleted flow '{name}'")
# -----------------------------------------------------------------------------
# Node source
# -----------------------------------------------------------------------------
@router.get("/{name}/nodes/{node_id}/source", response_model=NodeSource)
def read_node_source(
name: str,
node_id: str,
controller: FlowControllerDep,
) -> Any:
"""Read a node's Python source."""
_read_flow(controller, name)
return NodeSource(code=controller.store.read_node_source(name, node_id))
@router.put("/{name}/nodes/{node_id}/source", response_model=NodeStatusPublic)
async def save_node_source(
name: str,
node_id: str,
source: NodeSource,
controller: FlowControllerDep,
) -> Any:
"""Save a node's source and report whether it loads."""
_read_flow(controller, name)
changed = await run_in_threadpool(
controller.store.write_node_source, name, node_id, source.code
)
if changed:
await controller.reload()
status = next(
(s for s in controller.node_statuses(name) if s.id == f"{name}.{node_id}"),
None,
)
return status or NodeStatusPublic(id=f"{name}.{node_id}")
# -----------------------------------------------------------------------------
# Validation and execution
# -----------------------------------------------------------------------------
@router.post("/{name}/validate", response_model=ValidationResult)
def validate_flow(name: str, controller: FlowControllerDep) -> Any:
"""Report what would keep this flow from running."""
_read_flow(controller, name)
return ValidationResult(issues=controller.flow_issues(name))
@router.post("/{name}/run", response_model=FlowStatePublic)
async def run_flow(
name: str,
body: RunRequest,
controller: FlowControllerDep,
) -> Any:
"""Run every node of a flow once."""
_read_flow(controller, name)
inputs = {qualify(name, key): value for key, value in body.inputs.items()}
await run_in_threadpool(controller.run_flow, name, inputs)
return _flow_state(controller, name)
@router.post("/{name}/nodes/{node_id}/trigger", response_model=FlowStatePublic)
async def trigger_node(
name: str,
node_id: str,
body: TriggerRequest,
controller: FlowControllerDep,
) -> Any:
"""Feed values into a single node."""
try:
await run_in_threadpool(
controller.trigger_node, f"{name}.{node_id}", body.values
)
except KeyError:
raise HTTPException(
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
)
return _flow_state(controller, name)
@router.get("/{name}/state", response_model=FlowStatePublic)
def read_flow_state(name: str, controller: FlowControllerDep) -> Any:
"""The last value seen on every message of this flow."""
_read_flow(controller, name)
return _flow_state(controller, name)
# -----------------------------------------------------------------------------
# Live updates
# -----------------------------------------------------------------------------
@ws_router.websocket("/ws")
async def flow_events(websocket: WebSocket, token: str = "") -> None:
"""Stream values, node status and execution events as they happen.
The token goes in the query string because browsers cannot set headers on
a websocket handshake.
"""
with Session(engine) as session:
user = user_from_token(session, token)
if user is None:
await websocket.close(code=1008)
return
await websocket.accept()
controller: FlowController | None = getattr(
websocket.app.state, "flow_controller", None
)
if controller is not None:
await websocket.send_json(
{
"type": "snapshot",
"values": controller.values(),
"nodes": [s.model_dump() for s in controller.node_statuses()],
"issues": [i.model_dump() for i in controller.issues],
}
)
async with event_bus.subscribe() as queue:
receiver = asyncio.create_task(websocket.receive_text())
try:
while True:
sender = asyncio.create_task(queue.get())
done, _ = await asyncio.wait(
{sender, receiver}, return_when=asyncio.FIRST_COMPLETED
)
if receiver in done:
# The client went away.
sender.cancel()
break
await websocket.send_json(sender.result())
except WebSocketDisconnect:
pass
finally:
receiver.cancel()