Add the flow API: typed messages, git-backed store, REST and live events

Makes the flow engine reachable from the API, which is what M3 needs before
any of it can reach the browser.

- app/flow is a package now; the prototype's watch-dir scripts and the
  matplotlib/networkx visualiser are gone with their dependencies.
- Messages carry a serializable dtype instead of a live Python type, and a
  port name, so the graph can speak qualified names while node functions keep
  local arguments. Redis state is JSON, not pickle.
- Message names are namespaced per flow ("heating.temp"); a bare name resolves
  to its own flow, a dotted one crosses flows.
- Several nodes may provide the same message: producers are a list, so fan-in
  is a real edge instead of a silently dropped one.
- Flows are stored as flow.json plus node sources in a git repository, one
  commit per save, with identical saves skipped so autosave stays quiet.
- Node failures are isolated and reported per node; validate() returns cycles
  and unconnected inputs instead of raising deep in a run.
- Credentials live in an encrypted store and are referenced as {"$secret": …}.
- Engine events reach websocket clients through a bus, so values, node status
  and execution show up live.
This commit is contained in:
Melvin Strobl
2026-08-15 17:23:36 +02:00
parent ec19e2beda
commit e8895faa47
54 changed files with 2586 additions and 4978 deletions
+112
View File
@@ -0,0 +1,112 @@
"""Message specifications: the typed contract between nodes.
A node declares ports; each port binds to a message name. The message name is
the wiring: a node consuming ``heating.temperature`` receives whatever any node
provides under that name. Names are namespaced per flow — a bare name is
qualified with the owning flow (``temperature`` in flow ``heating`` becomes
``heating.temperature``), a dotted name is used as written, which is how flows
consume each other's messages.
"""
from __future__ import annotations
import json
from enum import Enum
from typing import Any
from pydantic import BaseModel, ConfigDict, model_validator
class DType(str, Enum):
"""Serializable payload types.
Binary payloads (tensors, images) will arrive later as explicitly declared
codec fields; until then everything on the wire is JSON.
"""
FLOAT = "float"
INT = "int"
STR = "str"
BOOL = "bool"
JSON = "json"
_JSON_TYPES = (dict, list, str, int, float, bool, type(None))
class MessageSpec(BaseModel):
"""A single port of a node, and the message it is bound to.
:param name: The message this port binds to. Bare names are qualified with
the flow name at load time; empty means the port is unbound.
:param port: The identifier the node function sees. Defaults to the last
segment of ``name``, so unqualified flows read naturally.
:param dtype: Payload type, validated on every message that passes through.
"""
model_config = ConfigDict(frozen=True)
name: str = ""
port: str = ""
dtype: DType = DType.FLOAT
@model_validator(mode="after")
def _default_port(self) -> MessageSpec:
if not self.port and self.name:
object.__setattr__(self, "port", self.name.rsplit(".", 1)[-1])
return self
def check(self, value: Any) -> None:
"""Raise if ``value`` does not match this port's declared type."""
if self.dtype is DType.BOOL:
ok = isinstance(value, bool)
elif self.dtype is DType.INT:
# bool is an int subclass; a flag is not a number here.
ok = isinstance(value, int) and not isinstance(value, bool)
elif self.dtype is DType.FLOAT:
ok = isinstance(value, (int, float)) and not isinstance(value, bool)
elif self.dtype is DType.STR:
ok = isinstance(value, str)
else:
ok = isinstance(value, _JSON_TYPES)
if not ok:
raise TypeError(
f"{self.name or self.port}: expected {self.dtype.value}, "
f"got {type(value).__name__}"
)
def coerce(self, value: Any) -> Any:
"""Best-effort conversion of an external value into this port's type.
Used where payloads arrive as text (HTTP query strings, MQTT), never on
the path between nodes — there a wrong type is an error, not a hint.
"""
if self.dtype is DType.FLOAT:
return float(value)
if self.dtype is DType.INT:
return int(value)
if self.dtype is DType.BOOL:
if isinstance(value, bool):
return value
return str(value).lower() in ("true", "1", "yes", "on")
if self.dtype is DType.STR:
return value if isinstance(value, str) else json.dumps(value)
return value
def __repr__(self) -> str:
return f"MessageSpec({self.name or self.port})"
def __hash__(self) -> int:
return hash((self.name, self.port))
def qualify(flow: str, name: str) -> str:
"""Resolve a message name against its flow namespace."""
if not name:
return ""
return name if "." in name else f"{flow}.{name}"
def flow_of(qualified: str) -> str:
"""The flow a qualified message name belongs to."""
return qualified.split(".", 1)[0]