Connection nodes speak of listener and sender, and of send_queue_size

mqtt and http take websocket's vocabulary: their Mode is LISTENER/SENDER,
default names read mqtt_listener_… and http_listener_…, and mqtt's
publish_queue_size is send_queue_size. No alias stays behind, so the flow
store rewrites the old setting in every published and draft mqtt node when it
opens, in the same migration as trigger → debounce. Every mqtt and http
setting carries a one-line description for the editor, and the node-types
reference uses one wording for direction and says that an http sender
publishes a JSON object response on its outputs.
This commit is contained in:
2026-10-15 09:44:22 +02:00
parent c032898135
commit c9fda50601
12 changed files with 241 additions and 160 deletions
+99 -70
View File
@@ -1,4 +1,4 @@
"""MQTT nodes: a subscriber that wakes the graph, a publisher that speaks for it."""
"""MQTT nodes: a listener that wakes the graph, a sender that speaks for it."""
from __future__ import annotations
@@ -62,15 +62,14 @@ def topic_matches(filter_: str, topic: str) -> bool:
class MqttNode(Node):
"""
MQTT node that can act as a subscriber (trigger) or publisher (sender).
MQTT node that acts as a listener or a sender.
This node integrates with an MQTT broker to either:
- **Trigger mode (Subscriber)**: Subscribe to MQTT topics and inject received
messages into the pipeline. Used when ``provides`` is specified but
``requires`` is empty.
- **Sender mode (Publisher)**: Publish pipeline data to MQTT topics. Used when
``requires`` is specified.
- **Listener**: Subscribe to MQTT topics and inject received messages into
the pipeline. Used when ``provides`` is specified but ``requires`` is empty.
- **Sender**: Publish pipeline data to MQTT topics. Used when ``requires``
is specified.
The ``topic`` parameter in ``params`` controls the mapping between pipeline
message names and MQTT topics:
@@ -78,11 +77,11 @@ class MqttNode(Node):
- **dict**: Explicit mapping from message name to MQTT topic, e.g.
``{"temperature": "sensors/room1/temp", "humidity": "sensors/room1/hum"}``.
- **str** (legacy): A single topic string. All messages are mapped to this
one topic (subscriber receives from it, publisher sends to it).
one topic (a listener subscribes to it, a sender publishes to it).
:param requires: Messages required by this node (makes it a publisher node).
:param requires: Messages required by this node (makes it a sender).
:type requires: MessageSpec | list[MessageSpec] | None
:param provides: Messages provided by this node (makes it a subscriber node).
:param provides: Messages provided by this node (makes it a listener).
:type provides: MessageSpec | list[MessageSpec] | None
:param params: Parameters dict containing:
- ``topic`` (str | dict): MQTT topic(s). A dict maps message names to
@@ -97,7 +96,7 @@ class MqttNode(Node):
- ``keepalive`` (int): Keepalive interval in seconds (default: 60)
- ``timeout`` (float): Deadline for a broker operation in seconds
(default: 10.0)
- ``publish_queue_size`` (int): Publisher backlog depth (default: 256)
- ``send_queue_size`` (int): Sender backlog depth (default: 256)
:type params: dict
:param name: Optional name for the node.
:type name: str | None
@@ -105,9 +104,9 @@ class MqttNode(Node):
:raises ValueError: If both ``requires`` and ``provides`` are empty.
:example:
Subscriber with per-message topics:
Listener with per-message topics:
>>> subscriber = MqttNode(
>>> listener = MqttNode(
... provides=[
... MessageSpec(name="inverter_input", dtype=DType.FLOAT),
... MessageSpec(name="inverter_output", dtype=DType.FLOAT),
@@ -121,9 +120,9 @@ class MqttNode(Node):
... },
... )
Publisher with per-message topics:
Sender with per-message topics:
>>> publisher = MqttNode(
>>> sender = MqttNode(
... requires=[
... MessageSpec(name="target_temp", dtype=DType.FLOAT),
... MessageSpec(name="fan_speed", dtype=DType.INT),
@@ -138,9 +137,9 @@ class MqttNode(Node):
... },
... )
Legacy single-topic subscriber:
Legacy single-topic listener:
>>> subscriber = MqttNode(
>>> listener = MqttNode(
... provides=[MessageSpec(name="temperature", dtype=DType.FLOAT)],
... params={"topic": "sensors/temperature", "broker_host": "localhost"},
... )
@@ -149,8 +148,8 @@ class MqttNode(Node):
class Mode(Enum):
"""Operating mode of the MQTT node."""
SUBSCRIBER = "subscriber" # Receives MQTT messages (trigger)
PUBLISHER = "publisher" # Sends MQTT messages (sender)
LISTENER = "listener" # Subscribes and injects what arrives
SENDER = "sender" # Publishes its inputs
# Publishing again is a second command to whatever is listening.
idempotent = False
@@ -167,14 +166,14 @@ class MqttNode(Node):
"retain",
"keepalive",
"timeout",
"publish_queue_size",
"send_queue_size",
"json_keys",
"_topic_to_ports",
"_wildcards",
"_subscription_task",
"_mqtt_client",
"_stop_event",
"_publish_queue",
"_send_queue",
"_publisher_task",
"_loop",
)
@@ -183,15 +182,41 @@ class MqttNode(Node):
model_config = ConfigDict(extra="allow")
# One topic for every port, or a per-port mapping.
topic: str | dict[str, str] = "*"
broker_host: str = "localhost"
broker_port: int = 1883
username: str | None = None
password: str | None = Field(default=None, json_schema_extra={"x-secret": True})
client_id: str | None = None
qos: int = 0
retain: bool = False
keepalive: int = 60
topic: str | dict[str, str] = Field(
default="*",
description='One topic for every port, or {"port": "some/topic"} per port.',
)
broker_host: str = Field(
default="localhost",
description="Hostname or IP of the broker, as the engine reaches it.",
)
broker_port: int = Field(
default=1883, description="The port the broker listens on."
)
username: str | None = Field(
default=None, description="Username for the broker, if it asks for one."
)
password: str | None = Field(
default=None,
description="Password for the broker, kept in the secrets store.",
json_schema_extra={"x-secret": True},
)
client_id: str | None = Field(
default=None,
description="Client identifier the broker sees. Empty picks a random one.",
)
qos: int = Field(
default=0,
description="Quality of service for subscribing and publishing, 0, 1 or 2.",
)
retain: bool = Field(
default=False,
description="Have the broker keep the last value sent for later subscribers.",
)
keepalive: int = Field(
default=60,
description="Seconds between pings that keep an idle connection open.",
)
# Bounds every broker operation: subscribe, publish, and the
# disconnect acknowledgement on the way out. Without one a client
# whose socket died waits for that ack forever, and the task never
@@ -202,9 +227,9 @@ class MqttNode(Node):
description="Give up on a broker operation after this many seconds.",
)
# Deep enough to ride out a broker hiccup, shallow enough that a
# publisher which cannot keep up drops old values instead of growing
# sender which cannot keep up drops old values instead of growing
# without bound. A node that bursts wants more than one that trickles.
publish_queue_size: int = Field(
send_queue_size: int = Field(
default=256,
gt=0,
description=(
@@ -215,17 +240,23 @@ class MqttNode(Node):
# Which key to lift out of a JSON object payload. A device that wraps
# its reading — Victron's ``{"value": 5}`` — is otherwise a Python node
# per port. One key for every port, or a per-port mapping.
json_key: str | dict[str, str] = ""
json_key: str | dict[str, str] = Field(
default="",
description=(
"Key to lift out of a JSON object payload, for every port or as "
'{"port": "key"}.'
),
)
@classmethod
def target_key(cls, params: dict[str, Any]) -> str | None:
"""The broker and topic, which is one physical thing.
A publisher and a subscriber on the same topic get the same key on
A sender and a listener on the same topic get the same key on
purpose: they are two ends of one wire, and drawing them as one neuron
is the only way the path through the broker shows up at all.
"""
# ponytail: publisher and subscriber merge into one neuron; key on mode
# ponytail: sender and listener merge into one neuron; key on mode
# as well if the two directions ever need telling apart.
fields = cls.Params.model_fields
topic = params.get("topic", fields["topic"].default)
@@ -253,9 +284,9 @@ class MqttNode(Node):
)
# Inputs mean this node publishes; outputs mean it subscribes.
self.mode = MqttNode.Mode.PUBLISHER if requires else MqttNode.Mode.SUBSCRIBER
self.mode = MqttNode.Mode.SENDER if requires else MqttNode.Mode.LISTENER
ports = provides if self.mode == MqttNode.Mode.SUBSCRIBER else requires
ports = provides if self.mode == MqttNode.Mode.LISTENER else requires
if isinstance(cfg.topic, dict):
self.topics: dict[str, str] = dict(cfg.topic)
else:
@@ -285,13 +316,13 @@ class MqttNode(Node):
self.retain = cfg.retain
self.keepalive = cfg.keepalive
self.timeout = cfg.timeout
self.publish_queue_size = cfg.publish_queue_size
self.send_queue_size = cfg.send_queue_size
# Runtime state
self._subscription_task: asyncio.Task[None] | None = None
self._mqtt_client = None
self._stop_event: asyncio.Event | None = None
self._publish_queue: asyncio.Queue[dict[str, Any]] | None = None
self._send_queue: asyncio.Queue[dict[str, Any]] | None = None
self._publisher_task: asyncio.Task[None] | None = None
self._loop: asyncio.AbstractEventLoop | None = None
@@ -311,13 +342,13 @@ class MqttNode(Node):
name = f"mqtt_{self.mode.value}_{safe_topic}"
# Initialize parent with appropriate function
# For subscriber mode, f is a no-op since data is injected via inject()
# For publisher mode, f handles the outgoing MQTT publish
# For a listener, f is a no-op since data is injected via inject()
# For a sender, f handles the outgoing MQTT publish
super().__init__(
f=(
self._noop_subscriber
if self.mode == MqttNode.Mode.SUBSCRIBER
else self._publisher_handler
self._noop_listener
if self.mode == MqttNode.Mode.LISTENER
else self._sender_handler
),
requires=requires,
provides=provides,
@@ -326,13 +357,11 @@ class MqttNode(Node):
)
@staticmethod
def _noop_subscriber(
params: dict[str, Any], **kwargs: Any
) -> dict[str, Any] | None:
def _noop_listener(params: dict[str, Any], **kwargs: Any) -> dict[str, Any] | None:
"""
No-op function for subscriber mode nodes.
No-op function for listener nodes.
Subscriber mode nodes inject data via :meth:`inject`, not :meth:`__call__`.
Listener nodes inject data via :meth:`inject`, not :meth:`__call__`.
This function exists only to satisfy the Node interface and should not
be called directly.
@@ -345,11 +374,11 @@ class MqttNode(Node):
"""
return None
def _publisher_handler(
def _sender_handler(
self, params: dict[str, Any], **kwargs: Any
) -> dict[str, Any] | None:
"""
Publish pipeline data to MQTT topic (publisher mode).
Publish pipeline data to MQTT topic (sender).
This method is called when upstream dependencies are satisfied.
Handing the payload to the node's publisher task is all that happens
@@ -363,7 +392,7 @@ class MqttNode(Node):
:returns: None (publishing is fire-and-forget).
:rtype: dict | None
"""
loop, queue = self._loop, self._publish_queue
loop, queue = self._loop, self._send_queue
if loop is not None and queue is not None:
loop.call_soon_threadsafe(self._enqueue, queue, dict(kwargs))
return None
@@ -386,21 +415,21 @@ class MqttNode(Node):
if queue.full():
try:
queue.get_nowait()
logger.warning("Publish queue full for node '%s', dropped", self.name)
self.report_health("degraded", "publish queue full")
logger.warning("Send queue full for node '%s', dropped", self.name)
self.report_health("degraded", "send queue full")
except asyncio.QueueEmpty:
pass
queue.put_nowait(data)
async def _publisher_loop(self) -> None:
"""Hold one connection and drain the publish queue over it.
"""Hold one connection and drain the send queue over it.
A dropped connection raises, and the supervisor decides when to
reconnect — the same arrangement the subscriber uses.
reconnect — the same arrangement the listener uses.
"""
aiomqtt = _aiomqtt()
queue = self._publish_queue
queue = self._send_queue
if queue is None:
return
@@ -461,7 +490,7 @@ class MqttNode(Node):
continue
# A string goes on the wire as it stands. Devices on a shared
# broker expect bare values, and the subscriber below already
# broker expect bare values, and the listener below already
# falls back to the raw text when it is not JSON, so a
# fluksio-to-fluksio round trip is unaffected.
payload = value if isinstance(value, str) else json.dumps(value)
@@ -479,8 +508,8 @@ class MqttNode(Node):
)
async def start(self, app: FastAPI | None = None) -> None:
"""A subscriber listens; a publisher opens the connection it will reuse."""
if self.mode is MqttNode.Mode.SUBSCRIBER:
"""A listener subscribes; a sender opens the connection it will reuse."""
if self.mode is MqttNode.Mode.LISTENER:
await self.start_subscription()
else:
await self.start_publisher()
@@ -491,41 +520,41 @@ class MqttNode(Node):
async def start_publisher(self) -> None:
"""Run the task that owns this node's connection to the broker."""
if self._publish_queue is not None:
if self._send_queue is not None:
return
self._publish_queue = asyncio.Queue(maxsize=self.publish_queue_size)
self._send_queue = asyncio.Queue(maxsize=self.send_queue_size)
self._loop = asyncio.get_running_loop()
self._publisher_task = self._run_supervised("mqtt-out", self._publisher_loop)
async def stop_publisher(self) -> None:
"""Drop the queue and let the connection go."""
if self._publish_queue is None:
if self._send_queue is None:
return
if self._publisher_task is not None:
await self._cancel_task(self._publisher_task)
self._publisher_task = None
self._publish_queue = None
self._send_queue = None
self._loop = None
async def start_subscription(self) -> None:
"""
Start the MQTT subscription for trigger mode nodes.
Start the MQTT subscription for listener nodes.
This method starts a background task that listens for messages
on the subscribed topic and triggers the pipeline when messages arrive.
:raises RuntimeError: If called on a publisher mode node.
:raises RuntimeError: If called on a sender.
:example:
>>> subscriber = MqttNode(
>>> listener = MqttNode(
... topic="sensors/#",
... provides=[MessageSpec(name="value", dtype=DType.FLOAT)],
... params={"broker_host": "localhost"},
... )
>>> await subscriber.start_subscription()
>>> await listener.start_subscription()
"""
if self.mode != MqttNode.Mode.SUBSCRIBER:
raise RuntimeError("Can only start subscription for subscriber mode nodes")
if self.mode != MqttNode.Mode.LISTENER:
raise RuntimeError("Can only start subscription for listener nodes")
if self._subscription_task is not None:
return # Already running