Connection nodes speak of listener and sender, and of send_queue_size
mqtt and http take websocket's vocabulary: their Mode is LISTENER/SENDER, default names read mqtt_listener_… and http_listener_…, and mqtt's publish_queue_size is send_queue_size. No alias stays behind, so the flow store rewrites the old setting in every published and draft mqtt node when it opens, in the same migration as trigger → debounce. Every mqtt and http setting carries a one-line description for the editor, and the node-types reference uses one wording for direction and says that an http sender publishes a JSON object response on its outputs.
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
"""MQTT nodes: a subscriber that wakes the graph, a publisher that speaks for it."""
|
||||
"""MQTT nodes: a listener that wakes the graph, a sender that speaks for it."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -62,15 +62,14 @@ def topic_matches(filter_: str, topic: str) -> bool:
|
||||
|
||||
class MqttNode(Node):
|
||||
"""
|
||||
MQTT node that can act as a subscriber (trigger) or publisher (sender).
|
||||
MQTT node that acts as a listener or a sender.
|
||||
|
||||
This node integrates with an MQTT broker to either:
|
||||
|
||||
- **Trigger mode (Subscriber)**: Subscribe to MQTT topics and inject received
|
||||
messages into the pipeline. Used when ``provides`` is specified but
|
||||
``requires`` is empty.
|
||||
- **Sender mode (Publisher)**: Publish pipeline data to MQTT topics. Used when
|
||||
``requires`` is specified.
|
||||
- **Listener**: Subscribe to MQTT topics and inject received messages into
|
||||
the pipeline. Used when ``provides`` is specified but ``requires`` is empty.
|
||||
- **Sender**: Publish pipeline data to MQTT topics. Used when ``requires``
|
||||
is specified.
|
||||
|
||||
The ``topic`` parameter in ``params`` controls the mapping between pipeline
|
||||
message names and MQTT topics:
|
||||
@@ -78,11 +77,11 @@ class MqttNode(Node):
|
||||
- **dict**: Explicit mapping from message name to MQTT topic, e.g.
|
||||
``{"temperature": "sensors/room1/temp", "humidity": "sensors/room1/hum"}``.
|
||||
- **str** (legacy): A single topic string. All messages are mapped to this
|
||||
one topic (subscriber receives from it, publisher sends to it).
|
||||
one topic (a listener subscribes to it, a sender publishes to it).
|
||||
|
||||
:param requires: Messages required by this node (makes it a publisher node).
|
||||
:param requires: Messages required by this node (makes it a sender).
|
||||
:type requires: MessageSpec | list[MessageSpec] | None
|
||||
:param provides: Messages provided by this node (makes it a subscriber node).
|
||||
:param provides: Messages provided by this node (makes it a listener).
|
||||
:type provides: MessageSpec | list[MessageSpec] | None
|
||||
:param params: Parameters dict containing:
|
||||
- ``topic`` (str | dict): MQTT topic(s). A dict maps message names to
|
||||
@@ -97,7 +96,7 @@ class MqttNode(Node):
|
||||
- ``keepalive`` (int): Keepalive interval in seconds (default: 60)
|
||||
- ``timeout`` (float): Deadline for a broker operation in seconds
|
||||
(default: 10.0)
|
||||
- ``publish_queue_size`` (int): Publisher backlog depth (default: 256)
|
||||
- ``send_queue_size`` (int): Sender backlog depth (default: 256)
|
||||
:type params: dict
|
||||
:param name: Optional name for the node.
|
||||
:type name: str | None
|
||||
@@ -105,9 +104,9 @@ class MqttNode(Node):
|
||||
:raises ValueError: If both ``requires`` and ``provides`` are empty.
|
||||
|
||||
:example:
|
||||
Subscriber with per-message topics:
|
||||
Listener with per-message topics:
|
||||
|
||||
>>> subscriber = MqttNode(
|
||||
>>> listener = MqttNode(
|
||||
... provides=[
|
||||
... MessageSpec(name="inverter_input", dtype=DType.FLOAT),
|
||||
... MessageSpec(name="inverter_output", dtype=DType.FLOAT),
|
||||
@@ -121,9 +120,9 @@ class MqttNode(Node):
|
||||
... },
|
||||
... )
|
||||
|
||||
Publisher with per-message topics:
|
||||
Sender with per-message topics:
|
||||
|
||||
>>> publisher = MqttNode(
|
||||
>>> sender = MqttNode(
|
||||
... requires=[
|
||||
... MessageSpec(name="target_temp", dtype=DType.FLOAT),
|
||||
... MessageSpec(name="fan_speed", dtype=DType.INT),
|
||||
@@ -138,9 +137,9 @@ class MqttNode(Node):
|
||||
... },
|
||||
... )
|
||||
|
||||
Legacy single-topic subscriber:
|
||||
Legacy single-topic listener:
|
||||
|
||||
>>> subscriber = MqttNode(
|
||||
>>> listener = MqttNode(
|
||||
... provides=[MessageSpec(name="temperature", dtype=DType.FLOAT)],
|
||||
... params={"topic": "sensors/temperature", "broker_host": "localhost"},
|
||||
... )
|
||||
@@ -149,8 +148,8 @@ class MqttNode(Node):
|
||||
class Mode(Enum):
|
||||
"""Operating mode of the MQTT node."""
|
||||
|
||||
SUBSCRIBER = "subscriber" # Receives MQTT messages (trigger)
|
||||
PUBLISHER = "publisher" # Sends MQTT messages (sender)
|
||||
LISTENER = "listener" # Subscribes and injects what arrives
|
||||
SENDER = "sender" # Publishes its inputs
|
||||
|
||||
# Publishing again is a second command to whatever is listening.
|
||||
idempotent = False
|
||||
@@ -167,14 +166,14 @@ class MqttNode(Node):
|
||||
"retain",
|
||||
"keepalive",
|
||||
"timeout",
|
||||
"publish_queue_size",
|
||||
"send_queue_size",
|
||||
"json_keys",
|
||||
"_topic_to_ports",
|
||||
"_wildcards",
|
||||
"_subscription_task",
|
||||
"_mqtt_client",
|
||||
"_stop_event",
|
||||
"_publish_queue",
|
||||
"_send_queue",
|
||||
"_publisher_task",
|
||||
"_loop",
|
||||
)
|
||||
@@ -183,15 +182,41 @@ class MqttNode(Node):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
# One topic for every port, or a per-port mapping.
|
||||
topic: str | dict[str, str] = "*"
|
||||
broker_host: str = "localhost"
|
||||
broker_port: int = 1883
|
||||
username: str | None = None
|
||||
password: str | None = Field(default=None, json_schema_extra={"x-secret": True})
|
||||
client_id: str | None = None
|
||||
qos: int = 0
|
||||
retain: bool = False
|
||||
keepalive: int = 60
|
||||
topic: str | dict[str, str] = Field(
|
||||
default="*",
|
||||
description='One topic for every port, or {"port": "some/topic"} per port.',
|
||||
)
|
||||
broker_host: str = Field(
|
||||
default="localhost",
|
||||
description="Hostname or IP of the broker, as the engine reaches it.",
|
||||
)
|
||||
broker_port: int = Field(
|
||||
default=1883, description="The port the broker listens on."
|
||||
)
|
||||
username: str | None = Field(
|
||||
default=None, description="Username for the broker, if it asks for one."
|
||||
)
|
||||
password: str | None = Field(
|
||||
default=None,
|
||||
description="Password for the broker, kept in the secrets store.",
|
||||
json_schema_extra={"x-secret": True},
|
||||
)
|
||||
client_id: str | None = Field(
|
||||
default=None,
|
||||
description="Client identifier the broker sees. Empty picks a random one.",
|
||||
)
|
||||
qos: int = Field(
|
||||
default=0,
|
||||
description="Quality of service for subscribing and publishing, 0, 1 or 2.",
|
||||
)
|
||||
retain: bool = Field(
|
||||
default=False,
|
||||
description="Have the broker keep the last value sent for later subscribers.",
|
||||
)
|
||||
keepalive: int = Field(
|
||||
default=60,
|
||||
description="Seconds between pings that keep an idle connection open.",
|
||||
)
|
||||
# Bounds every broker operation: subscribe, publish, and the
|
||||
# disconnect acknowledgement on the way out. Without one a client
|
||||
# whose socket died waits for that ack forever, and the task never
|
||||
@@ -202,9 +227,9 @@ class MqttNode(Node):
|
||||
description="Give up on a broker operation after this many seconds.",
|
||||
)
|
||||
# Deep enough to ride out a broker hiccup, shallow enough that a
|
||||
# publisher which cannot keep up drops old values instead of growing
|
||||
# sender which cannot keep up drops old values instead of growing
|
||||
# without bound. A node that bursts wants more than one that trickles.
|
||||
publish_queue_size: int = Field(
|
||||
send_queue_size: int = Field(
|
||||
default=256,
|
||||
gt=0,
|
||||
description=(
|
||||
@@ -215,17 +240,23 @@ class MqttNode(Node):
|
||||
# Which key to lift out of a JSON object payload. A device that wraps
|
||||
# its reading — Victron's ``{"value": 5}`` — is otherwise a Python node
|
||||
# per port. One key for every port, or a per-port mapping.
|
||||
json_key: str | dict[str, str] = ""
|
||||
json_key: str | dict[str, str] = Field(
|
||||
default="",
|
||||
description=(
|
||||
"Key to lift out of a JSON object payload, for every port or as "
|
||||
'{"port": "key"}.'
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def target_key(cls, params: dict[str, Any]) -> str | None:
|
||||
"""The broker and topic, which is one physical thing.
|
||||
|
||||
A publisher and a subscriber on the same topic get the same key on
|
||||
A sender and a listener on the same topic get the same key on
|
||||
purpose: they are two ends of one wire, and drawing them as one neuron
|
||||
is the only way the path through the broker shows up at all.
|
||||
"""
|
||||
# ponytail: publisher and subscriber merge into one neuron; key on mode
|
||||
# ponytail: sender and listener merge into one neuron; key on mode
|
||||
# as well if the two directions ever need telling apart.
|
||||
fields = cls.Params.model_fields
|
||||
topic = params.get("topic", fields["topic"].default)
|
||||
@@ -253,9 +284,9 @@ class MqttNode(Node):
|
||||
)
|
||||
|
||||
# Inputs mean this node publishes; outputs mean it subscribes.
|
||||
self.mode = MqttNode.Mode.PUBLISHER if requires else MqttNode.Mode.SUBSCRIBER
|
||||
self.mode = MqttNode.Mode.SENDER if requires else MqttNode.Mode.LISTENER
|
||||
|
||||
ports = provides if self.mode == MqttNode.Mode.SUBSCRIBER else requires
|
||||
ports = provides if self.mode == MqttNode.Mode.LISTENER else requires
|
||||
if isinstance(cfg.topic, dict):
|
||||
self.topics: dict[str, str] = dict(cfg.topic)
|
||||
else:
|
||||
@@ -285,13 +316,13 @@ class MqttNode(Node):
|
||||
self.retain = cfg.retain
|
||||
self.keepalive = cfg.keepalive
|
||||
self.timeout = cfg.timeout
|
||||
self.publish_queue_size = cfg.publish_queue_size
|
||||
self.send_queue_size = cfg.send_queue_size
|
||||
|
||||
# Runtime state
|
||||
self._subscription_task: asyncio.Task[None] | None = None
|
||||
self._mqtt_client = None
|
||||
self._stop_event: asyncio.Event | None = None
|
||||
self._publish_queue: asyncio.Queue[dict[str, Any]] | None = None
|
||||
self._send_queue: asyncio.Queue[dict[str, Any]] | None = None
|
||||
self._publisher_task: asyncio.Task[None] | None = None
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
@@ -311,13 +342,13 @@ class MqttNode(Node):
|
||||
name = f"mqtt_{self.mode.value}_{safe_topic}"
|
||||
|
||||
# Initialize parent with appropriate function
|
||||
# For subscriber mode, f is a no-op since data is injected via inject()
|
||||
# For publisher mode, f handles the outgoing MQTT publish
|
||||
# For a listener, f is a no-op since data is injected via inject()
|
||||
# For a sender, f handles the outgoing MQTT publish
|
||||
super().__init__(
|
||||
f=(
|
||||
self._noop_subscriber
|
||||
if self.mode == MqttNode.Mode.SUBSCRIBER
|
||||
else self._publisher_handler
|
||||
self._noop_listener
|
||||
if self.mode == MqttNode.Mode.LISTENER
|
||||
else self._sender_handler
|
||||
),
|
||||
requires=requires,
|
||||
provides=provides,
|
||||
@@ -326,13 +357,11 @@ class MqttNode(Node):
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _noop_subscriber(
|
||||
params: dict[str, Any], **kwargs: Any
|
||||
) -> dict[str, Any] | None:
|
||||
def _noop_listener(params: dict[str, Any], **kwargs: Any) -> dict[str, Any] | None:
|
||||
"""
|
||||
No-op function for subscriber mode nodes.
|
||||
No-op function for listener nodes.
|
||||
|
||||
Subscriber mode nodes inject data via :meth:`inject`, not :meth:`__call__`.
|
||||
Listener nodes inject data via :meth:`inject`, not :meth:`__call__`.
|
||||
This function exists only to satisfy the Node interface and should not
|
||||
be called directly.
|
||||
|
||||
@@ -345,11 +374,11 @@ class MqttNode(Node):
|
||||
"""
|
||||
return None
|
||||
|
||||
def _publisher_handler(
|
||||
def _sender_handler(
|
||||
self, params: dict[str, Any], **kwargs: Any
|
||||
) -> dict[str, Any] | None:
|
||||
"""
|
||||
Publish pipeline data to MQTT topic (publisher mode).
|
||||
Publish pipeline data to MQTT topic (sender).
|
||||
|
||||
This method is called when upstream dependencies are satisfied.
|
||||
Handing the payload to the node's publisher task is all that happens
|
||||
@@ -363,7 +392,7 @@ class MqttNode(Node):
|
||||
:returns: None (publishing is fire-and-forget).
|
||||
:rtype: dict | None
|
||||
"""
|
||||
loop, queue = self._loop, self._publish_queue
|
||||
loop, queue = self._loop, self._send_queue
|
||||
if loop is not None and queue is not None:
|
||||
loop.call_soon_threadsafe(self._enqueue, queue, dict(kwargs))
|
||||
return None
|
||||
@@ -386,21 +415,21 @@ class MqttNode(Node):
|
||||
if queue.full():
|
||||
try:
|
||||
queue.get_nowait()
|
||||
logger.warning("Publish queue full for node '%s', dropped", self.name)
|
||||
self.report_health("degraded", "publish queue full")
|
||||
logger.warning("Send queue full for node '%s', dropped", self.name)
|
||||
self.report_health("degraded", "send queue full")
|
||||
except asyncio.QueueEmpty:
|
||||
pass
|
||||
queue.put_nowait(data)
|
||||
|
||||
async def _publisher_loop(self) -> None:
|
||||
"""Hold one connection and drain the publish queue over it.
|
||||
"""Hold one connection and drain the send queue over it.
|
||||
|
||||
A dropped connection raises, and the supervisor decides when to
|
||||
reconnect — the same arrangement the subscriber uses.
|
||||
reconnect — the same arrangement the listener uses.
|
||||
"""
|
||||
aiomqtt = _aiomqtt()
|
||||
|
||||
queue = self._publish_queue
|
||||
queue = self._send_queue
|
||||
if queue is None:
|
||||
return
|
||||
|
||||
@@ -461,7 +490,7 @@ class MqttNode(Node):
|
||||
continue
|
||||
|
||||
# A string goes on the wire as it stands. Devices on a shared
|
||||
# broker expect bare values, and the subscriber below already
|
||||
# broker expect bare values, and the listener below already
|
||||
# falls back to the raw text when it is not JSON, so a
|
||||
# fluksio-to-fluksio round trip is unaffected.
|
||||
payload = value if isinstance(value, str) else json.dumps(value)
|
||||
@@ -479,8 +508,8 @@ class MqttNode(Node):
|
||||
)
|
||||
|
||||
async def start(self, app: FastAPI | None = None) -> None:
|
||||
"""A subscriber listens; a publisher opens the connection it will reuse."""
|
||||
if self.mode is MqttNode.Mode.SUBSCRIBER:
|
||||
"""A listener subscribes; a sender opens the connection it will reuse."""
|
||||
if self.mode is MqttNode.Mode.LISTENER:
|
||||
await self.start_subscription()
|
||||
else:
|
||||
await self.start_publisher()
|
||||
@@ -491,41 +520,41 @@ class MqttNode(Node):
|
||||
|
||||
async def start_publisher(self) -> None:
|
||||
"""Run the task that owns this node's connection to the broker."""
|
||||
if self._publish_queue is not None:
|
||||
if self._send_queue is not None:
|
||||
return
|
||||
self._publish_queue = asyncio.Queue(maxsize=self.publish_queue_size)
|
||||
self._send_queue = asyncio.Queue(maxsize=self.send_queue_size)
|
||||
self._loop = asyncio.get_running_loop()
|
||||
self._publisher_task = self._run_supervised("mqtt-out", self._publisher_loop)
|
||||
|
||||
async def stop_publisher(self) -> None:
|
||||
"""Drop the queue and let the connection go."""
|
||||
if self._publish_queue is None:
|
||||
if self._send_queue is None:
|
||||
return
|
||||
if self._publisher_task is not None:
|
||||
await self._cancel_task(self._publisher_task)
|
||||
self._publisher_task = None
|
||||
self._publish_queue = None
|
||||
self._send_queue = None
|
||||
self._loop = None
|
||||
|
||||
async def start_subscription(self) -> None:
|
||||
"""
|
||||
Start the MQTT subscription for trigger mode nodes.
|
||||
Start the MQTT subscription for listener nodes.
|
||||
|
||||
This method starts a background task that listens for messages
|
||||
on the subscribed topic and triggers the pipeline when messages arrive.
|
||||
|
||||
:raises RuntimeError: If called on a publisher mode node.
|
||||
:raises RuntimeError: If called on a sender.
|
||||
|
||||
:example:
|
||||
>>> subscriber = MqttNode(
|
||||
>>> listener = MqttNode(
|
||||
... topic="sensors/#",
|
||||
... provides=[MessageSpec(name="value", dtype=DType.FLOAT)],
|
||||
... params={"broker_host": "localhost"},
|
||||
... )
|
||||
>>> await subscriber.start_subscription()
|
||||
>>> await listener.start_subscription()
|
||||
"""
|
||||
if self.mode != MqttNode.Mode.SUBSCRIBER:
|
||||
raise RuntimeError("Can only start subscription for subscriber mode nodes")
|
||||
if self.mode != MqttNode.Mode.LISTENER:
|
||||
raise RuntimeError("Can only start subscription for listener nodes")
|
||||
|
||||
if self._subscription_task is not None:
|
||||
return # Already running
|
||||
|
||||
Reference in New Issue
Block a user