Give the Influx and MQTT nodes their two missing knobs
The Influx client was built with no timeout, so every query and write fell through to influxdb-client's own 10 s default — invisible to a flow and unchangeable. The param is in seconds like its peers; the client counts in milliseconds, so the call sites convert. The publisher backlog was a module constant, read once at import. It is the depth at which the oldest payload is dropped and the node goes degraded, and a node that bursts wants more than one that trickles, so it moves to Params and is read where the queue is built. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01K1moruzue2kTJd3uVisgNk
This commit is contained in:
@@ -58,6 +58,7 @@ class InfluxDbNode(Node):
|
|||||||
- ``bucket`` (str): Bucket name (required)
|
- ``bucket`` (str): Bucket name (required)
|
||||||
- ``write_precision`` (str): Write precision ("ns", "us", "ms", "s"), default "ms"
|
- ``write_precision`` (str): Write precision ("ns", "us", "ms", "s"), default "ms"
|
||||||
- ``query_range`` (str): Default time range for queries, e.g., "-1h", "-24h"
|
- ``query_range`` (str): Default time range for queries, e.g., "-1h", "-24h"
|
||||||
|
- ``timeout`` (float): Deadline for a request in seconds (default: 10.0)
|
||||||
- ``writes`` (dict): Write configurations keyed by message name, each with:
|
- ``writes`` (dict): Write configurations keyed by message name, each with:
|
||||||
- ``measurement`` (str): Measurement name to write to
|
- ``measurement`` (str): Measurement name to write to
|
||||||
- ``field`` (str): Field name to write (default: "value")
|
- ``field`` (str): Field name to write (default: "value")
|
||||||
@@ -154,6 +155,7 @@ class InfluxDbNode(Node):
|
|||||||
"bucket",
|
"bucket",
|
||||||
"write_precision",
|
"write_precision",
|
||||||
"query_range",
|
"query_range",
|
||||||
|
"timeout",
|
||||||
"writes",
|
"writes",
|
||||||
"queries",
|
"queries",
|
||||||
"_write_client",
|
"_write_client",
|
||||||
@@ -169,6 +171,14 @@ class InfluxDbNode(Node):
|
|||||||
bucket: str
|
bucket: str
|
||||||
write_precision: str = "ms"
|
write_precision: str = "ms"
|
||||||
query_range: str = "-1h"
|
query_range: str = "-1h"
|
||||||
|
# Bounds every request to the server. Without one the client falls
|
||||||
|
# back to its own default, which no flow can see or change. Held in
|
||||||
|
# seconds like every other node; the client counts in milliseconds.
|
||||||
|
timeout: float = Field(
|
||||||
|
default=10.0,
|
||||||
|
gt=0,
|
||||||
|
description="Give up on a query or write after this many seconds.",
|
||||||
|
)
|
||||||
# Per-port write and query configuration.
|
# Per-port write and query configuration.
|
||||||
writes: dict[str, dict[str, Any]] = {}
|
writes: dict[str, dict[str, Any]] = {}
|
||||||
queries: dict[str, dict[str, Any]] = {}
|
queries: dict[str, dict[str, Any]] = {}
|
||||||
@@ -201,6 +211,7 @@ class InfluxDbNode(Node):
|
|||||||
self.bucket = cfg.bucket
|
self.bucket = cfg.bucket
|
||||||
self.write_precision = cfg.write_precision
|
self.write_precision = cfg.write_precision
|
||||||
self.query_range = cfg.query_range
|
self.query_range = cfg.query_range
|
||||||
|
self.timeout = cfg.timeout
|
||||||
self.writes = cfg.writes
|
self.writes = cfg.writes
|
||||||
self.queries = cfg.queries
|
self.queries = cfg.queries
|
||||||
|
|
||||||
@@ -285,7 +296,12 @@ class InfluxDbNode(Node):
|
|||||||
echo = {key: value for key, value in request.items() if key != "flux"}
|
echo = {key: value for key, value in request.items() if key != "flux"}
|
||||||
|
|
||||||
logger.info("Running Flux for node '%s': %s", self.name, flux)
|
logger.info("Running Flux for node '%s': %s", self.name, flux)
|
||||||
with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client:
|
with InfluxDBClient(
|
||||||
|
url=self.url,
|
||||||
|
token=self.token,
|
||||||
|
org=self.org,
|
||||||
|
timeout=int(self.timeout * 1000),
|
||||||
|
) as client:
|
||||||
tables = client.query_api().query(flux, org=self.org)
|
tables = client.query_api().query(flux, org=self.org)
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
@@ -326,7 +342,12 @@ class InfluxDbNode(Node):
|
|||||||
from influxdb_client.client.write_api import SYNCHRONOUS
|
from influxdb_client.client.write_api import SYNCHRONOUS
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client:
|
with InfluxDBClient(
|
||||||
|
url=self.url,
|
||||||
|
token=self.token,
|
||||||
|
org=self.org,
|
||||||
|
timeout=int(self.timeout * 1000),
|
||||||
|
) as client:
|
||||||
write_api = client.write_api(write_options=SYNCHRONOUS)
|
write_api = client.write_api(write_options=SYNCHRONOUS)
|
||||||
|
|
||||||
precision_map = {
|
precision_map = {
|
||||||
@@ -422,7 +443,12 @@ class InfluxDbNode(Node):
|
|||||||
results = {}
|
results = {}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client:
|
with InfluxDBClient(
|
||||||
|
url=self.url,
|
||||||
|
token=self.token,
|
||||||
|
org=self.org,
|
||||||
|
timeout=int(self.timeout * 1000),
|
||||||
|
) as client:
|
||||||
query_api = client.query_api()
|
query_api = client.query_api()
|
||||||
|
|
||||||
for spec in self.output_ports:
|
for spec in self.output_ports:
|
||||||
|
|||||||
@@ -19,10 +19,6 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Deep enough to ride out a broker hiccup, shallow enough that a publisher
|
|
||||||
# which cannot keep up drops old values instead of growing without bound.
|
|
||||||
PUBLISH_QUEUE_SIZE = 256
|
|
||||||
|
|
||||||
|
|
||||||
def topic_matches(filter_: str, topic: str) -> bool:
|
def topic_matches(filter_: str, topic: str) -> bool:
|
||||||
"""Does an MQTT topic filter cover this topic?
|
"""Does an MQTT topic filter cover this topic?
|
||||||
@@ -85,6 +81,7 @@ class MqttNode(Node):
|
|||||||
- ``keepalive`` (int): Keepalive interval in seconds (default: 60)
|
- ``keepalive`` (int): Keepalive interval in seconds (default: 60)
|
||||||
- ``timeout`` (float): Deadline for a broker operation in seconds
|
- ``timeout`` (float): Deadline for a broker operation in seconds
|
||||||
(default: 10.0)
|
(default: 10.0)
|
||||||
|
- ``publish_queue_size`` (int): Publisher backlog depth (default: 256)
|
||||||
:type params: dict
|
:type params: dict
|
||||||
:param name: Optional name for the node.
|
:param name: Optional name for the node.
|
||||||
:type name: str | None
|
:type name: str | None
|
||||||
@@ -154,6 +151,7 @@ class MqttNode(Node):
|
|||||||
"retain",
|
"retain",
|
||||||
"keepalive",
|
"keepalive",
|
||||||
"timeout",
|
"timeout",
|
||||||
|
"publish_queue_size",
|
||||||
"json_keys",
|
"json_keys",
|
||||||
"_topic_to_ports",
|
"_topic_to_ports",
|
||||||
"_wildcards",
|
"_wildcards",
|
||||||
@@ -187,6 +185,17 @@ class MqttNode(Node):
|
|||||||
gt=0,
|
gt=0,
|
||||||
description="Give up on a broker operation after this many seconds.",
|
description="Give up on a broker operation after this many seconds.",
|
||||||
)
|
)
|
||||||
|
# Deep enough to ride out a broker hiccup, shallow enough that a
|
||||||
|
# publisher which cannot keep up drops old values instead of growing
|
||||||
|
# without bound. A node that bursts wants more than one that trickles.
|
||||||
|
publish_queue_size: int = Field(
|
||||||
|
default=256,
|
||||||
|
gt=0,
|
||||||
|
description=(
|
||||||
|
"How many payloads may wait for the broker. Past this the oldest "
|
||||||
|
"is dropped and the node reports degraded."
|
||||||
|
),
|
||||||
|
)
|
||||||
# Which key to lift out of a JSON object payload. A device that wraps
|
# Which key to lift out of a JSON object payload. A device that wraps
|
||||||
# its reading — Victron's ``{"value": 5}`` — is otherwise a Python node
|
# its reading — Victron's ``{"value": 5}`` — is otherwise a Python node
|
||||||
# per port. One key for every port, or a per-port mapping.
|
# per port. One key for every port, or a per-port mapping.
|
||||||
@@ -260,6 +269,7 @@ class MqttNode(Node):
|
|||||||
self.retain = cfg.retain
|
self.retain = cfg.retain
|
||||||
self.keepalive = cfg.keepalive
|
self.keepalive = cfg.keepalive
|
||||||
self.timeout = cfg.timeout
|
self.timeout = cfg.timeout
|
||||||
|
self.publish_queue_size = cfg.publish_queue_size
|
||||||
|
|
||||||
# Runtime state
|
# Runtime state
|
||||||
self._subscription_task: asyncio.Task[None] | None = None
|
self._subscription_task: asyncio.Task[None] | None = None
|
||||||
@@ -467,7 +477,7 @@ class MqttNode(Node):
|
|||||||
"""Run the task that owns this node's connection to the broker."""
|
"""Run the task that owns this node's connection to the broker."""
|
||||||
if self._publish_queue is not None:
|
if self._publish_queue is not None:
|
||||||
return
|
return
|
||||||
self._publish_queue = asyncio.Queue(maxsize=PUBLISH_QUEUE_SIZE)
|
self._publish_queue = asyncio.Queue(maxsize=self.publish_queue_size)
|
||||||
self._loop = asyncio.get_running_loop()
|
self._loop = asyncio.get_running_loop()
|
||||||
self._publisher_task = self._run_supervised("mqtt-out", self._publisher_loop)
|
self._publisher_task = self._run_supervised("mqtt-out", self._publisher_loop)
|
||||||
|
|
||||||
|
|||||||
@@ -197,6 +197,47 @@ def test_a_flux_request_is_run_rather_than_written(monkeypatch):
|
|||||||
assert out == {"answer": {"rows": [], "range_s": 3600}}
|
assert out == {"answer": {"rows": [], "range_s": 3600}}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_configured_timeout_reaches_the_influx_client(monkeypatch):
|
||||||
|
"""The client counts in milliseconds; the param is seconds like its peers."""
|
||||||
|
import influxdb_client
|
||||||
|
|
||||||
|
from fluksio.flow.nodes import InfluxDbNode
|
||||||
|
|
||||||
|
seen: dict = {}
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
seen.update(kwargs)
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def query_api(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def query(self, *_, **__):
|
||||||
|
return []
|
||||||
|
|
||||||
|
monkeypatch.setattr(influxdb_client, "InfluxDBClient", FakeClient)
|
||||||
|
|
||||||
|
node = InfluxDbNode(
|
||||||
|
provides=[MessageSpec(name="answer", dtype=DType.JSON)],
|
||||||
|
params={
|
||||||
|
"url": "http://influx",
|
||||||
|
"token": "t",
|
||||||
|
"org": "o",
|
||||||
|
"bucket": "b",
|
||||||
|
"timeout": 2.5,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
node._run_flux({"flux": 'from(bucket: "b")'})
|
||||||
|
|
||||||
|
assert seen["timeout"] == 2500
|
||||||
|
|
||||||
|
|
||||||
def test_a_falsy_return_is_a_mistake_not_silence():
|
def test_a_falsy_return_is_a_mistake_not_silence():
|
||||||
"""Only None means "nothing to publish"."""
|
"""Only None means "nothing to publish"."""
|
||||||
from fluksio.flow.nodes import Node
|
from fluksio.flow.nodes import Node
|
||||||
|
|||||||
@@ -119,3 +119,35 @@ def test_the_configured_timeout_reaches_the_broker_client(monkeypatch):
|
|||||||
asyncio.run(node._publish_once({"setpoint": 21.0}))
|
asyncio.run(node._publish_once({"setpoint": 21.0}))
|
||||||
|
|
||||||
assert seen["timeout"] == 2.5
|
assert seen["timeout"] == 2.5
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_configured_backlog_reaches_the_publish_queue(monkeypatch):
|
||||||
|
"""The depth is read when the queue is built, so it has to be per node."""
|
||||||
|
import aiomqtt
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *_):
|
||||||
|
return False
|
||||||
|
|
||||||
|
monkeypatch.setattr(aiomqtt, "Client", FakeClient)
|
||||||
|
|
||||||
|
node = MqttNode(
|
||||||
|
requires=[MessageSpec(name="setpoint", port="setpoint", dtype=DType.FLOAT)],
|
||||||
|
params={"topic": {"setpoint": "heating/setpoint"}, "publish_queue_size": 8},
|
||||||
|
)
|
||||||
|
node.assign_flow("heating", "out")
|
||||||
|
|
||||||
|
async def scenario() -> int:
|
||||||
|
await node.start_publisher()
|
||||||
|
assert node._publish_queue is not None
|
||||||
|
size = node._publish_queue.maxsize
|
||||||
|
await node.stop_publisher()
|
||||||
|
return size
|
||||||
|
|
||||||
|
assert asyncio.run(scenario()) == 8
|
||||||
|
|||||||
Reference in New Issue
Block a user