Give connectors a store that survives restarts, and a way to clear it

`ConnectorNode.store` keeps short strings per connector package in the
engine's database (`connector_state`, migration f8c3e5a91d27), so the nodes of
one package share them and `fluksio serve` needs nothing new: get, set and
delete, at most 64 kB a value and 256 keys a package. A superuser lists the
keys (never the values) and clears them through `/connectors/{node_type}/state`,
or under Stored state in a connector node's panel. Documented in the connector
contract.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013GAeg4Jo6xjZzjwy9QDWMQ
This commit is contained in:
2026-10-06 17:19:48 +02:00
co-authored by Claude Opus 5.5
parent b086d5d22b
commit 847abd67d6
15 changed files with 710 additions and 4 deletions
+20 -1
View File
@@ -20,7 +20,10 @@ What a connector gets from the base class:
reading it;
* :meth:`ConnectorNode.save_artifact`, for a device whose readings are bytes —
a camera frame, a recorded clip — which travel as a reference on a media
port rather than as the message itself.
port rather than as the message itself;
* :attr:`ConnectorNode.store`, for the few short strings that have to outlive
a restart — a certificate pinned on first contact, a pairing token — shared
by every node of the connector's package.
The message schemas and the parameter model are the rest of the contract, and
they are the same ones the built-in nodes use. See ``docs/connectors/`` for the
@@ -35,6 +38,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
from pydantic import BaseModel, Field
from fluksio.flow.connector_store import ConnectorStore, namespace_of
from fluksio.flow.nodes import Node
if TYPE_CHECKING:
@@ -92,6 +96,7 @@ class ConnectorNode(Node):
"_stop_event",
"_last_published",
"_artifacts",
"_store",
"_down",
)
@@ -102,6 +107,7 @@ class ConnectorNode(Node):
self._stop_event: asyncio.Event | None = None
self._last_published: dict[str, Any] = {}
self._artifacts: ArtifactStore | None = None
self._store: ConnectorStore | None = None
self._down = False
def _dispatch(self, params: dict[str, Any], **ports: Any) -> dict[str, Any] | None:
@@ -166,6 +172,19 @@ class ConnectorNode(Node):
[data], name=name, media_type=media_type, volatile=volatile
)
@property
def store(self) -> ConnectorStore:
"""Short strings this connector keeps across restarts.
``get(key)``, ``set(key, value)`` and ``delete(key)``, namespaced by the
connector's top-level package, so every node of the package sees the
same keys. Blocking: call it through ``asyncio.to_thread`` from the
event loop. Available before :meth:`start`, unlike the artifact store.
"""
if self._store is None:
self._store = ConnectorStore(namespace_of(type(self)))
return self._store
async def start(self, app: FastAPI | None = None) -> None:
self._artifacts = getattr(app.state, "artifact_store", None) if app else None
if self.config.poll_interval > 0 and self._stop_event is None:
+139
View File
@@ -0,0 +1,139 @@
"""What a connector keeps across restarts.
A connector holds connections and handles, and those are meant to go when the
process does. A few things are not: the certificate a device presented on first
contact, a token it paired with. :attr:`ConnectorNode.store` is where those go —
a handful of short strings per connector package, in the engine's own database,
so they survive a restart and need nothing ``fluksio serve`` does not have.
Keyed by the connector's top-level package rather than by node: the nodes of
one Hue bridge share the certificate pinned for it, and every node type a
package installs shares the same keys. A superuser lists and clears them
through ``/connectors/{node_type}/state``.
"""
from __future__ import annotations
import re
from datetime import UTC, datetime
from sqlalchemy import delete, func
from sqlmodel import Session, col, select
from fluksio.models import ConnectorState
#: A value is a short string — a fingerprint, a PEM, a token. Bytes worth
#: keeping belong in the artifact store.
MAX_VALUE_BYTES = 64 * 1024
#: Keys per connector package. Past this a connector is keeping something per
#: reading rather than per device.
MAX_KEYS = 256
#: What a top-level package can be called.
CONNECTOR_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,99}")
#: One URL path segment, so the reset route can name every key.
KEY = re.compile(r"[^/\s]{1,200}")
def namespace_of(cls: type) -> str:
"""The package a connector class keeps its state under."""
return cls.__module__.partition(".")[0]
def _checked(key: str) -> str:
if not KEY.fullmatch(key):
raise ValueError(
f"'{key}' is not a valid key: 1 to 200 characters, no slash or whitespace"
)
return key
class ConnectorStore:
"""Short strings one connector package keeps across restarts.
Every call is a database round trip, so from the event loop it goes
through ``asyncio.to_thread`` like any other blocking I/O.
"""
def __init__(self, connector: str) -> None:
if not CONNECTOR_NAME.fullmatch(connector):
raise ValueError(f"'{connector}' is not a connector package name")
self.connector = connector
def get(self, key: str) -> str | None:
"""The value stored under *key*, or ``None``."""
# Imported here: a connector imports this module, and its tests have
# no business opening the engine's database by importing it.
from fluksio.core.db import engine
with Session(engine) as session:
row = session.get(ConnectorState, (self.connector, _checked(key)))
return None if row is None else row.value
def set(self, key: str, value: str) -> None:
"""Store *value* under *key*, replacing what was there."""
from fluksio.core.db import engine, writing
_checked(key)
if len(value.encode()) > MAX_VALUE_BYTES:
raise ValueError(
f"'{key}' is too large to store: {MAX_VALUE_BYTES} bytes at most"
)
with Session(engine) as session, writing(session):
row = session.get(ConnectorState, (self.connector, key))
if row is None:
count = session.exec(
select(func.count())
.select_from(ConnectorState)
.where(ConnectorState.connector == self.connector)
).one()
if count >= MAX_KEYS:
raise ValueError(
f"{self.connector} already stores {MAX_KEYS} keys, the most "
"a connector may keep"
)
row = ConnectorState(connector=self.connector, key=key, value=value)
else:
row.value = value
row.updated_at = datetime.now(UTC)
session.add(row)
session.commit()
def delete(self, key: str) -> bool:
"""Forget *key*. Whether there was anything to forget."""
from fluksio.core.db import engine
with Session(engine) as session:
result = session.exec(
delete(ConnectorState).where(
col(ConnectorState.connector) == self.connector,
col(ConnectorState.key) == _checked(key),
)
)
session.commit()
return bool(result.rowcount)
def entries(self) -> list[ConnectorState]:
"""Every key this package keeps, in key order."""
from fluksio.core.db import engine
with Session(engine) as session:
return list(
session.exec(
select(ConnectorState)
.where(ConnectorState.connector == self.connector)
.order_by(col(ConnectorState.key))
).all()
)
def clear(self) -> int:
"""Forget every key. How many there were."""
from fluksio.core.db import engine
with Session(engine) as session:
result = session.exec(
delete(ConnectorState).where(
col(ConnectorState.connector) == self.connector
)
)
session.commit()
return int(result.rowcount)