Give connectors a store that survives restarts, and a way to clear it
`ConnectorNode.store` keeps short strings per connector package in the
engine's database (`connector_state`, migration f8c3e5a91d27), so the nodes of
one package share them and `fluksio serve` needs nothing new: get, set and
delete, at most 64 kB a value and 256 keys a package. A superuser lists the
keys (never the values) and clears them through `/connectors/{node_type}/state`,
or under Stored state in a connector node's panel. Documented in the connector
contract.
This commit is contained in:
@@ -20,7 +20,10 @@ What a connector gets from the base class:
|
||||
reading it;
|
||||
* :meth:`ConnectorNode.save_artifact`, for a device whose readings are bytes —
|
||||
a camera frame, a recorded clip — which travel as a reference on a media
|
||||
port rather than as the message itself.
|
||||
port rather than as the message itself;
|
||||
* :attr:`ConnectorNode.store`, for the few short strings that have to outlive
|
||||
a restart — a certificate pinned on first contact, a pairing token — shared
|
||||
by every node of the connector's package.
|
||||
|
||||
The message schemas and the parameter model are the rest of the contract, and
|
||||
they are the same ones the built-in nodes use. See ``docs/connectors/`` for the
|
||||
@@ -35,6 +38,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from fluksio.flow.connector_store import ConnectorStore, namespace_of
|
||||
from fluksio.flow.nodes import Node
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -92,6 +96,7 @@ class ConnectorNode(Node):
|
||||
"_stop_event",
|
||||
"_last_published",
|
||||
"_artifacts",
|
||||
"_store",
|
||||
"_down",
|
||||
)
|
||||
|
||||
@@ -102,6 +107,7 @@ class ConnectorNode(Node):
|
||||
self._stop_event: asyncio.Event | None = None
|
||||
self._last_published: dict[str, Any] = {}
|
||||
self._artifacts: ArtifactStore | None = None
|
||||
self._store: ConnectorStore | None = None
|
||||
self._down = False
|
||||
|
||||
def _dispatch(self, params: dict[str, Any], **ports: Any) -> dict[str, Any] | None:
|
||||
@@ -166,6 +172,19 @@ class ConnectorNode(Node):
|
||||
[data], name=name, media_type=media_type, volatile=volatile
|
||||
)
|
||||
|
||||
@property
|
||||
def store(self) -> ConnectorStore:
|
||||
"""Short strings this connector keeps across restarts.
|
||||
|
||||
``get(key)``, ``set(key, value)`` and ``delete(key)``, namespaced by the
|
||||
connector's top-level package, so every node of the package sees the
|
||||
same keys. Blocking: call it through ``asyncio.to_thread`` from the
|
||||
event loop. Available before :meth:`start`, unlike the artifact store.
|
||||
"""
|
||||
if self._store is None:
|
||||
self._store = ConnectorStore(namespace_of(type(self)))
|
||||
return self._store
|
||||
|
||||
async def start(self, app: FastAPI | None = None) -> None:
|
||||
self._artifacts = getattr(app.state, "artifact_store", None) if app else None
|
||||
if self.config.poll_interval > 0 and self._stop_event is None:
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
"""What a connector keeps across restarts.
|
||||
|
||||
A connector holds connections and handles, and those are meant to go when the
|
||||
process does. A few things are not: the certificate a device presented on first
|
||||
contact, a token it paired with. :attr:`ConnectorNode.store` is where those go —
|
||||
a handful of short strings per connector package, in the engine's own database,
|
||||
so they survive a restart and need nothing ``fluksio serve`` does not have.
|
||||
|
||||
Keyed by the connector's top-level package rather than by node: the nodes of
|
||||
one Hue bridge share the certificate pinned for it, and every node type a
|
||||
package installs shares the same keys. A superuser lists and clears them
|
||||
through ``/connectors/{node_type}/state``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import delete, func
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from fluksio.models import ConnectorState
|
||||
|
||||
#: A value is a short string — a fingerprint, a PEM, a token. Bytes worth
|
||||
#: keeping belong in the artifact store.
|
||||
MAX_VALUE_BYTES = 64 * 1024
|
||||
#: Keys per connector package. Past this a connector is keeping something per
|
||||
#: reading rather than per device.
|
||||
MAX_KEYS = 256
|
||||
#: What a top-level package can be called.
|
||||
CONNECTOR_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,99}")
|
||||
#: One URL path segment, so the reset route can name every key.
|
||||
KEY = re.compile(r"[^/\s]{1,200}")
|
||||
|
||||
|
||||
def namespace_of(cls: type) -> str:
|
||||
"""The package a connector class keeps its state under."""
|
||||
return cls.__module__.partition(".")[0]
|
||||
|
||||
|
||||
def _checked(key: str) -> str:
|
||||
if not KEY.fullmatch(key):
|
||||
raise ValueError(
|
||||
f"'{key}' is not a valid key: 1 to 200 characters, no slash or whitespace"
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
class ConnectorStore:
|
||||
"""Short strings one connector package keeps across restarts.
|
||||
|
||||
Every call is a database round trip, so from the event loop it goes
|
||||
through ``asyncio.to_thread`` like any other blocking I/O.
|
||||
"""
|
||||
|
||||
def __init__(self, connector: str) -> None:
|
||||
if not CONNECTOR_NAME.fullmatch(connector):
|
||||
raise ValueError(f"'{connector}' is not a connector package name")
|
||||
self.connector = connector
|
||||
|
||||
def get(self, key: str) -> str | None:
|
||||
"""The value stored under *key*, or ``None``."""
|
||||
# Imported here: a connector imports this module, and its tests have
|
||||
# no business opening the engine's database by importing it.
|
||||
from fluksio.core.db import engine
|
||||
|
||||
with Session(engine) as session:
|
||||
row = session.get(ConnectorState, (self.connector, _checked(key)))
|
||||
return None if row is None else row.value
|
||||
|
||||
def set(self, key: str, value: str) -> None:
|
||||
"""Store *value* under *key*, replacing what was there."""
|
||||
from fluksio.core.db import engine, writing
|
||||
|
||||
_checked(key)
|
||||
if len(value.encode()) > MAX_VALUE_BYTES:
|
||||
raise ValueError(
|
||||
f"'{key}' is too large to store: {MAX_VALUE_BYTES} bytes at most"
|
||||
)
|
||||
with Session(engine) as session, writing(session):
|
||||
row = session.get(ConnectorState, (self.connector, key))
|
||||
if row is None:
|
||||
count = session.exec(
|
||||
select(func.count())
|
||||
.select_from(ConnectorState)
|
||||
.where(ConnectorState.connector == self.connector)
|
||||
).one()
|
||||
if count >= MAX_KEYS:
|
||||
raise ValueError(
|
||||
f"{self.connector} already stores {MAX_KEYS} keys, the most "
|
||||
"a connector may keep"
|
||||
)
|
||||
row = ConnectorState(connector=self.connector, key=key, value=value)
|
||||
else:
|
||||
row.value = value
|
||||
row.updated_at = datetime.now(UTC)
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
def delete(self, key: str) -> bool:
|
||||
"""Forget *key*. Whether there was anything to forget."""
|
||||
from fluksio.core.db import engine
|
||||
|
||||
with Session(engine) as session:
|
||||
result = session.exec(
|
||||
delete(ConnectorState).where(
|
||||
col(ConnectorState.connector) == self.connector,
|
||||
col(ConnectorState.key) == _checked(key),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
def entries(self) -> list[ConnectorState]:
|
||||
"""Every key this package keeps, in key order."""
|
||||
from fluksio.core.db import engine
|
||||
|
||||
with Session(engine) as session:
|
||||
return list(
|
||||
session.exec(
|
||||
select(ConnectorState)
|
||||
.where(ConnectorState.connector == self.connector)
|
||||
.order_by(col(ConnectorState.key))
|
||||
).all()
|
||||
)
|
||||
|
||||
def clear(self) -> int:
|
||||
"""Forget every key. How many there were."""
|
||||
from fluksio.core.db import engine
|
||||
|
||||
with Session(engine) as session:
|
||||
result = session.exec(
|
||||
delete(ConnectorState).where(
|
||||
col(ConnectorState.connector) == self.connector
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return int(result.rowcount)
|
||||
Reference in New Issue
Block a user