Rename mapper: flow/namespace.py maps state keys, work items and alert dedup keys from an old flow key to a new one; the state key families and WorkItem fields are registered there, the key builders take their prefixes from it, and done markers read back with queue.done_marker

This commit is contained in:
2026-11-02 19:28:25 +01:00
parent fd637e0f1e
commit ebcc480a8c
6 changed files with 389 additions and 10 deletions
+187
View File
@@ -0,0 +1,187 @@
"""The names the engine keeps live state and queued work under, and how a
rename of a flow key maps them.
A flow's messages and nodes are named ``<key>.<name>``, and the engine's own
bookkeeping on them sits in state beside the values, under a reserved prefix
and the names it is about: ``__version__:heat.temp``,
``__last_seen__:heat.valve:hall.temp``. Renaming a flow key moves all of it to
the new name, so every family is registered here, beside the builders that
take their prefix from it, and a key of a family that is not is refused rather
than left behind.
Nothing here does I/O. It maps names; moving the keys and items is the state
and queue backends' work.
"""
from __future__ import annotations
from dataclasses import dataclass, replace
from typing import TYPE_CHECKING, Any
import orjson
from fluksio.flow.messages import flow_of
if TYPE_CHECKING:
from fluksio.flow.queue import WorkItem
#: The message names in state, as a set (`RedisState` only).
NAMES = "__names__"
#: A message's recorded points (`RedisState`; memory keeps them apart).
HISTORY = "__history__"
#: How often a message was published.
VERSION = "__version__"
#: When a message was last published.
TIMESTAMP = "__ts__"
#: The version of a message a node last ran on.
LAST_SEEN = "__last_seen__"
#: When a rate-limited input last woke a node.
DELIVERED = "__in_ts__"
#: The value a rate-limited output kept back.
HELD = "__held__"
#: When a node's rate-limit window is due to be let through.
FLUSH = "__flush__"
#: What an engine-shipped node type remembers (`Node.remember`).
NODE_STATE = "__node__"
#: A qualified message or node name, ``<flow key>.<name>``.
NAME = "name"
#: A node's own key, user data that may contain colons.
PRIVATE = "private"
#: A flow key.
FLOW = "flow"
#: A map keyed by qualified message names.
OUTPUTS = "outputs"
#: Every bookkeeping family in live state, by prefix, with the parts that
#: follow it. A key is ``<prefix>:<part>:<part>``. Names never contain a colon,
#: so the key splits at its first ones and the last part takes the rest, which
#: is how a private key keeps its own.
FAMILIES: dict[str, tuple[str, ...]] = {
NAMES: (),
HISTORY: (NAME,),
VERSION: (NAME,),
TIMESTAMP: (NAME,),
LAST_SEEN: (NAME, NAME),
DELIVERED: (NAME, NAME),
HELD: (NAME,),
FLUSH: (NAME,),
NODE_STATE: (NAME, PRIVATE),
}
#: What each `WorkItem` field holds that a rename changes, or None. A guard is
#: the node's own key, and the state it guards moves with the node.
ITEM_FIELDS: dict[str, str | None] = {
"kind": None,
"node": NAME,
"flow": FLOW,
"outputs": OUTPUTS,
"cause": None,
"not_before": None,
"guard_key": None,
"guard_value": None,
"entry_id": None,
"deliveries": None,
"enqueued_at": None,
"run_id": None,
"parent": None,
}
def _parts(key: str) -> tuple[str, list[tuple[str, str]]]:
"""A state key's family prefix and its parts, each with what it holds.
A key without a prefix is a message value, and its family is ``""``.
"""
if not key.startswith("__"):
return "", [(key, NAME)]
family, sep, rest = key.partition(":")
shape = FAMILIES.get(family)
parts = rest.split(":", len(shape) - 1) if shape is not None and sep else []
if shape is None or len(parts) != len(shape):
raise ValueError(f"State key '{key}' is in no registered family")
return family, list(zip(parts, shape, strict=True))
def touches(flow: str, key: str) -> bool:
"""Whether the state key *key* names anything of the flow key *flow*."""
_, parts = _parts(key)
return any(kind == NAME and flow_of(part) == flow for part, kind in parts)
@dataclass(frozen=True)
class Rename:
"""The flow key *old* becoming *new*, applied to the names that embed it."""
old: str
new: str
def name(self, qualified: str) -> str:
"""A qualified message or node name in the new namespace.
Only the part before the first dot is the flow, so renaming ``heat``
leaves ``heating.temp`` alone. A bare key is its own namespace.
"""
if flow_of(qualified) != self.old:
return qualified
return self.new + qualified[len(self.old) :]
def flow(self, key: str) -> str:
return self.new if key == self.old else key
def state_key(self, key: str) -> str | None:
"""Where the state key *key* moves to, or None if it stays.
:raises ValueError: For a bookkeeping key of an unregistered family.
"""
family, parts = _parts(key)
mapped = [self.name(part) if kind == NAME else part for part, kind in parts]
renamed = ":".join([family, *mapped]) if family else mapped[0]
return renamed if renamed != key else None
def item(self, item: WorkItem) -> WorkItem | None:
"""*item* with its names in the new namespace, or None if it has none.
A ``run`` item names its flow by id, which a rename leaves as it is.
"""
if item.kind == "run":
return None
changes = {}
for field, holds in ITEM_FIELDS.items():
if holds is not None:
value = getattr(item, field)
mapped = self._mapped(holds, value)
if mapped != value:
changes[field] = mapped
return replace(item, **changes) if changes else None
def fields(self, fields: dict[str, str]) -> dict[str, str] | None:
"""A serialized item (`WorkItem.to_fields`) in the new namespace, or
None if it names nothing renamed. Fields a rename does not touch keep
their exact text, so an unchanged item serializes as it did."""
if fields.get("kind") == "run":
return None
renamed = dict(fields)
for field, holds in ITEM_FIELDS.items():
if holds is None or field not in fields:
continue
if holds == OUTPUTS:
outputs = orjson.loads(fields[field] or "{}")
mapped = self._mapped(holds, outputs)
if mapped != outputs:
renamed[field] = orjson.dumps(mapped).decode()
else:
renamed[field] = self._mapped(holds, fields[field])
return renamed if renamed != fields else None
def dedup_key(self, key: str) -> str:
"""An alert's ``<event type>:<node or flow>`` (`alerts.dedup_key`)."""
event, sep, about = key.partition(":")
return f"{event}{sep}{self.name(about)}"
def _mapped(self, holds: str, value: Any) -> Any:
if holds == NAME:
return self.name(value)
if holds == FLOW:
return self.flow(value)
return {self.name(name): one for name, one in value.items()}
+2 -1
View File
@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any
from fluksio.flow import logs
from fluksio.flow.messages import MessageSpec, qualify
from fluksio.flow.namespace import NODE_STATE
from fluksio.flow.supervision import Supervisor
if TYPE_CHECKING:
@@ -283,7 +284,7 @@ class Node:
# -------------------------------------------------------------------------
def _state_key(self, key: str) -> str:
return f"__node__:{self.id}:{key}"
return f"{NODE_STATE}:{self.id}:{key}"
def remember(self, key: str, value: Any) -> None:
"""Keep a value for the next run of this node."""
+14 -6
View File
@@ -34,6 +34,14 @@ from fluksio.flow import logs
from fluksio.flow.artifacts import is_reference
from fluksio.flow.events import EventBus
from fluksio.flow.messages import flow_of, requalify
from fluksio.flow.namespace import (
DELIVERED,
FLUSH,
HELD,
LAST_SEEN,
TIMESTAMP,
VERSION,
)
from fluksio.flow.nodes import Node
from fluksio.flow.queue import WorkQueue
from fluksio.flow.state import MemoryState, StateBackend
@@ -633,25 +641,25 @@ class Pipeline:
# -------------------------------------------------------------------------
def _version_key(self, msg_name: str) -> str:
return f"__version__:{msg_name}"
return f"{VERSION}:{msg_name}"
def _last_seen_key(self, node_name: str, msg_name: str) -> str:
return f"__last_seen__:{node_name}:{msg_name}"
return f"{LAST_SEEN}:{node_name}:{msg_name}"
def _timestamp_key(self, msg_name: str) -> str:
return f"__ts__:{msg_name}"
return f"{TIMESTAMP}:{msg_name}"
def _delivered_key(self, node_name: str, msg_name: str) -> str:
"""When a rate-limited input last woke this node."""
return f"__in_ts__:{node_name}:{msg_name}"
return f"{DELIVERED}:{node_name}:{msg_name}"
def _held_key(self, msg_name: str) -> str:
"""The value a rate-limited output kept back, waiting for its window."""
return f"__held__:{msg_name}"
return f"{HELD}:{msg_name}"
def _flush_key(self, node_name: str) -> str:
"""When a rate-limit window of this node is due to be let through."""
return f"__flush__:{node_name}"
return f"{FLUSH}:{node_name}"
def _window_split(
self, node: Node, result: dict[str, Any], now: float
+11 -1
View File
@@ -52,6 +52,8 @@ STALE_CONSUMER_IDLE_MS = 3_600_000
# The Redis one expires each after an hour; this is the same idea sized by
# count, since redelivery happens seconds after the claim and never later.
DONE_MARKERS = 10_000
# The Redis queue keeps them as ``<namespace>:__done__:<entry id>:<node>``.
DONE = "__done__"
# A promotion is one Redis operation, including when the caller loses its
# connection before hearing the result. Write first: Lua does not roll back a
@@ -162,6 +164,14 @@ def owned_by(item: WorkItem, flow: str) -> bool:
)
def done_marker(key: str) -> tuple[str, str]:
"""The entry id and node a done marker is for: `RedisWorkQueue._done_key`
read back. Split at the last colon, since an entry id from the due lane
has one of its own and a node id never does."""
entry_id, node = key.split(f":{DONE}:", 1)[1].rsplit(":", 1)
return entry_id, node
class WorkQueue(ABC):
"""What the execution service pulls from."""
@@ -548,7 +558,7 @@ class RedisWorkQueue(WorkQueue):
return f"{self._ns}:__parked__:{flow}"
def _done_key(self, entry_id: str, node: str) -> str:
return f"{self._ns}:__done__:{entry_id}:{node}"
return f"{self._ns}:{DONE}:{entry_id}:{node}"
def add(self, item: WorkItem) -> None:
self._redis.xadd(
+4 -2
View File
@@ -17,6 +17,8 @@ from typing import Any, cast
import orjson
import redis
from fluksio.flow.namespace import HISTORY, NAMES
#: A node may return a dict keyed by something other than a string, and the
#: stdlib encoder this replaced turned those keys into strings rather than
#: refusing them. Two differences remain and are both improvements: a
@@ -533,7 +535,7 @@ class RedisState(StateBackend):
return self._deserialize(data) if data is not None else default
def _names_key(self) -> str:
return self._key("__names__")
return self._key(NAMES)
def message_names(self) -> list[str]:
"""The message names, from the set kept beside them.
@@ -770,7 +772,7 @@ class RedisState(StateBackend):
self._client.close()
def _history_key(self, key: str) -> str:
return self._key(f"__history__:{key}")
return self._key(f"{HISTORY}:{key}")
def append_history(
self, values: dict[str, Any], ts: float, limits: dict[str, int] | None = None