"""The persisted shape of a flow, shared by the store, the API and the editor. A flow is structure plus code: this module is the structure. Node logic for ``python`` nodes lives beside it as a plain ``.py`` file. """ from __future__ import annotations import re from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, field_validator from fluksio.flow.messages import MessageSpec NAME_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$") def _validate_name(value: str) -> str: if not NAME_PATTERN.match(value): raise ValueError( "Use lowercase letters, digits and underscores, starting with a letter" ) return value class Resources(BaseModel): """What one execution of a node needs to have to itself. Declaring nothing is the default and means what it always did: the node runs on the shared worker pool and nothing is accounted for it. That is right for the kind of node most flows are made of — a poll, a threshold, a message on its way somewhere. It is wrong for the other kind. A numerical library sizes its thread pool to every core it can see, so a handful of them at once oversubscribe the machine badly enough to starve the engine's own event loop, and a GPU library that preallocates most of the card deadlocks when a second one arrives. Both are a node saying how much of the machine it takes, which is what this is. """ model_config = ConfigDict(extra="forbid") cpus: int = Field( default=1, ge=1, description=( "Cores held for the whole execution. Also what the thread-pool " "variables are set to, so a library sizing itself to the machine " "sizes itself to this instead." ), ) gpus: int = Field( default=0, ge=0, description=( "Whole devices held for the whole execution, named to the node " "through CUDA_VISIBLE_DEVICES. Nothing else is given them while it " "runs, which is what keeps two preallocating processes apart." ), ) env: dict[str, str] = Field( default_factory=dict, description=( "Extra environment for the worker this node runs in, applied over " "what the allocation derives. Where a library's own tuning goes — " "XLA_FLAGS, XLA_PYTHON_CLIENT_MEM_FRACTION — since those are " "composed strings the engine must not invent." ), ) class NodeDef(BaseModel): """A node as stored: identity, configuration and ports. Deliberately no canvas position. The editor lays a flow out itself, so where a node sits is a fact about the drawing rather than about the flow — and a graph nobody can arrange is one worth keeping small. """ id: str type: str = "python" title: str = "" #: This node's settings: constants of its function, stored with the flow. #: A function node reads them as keyword arguments beside its ports, so a #: setting cannot share a name with one. params: dict[str, Any] = Field(default_factory=dict) requires: list[MessageSpec] = Field(default_factory=list) provides: list[MessageSpec] = Field(default_factory=list) #: Name of a shared source in the library, instead of this node's own file. #: Editing it edits the copy every flow using it runs. source_ref: str | None = None timeout: float | None = Field( default=None, ge=0, description=( "Seconds this node's code may be silent before it is stopped. A " "yield or an emit resets the clock, and the first call's imports " "are not charged to it. 0 disables the limit: the node runs until " "it finishes, and only a dead worker fails the call. Empty " "inherits the engine default." ), ) device: str | None = Field( default=None, description=( "Label of the worker this node's code must run on, such as 'gpu'. " "Empty means the engine's own workers. A run needing a label no " "attached worker carries waits rather than failing." ), ) device_policy: Literal["require", "prefer"] = Field( default="require", description=( "What to do when no worker carries `device`: wait for one, or run " "locally anyway." ), ) cache: bool = Field( default=True, description=( "Whether a batch run may reuse an earlier execution of this node " "with the same source, settings and inputs. Turn it off for a " "function whose answer can change on its own." ), ) resources: Resources | None = Field( default=None, description=( "What one execution of this node holds while it runs. Absent — the " "default — means it is not accounted for and shares the engine's " "workers, which is right for everything that is not compute-heavy." ), ) @field_validator("id") @classmethod def _check_id(cls, value: str) -> str: return _validate_name(value) class FlowOrigin(BaseModel): """Where a flow was declared, when that was somewhere other than here. A flow drawn on the canvas has no origin: the store is where it lives. One stamped with this was declared with the decorators in somebody's own repository and put here by ``fluksio sync``, so the node bodies below it are generated imports and the code they run is versioned twice — once here and once there. Its presence is what makes a flow code-defined. Deliberately no timestamp. The store commits every change it is given, so when a flow was last synced is a fact its own history already holds — and one that would otherwise change on every sync, making an unchanged upload look like a new version of the flow. """ kind: Literal["python"] = "python" #: The repository root on the machine that ran ``sync``. repo: str = "" #: Its commit, and whether the tree had uncommitted changes at the time — #: a run stamped with a dirty commit names code that was never stored. commit: str = "" dirty: bool = False class FlowInput(BaseModel): """A message the flow starts with rather than computes.""" spec: MessageSpec initial: Any | None = None class FlowDef(BaseModel): """One atomic flow.""" name: str title: str = "" nodes: list[NodeDef] = Field(default_factory=list) inputs: list[FlowInput] = Field(default_factory=list) version: int = 1 mode: Literal["live", "batch"] = Field( default="live", description=( "A live flow reacts to what arrives: its subscriptions, schedules " "and webhooks run until it is stopped. A batch flow only runs when " "a run asks it to, from its inputs to its outputs, and is never " "activated." ), ) outputs: list[str] = Field( default_factory=list, description=( "Messages a batch run reports as its result, unqualified. Empty " "means every message the flow ends up holding." ), ) origin: FlowOrigin | None = Field( default=None, description=( "Set when the flow was declared in code elsewhere and uploaded by " "`fluksio sync`. Absent for a flow drawn on the canvas." ), ) @field_validator("name") @classmethod def _check_name(cls, value: str) -> str: return _validate_name(value) class NodeSource(BaseModel): """The Python source of a node.""" code: str #: True when nothing is stored and `code` is the new-node template. An #: editor opens on it either way; a client deciding whether somebody wrote #: that code needs to know it was nobody. Read-only — set on the way out. missing: bool = False Health = Literal["ok", "degraded", "down"] class NodeStatusPublic(BaseModel): """Whether a node loaded, and how its connection is doing.""" id: str status: str = "active" error: str | None = None health: Health = "ok" health_detail: str | None = None #: The node's last runtime failure, kept after it runs again: a failure #: that fired an alert should leave a trace of what it was. last_error: str = "" last_error_ts: float | None = None class MessageValue(BaseModel): """The last payload seen on a message.""" value: Any = None ts: float | None = None class HistoryPoint(BaseModel): """One numeric value a message carried, and when.""" ts: float value: float class MessageHistory(BaseModel): """A message's recent numeric values, oldest first. Only numbers are recorded, so ``numeric`` tells the panel whether an empty series means "nothing plottable here" or "nothing has arrived yet". """ message: str numeric: bool = False points: list[HistoryPoint] = Field(default_factory=list) class FlowSummary(BaseModel): name: str title: str = "" node_count: int = 0 error_count: int = 0 has_draft: bool = False enabled: bool = True paused: bool = False # Its background tasks kept crashing, so the engine stopped restarting them. quarantined: bool = False #: Of the working copy, so publishing from a list needs no second read. version: int = 1 class FlowsPublic(BaseModel): data: list[FlowSummary] count: int class LibraryNode(BaseModel): """A node source shared across flows, and who is using it.""" name: str used_by: list[str] = Field(default_factory=list) class FlowStatePublic(BaseModel): values: dict[str, MessageValue] = Field(default_factory=dict) nodes: list[NodeStatusPublic] = Field(default_factory=list) class ModulePackage(BaseModel): """One package installed in the venv node code runs on.""" name: str version: str class ModulesInfo(BaseModel): """The venv node code imports from, and the manifest that describes it.""" python_version: str = "" venv_path: str = "" requirements: str = "" packages: list[ModulePackage] = Field(default_factory=list) #: Whether what is installed matches the manifest. applied: bool = False #: True when node code runs on the venv Fluksio was installed into rather #: than one the engine built. That venv belongs to whoever made it, so the #: manifest does not describe it and nothing here installs into it. adopted: bool = False class ApplyRequest(BaseModel): """A pip manifest, one requirement per line.""" requirements: str = "" class ApplyResult(BaseModel): ok: bool output: str = "" class BrainNode(BaseModel): """One neuron: a thing the engine talks to, or a node that only computes. Nodes of the same type pointing at the same outside thing — one broker topic, one URL, one bucket — are a single entry here, whichever flows they sit in. ``members`` are the ``flow.node_id`` names behind it, which is also what the live events are keyed by. """ id: str label: str kind: str members: list[str] = Field(default_factory=list) flows: list[str] = Field(default_factory=list) issue: str | None = Field( default=None, description=( "Why this neuron cannot run, if validation found something. A " "failure the engine hits while running arrives over the socket " "instead; this is the part that is already true before anything " "fires, and so has to travel with the graph." ), ) class BrainEdge(BaseModel): """Messages carrying values from one neuron to another.""" source: str target: str messages: list[str] = Field(default_factory=list) class BrainGraph(BaseModel): """Every published flow at once, merged on what its nodes talk to.""" nodes: list[BrainNode] = Field(default_factory=list) edges: list[BrainEdge] = Field(default_factory=list) class NodeTypeInfo(BaseModel): """A node type the editor can offer, with its parameter schema.""" type: str title: str description: str params_schema: dict[str, Any] = Field(default_factory=dict) has_source: bool = False #: Whether this type takes settings beyond the ones its schema declares. #: A function node's settings are its author's to name, and reach `process` #: as keyword arguments beside its ports. free_params: bool = False #: The package a connector came from; empty for the built-in types. plugin: str | None = None