seed: write the shared library before anything points at it

Sharing after the flows were created left every flow between the owner and
the end of the list pointing at a library entry that did not exist yet. A node
that fails to load provides nothing, so one missing entry reported as inputs
nothing carries, in eighteen places.

The library is written *from* a node, so this is one throwaway flow that
carries both sources in and is deleted again — the entries outlive it. Order
stops mattering, which also covers the case that would have broken the
previous fix: two instances of one shared node inside a single flow.

Verified against the installation: nineteen flows, zero validation issues.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-08-22 16:06:29 +02:00
co-authored by Claude Opus 5
parent 63fca5eaf3
commit 8734e51ef1
+112 -91
View File
@@ -113,76 +113,6 @@ def check(flows: list[Flow]) -> list[str]:
return problems return problems
def _check_schemas(flows: list[Flow]) -> list[str]:
"""Hand every node to the engine's own models before the API sees them.
The API would say the same thing, one flow at a time, after creating the
ones before it. Importing the models is free and answers it here — a
misspelled setting on a connector is otherwise found halfway through a
house.
"""
try:
from fluksio.flow.controller import NODE_TYPES
from fluksio.flow.schemas import FlowDef
except ImportError:
return []
problems = []
missing: set[str] = set()
for flow in flows:
try:
FlowDef.model_validate(
{
"name": flow.name,
"title": flow.title,
"nodes": flow.nodes,
"inputs": flow.inputs,
}
)
except Exception as exc: # noqa: BLE001 - reported, not raised
problems.append(f"{flow.name}: {exc}")
continue
for node in flow.nodes:
spec = NODE_TYPES.get(node["type"])
if spec is None:
# Connectors are installed into the image, not into the venv
# this runs in, so their absence here says nothing. A type that
# is neither built in nor a connector this house uses is a
# typo, and that is what is worth saying.
if node["type"] not in CONNECTORS:
problems.append(
f"{flow.name}.{node['id']}: no node type '{node['type']}'"
)
else:
missing.add(node["type"])
continue
if spec.free_params:
continue
params = getattr(spec.cls, "Params", None)
if params is None:
continue
try:
params.model_validate(_without_secrets(node.get("params", {})))
except Exception as exc: # noqa: BLE001 - reported, not raised
problems.append(f"{flow.name}.{node['id']} settings: {exc}")
if missing:
print(f" (from the image, not checked here: {', '.join(sorted(missing))})")
return problems
def _without_secrets(params: Any) -> Any:
"""A secret reference is an ordinary string by the time a node is built."""
if isinstance(params, dict):
if set(params) == {"$secret"}:
return "from-the-store"
return {k: _without_secrets(v) for k, v in params.items()}
if isinstance(params, list):
return [_without_secrets(v) for v in params]
return params
#: Node types that come from connector packages. `make connectors` installs #: Node types that come from connector packages. `make connectors` installs
#: them into the image, so a checkout does not have them and their absence #: them into the image, so a checkout does not have them and their absence
#: here is not a problem to report. #: here is not a problem to report.
@@ -353,34 +283,74 @@ def _check_widgets(known: set[str]) -> list[str]:
# ── pushing it ─────────────────────────────────────────────────────────── # ── pushing it ───────────────────────────────────────────────────────────
def push(api: Api, flows: list[Flow]) -> None: def _check_schemas(flows: list[Flow]) -> list[str]:
"""Create every flow, then share the two nodes the rest reference. """Hand every node to the engine's own models before the API sees them.
Sharing happens after the flows exist because the library is written from The API would say the same thing, one flow at a time, after creating the
a node that already holds the code — so the first flow to use each carries ones before it. Importing the models is free and answers it here — a
it in, and the others point at it. misspelled setting on a connector is otherwise found halfway through a
house.
""" """
library = {"arbiter": ARBITER, "motor": MOTOR} try:
owners: dict[str, tuple[str, str]] = {} from fluksio.flow.controller import NODE_TYPES
from fluksio.flow.schemas import FlowDef
except ImportError:
return []
problems = []
missing: set[str] = set()
for flow in flows: for flow in flows:
try:
FlowDef.model_validate(
{
"name": flow.name,
"title": flow.title,
"nodes": flow.nodes,
"inputs": flow.inputs,
}
)
except Exception as exc: # noqa: BLE001 - reported, not raised
problems.append(f"{flow.name}: {exc}")
continue
for node in flow.nodes: for node in flow.nodes:
ref = node.get("source_ref") spec = NODE_TYPES.get(node["type"])
if ref in library and ref not in owners: if spec is None:
owners[ref] = (flow.name, node["id"]) # Connectors are installed into the image, not into the venv
# This one carries the code in; the rest reference it. # this runs in, so their absence here says nothing. A type that
node.pop("source_ref") # is neither built in nor a connector this house uses is a
flow.sources[node["id"]] = library[ref] # typo, and that is what is worth saying.
if node["type"] not in CONNECTORS:
problems.append(
f"{flow.name}.{node['id']}: no node type '{node['type']}'"
)
else:
missing.add(node["type"])
continue
if spec.free_params:
continue
params = getattr(spec.cls, "Params", None)
if params is None:
continue
try:
params.model_validate(_without_secrets(node.get("params", {})))
except Exception as exc: # noqa: BLE001 - reported, not raised
problems.append(f"{flow.name}.{node['id']} settings: {exc}")
for flow in flows: if missing:
report(flow.name, len(flow.nodes), api.put_flow(flow)) print(f" (from the image, not checked here: {', '.join(sorted(missing))})")
return problems
for ref, (flow_name, node_id) in owners.items():
api.share(flow_name, node_id, ref)
print(f" library: '{ref}' shared from {flow_name}.{node_id}")
for name, title, glyph, widgets in dashboards.SCREENS: def _without_secrets(params: Any) -> Any:
api.put_dashboard(name, title, glyph, widgets) """A secret reference is an ordinary string by the time a node is built."""
print(f" dashboard '{name}': {len(widgets)} widgets, published") if isinstance(params, dict):
if set(params) == {"$secret"}:
return "from-the-store"
return {k: _without_secrets(v) for k, v in params.items()}
if isinstance(params, list):
return [_without_secrets(v) for v in params]
return params
#: Settings these flows depend on that a node type gained for this port. A node #: Settings these flows depend on that a node type gained for this port. A node
@@ -398,6 +368,9 @@ NEEDED = {
"http": { "http": {
"query": "the weather key would have to sit in the flow file in clear", "query": "the weather key would have to sit in the flow file in clear",
}, },
"artnet": {
"baseline": "the first frame would darken what it is not driving",
},
} }
@@ -416,6 +389,54 @@ def _engine_too_old(api: Api) -> list[tuple[str, str]]:
return stale return stale
LIBRARY = {"arbiter": ARBITER, "motor": MOTOR}
def seed_library(api: Api) -> None:
"""Write the shared sources before anything references them.
A node pointing at a library entry that does not exist yet fails to load,
and a node that failed to load provides nothing — so one missing entry
reports as inputs nothing carries, in every flow downstream of it. The
library is written *from* a node, though, so this is one throwaway flow
that carries the sources in and is deleted again; the entries outlive it.
"""
existing = {node["name"] for node in api("GET", "/flows/library")}
wanted = {name: code for name, code in LIBRARY.items() if name not in existing}
if not wanted:
return
bootstrap = Flow("librarybootstrap", "Library bootstrap")
for name in wanted:
bootstrap.add(
{
"id": name,
"type": "python",
"title": name,
"requires": [{"name": "unused", "dtype": "json"}],
"provides": [{"name": f"{name}_unused", "dtype": "json"}],
},
wanted[name],
)
bootstrap.input("unused", "json", {})
api.put_flow(bootstrap)
for name in wanted:
api.share(bootstrap.name, name, name)
print(f" library: '{name}' written")
api.drop(f"/flows/{bootstrap.name}")
def push(api: Api, flows: list[Flow]) -> None:
"""The library first, then every flow, then the screens."""
seed_library(api)
for flow in flows:
report(flow.name, len(flow.nodes), api.put_flow(flow))
for name, title, glyph, widgets in dashboards.SCREENS:
api.put_dashboard(name, title, glyph, widgets)
print(f" dashboard '{name}': {len(widgets)} widgets, published")
def main(argv: list[str]) -> int: def main(argv: list[str]) -> int:
dry = "--dry" in argv dry = "--dry" in argv
h = house() h = house()