Make documentation standalone and generalize examples

backend/Dockerfile

@@ -66,7 +66,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
       $(for e in $(echo "$EXTRAS" | tr , ' '); do printf -- '--extra %s ' "$e"; done)
 
 # Connectors are ordinary installed packages found through the
-# `fluksio.node_types` entry point. `make connectors` builds them into here;
+# `fluksio.node_types` entry point. Place built connector wheels here;
 # an image with none is the normal case.
 COPY ./backend/connector-wheels /tmp/connector-wheels
 RUN --mount=type=cache,target=/root/.cache/uv \

backend/README.md

@@ -1,60 +1,59 @@
 # Fluksio
 
-Fluksio is a node-based automation software that brings trust and reliability to your flow.
-It just works and looks good.
-Get started by running
+Turn Python functions into connected nodes, run experiments, and keep their
+metrics and artifacts together. Start an engine with Python 3.12 or newer
+on Linux or macOS:
 
 ```sh
 pip install fluksio
 fluksio serve
 ```
 
-and you're ready to go.
+The first start prints an admin password and opens a terminal dashboard.
+For a browser dashboard, connect the instance to the optional
+[portal](https://docs.fluksio.com/interface/portal/).
 
 ## For data science
 
-You can turn your existing data science project into a flow by decorating your functions with `@node` ...
+Save this small experiment as `study.py`. No dataset or GPU needed:
 
 ```python
-# myresearch/train.py
-import fluksio
-from fluksio import Port, node
+import math
+
+from fluksio import Flow, Port, node
 
 @node(
-    requires=["dataset", Port("lr", "float")],
-    provides=[Port("loss", "float", stream=True), Port("weights", "artifact")],
-    worker="gpu", worker_policy="prefer",
+    requires=[Port("lr", "float")],
+    provides=[Port("loss", "float", stream=True), Port("final_loss", "float")],
 )
-def fit(dataset, lr, epochs=25):
+def fit(lr, epochs=25):
     for epoch in range(epochs):
-        loss = step(...)
-        yield {"loss": loss}          # published as it happens, kept as a series
-    return {"weights": fluksio.save_artifact("weights.pt")}
+        loss = math.exp(-lr * (epoch + 1))
+        yield {"loss": loss}
+    return {"final_loss": loss}
+
+
+train = Flow(
+    "train",
+    nodes=[fit],
+    inputs=[Port("lr", "float", initial=0.01)],
+    outputs=["final_loss"],
+)
 ```
 
-... and passing them to a `Flow`:
-
-```python
-# myresearch/pipeline.py
-from fluksio import Flow, Port
-from myresearch.data import prepare
-from myresearch.evaluate import evaluate
-from myresearch.train import fit
-
-train = Flow("train", nodes=[prepare, fit, evaluate],
-             inputs=[Port("lr", "float", initial=0.01)], outputs=["score"])
-```
-
-Fluksio will automatically infer the order of nodes based on the inputs and outputs you defined.
-When everything is set, you can launch your first run as follows:
+Matching message names connect the nodes and determine their order. After
+syncing the project, start a run:
 
 ```sh
+fluksio sync study
 fluksio run train --lr 0.05 --wait
 ```
 
-Checkout our [documentation](https://docs.fluksio.com/getting-started/data-science/) for more infos.
+Each yield records a point on the loss curve. Replace the toy calculation
+with your own training code when you're ready. For artifacts and comparisons, try the
+[data science guide](https://docs.fluksio.com/getting-started/data-science/).
 
-## Some other features
+## More to explore
 
 - **Flows**: typed messages between nodes, wired by name, edited on a canvas
   or declared in code. Every change is a commit in a git repository you own.
@@ -65,8 +64,8 @@ Checkout our [documentation](https://docs.fluksio.com/getting-started/data-scien
 - **Remote workers**: `pip install fluksio-worker` on the GPU box; it dials
   *out* over one websocket, so nothing there has to be reachable.
 
-Fluksio can also be used for facility automation.
-Visit us on [fluksio.com](https://fluksio.com) or go straight to our [documentation](https://docs.fluksio.com).
+For sensors, devices and live automation, try the
+[facility automation guide](https://docs.fluksio.com/getting-started/facility-automation/).
 
 ## License
 

backend/connector-wheels/.gitkeep

@@ -1,3 +1,2 @@
-Connector wheels built by `make connectors` at the workspace root land here and
-are installed into the image. Kept empty in git: a connector is an installed
-package, not part of the app.
+Place connector wheels here before building the backend image. They are
+installed into the engine environment. This directory stays empty in git.

backend/fluksio/api/routes/cloud.py

@@ -38,7 +38,7 @@ router = APIRouter(prefix="/cloud", tags=["cloud"])
 
 
 class EnrollBody(BaseModel):
-    #: Where the portal's API lives, e.g. https://hub.fluksio.com
+    #: Where the portal's API lives, e.g. https://portal.example.com
     portal_url: str = Field(min_length=1, max_length=255)
     claim_code: str = Field(min_length=1, max_length=32)
 

backend/fluksio/api/socket_auth.py

@@ -50,7 +50,7 @@ def socket_token(headers: Mapping[str, str], ticket: str = "", token: str = "")
 
     `token` is the query parameter clients sent before the other two existed.
     It is read for one more release so an engine can be upgraded before its
-    workers; remove it after that (see NOTEPAD).
+    workers. New clients should use the Authorization header.
     """
     header = headers.get("authorization") or ""
     if header.lower().startswith("bearer "):

backend/fluksio/flow/executor.py

@@ -205,11 +205,8 @@ class ExecutionService:
     def _sleep_until_due(self) -> bool:
         """Wait for the soonest deadline, the housekeeping cap, or a new one.
 
-        A fixed poll here made every delayed item late by 0-1000ms whatever the
-        load — on a rollershutter driven for a measured 26 seconds, 2-4% of its
-        travel every time, accumulating in the position its node believes it is
-        at. Sleeping to the deadline instead leaves a wake-up and a promotion,
-        which is milliseconds.
+        Sleeping until the deadline avoids adding a fixed polling delay to
+        timed work. New deadlines wake the timer so earlier work is not missed.
         """
         self._timer_wake.clear()
         try:

backend/fluksio/flow/nodes/websocket.py

@@ -150,7 +150,7 @@ class WebSocketNode(Node):
                 "A WebSocket node needs either inputs (to send) or outputs (to listen)"
             )
         # ponytail: one direction per node, as mqtt and http. Inputs and
-        # outputs over one socket is the NOTEPAD follow-up.
+        # outputs over one socket is not supported yet.
         self.mode = (
             WebSocketNode.Mode.SENDER if requires else WebSocketNode.Mode.LISTENER
         )

backend/fluksio/flow/nodes/wren.py

@@ -79,17 +79,12 @@ _NUMBERS = {w: n for n, w in enumerate(_ONES)} | {
 
 
 def _plain(command: str) -> str:
-    """The command as the model was trained on it: lower case, no sentence marks.
-
-    A transcript's "Turn on the kitchen light." reads less sure than the same
-    words plain, 0.80 against 0.99 with every area shown.
-    """
+    """Normalize case and sentence punctuation to match the training format."""
     return re.sub(r"[.,!?;:]+(?=\s|$)", "", command).strip().lower()
 
 
 def _aliased(prompt: str, aliases: dict[str, str]) -> str:
-    """``prompt`` with the house's words in wren's, whole words in any case:
-    "the traverse" as "the living room light", "page too" as "patio"."""
+    """Replace whole-word aliases with target names, ignoring case."""
     if not aliases:
         return prompt
     words = {k.lower(): v for k, v in aliases.items()}
@@ -103,8 +98,7 @@ def _aliased(prompt: str, aliases: dict[str, str]) -> str:
 
 
 def _digits(command: str) -> str:
-    """Number words as digits: "in five minutes" kept its delay only as "in 5
-    minutes", and "twenty two degrees" read as 21."""
+    """Convert number words to digits, as expected by the model."""
     tens, ones = "|".join(_TENS), "|".join(_ONES[1:10])
     command = re.sub(
         rf"\b({tens})[ -]({ones})\b",
@@ -266,8 +260,8 @@ class WrenNode(Node):
         )
         aliases: dict[str, str] = Field(
             default_factory=dict,
-            description="Your own words, and words heard wrong, in the model's: "
-            '{"traverse": "living room light", "page too": "patio"}.',
+            description="Map your own names to the model's target names: "
+            '{"desk lamp": "office light"}.',
         )
         examples: list[dict[str, Any]] = Field(
             default_factory=list,

backend/fluksio/flow/supervision.py

@@ -6,11 +6,8 @@ listens any more. The supervisor restarts those tasks with a growing delay and
 stands a flow down when one keeps failing, because a flow crash-looping every
 second is worse than a flow that is visibly stopped.
 
-Standing down is a rest, not a verdict. It used to be the end of the task: the
-flow stayed dead until somebody published it or the engine restarted. On a
-house that runs unattended that turns a broker rebooting at four in the morning
-into a hole in the record until someone notices — so the rest expires, the
-budget comes back, and the task tries again on a much longer clock.
+Quarantined tasks retry after a longer cooldown with a fresh restart budget.
+This lets unattended flows recover from extended connection failures.
 
 Deliberately a plain task registry rather than a `TaskGroup`: a group cancels
 its siblings when one member fails, which is the opposite of what supervision
@@ -38,8 +35,7 @@ BACKOFF = settings.FLOW_RESTART_BACKOFF
 # Per task rather than per flow: a flow whose subscriber and its keepalive
 # publisher both go down with the same broker spends one shared budget twice as
 # fast as it was sized for, and gives up before the backoff above has reached
-# its longest step. That is what took the house's inverter readings out for
-# nearly five hours — six crashes across two tasks inside 41 seconds.
+# its longest step.
 FAILURE_BUDGET = 5
 FAILURE_WINDOW = 300.0
 # How long a task sits out before its budget is handed back. Minutes rather

backend/fluksio/sdk/stream.py

@@ -21,9 +21,8 @@ from urllib.parse import urlsplit, urlunsplit
 #: the engine's own supervision backs off with.
 BACKOFF = (1.0, 5.0, 30.0, 60.0)
 
-#: How long a read waits before looking at `stop`. The socket is otherwise
-#: silent for as long as the house is, and a dashboard that is closing should
-#: not wait for the next event to notice.
+#: How long a read waits before checking `stop`, so an idle connection does
+#: not delay closing the dashboard.
 POLL_S = 1.0
 
 

backend/scripts/bench_engine.py

@@ -6,11 +6,11 @@ arriving at a subscriber and reaching everything downstream of it. It builds a
 pipeline and an execution service in this process, so there is no HTTP, no
 container and no browser between the measurement and the engine.
 
-Three scenarios, each answering a different question:
+Four scenarios, each answering a different question:
 
 ``throttle``
-    What does a rate-limited port cost? A source publishing every 15s behind
-    seven relays, driven at the rate the house's inverter actually sends.
+    What does a rate-limited port cost? A source publishes through relays
+    while incoming readings exceed its publication rate.
     Reports cascades enqueued, node executions and bus events per injection —
     the numbers, not the wall clock, are the point.
 
@@ -119,7 +119,7 @@ def node(
 def relay(
     node_id: str, src: str, dst: str, interval: float, counters: Counters
 ) -> Node:
-    """A node that republishes what it read — the shape the house is full of.
+    """A relay node that republishes its input.
 
     ``src`` and ``dst`` are bare port names: the pipeline qualifies them with
     the flow, and the node function only ever sees ports.
@@ -238,9 +238,8 @@ def percentile(values: list[float], p: float) -> float:
 def scenario_throttle(args: argparse.Namespace) -> dict[str, Any]:
     """A rate-limited source behind relays, driven far faster than its window.
 
-    The house's shape: an inverter sending ~200 readings a minute into seven
-    relay nodes, each told to publish every 15s. What should cost seven
-    publishes a minute is what this counts.
+    Counts the work performed when rapid inputs are held by a publication
+    interval. Relay count and interval come from the benchmark arguments.
     """
     counters = Counters()
     source = node(
@@ -285,11 +284,8 @@ def scenario_throttle(args: argparse.Namespace) -> dict[str, Any]:
 def scenario_relay(args: argparse.Namespace) -> dict[str, Any]:
     """An *unlimited* source into limited relays, and one node reading them all.
 
-    The other half of the house's shape, and the one holding it at the source
-    cannot help with: the readings really do arrive, so a cascade is owed —
-    but the relays hold their outputs back, and what reads them has nothing
-    new to read. That node published 619 messages a minute off inputs that
-    changed six times.
+    Each input starts a cascade, but rate-limited relay outputs should wake
+    the consumer only when they publish a new value.
     """
     counters = Counters()
     source = node("source", lambda params: None, provides=[spec("raw")])

backend/scripts/bench_startup.py

@@ -1,13 +1,7 @@
-"""How long it takes to get a run started, which is the thing Kedro is slow at.
+"""Measure submission and completion latency against a running engine.
 
-A pipeline framework that boots the project per run pays that cost every time:
-``kedro run`` on a pipeline that does nothing takes about a second, and a sweep
-of five hundred configs therefore spends ten minutes doing nothing. Fluksio
-answers that by not booting anything — the engine is already up and its workers
-already have the code loaded, so submitting is one request.
-
-This measures that claim against a running stack, so it can be checked rather
-than asserted. Run it with the dev stack up::
+Uses a no-op flow to isolate scheduling overhead. Optionally compare it with
+an empty Kedro pipeline on the same machine. Run with the dev stack up::
 
     make bench-startup
     make bench-startup BENCH_ARGS="--runs 50 --kedro ../path/to/kedro/project"

backend/scripts/soak.py

@@ -12,9 +12,8 @@ the work queue — and then takes pieces of the stack away to see what survives.
 The redis scenario, which `all` includes, needs `--force`: the Redis it stops
 is the one the whole stack journals to, not a copy of its own.
 
-Every docker verb goes through :func:`docker`, which refuses any container that
-is not part of this project's compose stack. The machine this runs on hosts
-unrelated services; nothing here may ever reach them.
+Every Docker command goes through :func:`docker`, which limits operations to
+containers in this project's Compose stack.
 
 The engine's semantic is latest-value-wins with cascades running concurrently,
 so counter equality is not a promise and is not asserted. What is asserted is

backend/tests/api/routes/test_camera_access.py

@@ -1,6 +1,6 @@
 """Who may see a device's camera: superusers only, or every member too.
 
-openpanel ROADMAP C4. ``viewers`` on a device covers its ``image`` and
+``viewers`` on a device covers its ``image`` and
 ``video`` messages, their bytes, frames on the socket and its live view. A
 device credential is bounded by its dashboards instead.
 """

backend/tests/api/routes/test_cameras.py

@@ -1,6 +1,6 @@
 """Cameras on a device: frames over its socket, demand, and what a screen fetches.
 
-openpanel ROADMAP C1. A device reports a ``camera`` port that sends only while
+A device reports a ``camera`` port that sends only while
 somebody wants it. Demand is what the viewer sockets want plus what running
 flows read, mapped onto the device's on-demand ports and sent to its socket.
 """

backend/tests/api/routes/test_live.py

@@ -1,6 +1,6 @@
 """Live view: WebRTC signalling relayed between a viewer's socket and a device's.
 
-openpanel ROADMAP L1. The server routes by session, holds a device to two
+The server routes by session, holds a device to two
 viewers, ends a session when either socket goes or no answer comes, and hands
 out TURN credentials in coturn's REST scheme.
 """

backend/tests/flow/test_house_vocabulary.py

@@ -1,10 +1,4 @@
-"""What the house port needed from the built-in nodes.
-
-Three additions, each with a device behind it: an MQTT filter that actually
-routes what it subscribed to, a value lifted out of the object a device wraps
-it in, and a wait that differs per message because a shutter takes longer to
-come up than to go down.
-"""
+"""Device messaging with MQTT filters, wrapped values and per-message delays."""
 
 from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.nodes import MqttNode

backend/tests/flow/test_pipeline.py

@@ -211,11 +211,8 @@ def test_only_advisory_codes_are_flagged_advisory():
 def test_a_producer_that_cannot_run_does_not_strand_its_consumer():
     """One wave, two producers of one consumer, and one of them never ready.
 
-    The house's shape: a battery reading wakes the boiler arbiter and the
-    watering-pump trigger in the same wave, and the DMX encoder reads both. The
-    pump waits on a pulse that only exists at 02:00, so it is never ready — and
-    passing over it used to leave the encoder unreachable, which put every
-    boiler and plug command into its message and no further.
+    The ready producer must reach its consumer even when a sibling is
+    waiting for an input. The consumer can use that sibling's stored value.
     """
     ran = []
 
@@ -268,10 +265,8 @@ def test_a_producer_that_cannot_run_does_not_strand_its_consumer():
 def test_a_quiet_producer_does_not_veto_a_noisy_sibling():
     """Two report-by-exception producers, one consumer, and only one with news.
 
-    The house reads both boilers into one DMX encoder through `rbe` nodes. The
-    kitchen boiler is unchanged almost always, so it returns nothing — and
-    while that held the encoder's dependency count up, the main boiler's
-    command reached its message and got no further.
+    A producer that publishes nothing must not block a shared consumer
+    from processing its sibling's new value.
     """
     ran = []
 

backend/tests/flow/test_service_node.py

@@ -50,11 +50,11 @@ def test_a_service_comes_back_and_its_url_follows(monkeypatch):
     async def scenario() -> None:
         agent = Agent()
         agent.worker = RemoteWorker(
-            name="amcomp",
+            name="gpu-worker",
             labels=["gpu"],
             send=agent.send_json,
             loop=asyncio.get_running_loop(),
-            host="10.0.0.5",
+            host="192.0.2.5",
         )
         hub = RemoteWorkerHub()
         node = ServiceNode(
@@ -78,7 +78,7 @@ def test_a_service_comes_back_and_its_url_follows(monkeypatch):
 
         hub.attach(agent.worker)
         await until(lambda: bool(agent.serves()))
-        await until(lambda: url() == "http://10.0.0.5:8110")
+        await until(lambda: url() == "http://192.0.2.5:8110")
         first = agent.serves()[0]
         assert first["argv"] == ["serve", "--port", "8110"]
         assert health[-1] == "ok"
@@ -88,7 +88,7 @@ def test_a_service_comes_back_and_its_url_follows(monkeypatch):
         await until(lambda: url() == "")
         assert "degraded" in health[health.index("ok") :]
         await until(lambda: len(agent.serves()) == 2)
-        await until(lambda: url() == "http://10.0.0.5:8110")
+        await until(lambda: url() == "http://192.0.2.5:8110")
 
         # Stopping the flow ends the call on the worker, and empties the url.
         await node.stop()
@@ -110,7 +110,7 @@ def test_a_service_that_never_gets_healthy_reads_down(monkeypatch):
     async def scenario() -> None:
         agent = Agent()
         agent.worker = RemoteWorker(
-            name="amcomp",
+            name="gpu-worker",
             labels=["gpu"],
             send=agent.send_json,
             loop=asyncio.get_running_loop(),

backend/tests/flow/test_supervision.py

@@ -116,11 +116,7 @@ def test_one_flows_failure_does_not_stand_another_down():
 
 
 def test_two_tasks_on_one_flow_do_not_spend_each_others_budget():
-    """The house's power flow: an MQTT subscriber and its keepalive publisher.
-
-    Both die with the same broker. Sharing one budget meant the flow was stood
-    down after half the restarts either task was allowed on its own.
-    """
+    """A subscriber and publisher each get a full restart budget."""
     tries = {"sub": 0, "pub": 0}
 
     def crasher(which: str):