A node's numbers leave through its ports, not a logging call
The first cut had node code call fluksio.log_metric, which was a second, undeclared way for data to leave a node: invisible to validation, absent from the canvas, and stored where the graph could not see it. That is precisely the MLflow discrepancy this framework exists to avoid, so it is gone. A node that produces values over time is a generator. Every yield is a dict keyed by output port, published the instant it happens — same port, same type check, same place on the canvas as any other value — and what it returns is its result. A port doing this declares stream: true, and a run keeps every number one takes, so experiment tracking is a consequence of the graph rather than an API beside it: a chart binds to a training curve the way it binds to a temperature. fluksio.emit writes the same ports imperatively, for where a yield cannot reach — inside a training framework's callback. In a live flow an emission also wakes what is downstream, as a subscriber publishing does; in a run it does not, because a run's graph is scheduled once and mid-node cascades would leave 'finished' with nothing to mean. The enqueued item carries no payload: the value is already in state, and one carrying it would re-apply an old emission after the node returned. Verified on the stack: 30 loss values arrived live on the flow socket during a run, attributed to the node that produced them, and the same node run on the remote worker streamed its curve back across the socket. Also caches remote compile results per worker, so attaching a GPU box does not put a network round trip in every rebuild. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AD8SfVhzXBG2nAfFcVh3iD
This commit is contained in:
Claude Fable 5
@@ -99,11 +99,31 @@ class RunContext:
|
||||
A run builds nodes of its own, so which run a node's worker call belongs to
|
||||
is bound when the proxy is made rather than looked up while it runs. That
|
||||
is what lets one run's node be cancelled without touching the same node in
|
||||
another, and what gives a metric reported mid-training somewhere to go.
|
||||
another.
|
||||
"""
|
||||
|
||||
run_id: str
|
||||
on_event: Callable[[dict[str, Any]], None] | None = None
|
||||
|
||||
|
||||
class EmitSink:
|
||||
"""Turns a worker's mid-call frames back into the node's own outputs.
|
||||
|
||||
The proxy has to exist before the node that runs it does, so this stands
|
||||
between them: built empty, handed to the proxy, and pointed at the node as
|
||||
soon as there is one. What arrives is a dict keyed by output port, which is
|
||||
the same thing a return value is — so it goes through the node, gets
|
||||
checked against the ports it declared, and is published from there.
|
||||
"""
|
||||
|
||||
__slots__ = ("node",)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.node: Node | None = None
|
||||
|
||||
def handle(self, event: dict[str, Any]) -> None:
|
||||
if self.node is None or event.get("event") != "emit":
|
||||
return
|
||||
self.node.emit(event.get("outputs") or {})
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -468,6 +488,9 @@ class FlowController:
|
||||
owner, local = flow, node_def.id
|
||||
code = self.store.read_node_source(flow, node_def.id, draft=draft)
|
||||
|
||||
# What a node produces before it returns comes back as frames;
|
||||
# this puts them through the node's own ports.
|
||||
emissions = EmitSink()
|
||||
if self.workers is None:
|
||||
function = load_function(owner, local, code)
|
||||
else:
|
||||
@@ -500,7 +523,7 @@ class FlowController:
|
||||
node_id=node_id,
|
||||
timeout=timeout,
|
||||
run_id=run.run_id if run else "",
|
||||
on_event=run.on_event if run else None,
|
||||
on_event=emissions.handle,
|
||||
)
|
||||
if node_def.device and self.remote is not None:
|
||||
# A node with a device runs on a worker carrying that
|
||||
@@ -515,7 +538,7 @@ class FlowController:
|
||||
node_id=node_id,
|
||||
timeout=timeout,
|
||||
run_id=run.run_id if run else "",
|
||||
on_event=run.on_event if run else None,
|
||||
on_event=emissions.handle,
|
||||
fallback=(
|
||||
function if node_def.device_policy == "prefer" else None
|
||||
),
|
||||
@@ -527,6 +550,7 @@ class FlowController:
|
||||
params=params,
|
||||
name=node_def.id,
|
||||
)
|
||||
emissions.node = node
|
||||
else:
|
||||
node = node_type.cls(
|
||||
requires=_bound(node_def.requires),
|
||||
@@ -916,6 +940,7 @@ class FlowController:
|
||||
state: StateBackend,
|
||||
draft: bool = False,
|
||||
observer: Callable[[NodeOutcome], None] | None = None,
|
||||
emission_observer: Callable[[str, dict[str, Any]], None] | None = None,
|
||||
run: RunContext | None = None,
|
||||
) -> Pipeline:
|
||||
"""Build one flow as a pipeline of its own, for a single run.
|
||||
@@ -936,6 +961,7 @@ class FlowController:
|
||||
max_workers=self.max_workers,
|
||||
initial_values=initial_values,
|
||||
observer=observer,
|
||||
emission_observer=emission_observer,
|
||||
)
|
||||
pipeline.history_limits = self.history_limits
|
||||
return pipeline
|
||||
|
||||
@@ -152,6 +152,14 @@ class MessageSpec(BaseModel):
|
||||
off is read when the node runs for some other reason, but never causes
|
||||
a run and never makes the node wait — which is how a node reads a
|
||||
message it also produces without depending on itself.
|
||||
:param stream: On an output, that this port produces repeatedly *during* one
|
||||
execution rather than once at the end — a training loss, a progress
|
||||
fraction. A node emits on it by being a generator and yielding, or by
|
||||
calling ``fluksio.emit``. What it means downstream is nothing special:
|
||||
a value published mid-execution is a value like any other. What it
|
||||
means to a run is that the whole series is kept, which is how a run's
|
||||
metrics are simply its streaming outputs rather than something logged
|
||||
beside them.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
@@ -162,6 +170,7 @@ class MessageSpec(BaseModel):
|
||||
item: DType | None = None
|
||||
interval: float = Field(default=0, ge=0)
|
||||
trigger: bool = True
|
||||
stream: bool = False
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _default_port(self) -> MessageSpec:
|
||||
|
||||
@@ -7,8 +7,9 @@ modules beside this one add what talking to a particular outside world means.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import logging
|
||||
from collections.abc import Callable, Coroutine, Iterable
|
||||
from collections.abc import Callable, Coroutine, Iterable, Iterator
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias
|
||||
|
||||
from app.flow import logs
|
||||
@@ -301,7 +302,42 @@ class Node:
|
||||
Downstream nodes are not triggered — the pipeline schedules those.
|
||||
"""
|
||||
kwargs = self._to_kwargs(inputs or {})
|
||||
return self._to_messages(self.f(**kwargs, params=self.params))
|
||||
result = self.f(**kwargs, params=self.params)
|
||||
if inspect.isgenerator(result):
|
||||
# Only when the node runs in this process; out of process the
|
||||
# worker has already drained it and sent each yield on ahead.
|
||||
result = self._drain(result)
|
||||
return self._to_messages(result)
|
||||
|
||||
def _drain(self, generator: Iterator[Any]) -> Any:
|
||||
"""Publish each yield as it happens; the end of it is the result."""
|
||||
pending: Any = None
|
||||
have_pending = False
|
||||
try:
|
||||
while True:
|
||||
value = next(generator)
|
||||
if have_pending:
|
||||
self.emit(pending)
|
||||
pending, have_pending = value, True
|
||||
except StopIteration as stop:
|
||||
if stop.value is not None:
|
||||
if have_pending:
|
||||
self.emit(pending)
|
||||
return stop.value
|
||||
return pending if have_pending else None
|
||||
|
||||
def emit(self, values: dict[str, Any] | None) -> None:
|
||||
"""Publish on this node's output ports mid-execution.
|
||||
|
||||
A value produced while a node is still working is a value like any
|
||||
other: same ports, same type checking, same place on the canvas. What
|
||||
it is *not* is a log — nothing leaves a node except through a port it
|
||||
declared, so a training curve is an output of the graph rather than a
|
||||
side effect beside it.
|
||||
"""
|
||||
outputs = self._to_messages(values)
|
||||
if outputs and self._pipeline is not None:
|
||||
self._pipeline.publish_emission(self, outputs)
|
||||
|
||||
def trigger(
|
||||
self, inputs: dict[str, Any] | None = None, durable: bool | None = None
|
||||
|
||||
@@ -117,6 +117,7 @@ class Pipeline:
|
||||
"_node_pool",
|
||||
"history_limits",
|
||||
"observer",
|
||||
"emission_observer",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -130,6 +131,7 @@ class Pipeline:
|
||||
work_queue: WorkQueue | None = None,
|
||||
node_pool: ThreadPoolExecutor | None = None,
|
||||
observer: Callable[[NodeOutcome], None] | None = None,
|
||||
emission_observer: Callable[[str, dict[str, Any]], None] | None = None,
|
||||
) -> None:
|
||||
self._nodes = nodes or []
|
||||
# Stopped flows are stored and survive a restart; paused ones are a
|
||||
@@ -151,6 +153,9 @@ class Pipeline:
|
||||
self._node_pool = node_pool
|
||||
# Set by a run, which needs every node it executed written down.
|
||||
self.observer = observer
|
||||
# And every value a node produced on the way, which is what a
|
||||
# training curve is once it goes out a port rather than into a log.
|
||||
self.emission_observer = emission_observer
|
||||
# How deep to keep each message's series; a chart asking for more
|
||||
# than the default puts its message in here. Swapped, never mutated.
|
||||
self.history_limits: dict[str, int] = {}
|
||||
@@ -667,27 +672,7 @@ class Pipeline:
|
||||
result = self._throttled(node, result)
|
||||
|
||||
if result:
|
||||
ts = time.time()
|
||||
with state.lock():
|
||||
state.update(result)
|
||||
state.update(
|
||||
{self._timestamp_key(name): ts for name in result},
|
||||
)
|
||||
# Append-only, so it needs no lock of its own.
|
||||
state.append_history(result, ts, self.history_limits)
|
||||
self._increment_message_versions(result)
|
||||
origin = node_source(node)
|
||||
for name, value in result.items():
|
||||
self._publish(
|
||||
{
|
||||
"type": "message_value",
|
||||
"flow": flow_of(name),
|
||||
"name": name,
|
||||
"value": value,
|
||||
"ts": ts,
|
||||
"source": origin.model_dump(),
|
||||
}
|
||||
)
|
||||
self._record_outputs(node, result, state)
|
||||
|
||||
duration_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
self._publish(
|
||||
@@ -732,6 +717,64 @@ class Pipeline:
|
||||
)
|
||||
return None
|
||||
|
||||
def _record_outputs(
|
||||
self, node: Node, outputs: dict[str, Any], state: StateBackend
|
||||
) -> None:
|
||||
"""Put a node's outputs where everything downstream of them looks.
|
||||
|
||||
State, the timestamp beside it, the series, the version counter and the
|
||||
event the canvas draws from. Shared by a node returning and a node
|
||||
emitting mid-execution, because those are the same act: a value the
|
||||
node produced, leaving through a port it declared.
|
||||
"""
|
||||
ts = time.time()
|
||||
with state.lock():
|
||||
state.update(outputs)
|
||||
state.update({self._timestamp_key(name): ts for name in outputs})
|
||||
# Append-only, so it needs no lock of its own.
|
||||
state.append_history(outputs, ts, self.history_limits)
|
||||
self._increment_message_versions(outputs)
|
||||
origin = node_source(node)
|
||||
for name, value in outputs.items():
|
||||
self._publish(
|
||||
{
|
||||
"type": "message_value",
|
||||
"flow": flow_of(name),
|
||||
"name": name,
|
||||
"value": value,
|
||||
"ts": ts,
|
||||
"source": origin.model_dump(),
|
||||
}
|
||||
)
|
||||
|
||||
def publish_emission(self, node: Node, outputs: dict[str, Any]) -> None:
|
||||
"""Publish what a node produced while it is still running.
|
||||
|
||||
Recorded before it is throttled, and throttled before it is published:
|
||||
the run's history is the whole series, and a port declaring an interval
|
||||
is asking for the *canvas* not to be flooded, not for its curve to have
|
||||
holes in it.
|
||||
|
||||
Where there is a work queue — the live engine — an emission also wakes
|
||||
what is downstream of it, exactly as a subscriber publishing does. A
|
||||
run has no queue, and deliberately: its graph is scheduled once, and
|
||||
three thousand mid-node cascades would leave "the run has finished"
|
||||
with no meaning.
|
||||
"""
|
||||
self._observe_emission(node, outputs)
|
||||
passed = self._throttled(node, outputs)
|
||||
if not passed:
|
||||
return
|
||||
self._record_outputs(node, passed, self._state)
|
||||
if self._queue is not None:
|
||||
# Journalled with no payload of its own: the value is already in
|
||||
# state, published in the order it was produced. An item carrying
|
||||
# it would re-apply that value whenever it happened to be claimed,
|
||||
# which is how an emission from the middle of a node overwrites the
|
||||
# one it returned at the end. Downstream reads what is current,
|
||||
# which is what "the latest value wins" has always meant here.
|
||||
self._enqueue_cascade(node, None)
|
||||
|
||||
def _observe(self, outcome: NodeOutcome) -> None:
|
||||
"""Tell the run watching this pipeline, if there is one."""
|
||||
if self.observer is None:
|
||||
@@ -741,6 +784,14 @@ class Pipeline:
|
||||
except Exception:
|
||||
logger.exception("Run observer failed for '%s'", outcome.node)
|
||||
|
||||
def _observe_emission(self, node: Node, outputs: dict[str, Any]) -> None:
|
||||
if self.emission_observer is None:
|
||||
return
|
||||
try:
|
||||
self.emission_observer(node.id, outputs)
|
||||
except Exception:
|
||||
logger.exception("Run observer failed for an emission of '%s'", node.id)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Running, stopped, paused
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@@ -21,6 +21,7 @@ blocks on a queue of its own until the loop puts the answer there.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import queue
|
||||
import sys
|
||||
@@ -71,6 +72,12 @@ class RemoteWorker:
|
||||
self._pending: dict[str, queue.Queue[dict[str, Any] | None]] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._gone = False
|
||||
# Source this worker has already loaded, by digest. A rebuild asks
|
||||
# every device-bound node whether it compiles, and without this that
|
||||
# is a network round trip each time — which is how attaching a GPU box
|
||||
# makes rebuilding the flows slow enough to fail a health check. A
|
||||
# worker that reconnects is a new object, so this empties with it.
|
||||
self._compiled: set[str] = set()
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# From the socket's side, on the event loop
|
||||
@@ -150,6 +157,11 @@ class RemoteWorker:
|
||||
self._pending.pop(call_id, None)
|
||||
self._slots.release()
|
||||
|
||||
@property
|
||||
def compiled(self) -> set[str]:
|
||||
"""Digests of the source this worker has already loaded."""
|
||||
return self._compiled
|
||||
|
||||
@property
|
||||
def gone(self) -> bool:
|
||||
return self._gone
|
||||
@@ -292,6 +304,9 @@ class RemoteWorkerHub:
|
||||
worker = self.pick(label)
|
||||
if worker is None:
|
||||
return None
|
||||
digest = hashlib.md5(f"{flow}.{node}:{source}".encode()).hexdigest()
|
||||
if digest in worker.compiled:
|
||||
return None
|
||||
try:
|
||||
response = worker.request(
|
||||
{
|
||||
@@ -306,6 +321,7 @@ class RemoteWorkerHub:
|
||||
except RemoteError as exc:
|
||||
return f"{type(exc).__name__}: {exc}"
|
||||
if response.get("ok"):
|
||||
worker.compiled.add(digest)
|
||||
return None
|
||||
error = response.get("error") or {}
|
||||
return str(error.get("short") or "The node could not be loaded.")
|
||||
|
||||
+47
-59
@@ -73,8 +73,6 @@ LOG_CAP = 8000
|
||||
#: every step must not be a round trip every step.
|
||||
METRIC_BATCH = 500
|
||||
METRIC_FLUSH_S = 2.0
|
||||
#: Progress is for whoever is watching, so it is throttled hard.
|
||||
PROGRESS_INTERVAL_S = 1.0
|
||||
#: How often a run waiting for a worker looks again.
|
||||
WAIT_RETRY_S = 15.0
|
||||
|
||||
@@ -102,20 +100,25 @@ def digest_of(params: dict[str, Any], seed: int | None) -> str:
|
||||
def batch_issues(flow: FlowDef) -> list[str]:
|
||||
"""Why this flow cannot be run as a batch, if it cannot.
|
||||
|
||||
Only one thing genuinely breaks: a port with a discretization interval
|
||||
holds values back for a timer to release, and a run has no timer — the
|
||||
engine would drop them instead. A delay node is fine; without a queue to
|
||||
defer into it simply sleeps, which in a run is what was asked for.
|
||||
One thing genuinely breaks: a port with a discretization interval holds
|
||||
values back for a timer to release, and a run has no timer — the engine
|
||||
would drop them instead of delaying them. On a *streaming* port that is
|
||||
exactly right and is how you keep a chart from being flooded: the run's
|
||||
history keeps every value, and the interval only thins what is published.
|
||||
Anywhere else it is a message quietly going missing.
|
||||
|
||||
A delay node is fine; without a queue to defer into it simply sleeps,
|
||||
which in a run is what was asked for.
|
||||
"""
|
||||
issues: list[str] = []
|
||||
for node in flow.nodes:
|
||||
for spec in list(node.requires) + list(node.provides):
|
||||
if spec.interval > 0:
|
||||
if spec.interval > 0 and not spec.stream:
|
||||
issues.append(
|
||||
f"Node '{node.id}' rate-limits '{spec.port or spec.name}'. "
|
||||
"A run has no timer to release what that holds back, so "
|
||||
"the value would be dropped. Remove the interval to run "
|
||||
"this flow as a batch."
|
||||
"the value would be dropped. Remove the interval, or mark "
|
||||
"the port as streaming if it is a curve being thinned out."
|
||||
)
|
||||
return issues
|
||||
|
||||
@@ -165,81 +168,65 @@ def collect_result(flow: FlowDef, state: StateBackend) -> dict[str, Any]:
|
||||
|
||||
|
||||
class MetricSink:
|
||||
"""Collects what a run's nodes report, and writes it down in batches.
|
||||
"""Keeps the series a run's streaming outputs traced out.
|
||||
|
||||
Every call arrives on the thread of the node that made it, so this is
|
||||
locked. It is written synchronously rather than published: three thousand
|
||||
steps of a training curve is exactly the traffic the event bus is built to
|
||||
drop, and a curve with holes in it is not a result.
|
||||
A run's metrics are not logged; they are the numbers its nodes published on
|
||||
the way to finishing. This watches the emissions, keeps the numeric ones,
|
||||
and writes them in batches — synchronously rather than over the event bus,
|
||||
which drops what it cannot keep up with, and a training curve with holes in
|
||||
it is not a result.
|
||||
|
||||
The step is the count of emissions on that message. A node that publishes
|
||||
every tenth training step therefore has steps 0, 1, 2 rather than 0, 10,
|
||||
20 — a faithful x-axis of its own emissions, not of the loop inside it.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
run_id: str,
|
||||
publish: Callable[[dict[str, Any]], None] | None = None,
|
||||
batch: int = METRIC_BATCH,
|
||||
interval: float = METRIC_FLUSH_S,
|
||||
) -> None:
|
||||
self.run_id = run_id
|
||||
self._publish = publish
|
||||
self._batch = batch
|
||||
self._interval = interval
|
||||
self._rows: dict[tuple[str, int], RunMetric] = {}
|
||||
self._steps: dict[str, int] = {}
|
||||
self._last_flush = time.monotonic()
|
||||
self._last_progress = 0.0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def handle(self, event: dict[str, Any]) -> None:
|
||||
kind = event.get("event")
|
||||
if kind == "metric":
|
||||
self._metric(event)
|
||||
elif kind == "progress":
|
||||
self._progress(event)
|
||||
|
||||
def _metric(self, event: dict[str, Any]) -> None:
|
||||
node = str(event.get("call_id") or "").split(":", 1)[-1]
|
||||
row = RunMetric(
|
||||
run_id=self.run_id,
|
||||
name=str(event.get("name") or "")[:128],
|
||||
step=int(event.get("step", -1)),
|
||||
node=node[:255],
|
||||
ts=float(event.get("ts") or time.time()),
|
||||
value=float(event.get("value") or 0.0),
|
||||
)
|
||||
def handle(self, node_id: str, outputs: dict[str, Any]) -> None:
|
||||
"""One emission: every number in it belongs to this run's history."""
|
||||
now = time.time()
|
||||
rows: list[RunMetric] = []
|
||||
with self._lock:
|
||||
# Same name and step twice is the later value; the primary key says
|
||||
# so too, and colliding here is cheaper than colliding in Postgres.
|
||||
self._rows[(row.name, row.step)] = row
|
||||
for name, value in outputs.items():
|
||||
if not isinstance(value, (int, float)) or isinstance(value, bool):
|
||||
# A checkpoint or a record is on the run some other way —
|
||||
# as an artifact, or as its result. Only numbers are series.
|
||||
continue
|
||||
step = self._steps.get(name, -1) + 1
|
||||
self._steps[name] = step
|
||||
row = RunMetric(
|
||||
run_id=self.run_id,
|
||||
name=name[:128],
|
||||
step=step,
|
||||
node=node_id[:255],
|
||||
ts=now,
|
||||
value=float(value),
|
||||
)
|
||||
self._rows[(row.name, row.step)] = row
|
||||
due = (
|
||||
len(self._rows) >= self._batch
|
||||
or time.monotonic() - self._last_flush >= self._interval
|
||||
)
|
||||
rows = list(self._rows.values()) if due else []
|
||||
if due:
|
||||
rows = list(self._rows.values())
|
||||
self._rows.clear()
|
||||
self._last_flush = time.monotonic()
|
||||
if rows:
|
||||
self._write(rows)
|
||||
|
||||
def _progress(self, event: dict[str, Any]) -> None:
|
||||
"""Purely for whoever is watching: throttled, and never written down."""
|
||||
if self._publish is None:
|
||||
return
|
||||
now = time.monotonic()
|
||||
if now - self._last_progress < PROGRESS_INTERVAL_S:
|
||||
return
|
||||
self._last_progress = now
|
||||
self._publish(
|
||||
{
|
||||
"type": "run_progress",
|
||||
"run": self.run_id,
|
||||
"node": str(event.get("call_id") or "").split(":", 1)[-1],
|
||||
"fraction": event.get("fraction"),
|
||||
"message": event.get("message") or "",
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
|
||||
def flush(self) -> None:
|
||||
with self._lock:
|
||||
rows = list(self._rows.values())
|
||||
@@ -577,7 +564,7 @@ class RunService:
|
||||
errors += 1
|
||||
self._record_node(run_id, outcome)
|
||||
|
||||
sink = MetricSink(run_id, publish=self._publish_event)
|
||||
sink = MetricSink(run_id)
|
||||
try:
|
||||
flow = self.controller.store.read_flow(run.flow)
|
||||
state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}")
|
||||
@@ -585,7 +572,8 @@ class RunService:
|
||||
flow,
|
||||
state=state,
|
||||
observer=observe,
|
||||
run=RunContext(run_id=run_id, on_event=sink.handle),
|
||||
emission_observer=sink.handle,
|
||||
run=RunContext(run_id=run_id),
|
||||
)
|
||||
with self._lock:
|
||||
self._active[run_id] = pipeline
|
||||
|
||||
@@ -16,7 +16,15 @@ an ``event`` key is a report rather than the answer, and the engine keeps
|
||||
reading. That is what makes a training curve visible during the hours it takes
|
||||
to draw, and what tells the engine a long node is alive rather than hung —
|
||||
each event resets its deadline, so the timeout measures silence rather than
|
||||
duration. Node code reaches it by importing ``fluksio``.
|
||||
duration.
|
||||
|
||||
What travels that way is not a log. A node that produces values over time is a
|
||||
generator: every ``yield`` is a dict keyed by output port, published the moment
|
||||
it happens, and whatever the generator returns at the end is the node's result.
|
||||
Nothing leaves a node except through a port it declared, which is the whole
|
||||
point — a number worth keeping is an output, not a side effect. Where a yield
|
||||
cannot reach, because the value comes from inside somebody else's callback,
|
||||
``fluksio.emit(loss=0.3)`` writes the same ports the same way.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -34,6 +42,7 @@ sys.path[:] = [p for p in sys.path if os.path.abspath(p or ".") != _HERE]
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import inspect
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
@@ -74,38 +83,18 @@ def _emit(event: dict[str, Any]) -> None:
|
||||
|
||||
|
||||
class _Reporter(ModuleType):
|
||||
"""``import fluksio`` — what node code says while it is still running.
|
||||
"""``import fluksio`` — the parts of a node's job that need the engine."""
|
||||
|
||||
Deliberately tiny and deliberately not a return value: a training loop has
|
||||
numbers worth keeping thousands of steps before it has a result, and
|
||||
holding them until it returns is how they get lost when it does not.
|
||||
"""
|
||||
def emit(self, **ports: Any) -> None:
|
||||
"""Publish on this node's output ports without returning yet.
|
||||
|
||||
def log_metric(self, name: str, value: float, step: int = -1) -> None:
|
||||
"""Record one number, optionally at a step. Steps make a curve."""
|
||||
_emit(
|
||||
{
|
||||
"event": "metric",
|
||||
"name": str(name)[:128],
|
||||
"value": float(value),
|
||||
"step": int(step),
|
||||
}
|
||||
)
|
||||
|
||||
def log_metrics(self, values: dict[str, float], step: int = -1) -> None:
|
||||
"""Several at once, which is what a training loop usually has."""
|
||||
for name, value in values.items():
|
||||
self.log_metric(name, value, step)
|
||||
|
||||
def progress(self, fraction: float | None = None, message: str = "") -> None:
|
||||
"""How far along this node is, for whoever is watching it."""
|
||||
_emit(
|
||||
{
|
||||
"event": "progress",
|
||||
"fraction": None if fraction is None else float(fraction),
|
||||
"message": str(message)[:200],
|
||||
}
|
||||
)
|
||||
Yielding is the better way to say this and should be preferred; use
|
||||
this where a yield cannot reach — inside a training framework's
|
||||
callback, say, which calls you rather than the other way round. It is
|
||||
the same publication either way: the values go to the ports the node
|
||||
declared, and are checked against them.
|
||||
"""
|
||||
_emit({"event": "emit", "outputs": dict(ports)})
|
||||
|
||||
def save_artifact(
|
||||
self,
|
||||
@@ -316,6 +305,8 @@ def _handle(request: dict[str, Any], cache: dict[tuple[str, str], Any]) -> Any:
|
||||
result = function(
|
||||
**(request.get("kwargs") or {}), params=request.get("params") or {}
|
||||
)
|
||||
if inspect.isgenerator(result):
|
||||
result = _drain(result)
|
||||
try:
|
||||
json.dumps(result)
|
||||
except (TypeError, ValueError):
|
||||
@@ -328,6 +319,34 @@ def _handle(request: dict[str, Any], cache: dict[tuple[str, str], Any]) -> Any:
|
||||
return result
|
||||
|
||||
|
||||
def _drain(generator: Any) -> Any:
|
||||
"""Run a generator node, publishing each yield as it happens.
|
||||
|
||||
Two shapes work, and they mean the same thing. Yield throughout and
|
||||
``return`` the result at the end, which is the explicit one; or just yield,
|
||||
and the last one is the result. Either way what the node *produces over
|
||||
time* leaves through its ports while it is still running, and what it
|
||||
*ends up with* is its return value.
|
||||
"""
|
||||
pending: Any = None
|
||||
have_pending = False
|
||||
try:
|
||||
while True:
|
||||
value = next(generator)
|
||||
# Held one behind: until the next yield arrives this might be the
|
||||
# last one, and the last one is the result rather than an emission.
|
||||
if have_pending:
|
||||
_emit({"event": "emit", "outputs": pending})
|
||||
pending, have_pending = value, True
|
||||
except StopIteration as stop:
|
||||
if stop.value is not None:
|
||||
# It returned something, so every yield was an emission.
|
||||
if have_pending:
|
||||
_emit({"event": "emit", "outputs": pending})
|
||||
return stop.value
|
||||
return pending if have_pending else None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
global _RPC, _CALL_ID
|
||||
|
||||
|
||||
@@ -10,8 +10,9 @@ import pytest
|
||||
|
||||
from app.flow.messages import DType, MessageSpec
|
||||
from app.flow.nodes import Node
|
||||
from app.flow.pipeline import Pipeline
|
||||
from app.flow.pipeline import NodeOutcome, Pipeline
|
||||
from app.flow.runs import (
|
||||
MetricSink,
|
||||
RunRejected,
|
||||
batch_issues,
|
||||
collect_result,
|
||||
@@ -132,6 +133,137 @@ def test_a_failing_observer_does_not_take_the_node_down():
|
||||
assert pipeline.state["study.loss"] == 1.0
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Producing values before returning
|
||||
#
|
||||
# A number worth keeping is an output, not a log. A node that produces over
|
||||
# time is a generator, and each yield is published on the port it names — so
|
||||
# the run's metrics are its streaming outputs rather than something recorded
|
||||
# beside them.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def training_flow() -> FlowDef:
|
||||
return FlowDef(
|
||||
name="study",
|
||||
mode="batch",
|
||||
inputs=[FlowInput(spec=spec("steps", DType.INT), initial=3)],
|
||||
outputs=["final_loss"],
|
||||
nodes=[
|
||||
NodeDef(
|
||||
id="train",
|
||||
requires=[spec("steps", DType.INT)],
|
||||
provides=[
|
||||
spec("loss", stream=True),
|
||||
spec("final_loss"),
|
||||
],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def training_node(flow: str = "study") -> Node:
|
||||
def train(steps, params):
|
||||
loss = 1.0
|
||||
for _ in range(steps):
|
||||
loss = loss / 2
|
||||
yield {"loss": loss}
|
||||
return {"final_loss": loss}
|
||||
|
||||
return make_node(
|
||||
"train",
|
||||
flow,
|
||||
train,
|
||||
requires=[spec("steps", DType.INT)],
|
||||
provides=[spec("loss", stream=True), spec("final_loss")],
|
||||
)
|
||||
|
||||
|
||||
def test_each_yield_is_published_and_the_return_is_the_result():
|
||||
flow = training_flow()
|
||||
state = MemoryState()
|
||||
seen: list[tuple[str, dict]] = []
|
||||
pipeline = Pipeline(
|
||||
nodes=[training_node()],
|
||||
state=state,
|
||||
emission_observer=lambda node, outputs: seen.append((node, outputs)),
|
||||
)
|
||||
|
||||
pipeline.run(seed_values(flow, {"steps": 3}))
|
||||
|
||||
# It returned something, so every yield was a value produced on the way —
|
||||
# each published on `loss` the moment it happened.
|
||||
assert [outputs["study.loss"] for _node, outputs in seen] == [0.5, 0.25, 0.125]
|
||||
assert all(node == "study.train" for node, _ in seen)
|
||||
# The latest of them is what the message holds, as for any producer.
|
||||
assert state["study.loss"] == 0.125
|
||||
# And what it returned is the node's output, and so the run's result.
|
||||
assert collect_result(flow, state) == {"final_loss": 0.125}
|
||||
|
||||
|
||||
def test_without_a_return_the_last_yield_is_the_result():
|
||||
def train(params):
|
||||
yield {"loss": 1.0}
|
||||
yield {"loss": 0.5}
|
||||
|
||||
node = make_node("train", "study", train, provides=[spec("loss", stream=True)])
|
||||
seen: list[tuple[str, dict]] = []
|
||||
pipeline = Pipeline(
|
||||
nodes=[node],
|
||||
state=MemoryState(),
|
||||
emission_observer=lambda n, o: seen.append((n, o)),
|
||||
)
|
||||
pipeline.run()
|
||||
|
||||
# The last yield is the node's output rather than an emission, so it is
|
||||
# not counted twice.
|
||||
assert [outputs["study.loss"] for _node, outputs in seen] == [1.0]
|
||||
assert pipeline.state["study.loss"] == 0.5
|
||||
|
||||
|
||||
def test_emissions_are_checked_against_the_port_they_name():
|
||||
def wrong(params):
|
||||
yield {"loss": "not a number"}
|
||||
return {"final_loss": 1.0}
|
||||
|
||||
node = make_node(
|
||||
"train",
|
||||
"study",
|
||||
wrong,
|
||||
provides=[spec("loss", stream=True), spec("final_loss")],
|
||||
)
|
||||
seen: list[NodeOutcome] = []
|
||||
Pipeline(nodes=[node], state=MemoryState(), observer=seen.append).run()
|
||||
|
||||
# A wrong type is a failed node, exactly as it is for a return value —
|
||||
# which is the point of emissions going out through declared ports.
|
||||
assert not seen[0].ok
|
||||
assert "loss" in seen[0].error
|
||||
|
||||
|
||||
def test_an_emission_that_nothing_declares_is_ignored():
|
||||
def stray(params):
|
||||
yield {"undeclared": 1.0}
|
||||
return {"final_loss": 2.0}
|
||||
|
||||
node = make_node("train", "study", stray, provides=[spec("final_loss")])
|
||||
state = MemoryState()
|
||||
Pipeline(nodes=[node], state=state).run()
|
||||
|
||||
assert "study.undeclared" not in state
|
||||
assert state["study.final_loss"] == 2.0
|
||||
|
||||
|
||||
def test_emissions_reach_the_run_as_a_series_with_a_step_each():
|
||||
sink = MetricSink("run-1", batch=1)
|
||||
sink.handle("study.train", {"study.loss": 1.0, "study.tag": "ignored"})
|
||||
sink.handle("study.train", {"study.loss": 0.5})
|
||||
|
||||
# Numbers become the run's series; anything else is on the run some other
|
||||
# way — as its result, or as an artifact.
|
||||
assert sink._steps == {"study.loss": 1}
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# What a caller may ask for
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -158,6 +290,14 @@ def test_a_rate_limited_port_cannot_be_run_as_a_batch():
|
||||
assert not batch_issues(double_flow())
|
||||
|
||||
|
||||
def test_a_streaming_port_may_thin_itself_out():
|
||||
flow = double_flow()
|
||||
flow.nodes[0].provides = [spec("loss", interval=0.5, stream=True)]
|
||||
# On a curve, an interval is asking for the canvas not to be flooded —
|
||||
# the run's history still keeps every value.
|
||||
assert not batch_issues(flow)
|
||||
|
||||
|
||||
def test_the_digest_identifies_the_inputs_not_their_order():
|
||||
assert digest_of({"a": 1, "b": 2}, 3) == digest_of({"b": 2, "a": 1}, 3)
|
||||
assert digest_of({"a": 1}, 3) != digest_of({"a": 1}, 4)
|
||||
|
||||
@@ -181,21 +181,19 @@ def test_a_pool_can_stop_while_a_node_is_running(pool):
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Reporting from inside a node that has not returned yet
|
||||
# Producing values before returning: a generator node
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_node_reports_metrics_while_it_is_still_running(pool):
|
||||
def test_a_generator_node_publishes_each_yield_and_returns_the_end(pool):
|
||||
seen = []
|
||||
result = pool.run(
|
||||
"demo",
|
||||
"train",
|
||||
"import fluksio\n"
|
||||
"def process(params):\n"
|
||||
" for step in range(3):\n"
|
||||
" fluksio.log_metric('loss', 1.0 / (step + 1), step)\n"
|
||||
" fluksio.progress(0.5, 'halfway')\n"
|
||||
" return {'out': 1}\n",
|
||||
" yield {'loss': 1.0 / (step + 1)}\n"
|
||||
" return {'weights': 'w', 'final_loss': 0.25}\n",
|
||||
{},
|
||||
{},
|
||||
"demo.train",
|
||||
@@ -204,32 +202,80 @@ def test_a_node_reports_metrics_while_it_is_still_running(pool):
|
||||
on_event=seen.append,
|
||||
)
|
||||
|
||||
assert result == {"out": 1}
|
||||
metrics = [event for event in seen if event["event"] == "metric"]
|
||||
assert [(m["name"], m["step"]) for m in metrics] == [
|
||||
("loss", 0),
|
||||
("loss", 1),
|
||||
("loss", 2),
|
||||
]
|
||||
assert metrics[0]["value"] == 1.0
|
||||
# Every event says which call it belongs to, so a sweep can tell them apart.
|
||||
assert {m["call_id"] for m in metrics} == {"r1:demo.train"}
|
||||
assert [event["event"] for event in seen if event["event"] == "progress"] == [
|
||||
"progress"
|
||||
# What it returned is the node's output; what it yielded went out as it
|
||||
# happened, on the same ports.
|
||||
assert result == {"weights": "w", "final_loss": 0.25}
|
||||
assert [event["outputs"] for event in seen] == [
|
||||
{"loss": 1.0},
|
||||
{"loss": 0.5},
|
||||
{"loss": 1 / 3},
|
||||
]
|
||||
# Every frame says which call it belongs to, so a sweep can tell them apart.
|
||||
assert {event["call_id"] for event in seen} == {"r1:demo.train"}
|
||||
|
||||
|
||||
def test_without_a_return_the_last_yield_is_the_result(pool):
|
||||
seen = []
|
||||
result = pool.run(
|
||||
"demo",
|
||||
"count",
|
||||
"def process(params):\n"
|
||||
" yield {'out': 1}\n"
|
||||
" yield {'out': 2}\n"
|
||||
" yield {'out': 3}\n",
|
||||
{},
|
||||
{},
|
||||
"demo.count",
|
||||
timeout=5,
|
||||
on_event=seen.append,
|
||||
)
|
||||
|
||||
assert result == {"out": 3}
|
||||
assert [event["outputs"] for event in seen] == [{"out": 1}, {"out": 2}]
|
||||
|
||||
|
||||
def test_emit_reaches_the_same_ports_from_inside_a_callback(pool):
|
||||
# A value produced somewhere a yield cannot reach — inside a framework's
|
||||
# callback — is still an output rather than a log.
|
||||
seen = []
|
||||
result = pool.run(
|
||||
"demo",
|
||||
"fit",
|
||||
"import fluksio\n"
|
||||
"def process(params):\n"
|
||||
" def on_epoch(n):\n"
|
||||
" fluksio.emit(loss=1.0 / (n + 1))\n"
|
||||
" for epoch in range(2):\n"
|
||||
" on_epoch(epoch)\n"
|
||||
" return {'done': True}\n",
|
||||
{},
|
||||
{},
|
||||
"demo.fit",
|
||||
timeout=5,
|
||||
on_event=seen.append,
|
||||
)
|
||||
|
||||
assert result == {"done": True}
|
||||
assert [event["outputs"] for event in seen] == [{"loss": 1.0}, {"loss": 0.5}]
|
||||
|
||||
|
||||
def test_a_plain_function_still_just_returns(pool):
|
||||
seen = []
|
||||
assert run(pool, "def process(params):\n return {'out': 7}\n") == {"out": 7}
|
||||
assert seen == []
|
||||
|
||||
|
||||
def test_events_hold_off_the_timeout_but_silence_does_not(pool):
|
||||
# The deadline measures silence: a node reporting every 0.05s stays alive
|
||||
# The deadline measures silence: a node yielding every 0.05s stays alive
|
||||
# well past a 0.3s timeout, which is what a two-hour training needs.
|
||||
result = pool.run(
|
||||
"demo",
|
||||
"slow",
|
||||
"import time, fluksio\n"
|
||||
"import time\n"
|
||||
"def process(params):\n"
|
||||
" for step in range(12):\n"
|
||||
" time.sleep(0.05)\n"
|
||||
" fluksio.log_metric('beat', step, step)\n"
|
||||
" yield {'beat': step}\n"
|
||||
" return {'done': True}\n",
|
||||
{},
|
||||
{},
|
||||
|
||||
Reference in New Issue
Block a user