A node's numbers leave through its ports, not a logging call
The first cut had node code call fluksio.log_metric, which was a second, undeclared way for data to leave a node: invisible to validation, absent from the canvas, and stored where the graph could not see it. That is precisely the MLflow discrepancy this framework exists to avoid, so it is gone. A node that produces values over time is a generator. Every yield is a dict keyed by output port, published the instant it happens — same port, same type check, same place on the canvas as any other value — and what it returns is its result. A port doing this declares stream: true, and a run keeps every number one takes, so experiment tracking is a consequence of the graph rather than an API beside it: a chart binds to a training curve the way it binds to a temperature. fluksio.emit writes the same ports imperatively, for where a yield cannot reach — inside a training framework's callback. In a live flow an emission also wakes what is downstream, as a subscriber publishing does; in a run it does not, because a run's graph is scheduled once and mid-node cascades would leave 'finished' with nothing to mean. The enqueued item carries no payload: the value is already in state, and one carrying it would re-apply an old emission after the node returned. Verified on the stack: 30 loss values arrived live on the flow socket during a run, attributed to the node that produced them, and the same node run on the remote worker streamed its curve back across the socket. Also caches remote compile results per worker, so attaching a GPU box does not put a network round trip in every rebuild. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AD8SfVhzXBG2nAfFcVh3iD
This commit is contained in:
Claude Fable 5
@@ -10,8 +10,9 @@ import pytest
|
||||
|
||||
from app.flow.messages import DType, MessageSpec
|
||||
from app.flow.nodes import Node
|
||||
from app.flow.pipeline import Pipeline
|
||||
from app.flow.pipeline import NodeOutcome, Pipeline
|
||||
from app.flow.runs import (
|
||||
MetricSink,
|
||||
RunRejected,
|
||||
batch_issues,
|
||||
collect_result,
|
||||
@@ -132,6 +133,137 @@ def test_a_failing_observer_does_not_take_the_node_down():
|
||||
assert pipeline.state["study.loss"] == 1.0
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Producing values before returning
|
||||
#
|
||||
# A number worth keeping is an output, not a log. A node that produces over
|
||||
# time is a generator, and each yield is published on the port it names — so
|
||||
# the run's metrics are its streaming outputs rather than something recorded
|
||||
# beside them.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def training_flow() -> FlowDef:
|
||||
return FlowDef(
|
||||
name="study",
|
||||
mode="batch",
|
||||
inputs=[FlowInput(spec=spec("steps", DType.INT), initial=3)],
|
||||
outputs=["final_loss"],
|
||||
nodes=[
|
||||
NodeDef(
|
||||
id="train",
|
||||
requires=[spec("steps", DType.INT)],
|
||||
provides=[
|
||||
spec("loss", stream=True),
|
||||
spec("final_loss"),
|
||||
],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def training_node(flow: str = "study") -> Node:
|
||||
def train(steps, params):
|
||||
loss = 1.0
|
||||
for _ in range(steps):
|
||||
loss = loss / 2
|
||||
yield {"loss": loss}
|
||||
return {"final_loss": loss}
|
||||
|
||||
return make_node(
|
||||
"train",
|
||||
flow,
|
||||
train,
|
||||
requires=[spec("steps", DType.INT)],
|
||||
provides=[spec("loss", stream=True), spec("final_loss")],
|
||||
)
|
||||
|
||||
|
||||
def test_each_yield_is_published_and_the_return_is_the_result():
|
||||
flow = training_flow()
|
||||
state = MemoryState()
|
||||
seen: list[tuple[str, dict]] = []
|
||||
pipeline = Pipeline(
|
||||
nodes=[training_node()],
|
||||
state=state,
|
||||
emission_observer=lambda node, outputs: seen.append((node, outputs)),
|
||||
)
|
||||
|
||||
pipeline.run(seed_values(flow, {"steps": 3}))
|
||||
|
||||
# It returned something, so every yield was a value produced on the way —
|
||||
# each published on `loss` the moment it happened.
|
||||
assert [outputs["study.loss"] for _node, outputs in seen] == [0.5, 0.25, 0.125]
|
||||
assert all(node == "study.train" for node, _ in seen)
|
||||
# The latest of them is what the message holds, as for any producer.
|
||||
assert state["study.loss"] == 0.125
|
||||
# And what it returned is the node's output, and so the run's result.
|
||||
assert collect_result(flow, state) == {"final_loss": 0.125}
|
||||
|
||||
|
||||
def test_without_a_return_the_last_yield_is_the_result():
|
||||
def train(params):
|
||||
yield {"loss": 1.0}
|
||||
yield {"loss": 0.5}
|
||||
|
||||
node = make_node("train", "study", train, provides=[spec("loss", stream=True)])
|
||||
seen: list[tuple[str, dict]] = []
|
||||
pipeline = Pipeline(
|
||||
nodes=[node],
|
||||
state=MemoryState(),
|
||||
emission_observer=lambda n, o: seen.append((n, o)),
|
||||
)
|
||||
pipeline.run()
|
||||
|
||||
# The last yield is the node's output rather than an emission, so it is
|
||||
# not counted twice.
|
||||
assert [outputs["study.loss"] for _node, outputs in seen] == [1.0]
|
||||
assert pipeline.state["study.loss"] == 0.5
|
||||
|
||||
|
||||
def test_emissions_are_checked_against_the_port_they_name():
|
||||
def wrong(params):
|
||||
yield {"loss": "not a number"}
|
||||
return {"final_loss": 1.0}
|
||||
|
||||
node = make_node(
|
||||
"train",
|
||||
"study",
|
||||
wrong,
|
||||
provides=[spec("loss", stream=True), spec("final_loss")],
|
||||
)
|
||||
seen: list[NodeOutcome] = []
|
||||
Pipeline(nodes=[node], state=MemoryState(), observer=seen.append).run()
|
||||
|
||||
# A wrong type is a failed node, exactly as it is for a return value —
|
||||
# which is the point of emissions going out through declared ports.
|
||||
assert not seen[0].ok
|
||||
assert "loss" in seen[0].error
|
||||
|
||||
|
||||
def test_an_emission_that_nothing_declares_is_ignored():
|
||||
def stray(params):
|
||||
yield {"undeclared": 1.0}
|
||||
return {"final_loss": 2.0}
|
||||
|
||||
node = make_node("train", "study", stray, provides=[spec("final_loss")])
|
||||
state = MemoryState()
|
||||
Pipeline(nodes=[node], state=state).run()
|
||||
|
||||
assert "study.undeclared" not in state
|
||||
assert state["study.final_loss"] == 2.0
|
||||
|
||||
|
||||
def test_emissions_reach_the_run_as_a_series_with_a_step_each():
|
||||
sink = MetricSink("run-1", batch=1)
|
||||
sink.handle("study.train", {"study.loss": 1.0, "study.tag": "ignored"})
|
||||
sink.handle("study.train", {"study.loss": 0.5})
|
||||
|
||||
# Numbers become the run's series; anything else is on the run some other
|
||||
# way — as its result, or as an artifact.
|
||||
assert sink._steps == {"study.loss": 1}
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# What a caller may ask for
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -158,6 +290,14 @@ def test_a_rate_limited_port_cannot_be_run_as_a_batch():
|
||||
assert not batch_issues(double_flow())
|
||||
|
||||
|
||||
def test_a_streaming_port_may_thin_itself_out():
|
||||
flow = double_flow()
|
||||
flow.nodes[0].provides = [spec("loss", interval=0.5, stream=True)]
|
||||
# On a curve, an interval is asking for the canvas not to be flooded —
|
||||
# the run's history still keeps every value.
|
||||
assert not batch_issues(flow)
|
||||
|
||||
|
||||
def test_the_digest_identifies_the_inputs_not_their_order():
|
||||
assert digest_of({"a": 1, "b": 2}, 3) == digest_of({"b": 2, "a": 1}, 3)
|
||||
assert digest_of({"a": 1}, 3) != digest_of({"a": 1}, 4)
|
||||
|
||||
@@ -181,21 +181,19 @@ def test_a_pool_can_stop_while_a_node_is_running(pool):
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Reporting from inside a node that has not returned yet
|
||||
# Producing values before returning: a generator node
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_node_reports_metrics_while_it_is_still_running(pool):
|
||||
def test_a_generator_node_publishes_each_yield_and_returns_the_end(pool):
|
||||
seen = []
|
||||
result = pool.run(
|
||||
"demo",
|
||||
"train",
|
||||
"import fluksio\n"
|
||||
"def process(params):\n"
|
||||
" for step in range(3):\n"
|
||||
" fluksio.log_metric('loss', 1.0 / (step + 1), step)\n"
|
||||
" fluksio.progress(0.5, 'halfway')\n"
|
||||
" return {'out': 1}\n",
|
||||
" yield {'loss': 1.0 / (step + 1)}\n"
|
||||
" return {'weights': 'w', 'final_loss': 0.25}\n",
|
||||
{},
|
||||
{},
|
||||
"demo.train",
|
||||
@@ -204,32 +202,80 @@ def test_a_node_reports_metrics_while_it_is_still_running(pool):
|
||||
on_event=seen.append,
|
||||
)
|
||||
|
||||
assert result == {"out": 1}
|
||||
metrics = [event for event in seen if event["event"] == "metric"]
|
||||
assert [(m["name"], m["step"]) for m in metrics] == [
|
||||
("loss", 0),
|
||||
("loss", 1),
|
||||
("loss", 2),
|
||||
]
|
||||
assert metrics[0]["value"] == 1.0
|
||||
# Every event says which call it belongs to, so a sweep can tell them apart.
|
||||
assert {m["call_id"] for m in metrics} == {"r1:demo.train"}
|
||||
assert [event["event"] for event in seen if event["event"] == "progress"] == [
|
||||
"progress"
|
||||
# What it returned is the node's output; what it yielded went out as it
|
||||
# happened, on the same ports.
|
||||
assert result == {"weights": "w", "final_loss": 0.25}
|
||||
assert [event["outputs"] for event in seen] == [
|
||||
{"loss": 1.0},
|
||||
{"loss": 0.5},
|
||||
{"loss": 1 / 3},
|
||||
]
|
||||
# Every frame says which call it belongs to, so a sweep can tell them apart.
|
||||
assert {event["call_id"] for event in seen} == {"r1:demo.train"}
|
||||
|
||||
|
||||
def test_without_a_return_the_last_yield_is_the_result(pool):
|
||||
seen = []
|
||||
result = pool.run(
|
||||
"demo",
|
||||
"count",
|
||||
"def process(params):\n"
|
||||
" yield {'out': 1}\n"
|
||||
" yield {'out': 2}\n"
|
||||
" yield {'out': 3}\n",
|
||||
{},
|
||||
{},
|
||||
"demo.count",
|
||||
timeout=5,
|
||||
on_event=seen.append,
|
||||
)
|
||||
|
||||
assert result == {"out": 3}
|
||||
assert [event["outputs"] for event in seen] == [{"out": 1}, {"out": 2}]
|
||||
|
||||
|
||||
def test_emit_reaches_the_same_ports_from_inside_a_callback(pool):
|
||||
# A value produced somewhere a yield cannot reach — inside a framework's
|
||||
# callback — is still an output rather than a log.
|
||||
seen = []
|
||||
result = pool.run(
|
||||
"demo",
|
||||
"fit",
|
||||
"import fluksio\n"
|
||||
"def process(params):\n"
|
||||
" def on_epoch(n):\n"
|
||||
" fluksio.emit(loss=1.0 / (n + 1))\n"
|
||||
" for epoch in range(2):\n"
|
||||
" on_epoch(epoch)\n"
|
||||
" return {'done': True}\n",
|
||||
{},
|
||||
{},
|
||||
"demo.fit",
|
||||
timeout=5,
|
||||
on_event=seen.append,
|
||||
)
|
||||
|
||||
assert result == {"done": True}
|
||||
assert [event["outputs"] for event in seen] == [{"loss": 1.0}, {"loss": 0.5}]
|
||||
|
||||
|
||||
def test_a_plain_function_still_just_returns(pool):
|
||||
seen = []
|
||||
assert run(pool, "def process(params):\n return {'out': 7}\n") == {"out": 7}
|
||||
assert seen == []
|
||||
|
||||
|
||||
def test_events_hold_off_the_timeout_but_silence_does_not(pool):
|
||||
# The deadline measures silence: a node reporting every 0.05s stays alive
|
||||
# The deadline measures silence: a node yielding every 0.05s stays alive
|
||||
# well past a 0.3s timeout, which is what a two-hour training needs.
|
||||
result = pool.run(
|
||||
"demo",
|
||||
"slow",
|
||||
"import time, fluksio\n"
|
||||
"import time\n"
|
||||
"def process(params):\n"
|
||||
" for step in range(12):\n"
|
||||
" time.sleep(0.05)\n"
|
||||
" fluksio.log_metric('beat', step, step)\n"
|
||||
" yield {'beat': step}\n"
|
||||
" return {'done': True}\n",
|
||||
{},
|
||||
{},
|
||||
|
||||
Reference in New Issue
Block a user