fluksio.arrived(): a node can tell which inputs the wave behind it wrote, so a value written again unchanged is seen

This commit is contained in:
2026-11-05 21:24:21 +01:00
parent b7f1d21712
commit 7ddf8aa8f9
9 changed files with 127 additions and 7 deletions
+10
View File
@@ -38,6 +38,7 @@ __all__ = [
"FLOWS",
"Flow",
"Port",
"arrived",
"build",
"emit",
"load_artifact",
@@ -60,6 +61,15 @@ def emit(**ports: Any) -> None:
logger.debug("fluksio.emit(%s) outside a node", ", ".join(ports))
def arrived() -> set[str] | None:
"""The input ports the message that started this run wrote.
Outside a node nothing started it, so this cannot say and returns None,
the same answer a node gets when its flow starts.
"""
return None
def _local_store() -> Any:
"""The worker's own artifact code, pointed at a directory of this machine.
+6
View File
@@ -10,6 +10,7 @@ import asyncio
import inspect
import logging
from collections.abc import Callable, Coroutine, Iterable, Iterator
from contextvars import ContextVar
from typing import TYPE_CHECKING, Any
from fluksio.flow import logs
@@ -37,6 +38,11 @@ class NodeOutputError(TypeError):
#: Settings the engine reads itself rather than handing to the node's function.
RESERVED_SETTINGS = frozenset({"synchronous"})
#: The input ports the wave running a node has written, for the length of that
#: run. None when the run cannot say, such as a flow starting or a replay. The
#: worker request carries it to ``fluksio.arrived()`` inside the node.
ARRIVED: ContextVar[list[str] | None] = ContextVar("arrived", default=None)
class Node:
"""
+24 -3
View File
@@ -44,6 +44,7 @@ from fluksio.flow.namespace import (
VERSION,
)
from fluksio.flow.nodes import Node
from fluksio.flow.nodes.base import ARRIVED
from fluksio.flow.queue import WorkQueue
from fluksio.flow.state import MemoryState, StateBackend
@@ -1070,6 +1071,7 @@ class Pipeline:
entry_id: str = "",
overrides: dict[str, Any] | None = None,
inputs: dict[str, Any] | None = None,
arrived: list[str] | None = None,
) -> dict[str, Any] | None:
"""Run one node and record its outputs. Never raises.
@@ -1108,8 +1110,12 @@ class Pipeline:
if hit:
return restored
with logs.capture(collected):
result = node.execute(inputs)
token = ARRIVED.set(arrived)
try:
with logs.capture(collected):
result = node.execute(inputs)
finally:
ARRIVED.reset(token)
self.publish_log(node, collected, "")
if (
@@ -1483,8 +1489,23 @@ class Pipeline:
if replay and entry_id and self._already_done(entry_id, n):
complete(n)
continue
# Which of its inputs this wave wrote, so a node can tell a
# value written again from one that only sits in state.
arrived = (
sorted(
spec.port for name, spec in n.requires.items() if name in fresh
)
if fresh is not None
else None
)
future = executor.submit(
self._execute_node, n, state, entry_id, overrides, ready_inputs
self._execute_node,
n,
state,
entry_id,
overrides,
ready_inputs,
arrived,
)
node_futures[future] = n
future.add_done_callback(finished.put)
+2 -1
View File
@@ -31,7 +31,7 @@ from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from fluksio.flow.nodes.base import NodeOutputError
from fluksio.flow.nodes.base import ARRIVED, NodeOutputError
from fluksio.flow.resources import ResourceAccountant
from fluksio.flow.workers import NodeTimeout, RemoteError, _remote_class
@@ -402,6 +402,7 @@ class RemoteWorkerHub:
"node": node,
"source": source,
"kwargs": kwargs,
"arrived": ARRIVED.get(),
"run": {"id": run_id} if run_id else None,
"timeout": timeout,
}
+2 -1
View File
@@ -33,7 +33,7 @@ from fluksio_worker import worker_main as _worker_main
from fluksio.flow.artifacts import is_reference
from fluksio.flow.events import EventBus
from fluksio.flow.nodes.base import NodeOutputError
from fluksio.flow.nodes.base import ARRIVED, NodeOutputError
logger = logging.getLogger(__name__)
@@ -625,6 +625,7 @@ class PythonWorkerPool:
"node": node,
"source": source,
"kwargs": kwargs,
"arrived": ARRIVED.get(),
"run": {"id": run_id} if run_id else None,
},
timeout=timeout,