fluksio.arrived(): a node can tell which inputs the wave behind it wrote, so a value written again unchanged is seen
This commit is contained in:
@@ -38,6 +38,7 @@ __all__ = [
|
||||
"FLOWS",
|
||||
"Flow",
|
||||
"Port",
|
||||
"arrived",
|
||||
"build",
|
||||
"emit",
|
||||
"load_artifact",
|
||||
@@ -60,6 +61,15 @@ def emit(**ports: Any) -> None:
|
||||
logger.debug("fluksio.emit(%s) outside a node", ", ".join(ports))
|
||||
|
||||
|
||||
def arrived() -> set[str] | None:
|
||||
"""The input ports the message that started this run wrote.
|
||||
|
||||
Outside a node nothing started it, so this cannot say and returns None,
|
||||
the same answer a node gets when its flow starts.
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
def _local_store() -> Any:
|
||||
"""The worker's own artifact code, pointed at a directory of this machine.
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import asyncio
|
||||
import inspect
|
||||
import logging
|
||||
from collections.abc import Callable, Coroutine, Iterable, Iterator
|
||||
from contextvars import ContextVar
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fluksio.flow import logs
|
||||
@@ -37,6 +38,11 @@ class NodeOutputError(TypeError):
|
||||
#: Settings the engine reads itself rather than handing to the node's function.
|
||||
RESERVED_SETTINGS = frozenset({"synchronous"})
|
||||
|
||||
#: The input ports the wave running a node has written, for the length of that
|
||||
#: run. None when the run cannot say, such as a flow starting or a replay. The
|
||||
#: worker request carries it to ``fluksio.arrived()`` inside the node.
|
||||
ARRIVED: ContextVar[list[str] | None] = ContextVar("arrived", default=None)
|
||||
|
||||
|
||||
class Node:
|
||||
"""
|
||||
|
||||
@@ -44,6 +44,7 @@ from fluksio.flow.namespace import (
|
||||
VERSION,
|
||||
)
|
||||
from fluksio.flow.nodes import Node
|
||||
from fluksio.flow.nodes.base import ARRIVED
|
||||
from fluksio.flow.queue import WorkQueue
|
||||
from fluksio.flow.state import MemoryState, StateBackend
|
||||
|
||||
@@ -1070,6 +1071,7 @@ class Pipeline:
|
||||
entry_id: str = "",
|
||||
overrides: dict[str, Any] | None = None,
|
||||
inputs: dict[str, Any] | None = None,
|
||||
arrived: list[str] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Run one node and record its outputs. Never raises.
|
||||
|
||||
@@ -1108,8 +1110,12 @@ class Pipeline:
|
||||
if hit:
|
||||
return restored
|
||||
|
||||
with logs.capture(collected):
|
||||
result = node.execute(inputs)
|
||||
token = ARRIVED.set(arrived)
|
||||
try:
|
||||
with logs.capture(collected):
|
||||
result = node.execute(inputs)
|
||||
finally:
|
||||
ARRIVED.reset(token)
|
||||
self.publish_log(node, collected, "")
|
||||
|
||||
if (
|
||||
@@ -1483,8 +1489,23 @@ class Pipeline:
|
||||
if replay and entry_id and self._already_done(entry_id, n):
|
||||
complete(n)
|
||||
continue
|
||||
# Which of its inputs this wave wrote, so a node can tell a
|
||||
# value written again from one that only sits in state.
|
||||
arrived = (
|
||||
sorted(
|
||||
spec.port for name, spec in n.requires.items() if name in fresh
|
||||
)
|
||||
if fresh is not None
|
||||
else None
|
||||
)
|
||||
future = executor.submit(
|
||||
self._execute_node, n, state, entry_id, overrides, ready_inputs
|
||||
self._execute_node,
|
||||
n,
|
||||
state,
|
||||
entry_id,
|
||||
overrides,
|
||||
ready_inputs,
|
||||
arrived,
|
||||
)
|
||||
node_futures[future] = n
|
||||
future.add_done_callback(finished.put)
|
||||
|
||||
@@ -31,7 +31,7 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from fluksio.flow.nodes.base import NodeOutputError
|
||||
from fluksio.flow.nodes.base import ARRIVED, NodeOutputError
|
||||
from fluksio.flow.resources import ResourceAccountant
|
||||
from fluksio.flow.workers import NodeTimeout, RemoteError, _remote_class
|
||||
|
||||
@@ -402,6 +402,7 @@ class RemoteWorkerHub:
|
||||
"node": node,
|
||||
"source": source,
|
||||
"kwargs": kwargs,
|
||||
"arrived": ARRIVED.get(),
|
||||
"run": {"id": run_id} if run_id else None,
|
||||
"timeout": timeout,
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ from fluksio_worker import worker_main as _worker_main
|
||||
|
||||
from fluksio.flow.artifacts import is_reference
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.nodes.base import NodeOutputError
|
||||
from fluksio.flow.nodes.base import ARRIVED, NodeOutputError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -625,6 +625,7 @@ class PythonWorkerPool:
|
||||
"node": node,
|
||||
"source": source,
|
||||
"kwargs": kwargs,
|
||||
"arrived": ARRIVED.get(),
|
||||
"run": {"id": run_id} if run_id else None,
|
||||
},
|
||||
timeout=timeout,
|
||||
|
||||
Reference in New Issue
Block a user