Node settings arrive as keyword arguments, not a params dict
A python node's settings are constants of its own function, so they are passed the way its ports are: by name. The controller binds them to the compiled function, the `params` field is gone from the worker and remote protocols, and a setting sharing a port's name is reported as a node error rather than shadowing it. The panel's scaffold follows suit and keeps the header in step with both ports and settings. The demo's `pace` moves from a flow input to a setting of the training node, which is what it always was. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NUb8YpL2s3gmN9WTACTt4q
This commit is contained in:
@@ -25,6 +25,7 @@ from app.flow.events import EventBus
|
||||
from app.flow.executor import ExecutionService
|
||||
from app.flow.messages import MessageSpec, flow_of, qualify
|
||||
from app.flow.nodes import (
|
||||
RESERVED_SETTINGS,
|
||||
ChangeNode,
|
||||
DelayNode,
|
||||
ExecNode,
|
||||
@@ -479,6 +480,17 @@ class FlowController:
|
||||
params = resolve_params(node_def.params)
|
||||
|
||||
if node_type.has_source:
|
||||
# Settings and ports are both keyword arguments of the same
|
||||
# function, so one name cannot mean both.
|
||||
ports = {spec.port for spec in _bound(node_def.requires)}
|
||||
clash = sorted((set(params) - RESERVED_SETTINGS) & ports)
|
||||
if clash:
|
||||
raise ValueError(
|
||||
f"'{clash[0]}' is both an input and a setting of "
|
||||
f"'{node_def.id}'. A setting is an argument like a "
|
||||
"port, so rename one of them."
|
||||
)
|
||||
|
||||
# A shared node runs the library's copy, compiled once under
|
||||
# the library's own name so every flow using it agrees.
|
||||
if node_def.source_ref:
|
||||
@@ -544,7 +556,7 @@ class FlowController:
|
||||
),
|
||||
)
|
||||
node = Node(
|
||||
f=function,
|
||||
f=with_settings(function, params),
|
||||
requires=_bound(node_def.requires),
|
||||
provides=_bound(node_def.provides),
|
||||
params=params,
|
||||
@@ -1019,6 +1031,27 @@ def _bound(specs: list[MessageSpec]) -> list[MessageSpec]:
|
||||
return [spec for spec in specs if spec.name]
|
||||
|
||||
|
||||
def with_settings(
|
||||
function: Callable[..., Any], params: dict[str, Any]
|
||||
) -> Callable[..., Any]:
|
||||
"""A node's function with its settings bound as keyword arguments.
|
||||
|
||||
A setting is a constant of one node's function, so it is passed the way a
|
||||
port is: by name. The engine's own settings never reach the code, and the
|
||||
``params`` the pipeline offers is dropped here rather than travelling to a
|
||||
worker that has nothing to do with it.
|
||||
"""
|
||||
settings = {k: v for k, v in params.items() if k not in RESERVED_SETTINGS}
|
||||
|
||||
def call(
|
||||
params: dict[str, Any] | None = None, # noqa: ARG001 - absorbed here
|
||||
**ports: Any,
|
||||
) -> Any:
|
||||
return function(**ports, **settings)
|
||||
|
||||
return call
|
||||
|
||||
|
||||
def _collect_issues(
|
||||
loaded: dict[str, LoadedNode],
|
||||
pipeline: Pipeline,
|
||||
|
||||
@@ -4,7 +4,7 @@ Split by the outside world each one talks to. Importing from
|
||||
``app.flow.nodes`` keeps working, which is what every caller does.
|
||||
"""
|
||||
|
||||
from app.flow.nodes.base import Node
|
||||
from app.flow.nodes.base import RESERVED_SETTINGS, Node
|
||||
from app.flow.nodes.delay import DelayNode
|
||||
from app.flow.nodes.exec import ExecNode
|
||||
from app.flow.nodes.file import FileNode
|
||||
@@ -30,6 +30,7 @@ __all__ = [
|
||||
"MqttNode",
|
||||
"Node",
|
||||
"NtfyNode",
|
||||
"RESERVED_SETTINGS",
|
||||
"RbeNode",
|
||||
"SwitchNode",
|
||||
"TriggerNode",
|
||||
|
||||
@@ -33,6 +33,10 @@ class NodeOutputError(TypeError):
|
||||
"""A node function returned something that cannot be mapped onto ports."""
|
||||
|
||||
|
||||
#: Settings the engine reads itself rather than handing to the node's function.
|
||||
RESERVED_SETTINGS = frozenset({"synchronous"})
|
||||
|
||||
|
||||
class Node:
|
||||
"""
|
||||
A pipeline node that wraps a function with typed inputs/outputs.
|
||||
@@ -50,7 +54,9 @@ class Node:
|
||||
:param provides: Output messages this node produces. Can be a single Message
|
||||
or list of Messages.
|
||||
:type provides: MessageSpec | list[MessageSpec]
|
||||
:param params: Additional parameters passed to the function during execution.
|
||||
:param params: This node's settings — constants of its function, stored
|
||||
with the flow. A function node reads them as keyword arguments beside
|
||||
its ports; a built-in type validates them against its own ``Params``.
|
||||
:type params: dict
|
||||
:param name: Optional name for the node. Defaults to function name.
|
||||
:type name: str | None
|
||||
@@ -60,14 +66,14 @@ class Node:
|
||||
:vartype synchronous: bool
|
||||
|
||||
:example:
|
||||
>>> def process_temp(temperature, params):
|
||||
... return {"celsius": temperature * 0.5 + 32}
|
||||
>>> def process_temp(temperature, offset):
|
||||
... return {"celsius": temperature * 0.5 + offset}
|
||||
>>>
|
||||
>>> temp_node = Node(
|
||||
... f=process_temp,
|
||||
... requires=MessageSpec(name="temperature", dtype=DType.FLOAT),
|
||||
... provides=MessageSpec(name="celsius", dtype=DType.FLOAT),
|
||||
... params={},
|
||||
... params={"offset": 32},
|
||||
... )
|
||||
"""
|
||||
|
||||
|
||||
@@ -255,7 +255,6 @@ class RemoteWorkerHub:
|
||||
node: str,
|
||||
source: str,
|
||||
kwargs: dict[str, Any],
|
||||
params: dict[str, Any] | None,
|
||||
node_id: str,
|
||||
timeout: float,
|
||||
run_id: str = "",
|
||||
@@ -272,7 +271,6 @@ class RemoteWorkerHub:
|
||||
"node": node,
|
||||
"source": source,
|
||||
"kwargs": kwargs,
|
||||
"params": params or {},
|
||||
"run": {"id": run_id} if run_id else None,
|
||||
"timeout": timeout,
|
||||
},
|
||||
@@ -344,16 +342,15 @@ class RemoteWorkerHub:
|
||||
the label rather than requiring it.
|
||||
"""
|
||||
|
||||
def call(params: dict[str, Any] | None = None, **kwargs: Any) -> Any:
|
||||
def call(**kwargs: Any) -> Any:
|
||||
if fallback is not None and self.pick(label) is None:
|
||||
return fallback(params=params, **kwargs)
|
||||
return fallback(**kwargs)
|
||||
return self.run(
|
||||
label,
|
||||
flow,
|
||||
node,
|
||||
source,
|
||||
kwargs,
|
||||
params,
|
||||
node_id,
|
||||
timeout,
|
||||
run_id=run_id,
|
||||
|
||||
@@ -35,6 +35,9 @@ class NodeDef(BaseModel):
|
||||
id: str
|
||||
type: str = "python"
|
||||
title: str = ""
|
||||
#: This node's settings: constants of its function, stored with the flow.
|
||||
#: A function node reads them as keyword arguments beside its ports, so a
|
||||
#: setting cannot share a name with one.
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
requires: list[MessageSpec] = Field(default_factory=list)
|
||||
provides: list[MessageSpec] = Field(default_factory=list)
|
||||
@@ -264,8 +267,8 @@ class NodeTypeInfo(BaseModel):
|
||||
params_schema: dict[str, Any] = Field(default_factory=dict)
|
||||
has_source: bool = False
|
||||
#: Whether this type takes settings beyond the ones its schema declares.
|
||||
#: A function node's params are its author's to name, and reach `process`
|
||||
#: as whatever they put there.
|
||||
#: A function node's settings are its author's to name, and reach `process`
|
||||
#: as keyword arguments beside its ports.
|
||||
free_params: bool = False
|
||||
#: The package a connector came from; empty for the built-in types.
|
||||
plugin: str | None = None
|
||||
|
||||
@@ -31,7 +31,7 @@ LIB_DIR = "_lib"
|
||||
DEFAULT_SOURCE = '''"""A new node. Return a dict keyed by your output ports."""
|
||||
|
||||
|
||||
def process(params):
|
||||
def process():
|
||||
return {}
|
||||
'''
|
||||
|
||||
|
||||
@@ -302,9 +302,7 @@ def _handle(request: dict[str, Any], cache: dict[tuple[str, str], Any]) -> Any:
|
||||
if request["op"] == "compile":
|
||||
return None
|
||||
|
||||
result = function(
|
||||
**(request.get("kwargs") or {}), params=request.get("params") or {}
|
||||
)
|
||||
result = function(**(request.get("kwargs") or {}))
|
||||
if inspect.isgenerator(result):
|
||||
result = _drain(result)
|
||||
try:
|
||||
|
||||
@@ -379,7 +379,6 @@ class PythonWorkerPool:
|
||||
node: str,
|
||||
source: str,
|
||||
kwargs: dict[str, Any],
|
||||
params: dict[str, Any] | None,
|
||||
node_id: str,
|
||||
timeout: float,
|
||||
run_id: str = "",
|
||||
@@ -393,7 +392,6 @@ class PythonWorkerPool:
|
||||
"node": node,
|
||||
"source": source,
|
||||
"kwargs": kwargs,
|
||||
"params": params or {},
|
||||
"run": {"id": run_id} if run_id else None,
|
||||
},
|
||||
timeout=timeout,
|
||||
@@ -432,13 +430,12 @@ class PythonWorkerPool:
|
||||
node be told apart when one of them is cancelled.
|
||||
"""
|
||||
|
||||
def call(params: dict[str, Any] | None = None, **kwargs: Any) -> Any:
|
||||
def call(**kwargs: Any) -> Any:
|
||||
return self.run(
|
||||
flow,
|
||||
node,
|
||||
source,
|
||||
kwargs,
|
||||
params,
|
||||
node_id,
|
||||
timeout,
|
||||
run_id=run_id,
|
||||
|
||||
@@ -169,8 +169,8 @@ async def save_flow(name: str, definition: dict[str, Any]) -> Any:
|
||||
async def save_node_source(name: str, node_id: str, code: str) -> Any:
|
||||
"""Save a node's Python source and report whether it compiles.
|
||||
|
||||
A node defines ``process(...)``, taking one argument per input port plus
|
||||
``params``, and returns a dict keyed by output port.
|
||||
A node defines ``process(...)``, taking one argument per input port and
|
||||
one per setting, and returns a dict keyed by output port.
|
||||
"""
|
||||
return await _call(
|
||||
"PUT", f"/flows/{name}/nodes/{node_id}/source", json={"code": code}
|
||||
|
||||
@@ -5,12 +5,12 @@ from app.core.config import settings
|
||||
PREFIX = f"{settings.API_V1_STR}/flows"
|
||||
|
||||
WORKING_NODE = """
|
||||
def process(params):
|
||||
def process():
|
||||
return {"reading": 21.5}
|
||||
"""
|
||||
|
||||
BROKEN_NODE = """
|
||||
def process(params):
|
||||
def process():
|
||||
raise RuntimeError("boom")
|
||||
"""
|
||||
|
||||
@@ -75,7 +75,7 @@ def test_broken_node_is_reported_and_siblings_stay_active(
|
||||
response = client.put(
|
||||
f"{PREFIX}/demo/nodes/logger/source",
|
||||
headers=superuser_token_headers,
|
||||
json={"code": "def process(reading, params:\n"},
|
||||
json={"code": "def process(reading:\n"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "error"
|
||||
@@ -98,7 +98,7 @@ def test_running_a_flow_produces_values(
|
||||
client.put(
|
||||
f"{PREFIX}/demo/nodes/logger/source",
|
||||
headers=superuser_token_headers,
|
||||
json={"code": "def process(reading, params):\n return {}\n"},
|
||||
json={"code": "def process(reading):\n return {}\n"},
|
||||
)
|
||||
|
||||
response = client.post(
|
||||
|
||||
@@ -8,8 +8,8 @@ from app.flow.messages import MessageSpec
|
||||
from app.flow.schemas import FlowDef, NodeDef
|
||||
from app.flow.store import FlowStore, StaleVersion
|
||||
|
||||
SOURCE = "def process(params):\n return {}\n"
|
||||
EDITED = "def process(params):\n return {'temp': 1}\n"
|
||||
SOURCE = "def process():\n return {}\n"
|
||||
EDITED = "def process():\n return {'temp': 1}\n"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -8,7 +8,7 @@ from app.flow.messages import MessageSpec
|
||||
from app.flow.schemas import FlowDef, NodeDef
|
||||
from app.flow.store import FlowStore, LibExists, LibNotFound
|
||||
|
||||
SOURCE = "def process(params):\n return {'temp': 1}\n"
|
||||
SOURCE = "def process():\n return {'temp': 1}\n"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -4,6 +4,7 @@ import sys
|
||||
from typing import Any
|
||||
|
||||
from app.flow import logs
|
||||
from app.flow.controller import with_settings
|
||||
from app.flow.messages import DType, MessageSpec
|
||||
from app.flow.nodes import Node
|
||||
from app.flow.pipeline import Pipeline
|
||||
@@ -73,7 +74,7 @@ def test_a_failing_node_reports_its_traceback():
|
||||
namespace: dict[str, Any] = {}
|
||||
exec(
|
||||
compile(
|
||||
'def process(params):\n print("about to fail")\n'
|
||||
'def process():\n print("about to fail")\n'
|
||||
' raise RuntimeError("boom")\n',
|
||||
"<node demo.broken>",
|
||||
"exec",
|
||||
@@ -82,7 +83,11 @@ def test_a_failing_node_reports_its_traceback():
|
||||
)
|
||||
|
||||
bus = RecordingBus()
|
||||
run_with_capture([make_node("broken", namespace["process"])], bus)
|
||||
# Wrapped the way the controller wraps it, so the settings a node declares
|
||||
# arrive as keyword arguments and the frames match the real call.
|
||||
run_with_capture(
|
||||
[make_node("broken", with_settings(namespace["process"], {}))], bus
|
||||
)
|
||||
|
||||
captured = logs_of(bus)
|
||||
assert len(captured) == 1
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""A node's settings are arguments of its function, like its ports.
|
||||
|
||||
What distinguishes them is where the value comes from: a port carries whatever
|
||||
the graph last published, a setting is a constant stored with the flow. Both
|
||||
arrive by name, so one name cannot mean both.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.flow.controller import FlowController
|
||||
from app.flow.messages import DType, MessageSpec
|
||||
from app.flow.pipeline import Pipeline
|
||||
from app.flow.schemas import FlowDef, NodeDef
|
||||
from app.flow.state import MemoryState
|
||||
from app.flow.store import FlowStore
|
||||
|
||||
SOURCE = "def process(reading, factor):\n return {'scaled': reading * factor}\n"
|
||||
|
||||
|
||||
def a_flow(**params: object) -> FlowDef:
|
||||
return FlowDef(
|
||||
name="house",
|
||||
nodes=[
|
||||
NodeDef(
|
||||
id="scale",
|
||||
params=dict(params),
|
||||
requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
|
||||
provides=[MessageSpec(name="scaled", dtype=DType.FLOAT)],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> FlowStore:
|
||||
return FlowStore(tmp_path / "flows")
|
||||
|
||||
|
||||
def test_a_setting_reaches_the_function_as_a_keyword_argument(store: FlowStore):
|
||||
store.write_flow(a_flow(factor=3))
|
||||
store.write_node_source("house", "scale", SOURCE)
|
||||
controller = FlowController(store)
|
||||
|
||||
nodes, _loaded, _initial, _inputs = controller._build_flows(
|
||||
[(store.read_flow("house"), False)]
|
||||
)
|
||||
pipeline = Pipeline(nodes=nodes, state=MemoryState())
|
||||
pipeline.run({"house.reading": 2.0})
|
||||
|
||||
assert pipeline.values()["house.scaled"]["value"] == 6.0
|
||||
|
||||
|
||||
def test_a_setting_named_after_a_port_is_refused(store: FlowStore):
|
||||
store.write_draft(a_flow(reading=3), 0)
|
||||
store.write_node_source("house", "scale", SOURCE, draft=True)
|
||||
controller = FlowController(store)
|
||||
|
||||
preview = controller.preview("house")
|
||||
|
||||
assert [node.status for node in preview.nodes] == ["error"]
|
||||
assert "both an input and a setting" in (preview.nodes[0].error or "")
|
||||
@@ -64,7 +64,7 @@ def test_a_call_crosses_to_the_thread_and_the_answer_comes_back(loop):
|
||||
thread = call_in_thread(
|
||||
lambda: result.update(
|
||||
value=hub.run(
|
||||
"gpu", "flow", "node", "src", {"x": 1}, {}, "flow.node", timeout=5
|
||||
"gpu", "flow", "node", "src", {"x": 1}, "flow.node", timeout=5
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -93,7 +93,6 @@ def test_reports_arrive_before_the_answer_and_a_heartbeat_is_not_one(loop):
|
||||
"node",
|
||||
"src",
|
||||
{},
|
||||
{},
|
||||
"flow.node",
|
||||
timeout=5,
|
||||
run_id="r1",
|
||||
@@ -125,7 +124,7 @@ def test_a_failure_keeps_its_class_across_the_socket(loop):
|
||||
|
||||
def call() -> None:
|
||||
try:
|
||||
hub.run("gpu", "flow", "node", "src", {}, {}, "flow.node", timeout=5)
|
||||
hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=5)
|
||||
except Exception as exc:
|
||||
caught.append(exc)
|
||||
|
||||
@@ -150,7 +149,7 @@ def test_a_worker_that_goes_away_fails_the_call_rather_than_hanging(loop):
|
||||
|
||||
def call() -> None:
|
||||
try:
|
||||
hub.run("gpu", "flow", "node", "src", {}, {}, "flow.node", timeout=30)
|
||||
hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=30)
|
||||
except Exception as exc:
|
||||
caught.append(exc)
|
||||
|
||||
@@ -173,7 +172,7 @@ def test_silence_past_the_deadline_is_a_timeout(loop):
|
||||
|
||||
def call() -> None:
|
||||
try:
|
||||
hub.run("gpu", "flow", "node", "src", {}, {}, "flow.node", timeout=0.3)
|
||||
hub.run("gpu", "flow", "node", "src", {}, "flow.node", timeout=0.3)
|
||||
except Exception as exc:
|
||||
caught.append(exc)
|
||||
|
||||
@@ -186,7 +185,7 @@ def test_a_label_nothing_carries_is_named_rather_than_waited_on(loop):
|
||||
attach(hub, loop)
|
||||
|
||||
with pytest.raises(NoWorker, match="tpu"):
|
||||
hub.run("tpu", "flow", "node", "src", {}, {}, "flow.node", timeout=5)
|
||||
hub.run("tpu", "flow", "node", "src", {}, "flow.node", timeout=5)
|
||||
# Compiling against a machine that is not attached is not a broken node —
|
||||
# a node importing torch is correct there and missing here.
|
||||
assert hub.compile("tpu", "flow", "node", "src") is None
|
||||
@@ -208,9 +207,7 @@ def test_cancelling_a_run_reaches_only_that_run(loop):
|
||||
|
||||
def call(run_id: str) -> None:
|
||||
try:
|
||||
hub.run(
|
||||
"gpu", "flow", "node", "src", {}, {}, "flow.node", 30, run_id=run_id
|
||||
)
|
||||
hub.run("gpu", "flow", "node", "src", {}, "flow.node", 30, run_id=run_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@@ -43,9 +43,7 @@ def test_every_change_is_committed(store: FlowStore):
|
||||
store.write_flow(a_flow())
|
||||
assert commit_count(store) == before + 1
|
||||
|
||||
store.write_node_source(
|
||||
"heating", "sensor", "def process(params):\n return {}\n"
|
||||
)
|
||||
store.write_node_source("heating", "sensor", "def process():\n return {}\n")
|
||||
assert commit_count(store) == before + 2
|
||||
|
||||
|
||||
@@ -65,9 +63,7 @@ def test_missing_flow_is_reported(store: FlowStore):
|
||||
|
||||
def test_deleting_removes_flow_and_its_nodes(store: FlowStore):
|
||||
store.write_flow(a_flow())
|
||||
store.write_node_source(
|
||||
"heating", "sensor", "def process(params):\n return {}\n"
|
||||
)
|
||||
store.write_node_source("heating", "sensor", "def process():\n return {}\n")
|
||||
|
||||
store.delete_flow("heating")
|
||||
|
||||
@@ -77,9 +73,7 @@ def test_deleting_removes_flow_and_its_nodes(store: FlowStore):
|
||||
|
||||
def test_renaming_a_flow_carries_its_nodes(store: FlowStore):
|
||||
store.write_flow(a_flow())
|
||||
store.write_node_source(
|
||||
"heating", "sensor", "def process(params):\n return {}\n"
|
||||
)
|
||||
store.write_node_source("heating", "sensor", "def process():\n return {}\n")
|
||||
|
||||
renamed = store.rename_flow("heating", "warmth")
|
||||
|
||||
|
||||
@@ -23,18 +23,17 @@ def pool() -> Iterator[PythonWorkerPool]:
|
||||
|
||||
|
||||
def run(pool: PythonWorkerPool, code: str, node: str = "demo", **kwargs):
|
||||
return pool.run(
|
||||
"demo", node, code, kwargs, {"factor": 2}, f"demo.{node}", timeout=5
|
||||
)
|
||||
return pool.run("demo", node, code, kwargs, f"demo.{node}", timeout=5)
|
||||
|
||||
|
||||
def test_a_node_returns_its_value_and_what_it_printed(pool, capsys):
|
||||
result = run(
|
||||
pool,
|
||||
"def process(value, params):\n"
|
||||
"def process(value, factor):\n"
|
||||
" print('seen', value)\n"
|
||||
" return {'out': value * params['factor']}\n",
|
||||
" return {'out': value * factor}\n",
|
||||
value=21,
|
||||
factor=2,
|
||||
)
|
||||
assert result == {"out": 42}
|
||||
# The proxy writes them to stdout, which is where the engine's tee is.
|
||||
@@ -43,7 +42,7 @@ def test_a_node_returns_its_value_and_what_it_printed(pool, capsys):
|
||||
|
||||
def test_a_failure_keeps_its_class_and_points_at_the_node(pool):
|
||||
with pytest.raises(Exception) as caught:
|
||||
run(pool, "def process(params):\n raise ValueError('bad input')\n")
|
||||
run(pool, "def process():\n raise ValueError('bad input')\n")
|
||||
|
||||
# The engine renders a node error as "<class>: <message>", so both have to
|
||||
# survive the trip.
|
||||
@@ -55,9 +54,9 @@ def test_a_failure_keeps_its_class_and_points_at_the_node(pool):
|
||||
|
||||
def test_a_node_that_kills_its_worker_is_an_ordinary_error(pool):
|
||||
with pytest.raises(Exception, match="worker died"):
|
||||
run(pool, "import os\n\n\ndef process(params):\n os._exit(1)\n")
|
||||
run(pool, "import os\n\n\ndef process():\n os._exit(1)\n")
|
||||
|
||||
assert run(pool, "def process(params):\n return {'out': 1}\n") == {"out": 1}
|
||||
assert run(pool, "def process():\n return {'out': 1}\n") == {"out": 1}
|
||||
|
||||
|
||||
def test_a_node_that_runs_too_long_is_killed_and_the_pool_recovers(pool):
|
||||
@@ -66,8 +65,7 @@ def test_a_node_that_runs_too_long_is_killed_and_the_pool_recovers(pool):
|
||||
pool.run(
|
||||
"demo",
|
||||
"slow",
|
||||
"import time\n\n\ndef process(params):\n time.sleep(30)\n",
|
||||
{},
|
||||
"import time\n\n\ndef process():\n time.sleep(30)\n",
|
||||
{},
|
||||
"demo.slow",
|
||||
timeout=1,
|
||||
@@ -75,7 +73,7 @@ def test_a_node_that_runs_too_long_is_killed_and_the_pool_recovers(pool):
|
||||
assert time.monotonic() - started < 10
|
||||
|
||||
# The killed worker's slot is refilled on the next call.
|
||||
assert run(pool, "def process(params):\n return {'out': 2}\n") == {"out": 2}
|
||||
assert run(pool, "def process():\n return {'out': 2}\n") == {"out": 2}
|
||||
|
||||
|
||||
def test_a_running_node_can_be_cancelled(pool):
|
||||
@@ -92,8 +90,7 @@ def test_a_running_node_can_be_cancelled(pool):
|
||||
pool.run(
|
||||
"demo",
|
||||
"slow",
|
||||
"import time\n\n\ndef process(params):\n time.sleep(30)\n",
|
||||
{},
|
||||
"import time\n\n\ndef process():\n time.sleep(30)\n",
|
||||
{},
|
||||
"demo.slow",
|
||||
timeout=30,
|
||||
@@ -104,12 +101,12 @@ def test_a_running_node_can_be_cancelled(pool):
|
||||
|
||||
def test_a_result_that_is_not_json_is_refused(pool):
|
||||
with pytest.raises(Exception, match="cannot be sent back as JSON"):
|
||||
run(pool, "def process(params):\n return {'out': {1, 2}}\n")
|
||||
run(pool, "def process():\n return {'out': {1, 2}}\n")
|
||||
|
||||
|
||||
def test_compiling_reports_where_the_source_is_wrong(pool):
|
||||
assert pool.compile("demo", "broken", "def process(params)\n return {}\n")
|
||||
assert pool.compile("demo", "fine", "def process(params):\n return {}\n") is None
|
||||
assert pool.compile("demo", "broken", "def process()\n return {}\n")
|
||||
assert pool.compile("demo", "fine", "def process():\n return {}\n") is None
|
||||
|
||||
|
||||
def test_a_node_imports_the_standard_library_not_the_engines_own_modules(pool):
|
||||
@@ -118,7 +115,7 @@ def test_a_node_imports_the_standard_library_not_the_engines_own_modules(pool):
|
||||
result = run(
|
||||
pool,
|
||||
"import queue\nimport secrets\n\n\n"
|
||||
"def process(params):\n"
|
||||
"def process():\n"
|
||||
" return {'out': [queue.Queue().qsize(), len(secrets.token_hex(4))]}\n",
|
||||
)
|
||||
assert result == {"out": [0, 8]}
|
||||
@@ -134,7 +131,7 @@ def test_the_engines_secrets_are_not_in_a_workers_environment(pool, monkeypatch)
|
||||
result = run(
|
||||
pool,
|
||||
"import os\n\n\n"
|
||||
"def process(params):\n"
|
||||
"def process():\n"
|
||||
" return {'out': [k for k in ('SECRET_KEY', 'POSTGRES_PASSWORD',\n"
|
||||
" 'FLUKSIO_HARMLESS') if k in os.environ]}\n",
|
||||
)
|
||||
@@ -152,8 +149,7 @@ def test_a_pool_can_stop_while_a_node_is_running(pool):
|
||||
pool.run(
|
||||
"demo",
|
||||
node,
|
||||
"import time\n\n\ndef process(params):\n time.sleep(60)\n",
|
||||
{},
|
||||
"import time\n\n\ndef process():\n time.sleep(60)\n",
|
||||
{},
|
||||
f"demo.{node}",
|
||||
timeout=60,
|
||||
@@ -177,7 +173,7 @@ def test_a_pool_can_stop_while_a_node_is_running(pool):
|
||||
assert len(outcomes) == 2
|
||||
|
||||
with pytest.raises(Exception, match="shutting down"):
|
||||
run(pool, "def process(params):\n return {'out': 1}\n")
|
||||
run(pool, "def process():\n return {'out': 1}\n")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -190,12 +186,11 @@ def test_a_generator_node_publishes_each_yield_and_returns_the_end(pool):
|
||||
result = pool.run(
|
||||
"demo",
|
||||
"train",
|
||||
"def process(params):\n"
|
||||
"def process():\n"
|
||||
" for step in range(3):\n"
|
||||
" yield {'loss': 1.0 / (step + 1)}\n"
|
||||
" return {'weights': 'w', 'final_loss': 0.25}\n",
|
||||
{},
|
||||
{},
|
||||
"demo.train",
|
||||
timeout=5,
|
||||
run_id="r1",
|
||||
@@ -219,12 +214,11 @@ def test_without_a_return_the_last_yield_is_the_result(pool):
|
||||
result = pool.run(
|
||||
"demo",
|
||||
"count",
|
||||
"def process(params):\n"
|
||||
"def process():\n"
|
||||
" yield {'out': 1}\n"
|
||||
" yield {'out': 2}\n"
|
||||
" yield {'out': 3}\n",
|
||||
{},
|
||||
{},
|
||||
"demo.count",
|
||||
timeout=5,
|
||||
on_event=seen.append,
|
||||
@@ -242,14 +236,13 @@ def test_emit_reaches_the_same_ports_from_inside_a_callback(pool):
|
||||
"demo",
|
||||
"fit",
|
||||
"import fluksio\n"
|
||||
"def process(params):\n"
|
||||
"def process():\n"
|
||||
" def on_epoch(n):\n"
|
||||
" fluksio.emit(loss=1.0 / (n + 1))\n"
|
||||
" for epoch in range(2):\n"
|
||||
" on_epoch(epoch)\n"
|
||||
" return {'done': True}\n",
|
||||
{},
|
||||
{},
|
||||
"demo.fit",
|
||||
timeout=5,
|
||||
on_event=seen.append,
|
||||
@@ -261,7 +254,7 @@ def test_emit_reaches_the_same_ports_from_inside_a_callback(pool):
|
||||
|
||||
def test_a_plain_function_still_just_returns(pool):
|
||||
seen = []
|
||||
assert run(pool, "def process(params):\n return {'out': 7}\n") == {"out": 7}
|
||||
assert run(pool, "def process():\n return {'out': 7}\n") == {"out": 7}
|
||||
assert seen == []
|
||||
|
||||
|
||||
@@ -272,13 +265,12 @@ def test_events_hold_off_the_timeout_but_silence_does_not(pool):
|
||||
"demo",
|
||||
"slow",
|
||||
"import time\n"
|
||||
"def process(params):\n"
|
||||
"def process():\n"
|
||||
" for step in range(12):\n"
|
||||
" time.sleep(0.05)\n"
|
||||
" yield {'beat': step}\n"
|
||||
" return {'done': True}\n",
|
||||
{},
|
||||
{},
|
||||
"demo.slow",
|
||||
timeout=0.3,
|
||||
run_id="r2",
|
||||
@@ -290,8 +282,7 @@ def test_events_hold_off_the_timeout_but_silence_does_not(pool):
|
||||
pool.run(
|
||||
"demo",
|
||||
"quiet",
|
||||
"import time\ndef process(params):\n time.sleep(2)\n return {}\n",
|
||||
{},
|
||||
"import time\ndef process():\n time.sleep(2)\n return {}\n",
|
||||
{},
|
||||
"demo.quiet",
|
||||
timeout=0.3,
|
||||
@@ -309,8 +300,7 @@ def test_cancelling_one_run_leaves_the_same_node_in_another_alone(pool):
|
||||
pool.run(
|
||||
"demo",
|
||||
"hold",
|
||||
"import time\ndef process(params):\n time.sleep(5)\n return {}\n",
|
||||
{},
|
||||
"import time\ndef process():\n time.sleep(5)\n return {}\n",
|
||||
{},
|
||||
"demo.hold",
|
||||
timeout=10,
|
||||
@@ -344,10 +334,9 @@ def test_a_node_saves_and_loads_an_artifact(tmp_path):
|
||||
"demo",
|
||||
"save",
|
||||
"import fluksio\n"
|
||||
"def process(params):\n"
|
||||
"def process():\n"
|
||||
" return {'weights': fluksio.save_artifact(b'x' * 2048, 'w.npz')}\n",
|
||||
{},
|
||||
{},
|
||||
"demo.save",
|
||||
timeout=10,
|
||||
)["weights"]
|
||||
@@ -360,11 +349,10 @@ def test_a_node_saves_and_loads_an_artifact(tmp_path):
|
||||
"demo",
|
||||
"load",
|
||||
"import fluksio\n"
|
||||
"def process(weights, params):\n"
|
||||
"def process(weights):\n"
|
||||
" with open(fluksio.load_artifact(weights), 'rb') as f:\n"
|
||||
" return {'size': len(f.read())}\n",
|
||||
{"weights": ref},
|
||||
{},
|
||||
"demo.load",
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user