rootandClaude Fable 5
04329149b3
Execution was fire-and-forget: an MQTT message or webhook ran a cascade on a ThreadPoolExecutor built for that one wave, and an engine that died halfway through simply lost whatever was in flight. Concurrent triggers each built their own pool, so load meant unbounded threads. Every external trigger is now journaled to a Redis Streams queue before anything runs, and acknowledged only once its cascade finishes. A consumer thread drives cascades on one long-lived pool while node bodies run on another, so a cascade cannot starve the nodes it is waiting for. A reaper reclaims what a dead consumer never acknowledged — verified end to end: work journaled while the engine was stopped runs on restart, and work abandoned mid-cascade comes back as a second delivery. At-least-once needs a guard, so nodes that reach outside are marked non-idempotent and skipped on a redelivery they already completed. Without Redis the queue degrades to an in-memory one that does not pretend to be durable, and interactive callers still run inline. Also fixes two things this turned up: a delay node was sleeping on a worker thread, where a handful of them could occupy the whole pool, and webhooks 404'd whenever MCP was enabled because the app mounted at / answered first for every path. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011LF61rxW1FG5YCD2J9YqjY
134 lines
4.2 KiB
Python
134 lines
4.2 KiB
Python
"""Webhooks are mounted without authentication, so their secret is a boundary.
|
|
|
|
The secret travels as the last path segment; anything but an exact match must
|
|
be indistinguishable from a hook that does not exist.
|
|
"""
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
|
|
from app.flow.controller import HOOK_PREFIX, FlowController
|
|
from app.flow.messages import DType, MessageSpec
|
|
from app.flow.nodes import HttpNode
|
|
from app.flow.pipeline import Pipeline
|
|
from app.flow.schemas import FlowDef, NodeDef
|
|
from app.flow.secrets import init_secrets, resolve_params
|
|
from app.flow.store import FlowStore
|
|
|
|
SECRET = "d3vice-key"
|
|
HOOK = f"{HOOK_PREFIX}/house/sensor"
|
|
|
|
|
|
def hook_node(**params: object) -> HttpNode:
|
|
"""A trigger node mounted the way the controller mounts it."""
|
|
node = HttpNode(
|
|
provides=[MessageSpec(name="temp", dtype=DType.FLOAT)],
|
|
params={"url": "/sensor", "method": "POST", **params},
|
|
name="sensor",
|
|
)
|
|
node.assign_flow("house", "sensor")
|
|
node.url = HOOK
|
|
return node
|
|
|
|
|
|
def mount(node: HttpNode) -> tuple[TestClient, Pipeline]:
|
|
pipeline = Pipeline(nodes=[node])
|
|
app = FastAPI()
|
|
node.register_route(app)
|
|
return TestClient(app), pipeline
|
|
|
|
|
|
def test_the_right_secret_carries_the_payload_into_the_pipeline():
|
|
node = hook_node(secret=SECRET)
|
|
client, pipeline = mount(node)
|
|
|
|
response = client.post(f"{HOOK}/{SECRET}", json={"temp": 21.5})
|
|
|
|
assert response.status_code == 200
|
|
assert pipeline.values()["house.temp"]["value"] == 21.5
|
|
|
|
|
|
def test_a_hook_without_a_secret_still_answers_on_its_plain_url():
|
|
# Flows written before the parameter existed keep working unchanged.
|
|
node = hook_node()
|
|
client, pipeline = mount(node)
|
|
|
|
assert client.post(HOOK, json={"temp": 21.5}).status_code == 200
|
|
assert pipeline.values()["house.temp"]["value"] == 21.5
|
|
|
|
|
|
def test_a_hook_is_reachable_past_an_app_mounted_at_the_root():
|
|
"""The MCP app is mounted at "/" and answers for every path, so a hook
|
|
registered after it would never be reached."""
|
|
node = hook_node()
|
|
pipeline = Pipeline(nodes=[node])
|
|
app = FastAPI()
|
|
app.mount("/", FastAPI())
|
|
node.register_route(app)
|
|
|
|
response = TestClient(app).post(HOOK, json={"temp": 21.5})
|
|
|
|
assert response.status_code == 200
|
|
assert pipeline.values()["house.temp"]["value"] == 21.5
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"path",
|
|
[
|
|
f"{HOOK}/wrong",
|
|
# A prefix of the secret must not pass either.
|
|
f"{HOOK}/{SECRET[:-1]}",
|
|
# No secret at all: the route only exists with the segment.
|
|
HOOK,
|
|
f"{HOOK}/",
|
|
],
|
|
)
|
|
def test_anything_but_the_secret_looks_like_no_such_hook(path: str):
|
|
node = hook_node(secret=SECRET)
|
|
client, pipeline = mount(node)
|
|
|
|
assert client.post(path, json={"temp": 21.5}).status_code == 404
|
|
assert pipeline.values() == {}
|
|
|
|
|
|
def test_the_secret_can_be_read_from_the_secrets_store(tmp_path: Path):
|
|
init_secrets(tmp_path / "secrets.enc", "app-key").set("hook_key", SECRET)
|
|
|
|
# What the controller does before building the node.
|
|
params = resolve_params({"url": "/sensor", "secret": {"$secret": "hook_key"}})
|
|
node = hook_node(**params)
|
|
|
|
assert node.secret == SECRET
|
|
client, _ = mount(node)
|
|
assert client.post(f"{HOOK}/{SECRET}", json={"temp": 21.5}).status_code == 200
|
|
|
|
|
|
@pytest.mark.parametrize("secret, issues", [("", 1), (SECRET, 0)])
|
|
def test_a_hook_without_a_secret_is_reported(tmp_path: Path, secret: str, issues: int):
|
|
store = FlowStore(tmp_path / "flows")
|
|
store.write_flow(
|
|
FlowDef(
|
|
name="house",
|
|
nodes=[
|
|
NodeDef(
|
|
id="sensor",
|
|
type="http",
|
|
params={"url": "/sensor", "secret": secret},
|
|
provides=[MessageSpec(name="temp")],
|
|
)
|
|
],
|
|
)
|
|
)
|
|
controller = FlowController(store=store, fastapi_app=FastAPI())
|
|
|
|
asyncio.run(controller.reload())
|
|
|
|
open_hooks = [i for i in controller.issues if i.code == "unauthenticated_hook"]
|
|
assert len(open_hooks) == issues
|
|
# Either way the flow loads — an existing hook keeps working.
|
|
assert controller.get_node("house.sensor") is not None
|