"""HTTP nodes: an inbound webhook listener and an outbound request sender.""" from __future__ import annotations import asyncio import hmac import logging import threading from collections.abc import Iterable from enum import Enum from typing import TYPE_CHECKING, Any, Literal from urllib.parse import urlsplit, urlunsplit import httpx from pydantic import BaseModel, ConfigDict, Field from fluksio.flow.messages import MessageSpec from fluksio.flow.nodes.base import Node if TYPE_CHECKING: from fastapi import FastAPI logger = logging.getLogger(__name__) # One pooled client for every sender node: connections are the expensive part # of an HTTP request, and a node that fires every second should keep its own. # The ceiling is process-wide rather than per node, which is why it is a # constant here rather than a `Params` field on the node. HTTP_POOL_LIMIT = 32 _client: httpx.Client | None = None _client_lock = threading.Lock() def shared_client() -> httpx.Client: """The process-wide HTTP client, built on first use. Bounded on purpose: httpx's default pool is 100 connections with no per-host cap, so one endpoint that stops answering could take the whole pool and every other sender node with it. Read outside the lock once it exists — this is on the per-request path, and rebinding the global is what the lock is for. """ global _client if _client is not None: return _client with _client_lock: if _client is None: _client = httpx.Client( limits=httpx.Limits( max_connections=HTTP_POOL_LIMIT, max_keepalive_connections=HTTP_POOL_LIMIT // 2, keepalive_expiry=30.0, ), transport=httpx.HTTPTransport(retries=1), ) return _client def close_shared_client() -> None: """Release pooled connections at shutdown. Safe to call more than once.""" global _client with _client_lock: if _client is not None: _client.close() _client = None def _first_catch_all(app: FastAPI) -> int: """Where a webhook has to go in to be reachable. A mount at the root answers for every path, so anything registered after it is dead. Returns that mount's index, or the end of the table. """ from starlette.routing import Mount for index, route in enumerate(app.routes): if isinstance(route, Mount) and route.path in ("", "/"): return index return len(app.routes) class HttpNode(Node): """ HTTP node that acts as a listener (webhook) or a sender. This node integrates with FastAPI to either: - **Listener**: Receive incoming HTTP requests (GET/POST) and inject data into the pipeline. Used when ``provides`` is specified but ``requires`` is empty. - **Sender**: Make outgoing HTTP requests with pipeline data, and publish a JSON object response on the outputs. Used when ``requires`` is specified. :param url: The URL endpoint. For a listener, this is the route path (e.g., "/sensors/temperature"). For a sender, this is the full URL to send requests to. :type url: str :param method: HTTP method - "GET" or "POST". :type method: Literal["GET", "POST"] :param requires: Messages required by this node (makes it a sender). :type requires: MessageSpec | list[MessageSpec] | None :param provides: Messages provided by this node (makes it a listener). :type provides: MessageSpec | list[MessageSpec] | None :param params: Additional parameters for the node. :type params: dict :param name: Optional name for the node. :type name: str | None :param timeout: Request timeout in seconds (sender only). :type timeout: float :param headers: Additional HTTP headers. :type headers: dict[str, str] | None :param secret: Shared secret callers append to the webhook URL (listener only). :type secret: str :raises ValueError: If both ``requires`` and ``provides`` are empty, or if the configuration is invalid. :example: Listener (receives POST requests): >>> listener = HttpNode( ... url="/api/sensors/temperature", ... method="POST", ... provides=[MessageSpec(name="temperature", dtype=DType.FLOAT)], ... params={}, ... ) Sender (makes POST requests): >>> sender = HttpNode( ... url="https://api.example.com/data", ... method="POST", ... requires=[MessageSpec(name="temperature", dtype=DType.FLOAT)], ... params={}, ... ) """ class Mode(Enum): """Operating mode of the HTTP node.""" LISTENER = "listener" # Receives HTTP requests SENDER = "sender" # Sends HTTP requests # A repeated request is a repeated request, whatever the endpoint does # with it. idempotent = False __slots__ = ( "url", "method", "mode", "timeout", "headers", "query", "send_inputs", "secret", "_route_registered", ) class Params(BaseModel): model_config = ConfigDict(extra="allow") url: str = Field( description=( "A path such as /tick, served as a webhook at /hooks//, " "or the full URL a sender requests." ) ) method: Literal["GET", "POST"] = Field( default="POST", description="The method a webhook accepts or a sender uses.", ) timeout: float = Field( default=30.0, description="Seconds a sender waits for the response." ) headers: dict[str, str] = Field( default={}, description=( "Request headers, sender only. A value may be a secret reference." ), ) query: dict[str, Any] = Field( default_factory=dict, description=( "Fixed query parameters, sender only. A value may be a secret " "reference, which is how an API key stays out of the flow file." ), ) send_inputs: bool = Field( default=True, description=( "Send the node's inputs as the request body or query. Off when " "the inputs are only a trigger and the request is fully " "described by 'url' and 'query'." ), ) secret: str = Field( default="", description=( "Shared secret for a webhook, appended to its URL: " "/hooks///. Empty leaves the webhook open " "to anyone." ), json_schema_extra={"x-secret": True}, ) @classmethod def target_key(cls, params: dict[str, Any]) -> str | None: # ponytail: a webhook's stored url is the path before the flow name is # prefixed onto it, so two flows both receiving on "/tick" merge into # one neuron. Key on mode as well if that ever misleads. url = params.get("url") if not url: return None # A URL may legally carry credentials, and this key becomes a group id # in the brain graph — rendered into the response and into the DOM. parts = urlsplit(str(url)) if parts.username or parts.password: host = parts.netloc.rsplit("@", 1)[-1] return urlunsplit(parts._replace(netloc=host)) return str(url) def __init__( self, url: str | None = None, method: Literal["GET", "POST"] | None = None, requires: MessageSpec | Iterable[MessageSpec] = (), provides: MessageSpec | Iterable[MessageSpec] = (), params: dict[str, Any] | None = None, name: str | None = None, ): params = dict(params) if params else {} if url is not None: params["url"] = url if method is not None: params["method"] = method cfg = self.Params.model_validate(params) requires = Node._normalize_ports(requires) provides = Node._normalize_ports(provides) if not requires and not provides: raise ValueError( "An HTTP node needs either inputs (to send) or outputs (to receive)" ) # Inputs mean this node sends data out; outputs mean it receives. self.mode = HttpNode.Mode.SENDER if requires else HttpNode.Mode.LISTENER self.url = cfg.url self.method = cfg.method.upper() self.timeout = cfg.timeout self.headers = cfg.headers self.query = cfg.query self.send_inputs = cfg.send_inputs self.secret = cfg.secret self._route_registered = False # Set default name based on mode and URL if name is None: safe_url = self.url.replace("/", "_").replace(":", "").strip("_") name = f"http_{self.mode.value}_{safe_url}" # Initialize parent with appropriate function # For a listener, f is a no-op since data is injected via inject() # For a sender, f handles the outgoing HTTP request super().__init__( f=( self._noop_listener if self.mode == HttpNode.Mode.LISTENER else self._sender_handler ), requires=requires, provides=provides, params=params, name=name, ) @staticmethod def _noop_listener(params: dict[str, Any], **kwargs: Any) -> dict[str, Any] | None: """ No-op function for listener nodes. Listener nodes inject data via :meth:`inject`, not :meth:`__call__`. This function exists only to satisfy the Node interface. """ return None def _sender_handler( self, params: dict[str, Any], **kwargs: Any ) -> dict[str, Any] | None: """ Send HTTP request with pipeline data (sender). This method is called when upstream dependencies are satisfied. It sends the required data via HTTP request. Node functions already run on a worker thread, so the request is sent synchronously over the shared pooled client: a node publishing every second should reuse its connection, not open one per message. :param params: Node parameters. :type params: dict :param kwargs: Pipeline data to send (from required messages). :type kwargs: Any :returns: The reply's values for the declared output ports, or None. :rtype: dict | None """ client = shared_client() payload = dict(kwargs) if self.send_inputs else {} try: if self.method == "GET": response = client.get( self.url, params={**self.query, **payload}, headers=self.headers, timeout=self.timeout, ) else: # POST response = client.post( self.url, params=self.query or None, json=payload, headers=self.headers, timeout=self.timeout, ) response.raise_for_status() try: body = response.json() except Exception: return None # Outputs are keyed by port, so only an object can be one. A bare # scalar or list is a valid reply, just not something to publish, # and neither is a key no port declares: the reply is the # endpoint's, and a `status` beside the values is common. if not isinstance(body, dict): return None ports = {spec.port for spec in self.output_ports} return {key: value for key, value in body.items() if key in ports} or None except httpx.HTTPStatusError as e: logger.error( "HTTP error in node '%s': status=%s, url=%s", self.name, e.response.status_code, self.url, ) raise except httpx.RequestError as e: logger.error("Request error in node '%s': %s", self.name, e) raise async def start(self, app: FastAPI | None = None) -> None: """A webhook needs a route; a sender reaches out on its own.""" if self.mode is HttpNode.Mode.LISTENER and app is not None: self.register_route(app) async def stop(self, app: FastAPI | None = None) -> None: if self.mode is HttpNode.Mode.LISTENER and app is not None: self.unregister_route(app) def register_route(self, app: FastAPI) -> None: """ Register this node's HTTP endpoint with a FastAPI application. This method should only be called for listener nodes. It creates a route that, when called, triggers the node in the pipeline. :param app: The FastAPI application instance. :type app: FastAPI :raises RuntimeError: If called on a sender. :example: >>> from fastapi import FastAPI >>> app = FastAPI() >>> listener = HttpNode( ... url="/sensors/data", ... method="POST", ... provides=[MessageSpec(name="value", dtype=DType.FLOAT)], ... params={}, ... ) >>> listener.register_route(app) """ if self.mode != HttpNode.Mode.LISTENER: raise RuntimeError("Can only register routes for listener nodes") if self._route_registered: return from starlette.requests import Request from starlette.responses import JSONResponse from starlette.routing import Route async def handle_request(request: Request) -> JSONResponse: """ Handle incoming HTTP request and trigger the pipeline. :param request: The incoming Starlette request. :type request: Request :returns: JSON response with trigger result. :rtype: JSONResponse """ from fluksio.cloud.config import via_portal if ( self.secret and not hmac.compare_digest( str(request.path_params.get("secret", "")).encode(), self.secret.encode(), ) ) or (not self.secret and via_portal(request.headers)): # A wrong secret must look like no such hook at all, and so must # a hook without one reached through the portal, which would # otherwise be open to the whole internet. return JSONResponse(content={"detail": "Not Found"}, status_code=404) try: # Parse request data data: dict[str, Any] if self.method == "GET": data = dict(request.query_params) else: # POST content_type = request.headers.get("content-type", "") if "application/json" in content_type: data = await request.json() elif "application/x-www-form-urlencoded" in content_type: form = await request.form() data = dict(form) else: data = await request.json() # Default to JSON # Payload keys are port names; values arrive as text. typed_data = {} for spec in self.output_ports: if spec.port in data: typed_data[spec.port] = spec.coerce(data[spec.port]) # Inject data into the pipeline (listener nodes inject via # provides). Off the loop: journalling is a blocking Redis # round trip, and every webhook was making it on the thread # the whole API answers from. result = await asyncio.to_thread(self.inject, typed_data) return JSONResponse( content={ "status": "triggered", "node": self.name, "data": typed_data, "result": result if isinstance(result, dict) else None, } ) except ValueError as e: return JSONResponse( content={"error": str(e)}, status_code=400, ) except Exception as e: return JSONResponse( content={"error": str(e)}, status_code=500, ) # The secret is a path parameter, not part of the registered path, so # neither the route table nor the logs below carry its value. path = f"{self.url}/{{secret}}" if self.secret else self.url # Create a Starlette Route and add it directly to the app's routes route = Route( path, handle_request, methods=[self.method], name=self.id, ) # Ahead of any catch-all mount. Starlette takes the first route that # matches, and the MCP app is mounted at "/", so appending would put # every webhook behind something that answers for every path. app.routes.insert(_first_catch_all(app), route) self._route_registered = True logger.info( "Registered %s route for node '%s': %s", self.method, self.name, path, ) def unregister_route(self, app: FastAPI) -> None: """ Unregister this node's HTTP endpoint from a FastAPI application. :param app: The FastAPI application instance. :type app: FastAPI .. note:: FastAPI doesn't natively support route removal. This method removes the route from the internal routes list, but the change may not take effect until the application is restarted or the OpenAPI schema is regenerated. """ if not self._route_registered: return # FastAPI doesn't have a clean way to remove routes # We need to filter them out from the routes list app.routes[:] = [ route for route in app.routes if not (hasattr(route, "name") and route.name == self.id) ] self._route_registered = False logger.info("Unregistered route for node '%s': %s", self.name, self.url)