stroblme
29f4d93c2a
/search gains a `setting` category, `<node>.<setting>` under its flow with the node type and the value in a new `detail` field, so Ctrl-K finds which node uses a broker or a timeout and opens the node. `setting_text` is the one place that decides what may be said: a reference to a stored secret shows its name (to superusers only, as the secrets list does), a plain value in an `x-secret` field or under a key like password, token or api_key is masked and never indexed, nested credentials in headers or query are masked, long values are clipped. The global search ranks what contains the query as typed ahead of loose spellings before it caps a group, which long setting values would otherwise have crowded out.
283 lines
9.7 KiB
Python
283 lines
9.7 KiB
Python
"""One index of everything in this instance worth jumping to by name."""
|
|
|
|
import json
|
|
import re
|
|
from typing import Any, Literal
|
|
|
|
from fastapi import APIRouter, Depends, Request
|
|
from fastapi.concurrency import run_in_threadpool
|
|
from pydantic import BaseModel
|
|
|
|
from fluksio.api.deps import (
|
|
CurrentUser,
|
|
DashboardStoreDep,
|
|
FlowControllerDep,
|
|
get_current_user,
|
|
)
|
|
from fluksio.api.routes.alerts import read_config as read_alerts_config
|
|
from fluksio.flow import modules
|
|
from fluksio.flow.controller import NODE_TYPES, FlowController
|
|
from fluksio.flow.dashboards import DashboardNotFound, DashboardStore
|
|
from fluksio.flow.messages import qualify
|
|
from fluksio.flow.secrets import get_secrets
|
|
from fluksio.flow.store import FlowNotFound
|
|
|
|
# A device never reaches this route: ``deps._device_may`` is a whitelist that
|
|
# ends in a 403, and a whole-instance index is the opposite of what a screen
|
|
# on a wall is allowed to read.
|
|
router = APIRouter(
|
|
prefix="/search", tags=["search"], dependencies=[Depends(get_current_user)]
|
|
)
|
|
|
|
Category = Literal[
|
|
"flow",
|
|
"node",
|
|
"input",
|
|
"output",
|
|
"setting",
|
|
"dashboard",
|
|
"widget",
|
|
"device",
|
|
"secret",
|
|
"module",
|
|
"worker",
|
|
"alert",
|
|
]
|
|
|
|
|
|
class SearchEntry(BaseModel):
|
|
"""One thing somebody might be looking for.
|
|
|
|
Deliberately not a route: where a category lands is the frontend's business,
|
|
and it already owns the router. This says what the thing is and what it is
|
|
called, which is all the matching needs.
|
|
"""
|
|
|
|
category: Category
|
|
#: The id the frontend routes on. For an input or output, the qualified
|
|
#: message, which is what the canvas names its label by. For a setting,
|
|
#: ``<node>.<setting>``: a node id has no dot, so the node is what comes
|
|
#: before the first one.
|
|
name: str
|
|
#: Human title, often empty — a flow is usually only its name.
|
|
title: str = ""
|
|
#: The flow a node, input or output sits in, or the dashboard a widget
|
|
#: sits on.
|
|
parent: str = ""
|
|
#: Node type, input dtype, widget type, channel kind.
|
|
kind: str = ""
|
|
#: What a setting is set to, as text. Only ever what ``setting_text`` lets
|
|
#: through, so never a credential.
|
|
detail: str = ""
|
|
|
|
|
|
#: What stands in for a value that must not be shown, or searched for.
|
|
MASK = "\u2022\u2022\u2022\u2022"
|
|
|
|
#: How much of a value the index keeps. A setting can be a whole JSON document.
|
|
_DETAIL_CAP = 80
|
|
|
|
#: Keys that tend to hold a credential. Applied to every setting, declared or
|
|
#: not: a type's author can forget its ``x-secret``, and a masked value only
|
|
#: costs a search that could not have helped.
|
|
_SENSITIVE_KEY = re.compile(r"pass|token|secret|key|auth|credential", re.IGNORECASE)
|
|
|
|
|
|
def _is_secret_ref(value: Any) -> bool:
|
|
return isinstance(value, dict) and "$secret" in value
|
|
|
|
|
|
def _secret_ref(value: dict[str, Any], names: bool) -> str:
|
|
"""A reference to a stored secret: its name, to those who may read names."""
|
|
name = value.get("$secret")
|
|
return f"secret {name}" if names and isinstance(name, str) and name else "secret"
|
|
|
|
|
|
def _scrub(value: Any, names: bool) -> Any:
|
|
"""``value`` with every credential inside it replaced.
|
|
|
|
A stored secret is named, never read. Whatever sits under a key that looks
|
|
like a credential is masked, however deep: a header or a query parameter is
|
|
as good a place for a token as any.
|
|
"""
|
|
if _is_secret_ref(value):
|
|
return _secret_ref(value, names)
|
|
if isinstance(value, dict):
|
|
return {
|
|
key: MASK
|
|
if _SENSITIVE_KEY.search(str(key))
|
|
and item not in (None, "")
|
|
and not _is_secret_ref(item)
|
|
else _scrub(item, names)
|
|
for key, item in value.items()
|
|
}
|
|
if isinstance(value, list):
|
|
return [_scrub(item, names) for item in value]
|
|
return value
|
|
|
|
|
|
def setting_text(
|
|
key: str, value: Any, declared: dict[str, Any] | None, names: bool
|
|
) -> str:
|
|
"""What the search may say about one setting, never its secret value.
|
|
|
|
``declared`` is the node type's schema for this setting, None where the
|
|
type has none, as for every setting of a function node. A reference to a
|
|
stored secret is named (to those who may read names). A plain value in a
|
|
field the type flags ``x-secret``, or under a key that looks like a
|
|
credential, is masked. An empty setting is "" and gets no entry.
|
|
"""
|
|
if value is None or value == "":
|
|
return ""
|
|
if _is_secret_ref(value):
|
|
text = _secret_ref(value, names)
|
|
elif (declared or {}).get("x-secret") or _SENSITIVE_KEY.search(key):
|
|
text = MASK
|
|
elif isinstance(value, dict | list):
|
|
text = json.dumps(
|
|
_scrub(value, names), separators=(",", ":"), ensure_ascii=False
|
|
)
|
|
elif isinstance(value, bool):
|
|
text = "true" if value else "false"
|
|
else:
|
|
text = str(value)
|
|
return text if len(text) <= _DETAIL_CAP else text[: _DETAIL_CAP - 1] + "\u2026"
|
|
|
|
|
|
def _build(
|
|
controller: FlowController, dashboards: DashboardStore, hub: Any, secrets: bool
|
|
) -> list[SearchEntry]:
|
|
"""Read every store once. Blocking: disk and git throughout.
|
|
|
|
# ponytail: rebuilt per call. Key it on ``controller.store.revision`` if a
|
|
# store large enough to feel it ever shows up in a profile.
|
|
"""
|
|
entries: list[SearchEntry] = []
|
|
|
|
for name in controller.store.list_flows():
|
|
try:
|
|
flow = controller.store.read_flow(name, draft=True)
|
|
except FlowNotFound:
|
|
continue
|
|
entries.append(SearchEntry(category="flow", name=flow.name, title=flow.title))
|
|
entries.extend(
|
|
SearchEntry(
|
|
category="node",
|
|
name=node.id,
|
|
title=node.title,
|
|
parent=flow.name,
|
|
kind=node.type,
|
|
)
|
|
for node in flow.nodes
|
|
)
|
|
for node in flow.nodes:
|
|
declared = (
|
|
NODE_TYPES[node.type].params_schema.get("properties", {})
|
|
if node.type in NODE_TYPES
|
|
else {}
|
|
)
|
|
for key, value in node.params.items():
|
|
text = setting_text(key, value, declared.get(key), secrets)
|
|
if text:
|
|
entries.append(
|
|
SearchEntry(
|
|
category="setting",
|
|
name=f"{node.id}.{key}",
|
|
parent=flow.name,
|
|
kind=node.type,
|
|
detail=text,
|
|
)
|
|
)
|
|
# Its boundary, as the canvas draws it: a live flow never finishes, so
|
|
# only a batch one has outputs to report.
|
|
boundary: list[tuple[Category, str, str]] = [
|
|
("input", flow_input.spec.name, flow_input.spec.dtype.value)
|
|
for flow_input in flow.inputs
|
|
]
|
|
if flow.mode == "batch":
|
|
boundary += [("output", output, "") for output in flow.outputs]
|
|
for category, declared, kind in boundary:
|
|
message = qualify(flow.name, declared)
|
|
if message:
|
|
entries.append(
|
|
SearchEntry(
|
|
category=category,
|
|
name=message,
|
|
# The label the canvas draws: bare when it is the flow's own.
|
|
title=message.removeprefix(f"{flow.name}."),
|
|
parent=flow.name,
|
|
kind=kind,
|
|
)
|
|
)
|
|
|
|
for summary in dashboards.list():
|
|
try:
|
|
dashboard = dashboards.read(summary.name, draft=True)
|
|
except DashboardNotFound:
|
|
continue
|
|
entries.append(
|
|
SearchEntry(
|
|
category="dashboard", name=dashboard.name, title=dashboard.title
|
|
)
|
|
)
|
|
entries.extend(
|
|
SearchEntry(
|
|
category="widget",
|
|
name=widget.id,
|
|
title=widget.title,
|
|
parent=dashboard.name,
|
|
kind=widget.type,
|
|
)
|
|
for widget in dashboard.widgets
|
|
)
|
|
|
|
devices = getattr(controller, "devices", None)
|
|
entries.extend(
|
|
SearchEntry(category="device", name=device.id, title=device.title)
|
|
for device in (devices.list() if devices is not None else [])
|
|
)
|
|
|
|
if secrets:
|
|
entries.extend(
|
|
SearchEntry(category="secret", name=name) for name in get_secrets().list()
|
|
)
|
|
|
|
entries.extend(
|
|
SearchEntry(category="module", name=package.name, kind=package.version)
|
|
for package in modules.info(controller.store).packages
|
|
)
|
|
|
|
entries.extend(
|
|
SearchEntry(category="worker", name=worker.name)
|
|
for worker in (hub.workers() if hub is not None else [])
|
|
)
|
|
|
|
entries.extend(
|
|
SearchEntry(category="alert", name=channel.name, kind=channel.kind)
|
|
for channel in read_alerts_config().channels
|
|
)
|
|
|
|
return entries
|
|
|
|
|
|
@router.get("/", response_model=list[SearchEntry])
|
|
async def read_search_index(
|
|
current_user: CurrentUser,
|
|
request: Request,
|
|
controller: FlowControllerDep,
|
|
dashboards: DashboardStoreDep,
|
|
) -> Any:
|
|
"""Everything searchable, for the client to match against as it is typed.
|
|
|
|
The whole index rather than a query: it is a few hundred short rows for an
|
|
instance of any ordinary size, so one fetch when the panel opens beats a
|
|
round trip per keystroke — and the client already has a matcher.
|
|
|
|
Secrets are named only to a superuser, which is who ``/secrets`` answers to.
|
|
"""
|
|
# Absent when remote workers are switched off — not a reason to fail a search.
|
|
hub = getattr(request.app.state, "worker_hub", None)
|
|
return await run_in_threadpool(
|
|
_build, controller, dashboards, hub, current_user.is_superuser
|
|
)
|