Search: node settings are indexed, with credentials masked on the server
/search gains a `setting` category, `<node>.<setting>` under its flow with the node type and the value in a new `detail` field, so Ctrl-K finds which node uses a broker or a timeout and opens the node. `setting_text` is the one place that decides what may be said: a reference to a stored secret shows its name (to superusers only, as the secrets list does), a plain value in an `x-secret` field or under a key like password, token or api_key is masked and never indexed, nested credentials in headers or query are masked, long values are clipped. The global search ranks what contains the query as typed ahead of loose spellings before it caps a group, which long setting values would otherwise have crowded out.
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
"""One index of everything in this instance worth jumping to by name."""
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
@@ -14,7 +16,7 @@ from fluksio.api.deps import (
|
||||
)
|
||||
from fluksio.api.routes.alerts import read_config as read_alerts_config
|
||||
from fluksio.flow import modules
|
||||
from fluksio.flow.controller import FlowController
|
||||
from fluksio.flow.controller import NODE_TYPES, FlowController
|
||||
from fluksio.flow.dashboards import DashboardNotFound, DashboardStore
|
||||
from fluksio.flow.messages import qualify
|
||||
from fluksio.flow.secrets import get_secrets
|
||||
@@ -32,6 +34,7 @@ Category = Literal[
|
||||
"node",
|
||||
"input",
|
||||
"output",
|
||||
"setting",
|
||||
"dashboard",
|
||||
"widget",
|
||||
"device",
|
||||
@@ -52,7 +55,9 @@ class SearchEntry(BaseModel):
|
||||
|
||||
category: Category
|
||||
#: The id the frontend routes on. For an input or output, the qualified
|
||||
#: message, which is what the canvas names its label by.
|
||||
#: message, which is what the canvas names its label by. For a setting,
|
||||
#: ``<node>.<setting>``: a node id has no dot, so the node is what comes
|
||||
#: before the first one.
|
||||
name: str
|
||||
#: Human title, often empty — a flow is usually only its name.
|
||||
title: str = ""
|
||||
@@ -61,6 +66,82 @@ class SearchEntry(BaseModel):
|
||||
parent: str = ""
|
||||
#: Node type, input dtype, widget type, channel kind.
|
||||
kind: str = ""
|
||||
#: What a setting is set to, as text. Only ever what ``setting_text`` lets
|
||||
#: through, so never a credential.
|
||||
detail: str = ""
|
||||
|
||||
|
||||
#: What stands in for a value that must not be shown, or searched for.
|
||||
MASK = "\u2022\u2022\u2022\u2022"
|
||||
|
||||
#: How much of a value the index keeps. A setting can be a whole JSON document.
|
||||
_DETAIL_CAP = 80
|
||||
|
||||
#: Keys that tend to hold a credential. Applied to every setting, declared or
|
||||
#: not: a type's author can forget its ``x-secret``, and a masked value only
|
||||
#: costs a search that could not have helped.
|
||||
_SENSITIVE_KEY = re.compile(r"pass|token|secret|key|auth|credential", re.IGNORECASE)
|
||||
|
||||
|
||||
def _is_secret_ref(value: Any) -> bool:
|
||||
return isinstance(value, dict) and "$secret" in value
|
||||
|
||||
|
||||
def _secret_ref(value: dict[str, Any], names: bool) -> str:
|
||||
"""A reference to a stored secret: its name, to those who may read names."""
|
||||
name = value.get("$secret")
|
||||
return f"secret {name}" if names and isinstance(name, str) and name else "secret"
|
||||
|
||||
|
||||
def _scrub(value: Any, names: bool) -> Any:
|
||||
"""``value`` with every credential inside it replaced.
|
||||
|
||||
A stored secret is named, never read. Whatever sits under a key that looks
|
||||
like a credential is masked, however deep: a header or a query parameter is
|
||||
as good a place for a token as any.
|
||||
"""
|
||||
if _is_secret_ref(value):
|
||||
return _secret_ref(value, names)
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: MASK
|
||||
if _SENSITIVE_KEY.search(str(key))
|
||||
and item not in (None, "")
|
||||
and not _is_secret_ref(item)
|
||||
else _scrub(item, names)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [_scrub(item, names) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def setting_text(
|
||||
key: str, value: Any, declared: dict[str, Any] | None, names: bool
|
||||
) -> str:
|
||||
"""What the search may say about one setting, never its secret value.
|
||||
|
||||
``declared`` is the node type's schema for this setting, None where the
|
||||
type has none, as for every setting of a function node. A reference to a
|
||||
stored secret is named (to those who may read names). A plain value in a
|
||||
field the type flags ``x-secret``, or under a key that looks like a
|
||||
credential, is masked. An empty setting is "" and gets no entry.
|
||||
"""
|
||||
if value is None or value == "":
|
||||
return ""
|
||||
if _is_secret_ref(value):
|
||||
text = _secret_ref(value, names)
|
||||
elif (declared or {}).get("x-secret") or _SENSITIVE_KEY.search(key):
|
||||
text = MASK
|
||||
elif isinstance(value, dict | list):
|
||||
text = json.dumps(
|
||||
_scrub(value, names), separators=(",", ":"), ensure_ascii=False
|
||||
)
|
||||
elif isinstance(value, bool):
|
||||
text = "true" if value else "false"
|
||||
else:
|
||||
text = str(value)
|
||||
return text if len(text) <= _DETAIL_CAP else text[: _DETAIL_CAP - 1] + "\u2026"
|
||||
|
||||
|
||||
def _build(
|
||||
@@ -89,6 +170,24 @@ def _build(
|
||||
)
|
||||
for node in flow.nodes
|
||||
)
|
||||
for node in flow.nodes:
|
||||
declared = (
|
||||
NODE_TYPES[node.type].params_schema.get("properties", {})
|
||||
if node.type in NODE_TYPES
|
||||
else {}
|
||||
)
|
||||
for key, value in node.params.items():
|
||||
text = setting_text(key, value, declared.get(key), secrets)
|
||||
if text:
|
||||
entries.append(
|
||||
SearchEntry(
|
||||
category="setting",
|
||||
name=f"{node.id}.{key}",
|
||||
parent=flow.name,
|
||||
kind=node.type,
|
||||
detail=text,
|
||||
)
|
||||
)
|
||||
# Its boundary, as the canvas draws it: a live flow never finishes, so
|
||||
# only a batch one has outputs to report.
|
||||
boundary: list[tuple[Category, str, str]] = [
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
"""The one index the global search matches against."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from fluksio.api.routes.search import MASK, setting_text
|
||||
from fluksio.core.config import settings
|
||||
|
||||
PREFIX = f"{settings.API_V1_STR}/search"
|
||||
@@ -101,3 +104,89 @@ def test_secrets_are_named_only_to_a_superuser(
|
||||
|
||||
assert "broker_password" in secrets(superuser_token_headers)
|
||||
assert secrets(normal_user_token_headers) == set()
|
||||
|
||||
|
||||
def test_a_setting_never_shows_a_credential() -> None:
|
||||
"""The masking rules, one by one: nothing secret reaches the index."""
|
||||
plain = {"type": "number"}
|
||||
secret = {"type": "string", "x-secret": True}
|
||||
|
||||
assert setting_text("interval", 5, plain, False) == "5"
|
||||
assert setting_text("at_start", True, None, False) == "true"
|
||||
assert setting_text("topic", "", plain, False) == ""
|
||||
assert setting_text("topic", None, plain, False) == ""
|
||||
|
||||
# A declared credential: a reference is named for those who may read names,
|
||||
# a plain string never appears.
|
||||
ref = {"$secret": "ntfy_token"}
|
||||
assert setting_text("token", ref, secret, True) == "secret ntfy_token"
|
||||
assert setting_text("token", ref, secret, False) == "secret"
|
||||
assert setting_text("token", "hunter2", secret, True) == MASK
|
||||
|
||||
# An undeclared one is judged by its key, and a reference stays a reference.
|
||||
assert setting_text("api_key", "hunter2", None, True) == MASK
|
||||
assert setting_text("api_key", ref, None, False) == "secret"
|
||||
|
||||
# Nested, such as a header: what is neutral stays, what is not goes.
|
||||
headers: dict[str, Any] = {
|
||||
"Accept": "text/plain",
|
||||
"Authorization": "Bearer hunter2",
|
||||
"X-Api-Key": {"$secret": "key_name"},
|
||||
}
|
||||
text = setting_text("headers", headers, {"type": "object"}, False)
|
||||
assert "hunter2" not in text and "key_name" not in text
|
||||
assert '"Accept":"text/plain"' in text
|
||||
assert f'"Authorization":"{MASK}"' in text
|
||||
assert '"X-Api-Key":"secret"' in text
|
||||
|
||||
# A long value is clipped, so a document is not an index entry.
|
||||
clipped = setting_text("payload", "x" * 500, plain, False)
|
||||
assert len(clipped) == 80 and clipped.endswith("\u2026")
|
||||
|
||||
|
||||
def test_index_lists_a_nodes_settings(
|
||||
client: TestClient,
|
||||
superuser_token_headers: dict[str, str],
|
||||
normal_user_token_headers: dict[str, str],
|
||||
) -> None:
|
||||
"""A setting is found by its node and name, with its value beside it."""
|
||||
client.put(
|
||||
f"{FLOWS}/tuned",
|
||||
headers=superuser_token_headers,
|
||||
json={
|
||||
"name": "tuned",
|
||||
"nodes": [
|
||||
{"id": "gain", "type": "python", "params": {"factor": 2.5}},
|
||||
{
|
||||
"id": "hook",
|
||||
"type": "http",
|
||||
"params": {
|
||||
"url": "/tick",
|
||||
"secret": {"$secret": "hook_secret"},
|
||||
"headers": {"Authorization": "Bearer s3cr3t-value"},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
def settings_of(headers: dict[str, str]) -> tuple[dict[str, dict], str]:
|
||||
response = client.get(f"{PREFIX}/", headers=headers)
|
||||
found = {
|
||||
e["name"]: e
|
||||
for e in response.json()
|
||||
if e["category"] == "setting" and e["parent"] == "tuned"
|
||||
}
|
||||
return found, response.text
|
||||
|
||||
found, _ = settings_of(superuser_token_headers)
|
||||
assert found["gain.factor"]["detail"] == "2.5"
|
||||
assert found["gain.factor"]["kind"] == "python"
|
||||
assert found["hook.url"]["detail"] == "/tick"
|
||||
assert found["hook.secret"]["detail"] == "secret hook_secret"
|
||||
|
||||
# Who may not read a secret's name does not get it from here either.
|
||||
found, text = settings_of(normal_user_token_headers)
|
||||
assert found["hook.secret"]["detail"] == "secret"
|
||||
assert "hook_secret" not in text
|
||||
assert "s3cr3t-value" not in text
|
||||
|
||||
Reference in New Issue
Block a user