Search: node settings are indexed, with credentials masked on the server
/search gains a `setting` category, `<node>.<setting>` under its flow with the node type and the value in a new `detail` field, so Ctrl-K finds which node uses a broker or a timeout and opens the node. `setting_text` is the one place that decides what may be said: a reference to a stored secret shows its name (to superusers only, as the secrets list does), a plain value in an `x-secret` field or under a key like password, token or api_key is masked and never indexed, nested credentials in headers or query are masked, long values are clipped. The global search ranks what contains the query as typed ahead of loose spellings before it caps a group, which long setting values would otherwise have crowded out.
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
"""One index of everything in this instance worth jumping to by name."""
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
@@ -14,7 +16,7 @@ from fluksio.api.deps import (
|
||||
)
|
||||
from fluksio.api.routes.alerts import read_config as read_alerts_config
|
||||
from fluksio.flow import modules
|
||||
from fluksio.flow.controller import FlowController
|
||||
from fluksio.flow.controller import NODE_TYPES, FlowController
|
||||
from fluksio.flow.dashboards import DashboardNotFound, DashboardStore
|
||||
from fluksio.flow.messages import qualify
|
||||
from fluksio.flow.secrets import get_secrets
|
||||
@@ -32,6 +34,7 @@ Category = Literal[
|
||||
"node",
|
||||
"input",
|
||||
"output",
|
||||
"setting",
|
||||
"dashboard",
|
||||
"widget",
|
||||
"device",
|
||||
@@ -52,7 +55,9 @@ class SearchEntry(BaseModel):
|
||||
|
||||
category: Category
|
||||
#: The id the frontend routes on. For an input or output, the qualified
|
||||
#: message, which is what the canvas names its label by.
|
||||
#: message, which is what the canvas names its label by. For a setting,
|
||||
#: ``<node>.<setting>``: a node id has no dot, so the node is what comes
|
||||
#: before the first one.
|
||||
name: str
|
||||
#: Human title, often empty — a flow is usually only its name.
|
||||
title: str = ""
|
||||
@@ -61,6 +66,82 @@ class SearchEntry(BaseModel):
|
||||
parent: str = ""
|
||||
#: Node type, input dtype, widget type, channel kind.
|
||||
kind: str = ""
|
||||
#: What a setting is set to, as text. Only ever what ``setting_text`` lets
|
||||
#: through, so never a credential.
|
||||
detail: str = ""
|
||||
|
||||
|
||||
#: What stands in for a value that must not be shown, or searched for.
|
||||
MASK = "\u2022\u2022\u2022\u2022"
|
||||
|
||||
#: How much of a value the index keeps. A setting can be a whole JSON document.
|
||||
_DETAIL_CAP = 80
|
||||
|
||||
#: Keys that tend to hold a credential. Applied to every setting, declared or
|
||||
#: not: a type's author can forget its ``x-secret``, and a masked value only
|
||||
#: costs a search that could not have helped.
|
||||
_SENSITIVE_KEY = re.compile(r"pass|token|secret|key|auth|credential", re.IGNORECASE)
|
||||
|
||||
|
||||
def _is_secret_ref(value: Any) -> bool:
|
||||
return isinstance(value, dict) and "$secret" in value
|
||||
|
||||
|
||||
def _secret_ref(value: dict[str, Any], names: bool) -> str:
|
||||
"""A reference to a stored secret: its name, to those who may read names."""
|
||||
name = value.get("$secret")
|
||||
return f"secret {name}" if names and isinstance(name, str) and name else "secret"
|
||||
|
||||
|
||||
def _scrub(value: Any, names: bool) -> Any:
|
||||
"""``value`` with every credential inside it replaced.
|
||||
|
||||
A stored secret is named, never read. Whatever sits under a key that looks
|
||||
like a credential is masked, however deep: a header or a query parameter is
|
||||
as good a place for a token as any.
|
||||
"""
|
||||
if _is_secret_ref(value):
|
||||
return _secret_ref(value, names)
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: MASK
|
||||
if _SENSITIVE_KEY.search(str(key))
|
||||
and item not in (None, "")
|
||||
and not _is_secret_ref(item)
|
||||
else _scrub(item, names)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [_scrub(item, names) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def setting_text(
|
||||
key: str, value: Any, declared: dict[str, Any] | None, names: bool
|
||||
) -> str:
|
||||
"""What the search may say about one setting, never its secret value.
|
||||
|
||||
``declared`` is the node type's schema for this setting, None where the
|
||||
type has none, as for every setting of a function node. A reference to a
|
||||
stored secret is named (to those who may read names). A plain value in a
|
||||
field the type flags ``x-secret``, or under a key that looks like a
|
||||
credential, is masked. An empty setting is "" and gets no entry.
|
||||
"""
|
||||
if value is None or value == "":
|
||||
return ""
|
||||
if _is_secret_ref(value):
|
||||
text = _secret_ref(value, names)
|
||||
elif (declared or {}).get("x-secret") or _SENSITIVE_KEY.search(key):
|
||||
text = MASK
|
||||
elif isinstance(value, dict | list):
|
||||
text = json.dumps(
|
||||
_scrub(value, names), separators=(",", ":"), ensure_ascii=False
|
||||
)
|
||||
elif isinstance(value, bool):
|
||||
text = "true" if value else "false"
|
||||
else:
|
||||
text = str(value)
|
||||
return text if len(text) <= _DETAIL_CAP else text[: _DETAIL_CAP - 1] + "\u2026"
|
||||
|
||||
|
||||
def _build(
|
||||
@@ -89,6 +170,24 @@ def _build(
|
||||
)
|
||||
for node in flow.nodes
|
||||
)
|
||||
for node in flow.nodes:
|
||||
declared = (
|
||||
NODE_TYPES[node.type].params_schema.get("properties", {})
|
||||
if node.type in NODE_TYPES
|
||||
else {}
|
||||
)
|
||||
for key, value in node.params.items():
|
||||
text = setting_text(key, value, declared.get(key), secrets)
|
||||
if text:
|
||||
entries.append(
|
||||
SearchEntry(
|
||||
category="setting",
|
||||
name=f"{node.id}.{key}",
|
||||
parent=flow.name,
|
||||
kind=node.type,
|
||||
detail=text,
|
||||
)
|
||||
)
|
||||
# Its boundary, as the canvas draws it: a live flow never finishes, so
|
||||
# only a batch one has outputs to report.
|
||||
boundary: list[tuple[Category, str, str]] = [
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
"""The one index the global search matches against."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from fluksio.api.routes.search import MASK, setting_text
|
||||
from fluksio.core.config import settings
|
||||
|
||||
PREFIX = f"{settings.API_V1_STR}/search"
|
||||
@@ -101,3 +104,89 @@ def test_secrets_are_named_only_to_a_superuser(
|
||||
|
||||
assert "broker_password" in secrets(superuser_token_headers)
|
||||
assert secrets(normal_user_token_headers) == set()
|
||||
|
||||
|
||||
def test_a_setting_never_shows_a_credential() -> None:
|
||||
"""The masking rules, one by one: nothing secret reaches the index."""
|
||||
plain = {"type": "number"}
|
||||
secret = {"type": "string", "x-secret": True}
|
||||
|
||||
assert setting_text("interval", 5, plain, False) == "5"
|
||||
assert setting_text("at_start", True, None, False) == "true"
|
||||
assert setting_text("topic", "", plain, False) == ""
|
||||
assert setting_text("topic", None, plain, False) == ""
|
||||
|
||||
# A declared credential: a reference is named for those who may read names,
|
||||
# a plain string never appears.
|
||||
ref = {"$secret": "ntfy_token"}
|
||||
assert setting_text("token", ref, secret, True) == "secret ntfy_token"
|
||||
assert setting_text("token", ref, secret, False) == "secret"
|
||||
assert setting_text("token", "hunter2", secret, True) == MASK
|
||||
|
||||
# An undeclared one is judged by its key, and a reference stays a reference.
|
||||
assert setting_text("api_key", "hunter2", None, True) == MASK
|
||||
assert setting_text("api_key", ref, None, False) == "secret"
|
||||
|
||||
# Nested, such as a header: what is neutral stays, what is not goes.
|
||||
headers: dict[str, Any] = {
|
||||
"Accept": "text/plain",
|
||||
"Authorization": "Bearer hunter2",
|
||||
"X-Api-Key": {"$secret": "key_name"},
|
||||
}
|
||||
text = setting_text("headers", headers, {"type": "object"}, False)
|
||||
assert "hunter2" not in text and "key_name" not in text
|
||||
assert '"Accept":"text/plain"' in text
|
||||
assert f'"Authorization":"{MASK}"' in text
|
||||
assert '"X-Api-Key":"secret"' in text
|
||||
|
||||
# A long value is clipped, so a document is not an index entry.
|
||||
clipped = setting_text("payload", "x" * 500, plain, False)
|
||||
assert len(clipped) == 80 and clipped.endswith("\u2026")
|
||||
|
||||
|
||||
def test_index_lists_a_nodes_settings(
|
||||
client: TestClient,
|
||||
superuser_token_headers: dict[str, str],
|
||||
normal_user_token_headers: dict[str, str],
|
||||
) -> None:
|
||||
"""A setting is found by its node and name, with its value beside it."""
|
||||
client.put(
|
||||
f"{FLOWS}/tuned",
|
||||
headers=superuser_token_headers,
|
||||
json={
|
||||
"name": "tuned",
|
||||
"nodes": [
|
||||
{"id": "gain", "type": "python", "params": {"factor": 2.5}},
|
||||
{
|
||||
"id": "hook",
|
||||
"type": "http",
|
||||
"params": {
|
||||
"url": "/tick",
|
||||
"secret": {"$secret": "hook_secret"},
|
||||
"headers": {"Authorization": "Bearer s3cr3t-value"},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
def settings_of(headers: dict[str, str]) -> tuple[dict[str, dict], str]:
|
||||
response = client.get(f"{PREFIX}/", headers=headers)
|
||||
found = {
|
||||
e["name"]: e
|
||||
for e in response.json()
|
||||
if e["category"] == "setting" and e["parent"] == "tuned"
|
||||
}
|
||||
return found, response.text
|
||||
|
||||
found, _ = settings_of(superuser_token_headers)
|
||||
assert found["gain.factor"]["detail"] == "2.5"
|
||||
assert found["gain.factor"]["kind"] == "python"
|
||||
assert found["hook.url"]["detail"] == "/tick"
|
||||
assert found["hook.secret"]["detail"] == "secret hook_secret"
|
||||
|
||||
# Who may not read a secret's name does not get it from here either.
|
||||
found, text = settings_of(normal_user_token_headers)
|
||||
assert found["hook.secret"]["detail"] == "secret"
|
||||
assert "hook_secret" not in text
|
||||
assert "s3cr3t-value" not in text
|
||||
|
||||
+10
-3
@@ -31,9 +31,16 @@ instead: on Home it opens the Control tab, on any other page the assistant.
|
||||
|
||||
### Search
|
||||
|
||||
Use **Search**, or ⌘K / Ctrl-K, to find flows, nodes, flow inputs and outputs,
|
||||
dashboards, widgets, devices, secrets, modules, workers and alert channels.
|
||||
Selecting a result opens its editor or settings.
|
||||
Use **Search**, or ⌘K / Ctrl-K, to find flows, nodes, node settings, flow
|
||||
inputs and outputs, dashboards, widgets, devices, secrets, modules, workers and
|
||||
alert channels. Selecting a result opens its editor or settings.
|
||||
|
||||
A setting is listed as `node.setting` with its value beside it, so you can look
|
||||
up which node uses a broker, a topic or a timeout. A credential is never shown.
|
||||
A setting that points at a stored secret shows only the secret's name, and only to
|
||||
superusers. A plain value in a credential field, or under a key such
|
||||
as `password`, `token` or `api_key`, shows as `••••` and cannot be searched
|
||||
for.
|
||||
|
||||
Search covers the current instance. When connected through a portal,
|
||||
**All instances** returns to the instance list.
|
||||
|
||||
@@ -4112,7 +4112,7 @@ export const SearchEntrySchema = {
|
||||
properties: {
|
||||
category: {
|
||||
type: 'string',
|
||||
enum: ['flow', 'node', 'input', 'output', 'dashboard', 'widget', 'device', 'secret', 'module', 'worker', 'alert'],
|
||||
enum: ['flow', 'node', 'input', 'output', 'setting', 'dashboard', 'widget', 'device', 'secret', 'module', 'worker', 'alert'],
|
||||
title: 'Category'
|
||||
},
|
||||
name: {
|
||||
@@ -4133,6 +4133,11 @@ export const SearchEntrySchema = {
|
||||
type: 'string',
|
||||
title: 'Kind',
|
||||
default: ''
|
||||
},
|
||||
detail: {
|
||||
type: 'string',
|
||||
title: 'Detail',
|
||||
default: ''
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
|
||||
@@ -1350,14 +1350,15 @@ export type RunRequest = {
|
||||
* called, which is all the matching needs.
|
||||
*/
|
||||
export type SearchEntry = {
|
||||
category: 'flow' | 'node' | 'input' | 'output' | 'dashboard' | 'widget' | 'device' | 'secret' | 'module' | 'worker' | 'alert';
|
||||
category: 'flow' | 'node' | 'input' | 'output' | 'setting' | 'dashboard' | 'widget' | 'device' | 'secret' | 'module' | 'worker' | 'alert';
|
||||
name: string;
|
||||
title?: string;
|
||||
parent?: string;
|
||||
kind?: string;
|
||||
detail?: string;
|
||||
};
|
||||
|
||||
export type category = 'flow' | 'node' | 'input' | 'output' | 'dashboard' | 'widget' | 'device' | 'secret' | 'module' | 'worker' | 'alert';
|
||||
export type category = 'flow' | 'node' | 'input' | 'output' | 'setting' | 'dashboard' | 'widget' | 'device' | 'secret' | 'module' | 'worker' | 'alert';
|
||||
|
||||
export type SecretNames = {
|
||||
data: Array<(string)>;
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
MonitorSmartphone,
|
||||
Package,
|
||||
Server,
|
||||
SlidersHorizontal,
|
||||
Workflow,
|
||||
} from "lucide-react"
|
||||
import { useState } from "react"
|
||||
@@ -49,6 +50,7 @@ const GROUPS: {
|
||||
{ category: "node", label: "Nodes", icon: Box },
|
||||
{ category: "input", label: "Inputs", icon: LogIn },
|
||||
{ category: "output", label: "Outputs", icon: LogOut },
|
||||
{ category: "setting", label: "Settings", icon: SlidersHorizontal },
|
||||
{ category: "dashboard", label: "Dashboards", icon: LayoutDashboard },
|
||||
{ category: "widget", label: "Widgets", icon: LayoutGrid },
|
||||
{ category: "device", label: "Devices", icon: MonitorSmartphone },
|
||||
@@ -58,14 +60,17 @@ const GROUPS: {
|
||||
{ category: "alert", label: "Alerts", icon: Bell },
|
||||
]
|
||||
|
||||
/** The second line: where the thing lives, and what kind it is. */
|
||||
/**
|
||||
* The second line: where the thing lives, what kind it is, and for a setting
|
||||
* what it is set to.
|
||||
*/
|
||||
function hint(entry: SearchEntry): string {
|
||||
return [entry.parent, entry.kind].filter(Boolean).join(" · ")
|
||||
return [entry.parent, entry.kind, entry.detail].filter(Boolean).join(" · ")
|
||||
}
|
||||
|
||||
/** Everything about an entry that is worth matching against. */
|
||||
function searchValue(entry: SearchEntry): string {
|
||||
return `${entry.name} ${entry.title ?? ""} ${entry.parent ?? ""} ${entry.kind ?? ""}`
|
||||
return `${entry.name} ${entry.title ?? ""} ${entry.parent ?? ""} ${entry.kind ?? ""} ${entry.detail ?? ""}`
|
||||
}
|
||||
|
||||
/** cmdk lowercases and flattens whitespace and hyphens before it scores. */
|
||||
@@ -137,13 +142,21 @@ export function GlobalSearch({
|
||||
// every entry on each keystroke. Not memoised: the component returns early
|
||||
// while closed, so a hook cannot go here, and one pass is already the win.
|
||||
// The same pass drops what the query cannot reach, so each bucket is already
|
||||
// candidates by the time GROUP_CAP trims it.
|
||||
const byCategory = new Map<string, SearchEntry[]>()
|
||||
// candidates by the time GROUP_CAP trims it. What contains the query as
|
||||
// typed goes first: a long value (a setting's) is reachable by a loose
|
||||
// spelling of almost anything, and those must not crowd out the real hit.
|
||||
const wanted = normalise(typed)
|
||||
// Per category: what contains the query as typed, and what only spells it
|
||||
// loosely.
|
||||
const exact = new Map<string, SearchEntry[]>()
|
||||
const loose = new Map<string, SearchEntry[]>()
|
||||
for (const entry of entries) {
|
||||
if (!couldMatch(searchValue(entry), typed)) continue
|
||||
const bucket = byCategory.get(entry.category)
|
||||
const value = searchValue(entry)
|
||||
if (!couldMatch(value, typed)) continue
|
||||
const into = normalise(value).includes(wanted) ? exact : loose
|
||||
const bucket = into.get(entry.category)
|
||||
if (bucket) bucket.push(entry)
|
||||
else byCategory.set(entry.category, [entry])
|
||||
else into.set(entry.category, [entry])
|
||||
}
|
||||
|
||||
const go = (entry: SearchEntry) => {
|
||||
@@ -161,6 +174,13 @@ export function GlobalSearch({
|
||||
params: { flowName: entry.parent ?? "" },
|
||||
search: { node: entry.name },
|
||||
})
|
||||
// A setting is `<node>.<setting>`, and a node id has no dot.
|
||||
case "setting":
|
||||
return navigate({
|
||||
to: "/flows/$flowName",
|
||||
params: { flowName: entry.parent ?? "" },
|
||||
search: { node: entry.name.split(".")[0] },
|
||||
})
|
||||
// Arriving on the label's id does what clicking the label does.
|
||||
case "input":
|
||||
case "output":
|
||||
@@ -201,14 +221,14 @@ export function GlobalSearch({
|
||||
open={open}
|
||||
onOpenChange={onOpenChange}
|
||||
title="Search"
|
||||
description="Find a flow, node, dashboard or widget"
|
||||
description="Find a flow, node, setting, dashboard or widget"
|
||||
showCloseButton={false}
|
||||
className="top-[40%] bg-popover/80 shadow-e3 backdrop-blur-md sm:max-w-xl [&_[data-slot=command]]:bg-transparent"
|
||||
>
|
||||
<CommandInput
|
||||
value={query}
|
||||
onValueChange={setQuery}
|
||||
placeholder="Search flows, nodes, dashboards, widgets…"
|
||||
placeholder="Search flows, nodes, settings, dashboards, widgets…"
|
||||
data-testid="global-search-input"
|
||||
/>
|
||||
<CommandList className="max-h-[min(24rem,60svh)]">
|
||||
@@ -216,7 +236,10 @@ export function GlobalSearch({
|
||||
<>
|
||||
<CommandEmpty>Nothing matches that.</CommandEmpty>
|
||||
{GROUPS.map(({ category, label, icon: Icon }) => {
|
||||
const found = (byCategory.get(category) ?? []).slice(0, GROUP_CAP)
|
||||
const found = [
|
||||
...(exact.get(category) ?? []),
|
||||
...(loose.get(category) ?? []),
|
||||
].slice(0, GROUP_CAP)
|
||||
if (found.length === 0) return null
|
||||
return (
|
||||
<CommandGroup key={category} heading={label}>
|
||||
|
||||
@@ -21,7 +21,14 @@ test.beforeAll(async ({ browser }) => {
|
||||
data: {
|
||||
name: flowName,
|
||||
title: "Searchable flow",
|
||||
nodes: [{ id: "findme", type: "python", title: "Find me" }],
|
||||
nodes: [
|
||||
{
|
||||
id: "findme",
|
||||
type: "python",
|
||||
title: "Find me",
|
||||
params: { tuning_gain: 7, api_token: "s3cr3t-token" },
|
||||
},
|
||||
],
|
||||
inputs: [{ spec: { name: "findme_input", dtype: "float" } }],
|
||||
},
|
||||
})
|
||||
@@ -102,3 +109,26 @@ test("opens on the keyboard from anywhere", async ({ page }) => {
|
||||
|
||||
await expect(page.getByTestId("global-search-input")).toBeFocused()
|
||||
})
|
||||
|
||||
test("finds a setting with its value, and never a credential", async ({
|
||||
page,
|
||||
}) => {
|
||||
await page.goto("/")
|
||||
await page.getByRole("button", { name: "Search", exact: true }).click()
|
||||
const input = page.getByTestId("global-search-input")
|
||||
|
||||
await input.fill("tuning_gain")
|
||||
const gain = page.getByRole("option", { name: /findme\.tuning_gain/ })
|
||||
await expect(gain).toContainText(`${flowName} · python · 7`)
|
||||
|
||||
// The token is masked on the server, so it never reaches the page.
|
||||
await input.fill("api_token")
|
||||
await expect(
|
||||
page.getByRole("option", { name: /findme\.api_token/ }),
|
||||
).toContainText("••••")
|
||||
await expect(page.getByRole("dialog")).not.toContainText("s3cr3t-token")
|
||||
|
||||
await input.fill("tuning_gain")
|
||||
await gain.click()
|
||||
await expect(page).toHaveURL(new RegExp(`/flows/${flowName}\\?node=findme$`))
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user