diff --git a/backend/fluksio/api/routes/search.py b/backend/fluksio/api/routes/search.py index 484993e..c5d124b 100644 --- a/backend/fluksio/api/routes/search.py +++ b/backend/fluksio/api/routes/search.py @@ -1,5 +1,7 @@ """One index of everything in this instance worth jumping to by name.""" +import json +import re from typing import Any, Literal from fastapi import APIRouter, Depends, Request @@ -14,7 +16,7 @@ from fluksio.api.deps import ( ) from fluksio.api.routes.alerts import read_config as read_alerts_config from fluksio.flow import modules -from fluksio.flow.controller import FlowController +from fluksio.flow.controller import NODE_TYPES, FlowController from fluksio.flow.dashboards import DashboardNotFound, DashboardStore from fluksio.flow.messages import qualify from fluksio.flow.secrets import get_secrets @@ -32,6 +34,7 @@ Category = Literal[ "node", "input", "output", + "setting", "dashboard", "widget", "device", @@ -52,7 +55,9 @@ class SearchEntry(BaseModel): category: Category #: The id the frontend routes on. For an input or output, the qualified - #: message, which is what the canvas names its label by. + #: message, which is what the canvas names its label by. For a setting, + #: ``.``: a node id has no dot, so the node is what comes + #: before the first one. name: str #: Human title, often empty — a flow is usually only its name. title: str = "" @@ -61,6 +66,82 @@ class SearchEntry(BaseModel): parent: str = "" #: Node type, input dtype, widget type, channel kind. kind: str = "" + #: What a setting is set to, as text. Only ever what ``setting_text`` lets + #: through, so never a credential. + detail: str = "" + + +#: What stands in for a value that must not be shown, or searched for. +MASK = "\u2022\u2022\u2022\u2022" + +#: How much of a value the index keeps. A setting can be a whole JSON document. +_DETAIL_CAP = 80 + +#: Keys that tend to hold a credential. Applied to every setting, declared or +#: not: a type's author can forget its ``x-secret``, and a masked value only +#: costs a search that could not have helped. +_SENSITIVE_KEY = re.compile(r"pass|token|secret|key|auth|credential", re.IGNORECASE) + + +def _is_secret_ref(value: Any) -> bool: + return isinstance(value, dict) and "$secret" in value + + +def _secret_ref(value: dict[str, Any], names: bool) -> str: + """A reference to a stored secret: its name, to those who may read names.""" + name = value.get("$secret") + return f"secret {name}" if names and isinstance(name, str) and name else "secret" + + +def _scrub(value: Any, names: bool) -> Any: + """``value`` with every credential inside it replaced. + + A stored secret is named, never read. Whatever sits under a key that looks + like a credential is masked, however deep: a header or a query parameter is + as good a place for a token as any. + """ + if _is_secret_ref(value): + return _secret_ref(value, names) + if isinstance(value, dict): + return { + key: MASK + if _SENSITIVE_KEY.search(str(key)) + and item not in (None, "") + and not _is_secret_ref(item) + else _scrub(item, names) + for key, item in value.items() + } + if isinstance(value, list): + return [_scrub(item, names) for item in value] + return value + + +def setting_text( + key: str, value: Any, declared: dict[str, Any] | None, names: bool +) -> str: + """What the search may say about one setting, never its secret value. + + ``declared`` is the node type's schema for this setting, None where the + type has none, as for every setting of a function node. A reference to a + stored secret is named (to those who may read names). A plain value in a + field the type flags ``x-secret``, or under a key that looks like a + credential, is masked. An empty setting is "" and gets no entry. + """ + if value is None or value == "": + return "" + if _is_secret_ref(value): + text = _secret_ref(value, names) + elif (declared or {}).get("x-secret") or _SENSITIVE_KEY.search(key): + text = MASK + elif isinstance(value, dict | list): + text = json.dumps( + _scrub(value, names), separators=(",", ":"), ensure_ascii=False + ) + elif isinstance(value, bool): + text = "true" if value else "false" + else: + text = str(value) + return text if len(text) <= _DETAIL_CAP else text[: _DETAIL_CAP - 1] + "\u2026" def _build( @@ -89,6 +170,24 @@ def _build( ) for node in flow.nodes ) + for node in flow.nodes: + declared = ( + NODE_TYPES[node.type].params_schema.get("properties", {}) + if node.type in NODE_TYPES + else {} + ) + for key, value in node.params.items(): + text = setting_text(key, value, declared.get(key), secrets) + if text: + entries.append( + SearchEntry( + category="setting", + name=f"{node.id}.{key}", + parent=flow.name, + kind=node.type, + detail=text, + ) + ) # Its boundary, as the canvas draws it: a live flow never finishes, so # only a batch one has outputs to report. boundary: list[tuple[Category, str, str]] = [ diff --git a/backend/tests/api/routes/test_search.py b/backend/tests/api/routes/test_search.py index 5ce875e..7eff20a 100644 --- a/backend/tests/api/routes/test_search.py +++ b/backend/tests/api/routes/test_search.py @@ -1,7 +1,10 @@ """The one index the global search matches against.""" +from typing import Any + from fastapi.testclient import TestClient +from fluksio.api.routes.search import MASK, setting_text from fluksio.core.config import settings PREFIX = f"{settings.API_V1_STR}/search" @@ -101,3 +104,89 @@ def test_secrets_are_named_only_to_a_superuser( assert "broker_password" in secrets(superuser_token_headers) assert secrets(normal_user_token_headers) == set() + + +def test_a_setting_never_shows_a_credential() -> None: + """The masking rules, one by one: nothing secret reaches the index.""" + plain = {"type": "number"} + secret = {"type": "string", "x-secret": True} + + assert setting_text("interval", 5, plain, False) == "5" + assert setting_text("at_start", True, None, False) == "true" + assert setting_text("topic", "", plain, False) == "" + assert setting_text("topic", None, plain, False) == "" + + # A declared credential: a reference is named for those who may read names, + # a plain string never appears. + ref = {"$secret": "ntfy_token"} + assert setting_text("token", ref, secret, True) == "secret ntfy_token" + assert setting_text("token", ref, secret, False) == "secret" + assert setting_text("token", "hunter2", secret, True) == MASK + + # An undeclared one is judged by its key, and a reference stays a reference. + assert setting_text("api_key", "hunter2", None, True) == MASK + assert setting_text("api_key", ref, None, False) == "secret" + + # Nested, such as a header: what is neutral stays, what is not goes. + headers: dict[str, Any] = { + "Accept": "text/plain", + "Authorization": "Bearer hunter2", + "X-Api-Key": {"$secret": "key_name"}, + } + text = setting_text("headers", headers, {"type": "object"}, False) + assert "hunter2" not in text and "key_name" not in text + assert '"Accept":"text/plain"' in text + assert f'"Authorization":"{MASK}"' in text + assert '"X-Api-Key":"secret"' in text + + # A long value is clipped, so a document is not an index entry. + clipped = setting_text("payload", "x" * 500, plain, False) + assert len(clipped) == 80 and clipped.endswith("\u2026") + + +def test_index_lists_a_nodes_settings( + client: TestClient, + superuser_token_headers: dict[str, str], + normal_user_token_headers: dict[str, str], +) -> None: + """A setting is found by its node and name, with its value beside it.""" + client.put( + f"{FLOWS}/tuned", + headers=superuser_token_headers, + json={ + "name": "tuned", + "nodes": [ + {"id": "gain", "type": "python", "params": {"factor": 2.5}}, + { + "id": "hook", + "type": "http", + "params": { + "url": "/tick", + "secret": {"$secret": "hook_secret"}, + "headers": {"Authorization": "Bearer s3cr3t-value"}, + }, + }, + ], + }, + ) + + def settings_of(headers: dict[str, str]) -> tuple[dict[str, dict], str]: + response = client.get(f"{PREFIX}/", headers=headers) + found = { + e["name"]: e + for e in response.json() + if e["category"] == "setting" and e["parent"] == "tuned" + } + return found, response.text + + found, _ = settings_of(superuser_token_headers) + assert found["gain.factor"]["detail"] == "2.5" + assert found["gain.factor"]["kind"] == "python" + assert found["hook.url"]["detail"] == "/tick" + assert found["hook.secret"]["detail"] == "secret hook_secret" + + # Who may not read a secret's name does not get it from here either. + found, text = settings_of(normal_user_token_headers) + assert found["hook.secret"]["detail"] == "secret" + assert "hook_secret" not in text + assert "s3cr3t-value" not in text diff --git a/docs/interface/index.md b/docs/interface/index.md index 2e09f38..0151090 100644 --- a/docs/interface/index.md +++ b/docs/interface/index.md @@ -31,9 +31,16 @@ instead: on Home it opens the Control tab, on any other page the assistant. ### Search -Use **Search**, or ⌘K / Ctrl-K, to find flows, nodes, flow inputs and outputs, -dashboards, widgets, devices, secrets, modules, workers and alert channels. -Selecting a result opens its editor or settings. +Use **Search**, or ⌘K / Ctrl-K, to find flows, nodes, node settings, flow +inputs and outputs, dashboards, widgets, devices, secrets, modules, workers and +alert channels. Selecting a result opens its editor or settings. + +A setting is listed as `node.setting` with its value beside it, so you can look +up which node uses a broker, a topic or a timeout. A credential is never shown. +A setting that points at a stored secret shows only the secret's name, and only to +superusers. A plain value in a credential field, or under a key such +as `password`, `token` or `api_key`, shows as `••••` and cannot be searched +for. Search covers the current instance. When connected through a portal, **All instances** returns to the instance list. diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index c740387..0c4bd23 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -4112,7 +4112,7 @@ export const SearchEntrySchema = { properties: { category: { type: 'string', - enum: ['flow', 'node', 'input', 'output', 'dashboard', 'widget', 'device', 'secret', 'module', 'worker', 'alert'], + enum: ['flow', 'node', 'input', 'output', 'setting', 'dashboard', 'widget', 'device', 'secret', 'module', 'worker', 'alert'], title: 'Category' }, name: { @@ -4133,6 +4133,11 @@ export const SearchEntrySchema = { type: 'string', title: 'Kind', default: '' + }, + detail: { + type: 'string', + title: 'Detail', + default: '' } }, type: 'object', diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index d00fe4d..894d116 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -1350,14 +1350,15 @@ export type RunRequest = { * called, which is all the matching needs. */ export type SearchEntry = { - category: 'flow' | 'node' | 'input' | 'output' | 'dashboard' | 'widget' | 'device' | 'secret' | 'module' | 'worker' | 'alert'; + category: 'flow' | 'node' | 'input' | 'output' | 'setting' | 'dashboard' | 'widget' | 'device' | 'secret' | 'module' | 'worker' | 'alert'; name: string; title?: string; parent?: string; kind?: string; + detail?: string; }; -export type category = 'flow' | 'node' | 'input' | 'output' | 'dashboard' | 'widget' | 'device' | 'secret' | 'module' | 'worker' | 'alert'; +export type category = 'flow' | 'node' | 'input' | 'output' | 'setting' | 'dashboard' | 'widget' | 'device' | 'secret' | 'module' | 'worker' | 'alert'; export type SecretNames = { data: Array<(string)>; diff --git a/frontend/src/components/Common/GlobalSearch.tsx b/frontend/src/components/Common/GlobalSearch.tsx index e056b7c..e47eb6b 100644 --- a/frontend/src/components/Common/GlobalSearch.tsx +++ b/frontend/src/components/Common/GlobalSearch.tsx @@ -12,6 +12,7 @@ import { MonitorSmartphone, Package, Server, + SlidersHorizontal, Workflow, } from "lucide-react" import { useState } from "react" @@ -49,6 +50,7 @@ const GROUPS: { { category: "node", label: "Nodes", icon: Box }, { category: "input", label: "Inputs", icon: LogIn }, { category: "output", label: "Outputs", icon: LogOut }, + { category: "setting", label: "Settings", icon: SlidersHorizontal }, { category: "dashboard", label: "Dashboards", icon: LayoutDashboard }, { category: "widget", label: "Widgets", icon: LayoutGrid }, { category: "device", label: "Devices", icon: MonitorSmartphone }, @@ -58,14 +60,17 @@ const GROUPS: { { category: "alert", label: "Alerts", icon: Bell }, ] -/** The second line: where the thing lives, and what kind it is. */ +/** + * The second line: where the thing lives, what kind it is, and for a setting + * what it is set to. + */ function hint(entry: SearchEntry): string { - return [entry.parent, entry.kind].filter(Boolean).join(" · ") + return [entry.parent, entry.kind, entry.detail].filter(Boolean).join(" · ") } /** Everything about an entry that is worth matching against. */ function searchValue(entry: SearchEntry): string { - return `${entry.name} ${entry.title ?? ""} ${entry.parent ?? ""} ${entry.kind ?? ""}` + return `${entry.name} ${entry.title ?? ""} ${entry.parent ?? ""} ${entry.kind ?? ""} ${entry.detail ?? ""}` } /** cmdk lowercases and flattens whitespace and hyphens before it scores. */ @@ -137,13 +142,21 @@ export function GlobalSearch({ // every entry on each keystroke. Not memoised: the component returns early // while closed, so a hook cannot go here, and one pass is already the win. // The same pass drops what the query cannot reach, so each bucket is already - // candidates by the time GROUP_CAP trims it. - const byCategory = new Map() + // candidates by the time GROUP_CAP trims it. What contains the query as + // typed goes first: a long value (a setting's) is reachable by a loose + // spelling of almost anything, and those must not crowd out the real hit. + const wanted = normalise(typed) + // Per category: what contains the query as typed, and what only spells it + // loosely. + const exact = new Map() + const loose = new Map() for (const entry of entries) { - if (!couldMatch(searchValue(entry), typed)) continue - const bucket = byCategory.get(entry.category) + const value = searchValue(entry) + if (!couldMatch(value, typed)) continue + const into = normalise(value).includes(wanted) ? exact : loose + const bucket = into.get(entry.category) if (bucket) bucket.push(entry) - else byCategory.set(entry.category, [entry]) + else into.set(entry.category, [entry]) } const go = (entry: SearchEntry) => { @@ -161,6 +174,13 @@ export function GlobalSearch({ params: { flowName: entry.parent ?? "" }, search: { node: entry.name }, }) + // A setting is `.`, and a node id has no dot. + case "setting": + return navigate({ + to: "/flows/$flowName", + params: { flowName: entry.parent ?? "" }, + search: { node: entry.name.split(".")[0] }, + }) // Arriving on the label's id does what clicking the label does. case "input": case "output": @@ -201,14 +221,14 @@ export function GlobalSearch({ open={open} onOpenChange={onOpenChange} title="Search" - description="Find a flow, node, dashboard or widget" + description="Find a flow, node, setting, dashboard or widget" showCloseButton={false} className="top-[40%] bg-popover/80 shadow-e3 backdrop-blur-md sm:max-w-xl [&_[data-slot=command]]:bg-transparent" > @@ -216,7 +236,10 @@ export function GlobalSearch({ <> Nothing matches that. {GROUPS.map(({ category, label, icon: Icon }) => { - const found = (byCategory.get(category) ?? []).slice(0, GROUP_CAP) + const found = [ + ...(exact.get(category) ?? []), + ...(loose.get(category) ?? []), + ].slice(0, GROUP_CAP) if (found.length === 0) return null return ( diff --git a/frontend/tests/search.spec.ts b/frontend/tests/search.spec.ts index db9744d..55138d9 100644 --- a/frontend/tests/search.spec.ts +++ b/frontend/tests/search.spec.ts @@ -21,7 +21,14 @@ test.beforeAll(async ({ browser }) => { data: { name: flowName, title: "Searchable flow", - nodes: [{ id: "findme", type: "python", title: "Find me" }], + nodes: [ + { + id: "findme", + type: "python", + title: "Find me", + params: { tuning_gain: 7, api_token: "s3cr3t-token" }, + }, + ], inputs: [{ spec: { name: "findme_input", dtype: "float" } }], }, }) @@ -102,3 +109,26 @@ test("opens on the keyboard from anywhere", async ({ page }) => { await expect(page.getByTestId("global-search-input")).toBeFocused() }) + +test("finds a setting with its value, and never a credential", async ({ + page, +}) => { + await page.goto("/") + await page.getByRole("button", { name: "Search", exact: true }).click() + const input = page.getByTestId("global-search-input") + + await input.fill("tuning_gain") + const gain = page.getByRole("option", { name: /findme\.tuning_gain/ }) + await expect(gain).toContainText(`${flowName} · python · 7`) + + // The token is masked on the server, so it never reaches the page. + await input.fill("api_token") + await expect( + page.getByRole("option", { name: /findme\.api_token/ }), + ).toContainText("••••") + await expect(page.getByRole("dialog")).not.toContainText("s3cr3t-token") + + await input.fill("tuning_gain") + await gain.click() + await expect(page).toHaveURL(new RegExp(`/flows/${flowName}\\?node=findme$`)) +})