692 lines
24 KiB
Python
692 lines
24 KiB
Python
"""Renaming a flow's or a dashboard's key.
|
|
|
|
A rename moves everything named under a flow's key, in documents, history,
|
|
live state and queued work, so nothing may write through the API meanwhile.
|
|
`RenameGate` is the door that keeps it out. `plan_rename` says beforehand what
|
|
a rename would change and what refuses it, and `rewrite_documents` is its
|
|
document part. ``docs/private/id_refac_rename.md`` has the whole protocol.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import threading
|
|
from collections import Counter
|
|
from collections.abc import Callable, Iterable, Iterator
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Literal, cast
|
|
|
|
import orjson
|
|
from pydantic import BaseModel
|
|
from sqlalchemy import String
|
|
from sqlmodel import Session, col, func, select
|
|
from starlette.types import Scope
|
|
|
|
from fluksio.core.config import settings
|
|
from fluksio.core.db import engine as db_engine
|
|
from fluksio.flow.alerts import AlertsConfig
|
|
from fluksio.flow.controller import HOOK_PREFIX, FlowController
|
|
from fluksio.flow.dashboards import (
|
|
DASHBOARD_BINDINGS,
|
|
DASHBOARD_DIR,
|
|
DashboardDef,
|
|
DashboardStore,
|
|
results_name,
|
|
)
|
|
from fluksio.flow.executor import wait_out
|
|
from fluksio.flow.messages import flow_of
|
|
from fluksio.flow.namespace import FLOW_FIELDS, NAME, OUTPUTS, Rename, names, rebind
|
|
from fluksio.flow.schemas import FlowDef, _validate_name
|
|
from fluksio.flow.state import MemoryState
|
|
from fluksio.flow.store import ISSUED_ID, FlowStore, InvalidName, ReservedKey
|
|
from fluksio.models import AlertSent, Run, RunArtifact, RunMetric, RunNode
|
|
|
|
Kind = Literal["flow", "dashboard"]
|
|
|
|
#: What reading looks like. Everything else under the API changes something.
|
|
_READS = frozenset({"GET", "HEAD", "OPTIONS"})
|
|
|
|
#: Mutations that cannot reach a flow's namespace, under the API prefix: signing
|
|
#: in and accounts, the assistant (whose tool calls come back through the API
|
|
#: and are held there), a browser's push subscription, the portal, workers
|
|
#: attaching, artifact bytes (which are named by digest, and which a node on a
|
|
#: remote worker writes while the rename waits for it), the socket ticket and
|
|
#: the rename routes, which keep their own order.
|
|
_EXEMPT = re.compile(
|
|
r"/(?:login|password-recovery|password-recovery-html-content|reset-password"
|
|
r"|oauth|users|ai|cloud|workers|artifacts)(?:/|$)"
|
|
r"|/alerts/webpush(?:/|$)"
|
|
r"|/flows/ws-ticket$"
|
|
r"|/(?:flows|dashboards)/[^/]+/rename(?:/plan)?$"
|
|
)
|
|
|
|
|
|
class RenameGate:
|
|
"""The door every request passes, which a rename closes.
|
|
|
|
Open, it counts the mutating API requests it lets in, for as long as each
|
|
takes. Closed, it turns new ones away and the rename waits for those
|
|
already in, so it starts from a namespace nothing is writing to through
|
|
the API. Hooks of the flows being renamed are turned away too, since their
|
|
nodes are torn down meanwhile. Reads, websockets and the exempt routes
|
|
pass either way, uncounted.
|
|
|
|
Thread-safe: requests pass on the event loop and `close` waits on a
|
|
thread, as the engine holds do.
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
self._lock = threading.Condition()
|
|
self._admitted: Counter[str] = Counter()
|
|
self._closed = False
|
|
self._affected: frozenset[str] = frozenset()
|
|
#: The rename that closed it, for whoever is turned away.
|
|
self.op = ""
|
|
|
|
@property
|
|
def closed(self) -> bool:
|
|
with self._lock:
|
|
return self._closed
|
|
|
|
def admit(self, scope: Scope) -> tuple[bool, str]:
|
|
"""Whether *scope* may pass, and the name it is counted under until
|
|
`leave`, empty when it is not counted."""
|
|
path: str = scope["path"]
|
|
with self._lock:
|
|
if path.startswith(f"{HOOK_PREFIX}/"):
|
|
flow_id = path[len(HOOK_PREFIX) + 1 :].split("/", 1)[0]
|
|
return not (self._closed and flow_id in self._affected), ""
|
|
api = settings.API_V1_STR
|
|
if (
|
|
scope["type"] != "http"
|
|
or scope["method"] in _READS
|
|
or not path.startswith(f"{api}/")
|
|
or _EXEMPT.match(path, len(api))
|
|
):
|
|
return True, ""
|
|
if self._closed:
|
|
return False, ""
|
|
name = f"{scope['method']} {path}"
|
|
self._admitted[name] += 1
|
|
return True, name
|
|
|
|
def leave(self, name: str) -> None:
|
|
"""A request counted under *name* is done."""
|
|
with self._lock:
|
|
self._admitted -= Counter([name])
|
|
self._lock.notify_all()
|
|
|
|
def close(
|
|
self,
|
|
op: str,
|
|
affected: Iterable[str],
|
|
abort: threading.Event,
|
|
report: Callable[[list[str]], None] | None = None,
|
|
) -> bool:
|
|
"""Turn mutations and the hooks of the *affected* flow ids away, then
|
|
wait for the requests already in to finish. Blocking.
|
|
|
|
Without a limit: ``report`` hears the requests still running each time
|
|
they change, and setting ``abort`` gives up, opens the gate again and
|
|
answers False. True means closed until `open`.
|
|
"""
|
|
with self._lock:
|
|
self._closed = True
|
|
self._affected = frozenset(affected)
|
|
self.op = op
|
|
if wait_out(self._lock, lambda: sorted(self._admitted), abort, report):
|
|
return True
|
|
self.open()
|
|
return False
|
|
|
|
def open(self) -> None:
|
|
with self._lock:
|
|
self._closed = False
|
|
self._affected = frozenset()
|
|
self.op = ""
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Documents
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class Rewrite:
|
|
"""A stored document a rename changes, as texts before and after."""
|
|
|
|
kind: Kind
|
|
id: str
|
|
name: str
|
|
published_version: int | None
|
|
draft_version: int | None
|
|
#: Bindings moved, in whichever variant moves more of them.
|
|
fields: int
|
|
#: Bindings moved in the draft, which stays unpublished.
|
|
draft_fields: int
|
|
#: Declared in code (`FlowDef.origin`), so a sync puts the old names back.
|
|
code_defined: bool
|
|
#: ``(path, before, after)`` per variant, the path relative to the store.
|
|
files: list[tuple[str, str, str]]
|
|
|
|
|
|
_VARIANTS = (("published", ".json"), ("draft", ".draft.json"))
|
|
|
|
|
|
def _documents(
|
|
store: FlowStore, dashboards: DashboardStore
|
|
) -> Iterator[tuple[Kind, str, str, tuple[tuple[str, ...], ...]]]:
|
|
"""Every stored flow and dashboard: kind, id, path without its variant's
|
|
suffix, and where it names messages."""
|
|
for flow_id in store.list_flows():
|
|
yield "flow", flow_id, f"{flow_id}/flow", FLOW_FIELDS
|
|
found = {path.parent.name for path in dashboards.root.glob("*/dashboard*.json")}
|
|
for dash_id in sorted(found):
|
|
yield (
|
|
"dashboard",
|
|
dash_id,
|
|
f"{DASHBOARD_DIR}/{dash_id}/dashboard",
|
|
DASHBOARD_BINDINGS,
|
|
)
|
|
|
|
|
|
def _variants(root: Path, stem: str) -> dict[str, tuple[str, dict[str, Any]]]:
|
|
"""A document's published and draft variants, as text and as JSON.
|
|
|
|
Plain JSON rather than the model, so a field this version does not model,
|
|
or a dashboard still written as pages, comes through as it was. Empty for
|
|
one that does not parse, which the engine cannot read either.
|
|
"""
|
|
found: dict[str, tuple[str, dict[str, Any]]] = {}
|
|
for variant, suffix in _VARIANTS:
|
|
path = root / f"{stem}{suffix}"
|
|
if not path.exists():
|
|
continue
|
|
text = path.read_text()
|
|
try:
|
|
data = orjson.loads(text)
|
|
except orjson.JSONDecodeError:
|
|
return {}
|
|
if not isinstance(data, dict):
|
|
return {}
|
|
found[variant] = (text, data)
|
|
return found
|
|
|
|
|
|
def rewrite_documents(
|
|
store: FlowStore,
|
|
dashboards: DashboardStore,
|
|
kind: Kind,
|
|
obj_id: str,
|
|
rename: Rename,
|
|
) -> list[Rewrite]:
|
|
"""Every document the rename of *obj_id* changes, published and draft.
|
|
Reads only; writing the texts is the rename's.
|
|
|
|
The object itself takes the new key. Renaming a flow also moves every
|
|
qualified name of its namespace that a flow names (`FLOW_FIELDS`, its own
|
|
included) or a dashboard binds (`DASHBOARD_BINDINGS`). Bare names are a
|
|
flow's own and follow its key by themselves, and no other field changes.
|
|
Both variants of a document that changes go to one version above either,
|
|
so an editor holding the old text is refused rather than saving it back,
|
|
and a draft discarded later cannot bring an older version back.
|
|
"""
|
|
rewrites = []
|
|
for doc_kind, doc_id, stem, paths in _documents(store, dashboards):
|
|
own = doc_kind == kind and doc_id == obj_id
|
|
if kind == "dashboard" and not own:
|
|
continue
|
|
variants = _variants(store.root, stem)
|
|
if not variants:
|
|
continue
|
|
moved = {
|
|
variant: rebind(data, paths, rename.binding) if kind == "flow" else 0
|
|
for variant, (_, data) in variants.items()
|
|
}
|
|
if not own and not any(moved.values()):
|
|
continue
|
|
versions = {
|
|
variant: int(data.get("version", 1))
|
|
for variant, (_, data) in variants.items()
|
|
}
|
|
working = variants.get("draft") or variants["published"]
|
|
rewrite = Rewrite(
|
|
kind=doc_kind,
|
|
id=doc_id,
|
|
name=str(working[1].get("name", doc_id)),
|
|
published_version=versions.get("published"),
|
|
draft_version=versions.get("draft"),
|
|
fields=max(moved.values()),
|
|
draft_fields=moved.get("draft", 0),
|
|
code_defined=any(data.get("origin") for _, data in variants.values()),
|
|
files=[],
|
|
)
|
|
top = max(versions.values()) + 1
|
|
for variant, suffix in _VARIANTS:
|
|
if variant not in variants:
|
|
continue
|
|
text, data = variants[variant]
|
|
if own:
|
|
data["name"] = rename.new
|
|
data["version"] = top
|
|
# The stores write what pydantic writes, which this matches.
|
|
after = orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()
|
|
if text.endswith("\n"):
|
|
after += "\n"
|
|
rewrite.files.append((f"{stem}{suffix}", text, after))
|
|
rewrites.append(rewrite)
|
|
return rewrites
|
|
|
|
|
|
def references(store: FlowStore, dashboards: DashboardStore, key: str) -> list[str]:
|
|
"""The flows and dashboards that name a message of the flow key *key*,
|
|
published or draft, by id.
|
|
|
|
A rename onto *key* would hand those bindings to the renamed flow, and a
|
|
rename back would take them along. Creating a flow under *key* is another
|
|
matter: that is how bindings declared ahead of their flow are met.
|
|
"""
|
|
found = []
|
|
for _, doc_id, stem, paths in _documents(store, dashboards):
|
|
bound = (
|
|
name
|
|
for _, data in _variants(store.root, stem).values()
|
|
for name in names(data, paths)
|
|
)
|
|
if any("." in name and flow_of(name) == key for name in bound):
|
|
found.append(doc_id)
|
|
return found
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Plan
|
|
# -----------------------------------------------------------------------------
|
|
|
|
#: An alert's dedup key (`alerts.dedup_key`), whose tail is a node or a flow.
|
|
DEDUP = "dedup"
|
|
|
|
#: Where the history names messages, by table and column: a qualified name
|
|
#: (`namespace.NAME`), a JSON object keyed by them (`namespace.OUTPUTS`) or an
|
|
#: alert's dedup key. A rename rewrites exactly these. Every other column
|
|
#: holds ids, which never change, or payload and free text, which are history,
|
|
#: and a registry test fails when a column is added without saying which.
|
|
SQL_NAMES: dict[tuple[str, str], str] = {
|
|
("run_metric", "name"): NAME,
|
|
("run_artifact", "name"): NAME,
|
|
("run_node", "outputs"): OUTPUTS,
|
|
("alert_sent", "key"): DEDUP,
|
|
}
|
|
|
|
#: How recently another consumer must have read the queue to count as an
|
|
#: engine still running: it blocks a second at a time.
|
|
OTHER_ENGINE_MS = 10_000
|
|
|
|
#: Run states a rename refuses: the run would read names mid-move.
|
|
_ACTIVE_RUNS = ("queued", "running")
|
|
|
|
|
|
class RenamePlanRequest(BaseModel):
|
|
"""The key a flow or dashboard would be renamed to."""
|
|
|
|
new_name: str
|
|
|
|
|
|
class RenameRefusal(BaseModel):
|
|
"""Why a rename cannot go ahead, with the ids of what is in the way."""
|
|
|
|
code: str
|
|
detail: str
|
|
ids: list[str] = []
|
|
|
|
|
|
class RenameWarning(BaseModel):
|
|
"""Something a rename changes, or leaves, that its caller should know."""
|
|
|
|
code: str
|
|
detail: str
|
|
|
|
|
|
class PlannedDocument(BaseModel):
|
|
"""A document a rename rewrites, and the versions it was planned against."""
|
|
|
|
kind: Kind
|
|
id: str
|
|
name: str
|
|
published_version: int | None
|
|
draft_version: int | None
|
|
#: Bindings moved, in whichever variant moves more of them.
|
|
fields: int
|
|
|
|
|
|
class AlertCounts(BaseModel):
|
|
channels: int
|
|
ledger_rows: int
|
|
|
|
|
|
class HistoryCounts(BaseModel):
|
|
runs: int
|
|
metric_rows: int
|
|
artifact_rows: int
|
|
output_rows: int
|
|
|
|
|
|
class RuntimeCounts(BaseModel):
|
|
#: False with memory backends, whose state and queue a restart loses anyway.
|
|
durable: bool
|
|
state_keys: int
|
|
queue_items: int
|
|
delayed: int
|
|
parked: int
|
|
done_markers: int
|
|
|
|
|
|
class RenamePlan(BaseModel):
|
|
"""What renaming a flow's or dashboard's key would change, and whether it
|
|
may. Read-only; the rename request repeats ``version`` and
|
|
``fingerprint``, and is refused when either moved since."""
|
|
|
|
kind: Kind
|
|
id: str
|
|
name: str
|
|
new_name: str
|
|
#: The working copy's version, the draft's when there is one.
|
|
version: int
|
|
#: Over the kind, id and versions of every listed document and both
|
|
#: keys, so a document changing between plan and rename is noticed.
|
|
fingerprint: str = ""
|
|
allowed: bool = False
|
|
refusals: list[RenameRefusal] = []
|
|
documents: list[PlannedDocument] = []
|
|
#: Dry-run counts, a flow's only, which inform and are not checked again.
|
|
alerts: AlertCounts | None = None
|
|
history: HistoryCounts | None = None
|
|
runtime: RuntimeCounts | None = None
|
|
warnings: list[RenameWarning] = []
|
|
|
|
|
|
def plan_rename(
|
|
controller: FlowController,
|
|
dashboards: DashboardStore,
|
|
kind: Kind,
|
|
obj_id: str,
|
|
new_name: str,
|
|
alerts: AlertsConfig,
|
|
) -> RenamePlan:
|
|
"""What renaming the flow or dashboard *obj_id* to *new_name* would change,
|
|
and what refuses it. Blocking, and changes nothing.
|
|
|
|
:raises FlowNotFound: For a flow that does not exist.
|
|
:raises DashboardNotFound: For a dashboard that does not exist.
|
|
"""
|
|
store = controller.store
|
|
current: FlowDef | DashboardDef
|
|
if kind == "flow":
|
|
current = store.read_flow(obj_id, draft=True)
|
|
keys, ids = store.key_index(), set(store.list_flows())
|
|
else:
|
|
current = dashboards.read(obj_id, draft=True)
|
|
keys = dashboards.key_index()
|
|
ids = {path.parent.name for path in dashboards.root.glob("*/dashboard*.json")}
|
|
old = current.name
|
|
plan = RenamePlan(
|
|
kind=kind, id=obj_id, name=old, new_name=new_name, version=current.version
|
|
)
|
|
|
|
def refuse(code: str, detail: str, refused: Iterable[str] = ()) -> None:
|
|
plan.refusals.append(RenameRefusal(code=code, detail=detail, ids=[*refused]))
|
|
|
|
def warn(code: str, detail: str) -> None:
|
|
plan.warnings.append(RenameWarning(code=code, detail=detail))
|
|
|
|
try:
|
|
_validate_name(new_name)
|
|
except ValueError:
|
|
refuse("invalid_name", str(InvalidName(new_name)))
|
|
return plan
|
|
if ISSUED_ID.fullmatch(new_name):
|
|
refuse("reserved_shape", str(ReservedKey(new_name)))
|
|
return plan
|
|
if new_name == old:
|
|
refuse("same_name", f"The {kind} is already called '{old}'")
|
|
return plan
|
|
|
|
rename = Rename(old, new_name)
|
|
rewrites = rewrite_documents(store, dashboards, kind, obj_id, rename)
|
|
plan.documents = [
|
|
PlannedDocument(
|
|
kind=r.kind,
|
|
id=r.id,
|
|
name=r.name,
|
|
published_version=r.published_version,
|
|
draft_version=r.draft_version,
|
|
fields=r.fields,
|
|
)
|
|
for r in rewrites
|
|
]
|
|
plan.fingerprint = _fingerprint(plan)
|
|
if new_name in keys:
|
|
refuse("name_taken", f"Another {kind} has the key '{new_name}'")
|
|
if new_name in ids - {obj_id}:
|
|
warn(
|
|
"new_name_is_id",
|
|
f"'{new_name}' is also the id of another {kind}, so a client naming "
|
|
"it by either is asked which one it means",
|
|
)
|
|
if kind == "flow":
|
|
_plan_flow(plan, controller, dashboards, rename, rewrites, alerts, refuse, warn)
|
|
plan.allowed = not plan.refusals
|
|
return plan
|
|
|
|
|
|
def _plan_flow(
|
|
plan: RenamePlan,
|
|
controller: FlowController,
|
|
dashboards: DashboardStore,
|
|
rename: Rename,
|
|
rewrites: list[Rewrite],
|
|
alerts: AlertsConfig,
|
|
refuse: Callable[..., None],
|
|
warn: Callable[[str, str], None],
|
|
) -> None:
|
|
"""The part of a plan only a flow has: what holds its namespace, and
|
|
what moves outside the documents."""
|
|
old, new = rename.old, rename.new
|
|
own = next(r for r in rewrites if r.kind == "flow" and r.id == plan.id)
|
|
if own.code_defined:
|
|
refuse(
|
|
"code_defined",
|
|
f"'{old}' is declared in code: rename it there and sync it again",
|
|
)
|
|
coded = [r.id for r in rewrites if r is not own and r.code_defined]
|
|
if coded:
|
|
refuse(
|
|
"code_defined_dependent",
|
|
f"{len(coded)} flows declared in code name messages of '{old}', and "
|
|
"a sync would put the old names back: rename them in code first",
|
|
coded,
|
|
)
|
|
flow_ids = [r.id for r in rewrites if r.kind == "flow"]
|
|
with Session(db_engine) as session:
|
|
active = session.exec(
|
|
select(Run.id, Run.flow_id, Run.for_node).where(
|
|
col(Run.status).in_(_ACTIVE_RUNS)
|
|
)
|
|
).all()
|
|
running = [
|
|
run_id
|
|
for run_id, flow_id, for_node in active
|
|
if flow_id in flow_ids or flow_of(for_node or "") in flow_ids
|
|
]
|
|
if running:
|
|
refuse(
|
|
"run_active",
|
|
f"{len(running)} runs of affected flows are queued or running",
|
|
running,
|
|
)
|
|
history, collisions, too_long = _history(session, rename, flow_ids)
|
|
ledger = [
|
|
rename.dedup_key(key)
|
|
for key in session.exec(select(AlertSent.key)).all()
|
|
if rename.dedup_key(key) != key
|
|
]
|
|
if collisions:
|
|
refuse(
|
|
"history_collision",
|
|
f"{collisions} history rows of the affected flows are already "
|
|
f"named '{new}.*'",
|
|
)
|
|
too_long += [key for key in ledger if len(key) > _length(col(AlertSent.key))]
|
|
if too_long:
|
|
refuse(
|
|
"name_too_long",
|
|
f"'{too_long[0]}' would be longer than its column holds",
|
|
)
|
|
|
|
residue = controller.key_residue(new)
|
|
referencing = references(controller.store, dashboards, new)
|
|
if residue or referencing:
|
|
named = [*residue, *(f"bindings in '{one}'" for one in referencing)]
|
|
refuse(
|
|
"target_in_use",
|
|
f"'{new}' still has {', '.join(named[:3])} behind it",
|
|
referencing,
|
|
)
|
|
|
|
state = controller.state
|
|
queue = controller.execution.queue if controller.execution else None
|
|
counts = queue.count(rename) if queue else {}
|
|
plan.runtime = RuntimeCounts(
|
|
durable=not isinstance(state, MemoryState),
|
|
state_keys=state.count(rename),
|
|
queue_items=counts.get("queue_items", 0),
|
|
delayed=counts.get("delayed", 0),
|
|
parked=counts.get("parked", 0),
|
|
done_markers=counts.get("done_markers", 0),
|
|
)
|
|
if queue is not None:
|
|
others = queue.other_consumers(OTHER_ENGINE_MS)
|
|
if others:
|
|
refuse(
|
|
"other_engine",
|
|
f"Another engine reads the queue ({', '.join(others)}), which a "
|
|
"rename cannot hold",
|
|
)
|
|
version = queue.server_version()
|
|
if version is not None and version < (7, 0):
|
|
refuse(
|
|
"redis_too_old",
|
|
f"Redis {version[0]}.{version[1]} is older than 7.0, which "
|
|
"a rename needs",
|
|
)
|
|
|
|
plan.history = history
|
|
plan.alerts = AlertCounts(
|
|
channels=sum(
|
|
channel.kind == "dashboard"
|
|
and rename.binding(str(channel.config.get("message") or ""))
|
|
!= channel.config.get("message")
|
|
for channel in alerts.channels
|
|
),
|
|
ledger_rows=len(ledger),
|
|
)
|
|
|
|
warn(
|
|
"external_references",
|
|
f"REST /messages/{old}.*, scripts, CLI, SDK and MCP key arguments and "
|
|
"hand-written bindings keep the old name",
|
|
)
|
|
if results_name(old) in dashboards.key_index():
|
|
warn(
|
|
"results_dashboard",
|
|
f"The dashboard '{results_name(old)}' keeps its name",
|
|
)
|
|
drafts = [r.id for r in rewrites if r.draft_fields]
|
|
if drafts:
|
|
warn(
|
|
"drafts_rewritten",
|
|
f"{len(drafts)} unpublished drafts change and stay unpublished",
|
|
)
|
|
pipeline = controller.pipeline
|
|
if (pipeline is not None and pipeline.is_paused(old)) or plan.runtime.parked:
|
|
warn("paused", f"'{old}' is paused, and its parked work moves with it")
|
|
if not plan.runtime.durable:
|
|
warn(
|
|
"not_durable",
|
|
"State and queued work are kept in memory, so a restart during the "
|
|
"rename loses them, as any restart does",
|
|
)
|
|
rows = history.metric_rows + history.artifact_rows + history.output_rows
|
|
warn(
|
|
"engine_pause",
|
|
f"Automations wait while {rows} history rows and "
|
|
f"{plan.runtime.state_keys} state keys move",
|
|
)
|
|
|
|
|
|
def _history(
|
|
session: Session, rename: Rename, flow_ids: list[str]
|
|
) -> tuple[HistoryCounts, int, list[str]]:
|
|
"""What moves in the history of the flows *flow_ids*, how many of its
|
|
names are in the new namespace already, and the names that would no
|
|
longer fit their column."""
|
|
old, new = f"{rename.old}.", f"{rename.new}."
|
|
grow = len(rename.new) - len(rename.old)
|
|
runs = select(Run.id).where(col(Run.flow_id).in_(flow_ids))
|
|
counts = {}
|
|
collisions = 0
|
|
too_long = []
|
|
for model in (RunMetric, RunArtifact):
|
|
name = col(model.name)
|
|
scope = col(model.run_id).in_(runs)
|
|
moving = func.substr(name, 1, len(old)) == old
|
|
counts[model] = session.exec(
|
|
select(func.count()).select_from(model).where(scope, moving)
|
|
).one()
|
|
collisions += session.exec(
|
|
select(func.count())
|
|
.select_from(model)
|
|
.where(scope, func.substr(name, 1, len(new)) == new)
|
|
).one()
|
|
longest = session.exec(
|
|
select(name).where(scope, moving).order_by(func.length(name).desc())
|
|
).first()
|
|
if longest is not None and len(longest) + grow > _length(name):
|
|
too_long.append(rename.name(longest))
|
|
outputs = session.exec(
|
|
select(RunNode.outputs).where(
|
|
col(RunNode.run_id).in_(runs),
|
|
col(RunNode.outputs).contains(f'"{old}', autoescape=True),
|
|
)
|
|
).all()
|
|
moved_outputs = sum(
|
|
any(rename.binding(key) != key for key in json.loads(text or "null") or {})
|
|
for text in outputs
|
|
)
|
|
total = session.exec(
|
|
select(func.count()).select_from(Run).where(col(Run.flow_id).in_(flow_ids))
|
|
).one()
|
|
history = HistoryCounts(
|
|
runs=total,
|
|
metric_rows=counts[RunMetric],
|
|
artifact_rows=counts[RunArtifact],
|
|
output_rows=moved_outputs,
|
|
)
|
|
return history, collisions, too_long
|
|
|
|
|
|
def _length(column: Any) -> int:
|
|
"""How many characters a text column holds."""
|
|
return cast(int, cast(String, column.type).length)
|
|
|
|
|
|
def _fingerprint(plan: RenamePlan) -> str:
|
|
seen = sorted(
|
|
(d.kind, d.id, d.published_version or 0, d.draft_version or 0)
|
|
for d in plan.documents
|
|
)
|
|
body = json.dumps([seen, plan.name, plan.new_name]).encode()
|
|
return f"sha256:{hashlib.sha256(body).hexdigest()}"
|