Separate editing from running with a draft/publish split
Edits autosave to flow.draft.json and nodes.draft/ instead of the files the engine reads, so the pipeline keeps running the published version until someone publishes. Every save carries the version it was based on: a second client editing the same flow is refused with 409 and offered the choice between their version and its own, rather than silently overwriting. Draft saves no longer rebuild the pipeline; validation and node status for a draft come from a throwaway build that never touches live state. Also fixes a latent bug where an empty state backend is falsy, so Pipeline quietly built itself a second, private state and left message history empty. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
36be6f1081
commit
606ab3c423
+125
-27
@@ -34,7 +34,7 @@ from app.flow.schemas import (
|
||||
NodeTypeInfo,
|
||||
)
|
||||
from app.flow.state import as_number
|
||||
from app.flow.store import FlowExists, FlowNotFound
|
||||
from app.flow.store import FlowExists, FlowNotFound, StaleVersion
|
||||
from app.models import Message
|
||||
|
||||
router = APIRouter(
|
||||
@@ -46,11 +46,17 @@ ws_router = APIRouter(prefix="/flows", tags=["flows"])
|
||||
|
||||
|
||||
class FlowDetail(BaseModel):
|
||||
"""A flow plus how it is currently doing."""
|
||||
"""A flow plus how it is currently doing.
|
||||
|
||||
``definition`` is the working copy — the unpublished draft when there is
|
||||
one — because that is what the editor shows. ``nodes`` reports the
|
||||
published flow, which is what is actually running.
|
||||
"""
|
||||
|
||||
definition: FlowDef
|
||||
nodes: list[NodeStatusPublic] = []
|
||||
issues: list[ValidationIssue] = []
|
||||
has_draft: bool = False
|
||||
|
||||
|
||||
class ValidationResult(BaseModel):
|
||||
@@ -61,6 +67,10 @@ class RenameRequest(BaseModel):
|
||||
new_name: str
|
||||
|
||||
|
||||
class PublishRequest(BaseModel):
|
||||
version: int
|
||||
|
||||
|
||||
class RunRequest(BaseModel):
|
||||
inputs: dict[str, Any] = {}
|
||||
|
||||
@@ -70,16 +80,28 @@ class TriggerRequest(BaseModel):
|
||||
|
||||
|
||||
def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail:
|
||||
name = definition.name
|
||||
if controller.store.has_draft(name):
|
||||
# Report the draft the editor is showing, not the version running
|
||||
# underneath it — otherwise a node the author just broke looks fine.
|
||||
preview = controller.preview(name)
|
||||
return FlowDetail(
|
||||
definition=definition,
|
||||
nodes=preview.nodes,
|
||||
issues=preview.issues,
|
||||
has_draft=True,
|
||||
)
|
||||
return FlowDetail(
|
||||
definition=definition,
|
||||
nodes=controller.node_statuses(definition.name),
|
||||
issues=controller.flow_issues(definition.name),
|
||||
nodes=controller.node_statuses(name),
|
||||
issues=controller.flow_issues(name),
|
||||
)
|
||||
|
||||
|
||||
def _read_flow(controller: FlowController, name: str) -> FlowDef:
|
||||
"""The working copy: the draft when there is one, else what is published."""
|
||||
try:
|
||||
return controller.store.read_flow(name)
|
||||
return controller.store.read_flow(name, draft=True)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
|
||||
|
||||
@@ -102,14 +124,19 @@ def _flow_state(controller: FlowController, name: str) -> FlowStatePublic:
|
||||
def read_flows(controller: FlowControllerDep) -> Any:
|
||||
"""List every flow."""
|
||||
summaries = []
|
||||
for definition in controller.store.read_all():
|
||||
statuses = controller.node_statuses(definition.name)
|
||||
for name in controller.store.list_flows():
|
||||
try:
|
||||
definition = controller.store.read_flow(name, draft=True)
|
||||
except FlowNotFound:
|
||||
continue
|
||||
statuses = controller.node_statuses(name)
|
||||
summaries.append(
|
||||
FlowSummary(
|
||||
name=definition.name,
|
||||
title=definition.title,
|
||||
node_count=len(definition.nodes),
|
||||
error_count=sum(1 for s in statuses if s.status == "error"),
|
||||
has_draft=controller.store.has_draft(name),
|
||||
)
|
||||
)
|
||||
return FlowsPublic(data=summaries, count=len(summaries))
|
||||
@@ -135,7 +162,13 @@ async def save_flow(
|
||||
definition: FlowDef,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Create or replace a flow. Saving the same content again changes nothing."""
|
||||
"""Save unpublished changes to a flow.
|
||||
|
||||
This writes a draft: the running pipeline keeps the published version until
|
||||
someone publishes. ``version`` is the one the editor last saw — a mismatch
|
||||
means another client saved in between and answers 409 rather than throwing
|
||||
their work away.
|
||||
"""
|
||||
if definition.name != name:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="The flow name in the body must match the URL"
|
||||
@@ -145,10 +178,61 @@ async def save_flow(
|
||||
if len(duplicates) != len(definition.nodes):
|
||||
raise HTTPException(status_code=400, detail="Node names must be unique")
|
||||
|
||||
changed = await run_in_threadpool(controller.store.write_flow, definition)
|
||||
if changed:
|
||||
await controller.reload()
|
||||
return _detail(controller, definition)
|
||||
try:
|
||||
stored = await run_in_threadpool(
|
||||
controller.store.write_draft, definition, definition.version
|
||||
)
|
||||
except StaleVersion as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"message": str(exc), "current_version": exc.current},
|
||||
)
|
||||
return _detail(controller, stored)
|
||||
|
||||
|
||||
@router.post("/{name}/publish", response_model=FlowDetail)
|
||||
async def publish_flow(
|
||||
name: str,
|
||||
body: PublishRequest,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Deploy the unpublished changes: the engine picks them up from here."""
|
||||
_read_flow(controller, name)
|
||||
if not controller.store.has_draft(name):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Flow '{name}' has no unpublished changes"
|
||||
)
|
||||
try:
|
||||
published = await run_in_threadpool(
|
||||
controller.store.publish_flow, name, body.version
|
||||
)
|
||||
except StaleVersion as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"message": str(exc), "current_version": exc.current},
|
||||
)
|
||||
await controller.reload()
|
||||
return _detail(controller, published)
|
||||
|
||||
|
||||
@router.post("/{name}/discard-draft", response_model=FlowDetail)
|
||||
async def discard_draft(name: str, controller: FlowControllerDep) -> Any:
|
||||
"""Throw the unpublished changes away and go back to what is running."""
|
||||
_read_flow(controller, name)
|
||||
if not controller.store.has_draft(name):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Flow '{name}' has no unpublished changes"
|
||||
)
|
||||
if not controller.store.is_published(name):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Flow '{name}' has never been published — delete it instead of "
|
||||
"discarding it"
|
||||
),
|
||||
)
|
||||
published = await run_in_threadpool(controller.store.discard_draft, name)
|
||||
return _detail(controller, published)
|
||||
|
||||
|
||||
@router.delete("/{name}", response_model=Message)
|
||||
@@ -200,9 +284,9 @@ def read_node_source(
|
||||
node_id: str,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Read a node's Python source."""
|
||||
"""Read a node's Python source, including unpublished edits."""
|
||||
_read_flow(controller, name)
|
||||
return NodeSource(code=controller.store.read_node_source(name, node_id))
|
||||
return NodeSource(code=controller.store.read_node_source(name, node_id, draft=True))
|
||||
|
||||
|
||||
@router.put("/{name}/nodes/{node_id}/source", response_model=NodeStatusPublic)
|
||||
@@ -212,19 +296,24 @@ async def save_node_source(
|
||||
source: NodeSource,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Save a node's source and report whether it loads."""
|
||||
_read_flow(controller, name)
|
||||
changed = await run_in_threadpool(
|
||||
controller.store.write_node_source, name, node_id, source.code
|
||||
)
|
||||
if changed:
|
||||
await controller.reload()
|
||||
"""Save a node's source as an unpublished edit and report whether it loads.
|
||||
|
||||
status = next(
|
||||
(s for s in controller.node_statuses(name) if s.id == f"{name}.{node_id}"),
|
||||
None,
|
||||
The answer comes from compiling the code rather than from the running
|
||||
pipeline: a draft is not deployed, and compiling is both faster and more
|
||||
precise about what the author just typed.
|
||||
"""
|
||||
_read_flow(controller, name)
|
||||
await run_in_threadpool(
|
||||
controller.store.write_node_source, name, node_id, source.code, True
|
||||
)
|
||||
error = await run_in_threadpool(
|
||||
controller.compile_check, name, node_id, source.code
|
||||
)
|
||||
return NodeStatusPublic(
|
||||
id=f"{name}.{node_id}",
|
||||
status="error" if error else "active",
|
||||
error=error,
|
||||
)
|
||||
return status or NodeStatusPublic(id=f"{name}.{node_id}")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -236,6 +325,8 @@ async def save_node_source(
|
||||
def validate_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
"""Report what would keep this flow from running."""
|
||||
_read_flow(controller, name)
|
||||
if controller.store.has_draft(name):
|
||||
return ValidationResult(issues=controller.preview(name).issues)
|
||||
return ValidationResult(issues=controller.flow_issues(name))
|
||||
|
||||
|
||||
@@ -245,10 +336,17 @@ async def run_flow(
|
||||
body: RunRequest,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Run every node of a flow once."""
|
||||
"""Run every node of a flow once.
|
||||
|
||||
With unpublished changes this runs the draft, so the button matches what is
|
||||
on the canvas. Nothing is deployed by running it.
|
||||
"""
|
||||
_read_flow(controller, name)
|
||||
inputs = {qualify(name, key): value for key, value in body.inputs.items()}
|
||||
await run_in_threadpool(controller.run_flow, name, inputs)
|
||||
if controller.store.has_draft(name):
|
||||
await run_in_threadpool(controller.run_preview, name, inputs)
|
||||
else:
|
||||
await run_in_threadpool(controller.run_flow, name, inputs)
|
||||
return _flow_state(controller, name)
|
||||
|
||||
|
||||
|
||||
+147
-53
@@ -32,10 +32,10 @@ from app.flow.nodes import (
|
||||
Node,
|
||||
)
|
||||
from app.flow.pipeline import Pipeline, ValidationIssue
|
||||
from app.flow.schemas import NodeDef, NodeStatusPublic, NodeTypeInfo
|
||||
from app.flow.schemas import FlowDef, NodeDef, NodeStatusPublic, NodeTypeInfo
|
||||
from app.flow.secrets import SecretNotFound, resolve_params
|
||||
from app.flow.state import MemoryState, StateBackend
|
||||
from app.flow.store import FlowStore
|
||||
from app.flow.store import FlowNotFound, FlowStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -58,6 +58,14 @@ class LoadedNode:
|
||||
error: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Preview:
|
||||
"""A draft as it would run, without running it."""
|
||||
|
||||
nodes: list[NodeStatusPublic]
|
||||
issues: list[ValidationIssue]
|
||||
|
||||
|
||||
@dataclass
|
||||
class NodeType:
|
||||
"""A node type the editor can place on a canvas."""
|
||||
@@ -172,7 +180,7 @@ class FlowController:
|
||||
fastapi_app: FastAPI | None = None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.state = state or MemoryState()
|
||||
self.state = state if state is not None else MemoryState()
|
||||
self.events = events
|
||||
self.max_workers = max_workers
|
||||
self.app = fastapi_app
|
||||
@@ -197,25 +205,9 @@ class FlowController:
|
||||
async with self._lock:
|
||||
await self._teardown()
|
||||
|
||||
nodes: list[Node] = []
|
||||
loaded: dict[str, LoadedNode] = {}
|
||||
initial_values: dict[str, Any] = {}
|
||||
# Declared flow inputs, mapped to whether they start with a value.
|
||||
flow_inputs: dict[str, bool] = {}
|
||||
|
||||
for flow in self.store.read_all():
|
||||
for node_def in flow.nodes:
|
||||
entry = self._build_node(flow.name, node_def)
|
||||
loaded[entry.id] = entry
|
||||
if entry.node is not None:
|
||||
nodes.append(entry.node)
|
||||
for flow_input in flow.inputs:
|
||||
name = qualify(flow.name, flow_input.spec.name)
|
||||
if not name:
|
||||
continue
|
||||
flow_inputs[name] = flow_input.initial is not None
|
||||
if flow_input.initial is not None:
|
||||
initial_values[name] = flow_input.initial
|
||||
nodes, loaded, initial_values, flow_inputs = self._build_flows(
|
||||
[(flow, False) for flow in self.store.read_all()]
|
||||
)
|
||||
|
||||
self.loaded = loaded
|
||||
self.pipeline = Pipeline(
|
||||
@@ -225,35 +217,7 @@ class FlowController:
|
||||
max_workers=self.max_workers,
|
||||
initial_values=initial_values,
|
||||
)
|
||||
self.issues = self.pipeline.validate(flow_inputs)
|
||||
self.issues += [
|
||||
ValidationIssue(
|
||||
code="node_error",
|
||||
message=entry.error or "This node could not be loaded.",
|
||||
flow=entry.flow,
|
||||
node=entry.id,
|
||||
)
|
||||
for entry in loaded.values()
|
||||
if entry.status is NodeStatus.ERROR
|
||||
]
|
||||
# Webhooks are mounted without authentication, so a hook without a
|
||||
# secret is open to anyone who guesses its URL. Flows written before
|
||||
# the parameter existed keep running, but say so.
|
||||
self.issues += [
|
||||
ValidationIssue(
|
||||
code="unauthenticated_hook",
|
||||
message=(
|
||||
f"Webhook '{entry.node.local_id}' has no secret — "
|
||||
"anyone who knows its URL can trigger it."
|
||||
),
|
||||
flow=entry.flow,
|
||||
node=entry.id,
|
||||
)
|
||||
for entry in loaded.values()
|
||||
if isinstance(entry.node, HttpNode)
|
||||
and entry.node.mode == HttpNode.Mode.TRIGGER
|
||||
and not entry.node.secret
|
||||
]
|
||||
self.issues = _collect_issues(loaded, self.pipeline, flow_inputs)
|
||||
|
||||
await self._activate()
|
||||
|
||||
@@ -311,7 +275,35 @@ class FlowController:
|
||||
# Building
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def _build_node(self, flow: str, node_def: NodeDef) -> LoadedNode:
|
||||
def _build_flows(
|
||||
self, flows: list[tuple[FlowDef, bool]]
|
||||
) -> tuple[list[Node], dict[str, LoadedNode], dict[str, Any], dict[str, bool]]:
|
||||
"""Instantiate the nodes of several flows, each published or draft."""
|
||||
nodes: list[Node] = []
|
||||
loaded: dict[str, LoadedNode] = {}
|
||||
initial_values: dict[str, Any] = {}
|
||||
# Declared flow inputs, mapped to whether they start with a value.
|
||||
flow_inputs: dict[str, bool] = {}
|
||||
|
||||
for flow, draft in flows:
|
||||
for node_def in flow.nodes:
|
||||
entry = self._build_node(flow.name, node_def, draft=draft)
|
||||
loaded[entry.id] = entry
|
||||
if entry.node is not None:
|
||||
nodes.append(entry.node)
|
||||
for flow_input in flow.inputs:
|
||||
name = qualify(flow.name, flow_input.spec.name)
|
||||
if not name:
|
||||
continue
|
||||
flow_inputs[name] = flow_input.initial is not None
|
||||
if flow_input.initial is not None:
|
||||
initial_values[name] = flow_input.initial
|
||||
|
||||
return nodes, loaded, initial_values, flow_inputs
|
||||
|
||||
def _build_node(
|
||||
self, flow: str, node_def: NodeDef, draft: bool = False
|
||||
) -> LoadedNode:
|
||||
node_id = f"{flow}.{node_def.id}"
|
||||
entry = LoadedNode(id=node_id, flow=flow)
|
||||
try:
|
||||
@@ -322,7 +314,7 @@ class FlowController:
|
||||
params = resolve_params(node_def.params)
|
||||
|
||||
if node_type.has_source:
|
||||
code = self.store.read_node_source(flow, node_def.id)
|
||||
code = self.store.read_node_source(flow, node_def.id, draft=draft)
|
||||
node = Node(
|
||||
f=_load_function(flow, node_def.id, code),
|
||||
requires=_bound(node_def.requires),
|
||||
@@ -368,6 +360,51 @@ class FlowController:
|
||||
def flow_issues(self, flow: str) -> list[ValidationIssue]:
|
||||
return [issue for issue in self.issues if not issue.flow or issue.flow == flow]
|
||||
|
||||
def preview(self, name: str) -> Preview:
|
||||
"""Build a flow's unpublished draft without deploying it.
|
||||
|
||||
The draft stands in for its published self among the other flows, so
|
||||
cross-flow messages resolve the way they would after publishing. The
|
||||
graph is thrown away afterwards and seeded into a throwaway state, so
|
||||
nothing here touches what is running.
|
||||
"""
|
||||
flows: list[tuple[FlowDef, bool]] = [
|
||||
(flow, False) for flow in self.store.read_all() if flow.name != name
|
||||
]
|
||||
try:
|
||||
flows.append((self.store.read_flow(name, draft=True), True))
|
||||
except FlowNotFound:
|
||||
return Preview(nodes=[], issues=[])
|
||||
|
||||
nodes, loaded, initial_values, flow_inputs = self._build_flows(flows)
|
||||
pipeline = Pipeline(
|
||||
nodes=nodes,
|
||||
state=MemoryState(),
|
||||
max_workers=self.max_workers,
|
||||
initial_values=initial_values,
|
||||
)
|
||||
issues = _collect_issues(loaded, pipeline, flow_inputs)
|
||||
return Preview(
|
||||
nodes=[
|
||||
NodeStatusPublic(
|
||||
id=entry.id, status=entry.status.value, error=entry.error
|
||||
)
|
||||
for entry in loaded.values()
|
||||
if entry.flow == name
|
||||
],
|
||||
issues=[
|
||||
issue for issue in issues if not issue.flow or issue.flow == name
|
||||
],
|
||||
)
|
||||
|
||||
def compile_check(self, flow: str, node_id: str, code: str) -> str | None:
|
||||
"""Does this source load? Returns what to show the author, or None."""
|
||||
try:
|
||||
_load_function(flow, node_id, code)
|
||||
except Exception as exc:
|
||||
return _short_error(exc)
|
||||
return None
|
||||
|
||||
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
|
||||
return self.pipeline.values(flow) if self.pipeline else {}
|
||||
|
||||
@@ -385,6 +422,25 @@ class FlowController:
|
||||
return
|
||||
self.pipeline.run(inputs or {}, nodes=self.pipeline.flow_nodes(flow))
|
||||
|
||||
def run_preview(self, flow_name: str, inputs: dict[str, Any] | None = None) -> None:
|
||||
"""Run a flow's unpublished draft once, as the editor shows it.
|
||||
|
||||
The nodes are built and run but never activated, so a draft never
|
||||
acquires subscriptions, schedules or webhooks — only publishing does
|
||||
that. State and events are the live ones, so values land on the canvas
|
||||
exactly as they do for a published run.
|
||||
"""
|
||||
flow = self.store.read_flow(flow_name, draft=True)
|
||||
nodes, _, initial_values, _ = self._build_flows([(flow, True)])
|
||||
pipeline = Pipeline(
|
||||
nodes=nodes,
|
||||
state=self.state,
|
||||
events=self.events,
|
||||
max_workers=self.max_workers,
|
||||
initial_values=initial_values,
|
||||
)
|
||||
pipeline.run(inputs or {})
|
||||
|
||||
def trigger_node(self, node_id: str, values: dict[str, Any] | None = None) -> None:
|
||||
"""Feed values into one node. Blocking — call from a worker thread."""
|
||||
node = self.get_node(node_id)
|
||||
@@ -405,6 +461,44 @@ def _bound(specs: list[MessageSpec]) -> list[MessageSpec]:
|
||||
return [spec for spec in specs if spec.name]
|
||||
|
||||
|
||||
def _collect_issues(
|
||||
loaded: dict[str, LoadedNode],
|
||||
pipeline: Pipeline,
|
||||
flow_inputs: dict[str, bool],
|
||||
) -> list[ValidationIssue]:
|
||||
"""Everything wrong with a built graph, whether it is running or not."""
|
||||
issues = pipeline.validate(flow_inputs)
|
||||
issues += [
|
||||
ValidationIssue(
|
||||
code="node_error",
|
||||
message=entry.error or "This node could not be loaded.",
|
||||
flow=entry.flow,
|
||||
node=entry.id,
|
||||
)
|
||||
for entry in loaded.values()
|
||||
if entry.status is NodeStatus.ERROR
|
||||
]
|
||||
# Webhooks are mounted without authentication, so a hook without a secret is
|
||||
# open to anyone who guesses its URL. Flows written before the parameter
|
||||
# existed keep running, but say so.
|
||||
issues += [
|
||||
ValidationIssue(
|
||||
code="unauthenticated_hook",
|
||||
message=(
|
||||
f"Webhook '{entry.node.local_id}' has no secret — "
|
||||
"anyone who knows its URL can trigger it."
|
||||
),
|
||||
flow=entry.flow,
|
||||
node=entry.id,
|
||||
)
|
||||
for entry in loaded.values()
|
||||
if isinstance(entry.node, HttpNode)
|
||||
and entry.node.mode == HttpNode.Mode.TRIGGER
|
||||
and not entry.node.secret
|
||||
]
|
||||
return issues
|
||||
|
||||
|
||||
def _short_error(exc: Exception) -> str:
|
||||
"""One line a node author can act on: what broke, and where in their code.
|
||||
|
||||
|
||||
@@ -66,7 +66,10 @@ class Pipeline:
|
||||
initial_values: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
self._nodes = nodes or []
|
||||
self._state: StateBackend = state or MemoryState()
|
||||
# An empty state backend is falsy, so this cannot be ``state or ...``:
|
||||
# that would quietly hand the pipeline a second, private state and
|
||||
# leave everyone reading the shared one seeing nothing.
|
||||
self._state: StateBackend = state if state is not None else MemoryState()
|
||||
self._events = events
|
||||
self._max_workers = max_workers
|
||||
|
||||
|
||||
@@ -115,6 +115,7 @@ class FlowSummary(BaseModel):
|
||||
title: str = ""
|
||||
node_count: int = 0
|
||||
error_count: int = 0
|
||||
has_draft: bool = False
|
||||
|
||||
|
||||
class FlowsPublic(BaseModel):
|
||||
|
||||
+162
-21
@@ -4,6 +4,12 @@ Every flow is a directory: ``flow.json`` for the structure, ``nodes/*.py`` for
|
||||
node logic. The whole tree is a git repository and each saved change is a
|
||||
commit, so a flow's history is readable with ordinary git tooling and two
|
||||
flows never collide in one file.
|
||||
|
||||
Editing is separated from running. The editor writes ``flow.draft.json`` and
|
||||
``nodes.draft/*.py``; the engine reads only the published ``flow.json`` and
|
||||
``nodes/*.py``. Publishing promotes the draft over the published files and
|
||||
removes it, so a flow directory without draft files is simply one with nothing
|
||||
unpublished — which is what every flow written before this existed looks like.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,6 +17,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from app.flow.schemas import FlowDef
|
||||
@@ -43,12 +50,34 @@ class FlowExists(ValueError):
|
||||
return f"There is already a flow named '{self.name}'"
|
||||
|
||||
|
||||
class StaleVersion(ValueError):
|
||||
"""Someone else saved this flow since the version being written was read."""
|
||||
|
||||
def __init__(self, name: str, current: int) -> None:
|
||||
super().__init__(name)
|
||||
self.name = name
|
||||
self.current = current
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"Flow '{self.name}' has changed since you loaded it"
|
||||
|
||||
|
||||
def _same_content(left: FlowDef, right: FlowDef) -> bool:
|
||||
"""Equal but for the version counter, which the server owns."""
|
||||
return left.model_copy(update={"version": 0}) == right.model_copy(
|
||||
update={"version": 0}
|
||||
)
|
||||
|
||||
|
||||
class FlowStore:
|
||||
"""Reads and writes flows, committing every change."""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
# Draft writes are check-and-set, so two clients saving at once must not
|
||||
# interleave between reading the current version and writing the next.
|
||||
self._write_lock = threading.Lock()
|
||||
if not (self.root / ".git").exists():
|
||||
self._git("init", "-q")
|
||||
self._commit("Initialise flow store", allow_empty=True)
|
||||
@@ -91,28 +120,57 @@ class FlowStore:
|
||||
def _flow_file(self, name: str) -> Path:
|
||||
return self._flow_dir(name) / "flow.json"
|
||||
|
||||
def _draft_file(self, name: str) -> Path:
|
||||
return self._flow_dir(name) / "flow.draft.json"
|
||||
|
||||
def _node_file(self, flow: str, node_id: str) -> Path:
|
||||
return self._flow_dir(flow) / "nodes" / f"{node_id}.py"
|
||||
|
||||
def _draft_nodes_dir(self, flow: str) -> Path:
|
||||
return self._flow_dir(flow) / "nodes.draft"
|
||||
|
||||
def _draft_node_file(self, flow: str, node_id: str) -> Path:
|
||||
return self._draft_nodes_dir(flow) / f"{node_id}.py"
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Flows
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def list_flows(self) -> list[str]:
|
||||
return sorted(path.parent.name for path in self.root.glob("*/flow.json"))
|
||||
"""Every flow the editor knows, published or not."""
|
||||
names = {path.parent.name for path in self.root.glob("*/flow.json")}
|
||||
names |= {path.parent.name for path in self.root.glob("*/flow.draft.json")}
|
||||
return sorted(names)
|
||||
|
||||
def exists(self, name: str) -> bool:
|
||||
return self._flow_file(name).exists() or self._draft_file(name).exists()
|
||||
|
||||
def is_published(self, name: str) -> bool:
|
||||
return self._flow_file(name).exists()
|
||||
|
||||
def read_flow(self, name: str) -> FlowDef:
|
||||
def has_draft(self, name: str) -> bool:
|
||||
"""Are there unpublished changes — to the structure or to any source?"""
|
||||
if self._draft_file(name).exists():
|
||||
return True
|
||||
drafts = self._draft_nodes_dir(name)
|
||||
return drafts.exists() and any(drafts.glob("*.py"))
|
||||
|
||||
def read_flow(self, name: str, draft: bool = False) -> FlowDef:
|
||||
"""The published flow, or with ``draft`` the working copy."""
|
||||
if draft:
|
||||
path = self._draft_file(name)
|
||||
if path.exists():
|
||||
return FlowDef.model_validate_json(path.read_text())
|
||||
path = self._flow_file(name)
|
||||
if not path.exists():
|
||||
raise FlowNotFound(name)
|
||||
return FlowDef.model_validate_json(path.read_text())
|
||||
|
||||
def read_all(self) -> list[FlowDef]:
|
||||
"""Every published flow — what the engine runs."""
|
||||
flows = []
|
||||
for name in self.list_flows():
|
||||
for path in sorted(self.root.glob("*/flow.json")):
|
||||
name = path.parent.name
|
||||
try:
|
||||
flows.append(self.read_flow(name))
|
||||
except Exception:
|
||||
@@ -120,7 +178,7 @@ class FlowStore:
|
||||
return flows
|
||||
|
||||
def write_flow(self, flow: FlowDef) -> bool:
|
||||
"""Save a flow. Returns False when nothing actually changed."""
|
||||
"""Publish a flow directly. Returns False when nothing actually changed."""
|
||||
path = self._flow_file(flow.name)
|
||||
content = flow.model_dump_json(indent=2) + "\n"
|
||||
if path.exists() and path.read_text() == content:
|
||||
@@ -131,6 +189,68 @@ class FlowStore:
|
||||
self._commit(f"Update flow '{flow.name}'")
|
||||
return True
|
||||
|
||||
def write_draft(self, flow: FlowDef, base_version: int) -> FlowDef:
|
||||
"""Save unpublished changes, refusing to overwrite someone else's.
|
||||
|
||||
``base_version`` is the version the editor last saw. It has to match
|
||||
what is stored, otherwise another client saved in between and this
|
||||
write would silently discard their work.
|
||||
"""
|
||||
with self._write_lock:
|
||||
current: FlowDef | None = None
|
||||
if self.exists(flow.name):
|
||||
current = self.read_flow(flow.name, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(flow.name, current.version)
|
||||
if _same_content(current, flow):
|
||||
return current
|
||||
|
||||
stored = flow.model_copy(
|
||||
update={"version": (current.version if current else 0) + 1}
|
||||
)
|
||||
path = self._draft_file(flow.name)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(stored.model_dump_json(indent=2) + "\n")
|
||||
self._commit(f"Update draft of flow '{flow.name}'")
|
||||
return stored
|
||||
|
||||
def publish_flow(self, name: str, base_version: int) -> FlowDef:
|
||||
"""Promote the working copy to what the engine runs."""
|
||||
with self._write_lock:
|
||||
current = self.read_flow(name, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(name, current.version)
|
||||
|
||||
draft = self._draft_file(name)
|
||||
if draft.exists():
|
||||
self._flow_file(name).write_text(
|
||||
current.model_dump_json(indent=2) + "\n"
|
||||
)
|
||||
draft.unlink()
|
||||
|
||||
drafts = self._draft_nodes_dir(name)
|
||||
if drafts.exists():
|
||||
target = self._flow_dir(name) / "nodes"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for source in sorted(drafts.glob("*.py")):
|
||||
shutil.copy2(source, target / source.name)
|
||||
shutil.rmtree(drafts)
|
||||
|
||||
self._commit(f"Publish flow '{name}'")
|
||||
return current
|
||||
|
||||
def discard_draft(self, name: str) -> FlowDef:
|
||||
"""Throw the unpublished changes away and go back to what is running."""
|
||||
with self._write_lock:
|
||||
draft = self._draft_file(name)
|
||||
if draft.exists():
|
||||
draft.unlink()
|
||||
drafts = self._draft_nodes_dir(name)
|
||||
if drafts.exists():
|
||||
shutil.rmtree(drafts)
|
||||
self._commit(f"Discard draft of flow '{name}'")
|
||||
return self.read_flow(name)
|
||||
|
||||
def delete_flow(self, name: str) -> None:
|
||||
directory = self._flow_dir(name)
|
||||
if not directory.exists():
|
||||
@@ -150,22 +270,30 @@ class FlowStore:
|
||||
if self.exists(new_name):
|
||||
raise FlowExists(new_name)
|
||||
|
||||
flow = self.read_flow(name)
|
||||
self._flow_dir(name).rename(self._flow_dir(new_name))
|
||||
|
||||
renamed = flow.model_copy(update={"name": new_name})
|
||||
self._flow_file(new_name).write_text(renamed.model_dump_json(indent=2) + "\n")
|
||||
|
||||
for other in self.read_all():
|
||||
if other.name == new_name:
|
||||
# Published and draft alike carry the name, and both have to move.
|
||||
for path in (self._flow_file(new_name), self._draft_file(new_name)):
|
||||
if not path.exists():
|
||||
continue
|
||||
if self._retarget(other, f"{name}.", f"{new_name}."):
|
||||
self._flow_file(other.name).write_text(
|
||||
other.model_dump_json(indent=2) + "\n"
|
||||
)
|
||||
flow = FlowDef.model_validate_json(path.read_text())
|
||||
path.write_text(
|
||||
flow.model_copy(update={"name": new_name}).model_dump_json(indent=2)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
for other in self.list_flows():
|
||||
if other == new_name:
|
||||
continue
|
||||
for path in (self._flow_file(other), self._draft_file(other)):
|
||||
if not path.exists():
|
||||
continue
|
||||
flow = FlowDef.model_validate_json(path.read_text())
|
||||
if self._retarget(flow, f"{name}.", f"{new_name}."):
|
||||
path.write_text(flow.model_dump_json(indent=2) + "\n")
|
||||
|
||||
self._commit(f"Rename flow '{name}' to '{new_name}'")
|
||||
return renamed
|
||||
return self.read_flow(new_name, draft=True)
|
||||
|
||||
@staticmethod
|
||||
def _retarget(flow: FlowDef, old_prefix: str, new_prefix: str) -> bool:
|
||||
@@ -186,18 +314,31 @@ class FlowStore:
|
||||
# Node source
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def read_node_source(self, flow: str, node_id: str) -> str:
|
||||
def read_node_source(self, flow: str, node_id: str, draft: bool = False) -> str:
|
||||
if draft:
|
||||
path = self._draft_node_file(flow, node_id)
|
||||
if path.exists():
|
||||
return path.read_text()
|
||||
path = self._node_file(flow, node_id)
|
||||
return path.read_text() if path.exists() else DEFAULT_SOURCE
|
||||
|
||||
def write_node_source(self, flow: str, node_id: str, code: str) -> bool:
|
||||
path = self._node_file(flow, node_id)
|
||||
if path.exists() and path.read_text() == code:
|
||||
return False
|
||||
def write_node_source(
|
||||
self, flow: str, node_id: str, code: str, draft: bool = False
|
||||
) -> bool:
|
||||
if draft:
|
||||
# Compared against the working copy, so re-saving what is already
|
||||
# published does not create a draft file that never gets promoted.
|
||||
if self.read_node_source(flow, node_id, draft=True) == code:
|
||||
return False
|
||||
path = self._draft_node_file(flow, node_id)
|
||||
else:
|
||||
path = self._node_file(flow, node_id)
|
||||
if path.exists() and path.read_text() == code:
|
||||
return False
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(code)
|
||||
self._commit(f"Update node '{flow}.{node_id}'")
|
||||
self._commit(f"Update {'draft of ' if draft else ''}node '{flow}.{node_id}'")
|
||||
return True
|
||||
|
||||
def delete_node_source(self, flow: str, node_id: str) -> None:
|
||||
|
||||
Reference in New Issue
Block a user