Warm a worker before the node's clock starts, and refuse to delete a live flow
A node's timeout now covers its body only: the pool loads the source into the
worker it picked, off the node's budget, so imports that outlast the timeout no
longer make a node impossible to run. Draft checks compile without caching, so
saving does not evict what a busy node is serving calls from. Requests carry an
id the worker echoes and the pool checks, a reply is encoded once, and the
remote-exception cache is bounded.
DELETE /flows/{name} answers 409 while the flow has a running or queued run,
which is what was letting run_node rows outlive their run.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01StpRc2C6au1WJ1EUU7fsfu
This commit is contained in:
@@ -281,7 +281,9 @@ def test_delete_flow(
|
||||
client: TestClient, superuser_token_headers: dict[str, str], db: Session
|
||||
) -> None:
|
||||
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
|
||||
db.add(Run(id="run-1", flow="demo", created_at=datetime.now(UTC)))
|
||||
# Finished: a flow with a live run refuses to be deleted, which is its own
|
||||
# test below.
|
||||
db.add(Run(id="run-1", flow="demo", status="ok", created_at=datetime.now(UTC)))
|
||||
db.add(RunNode(run_id="run-1", node="sensor"))
|
||||
db.add(RunMetric(run_id="run-1", name="loss", step=-1))
|
||||
db.add(RunArtifact(run_id="run-1", name="model.pt"))
|
||||
@@ -301,6 +303,24 @@ def test_delete_flow(
|
||||
assert db.exec(select(func.count()).select_from(model)).one() == 0
|
||||
|
||||
|
||||
def test_delete_flow_is_refused_while_a_run_is_live(
|
||||
client: TestClient, superuser_token_headers: dict[str, str], db: Session
|
||||
) -> None:
|
||||
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
|
||||
db.add(
|
||||
Run(id="run-live", flow="demo", status="running", created_at=datetime.now(UTC))
|
||||
)
|
||||
db.commit()
|
||||
|
||||
response = client.delete(f"{PREFIX}/demo", headers=superuser_token_headers)
|
||||
assert response.status_code == 409
|
||||
assert "run-live" in response.json()["detail"]
|
||||
# Its driver is still writing node rows against that id, so nothing may go.
|
||||
assert (
|
||||
client.get(f"{PREFIX}/demo", headers=superuser_token_headers).status_code == 200
|
||||
)
|
||||
|
||||
|
||||
def test_node_types_are_listed(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
|
||||
@@ -76,6 +76,20 @@ def test_a_node_that_runs_too_long_is_killed_and_the_pool_recovers(pool):
|
||||
assert run(pool, "def process():\n return {'out': 2}\n") == {"out": 2}
|
||||
|
||||
|
||||
def test_a_slow_import_is_not_charged_to_the_nodes_timeout(pool):
|
||||
# Module-level work is what a node's imports are, and it happens once, on a
|
||||
# cold worker. Paying for it out of the per-call budget makes a node with
|
||||
# heavy imports impossible to run at all: the timeout kills the worker, so
|
||||
# the next attempt is cold again and starts over.
|
||||
code = "import time\n\ntime.sleep(1.5)\n\n\ndef process():\n return {'out': 1}\n"
|
||||
assert pool.run("demo", "heavy", code, {}, "demo.heavy", timeout=0.5) == {"out": 1}
|
||||
|
||||
# And the same worker does not pay for them a second time.
|
||||
started = time.monotonic()
|
||||
assert pool.run("demo", "heavy", code, {}, "demo.heavy", timeout=0.5) == {"out": 1}
|
||||
assert time.monotonic() - started < 1.0
|
||||
|
||||
|
||||
def test_a_running_node_can_be_cancelled(pool):
|
||||
def stop_it() -> None:
|
||||
for _ in range(100):
|
||||
|
||||
Reference in New Issue
Block a user