Refuse what a node cannot publish, and stop timing out work that is fine
Four things the python SDK turned up, each fixed where every client sees it. A key no port declares is now an error rather than a silent drop, on the return, the yield and the emit alike — the contract the docs already stated. The SDK reads literal yields at sync time, so a typo fails before anything runs, and an emission of one fails the call rather than being logged where nobody looks. NaN and infinity are refused at the port. JSON cannot spell either, so one that travelled came back as a 500, a socket frame that stopped the canvas, or a metric batch the database dropped whole. An artifact input takes `@run:<id>.<output>` or a bare digest, resolved on the engine — so the CLI, the run dialog and a python caller mean the same thing, and a sweep can pass one at all. Node timeouts are off by default. The clock measured silence, which a training node is full of, and remote workers had already stopped enforcing it — their heartbeat reset it. Now a heartbeat proves the agent rather than the node, ninety seconds of nothing fails the call either way, and the engine touches work it is still running so a long node is not redelivered at sixty seconds. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019V5bsYGNxcgPs4xXmTPx69
This commit is contained in:
Claude Opus 5
@@ -877,7 +877,13 @@ class FlowController:
|
||||
entry.status = NodeStatus.ERROR
|
||||
entry.error = problem
|
||||
return entry
|
||||
timeout = node_def.timeout or settings.FLOW_NODE_TIMEOUT
|
||||
# Not ``or``: 0 is a node saying it has no limit, which is
|
||||
# exactly the value that would fall through to the default.
|
||||
timeout = (
|
||||
node_def.timeout
|
||||
if node_def.timeout is not None
|
||||
else settings.FLOW_NODE_TIMEOUT
|
||||
)
|
||||
function = self.workers.proxy(
|
||||
owner,
|
||||
local,
|
||||
|
||||
@@ -31,6 +31,10 @@ CLAIM_BLOCK_MS = 1000
|
||||
# Long enough that a busy cascade is not mistaken for a dead one.
|
||||
RECLAIM_IDLE_MS = 60_000
|
||||
RECLAIM_INTERVAL_S = 30.0
|
||||
# How often to tell the queue that what we hold is still being worked on. A
|
||||
# node may run for as long as it likes, so what marks an item abandoned is this
|
||||
# stopping — which is what an engine that died does.
|
||||
TOUCH_INTERVAL_S = 20.0
|
||||
DELAYED_INTERVAL_S = 1.0
|
||||
MAX_CASCADES = 4
|
||||
# How long a reload waits for claimed work to finish before rebuilding anyway.
|
||||
@@ -54,6 +58,8 @@ class ExecutionService:
|
||||
self._intake.set()
|
||||
self._inflight = 0
|
||||
self._inflight_lock = threading.Condition()
|
||||
# Entry ids claimed and still running, under _inflight_lock.
|
||||
self._active: set[str] = set()
|
||||
self.node_pool = ThreadPoolExecutor(
|
||||
max_workers=max_workers or 4, thread_name_prefix="node"
|
||||
)
|
||||
@@ -145,6 +151,7 @@ class ExecutionService:
|
||||
def _tick(self) -> None:
|
||||
"""Promote delayed items, and take back what a dead engine dropped."""
|
||||
last_reclaim = 0.0
|
||||
last_touch = 0.0
|
||||
while not self._stop.is_set():
|
||||
self._stop.wait(DELAYED_INTERVAL_S)
|
||||
if self._stop.is_set():
|
||||
@@ -155,6 +162,16 @@ class ExecutionService:
|
||||
logger.error("Could not promote delayed work: %s", exc)
|
||||
|
||||
now = time.monotonic()
|
||||
if now - last_touch >= TOUCH_INTERVAL_S:
|
||||
last_touch = now
|
||||
with self._inflight_lock:
|
||||
running = list(self._active)
|
||||
try:
|
||||
self.queue.touch(running)
|
||||
except Exception as exc:
|
||||
logger.error("Could not touch claimed work: %s", exc)
|
||||
|
||||
|
||||
if now - last_reclaim < RECLAIM_INTERVAL_S:
|
||||
continue
|
||||
last_reclaim = now
|
||||
@@ -172,15 +189,18 @@ class ExecutionService:
|
||||
def _dispatch(self, item: WorkItem) -> None:
|
||||
with self._inflight_lock:
|
||||
self._inflight += 1
|
||||
if item.entry_id:
|
||||
self._active.add(item.entry_id)
|
||||
try:
|
||||
self._cascade_pool.submit(self._handle, item)
|
||||
except RuntimeError:
|
||||
# Pool already shutting down.
|
||||
self._done()
|
||||
self._done(item)
|
||||
|
||||
def _done(self) -> None:
|
||||
def _done(self, item: WorkItem) -> None:
|
||||
with self._inflight_lock:
|
||||
self._inflight -= 1
|
||||
self._active.discard(item.entry_id)
|
||||
self._inflight_lock.notify_all()
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
@@ -222,7 +242,7 @@ class ExecutionService:
|
||||
logger.error(
|
||||
"Could not acknowledge work for '%s': %s", item.node, exc
|
||||
)
|
||||
self._done()
|
||||
self._done(item)
|
||||
|
||||
def _run_item(self, item: WorkItem) -> bool:
|
||||
"""Run one item. False means it was not handled and must come back."""
|
||||
|
||||
@@ -11,6 +11,7 @@ consume each other's messages.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
@@ -65,11 +66,42 @@ _ITEM_TYPES = frozenset(
|
||||
)
|
||||
|
||||
|
||||
#: How deep to look for a non-finite number. Deeper than any payload that
|
||||
#: reads well on a canvas, and a bound on a value that refers to itself.
|
||||
_WALK_DEPTH = 8
|
||||
|
||||
|
||||
def _is_number(value: Any) -> bool:
|
||||
"""A measurement. bool is an int subclass; a flag is not a number here."""
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool)
|
||||
|
||||
|
||||
def _nonfinite(value: Any, depth: int = 0) -> float | None:
|
||||
"""The first NaN or infinity in a value, however deeply it sits.
|
||||
|
||||
JSON cannot spell either: ``json.dumps`` writes a bare ``NaN``, which a
|
||||
strict parser refuses. So a metric that goes non-finite leaves the engine
|
||||
as a response nobody can read, a socket frame that stops a canvas, or a row
|
||||
the database rejects — all of them a long way from the node that produced
|
||||
it. Naming it here costs one walk of a value already about to be encoded.
|
||||
"""
|
||||
if isinstance(value, float) and not math.isfinite(value):
|
||||
return value
|
||||
if depth >= _WALK_DEPTH:
|
||||
return None
|
||||
if isinstance(value, dict):
|
||||
items: Any = value.values()
|
||||
elif isinstance(value, (list, tuple)):
|
||||
items = value
|
||||
else:
|
||||
return None
|
||||
for item in items:
|
||||
found = _nonfinite(item, depth + 1)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _is_record(value: Any) -> bool:
|
||||
return isinstance(value, dict) and all(
|
||||
isinstance(key, str) and (item is None or isinstance(item, _SCALARS))
|
||||
@@ -188,6 +220,13 @@ class MessageSpec(BaseModel):
|
||||
def check(self, value: Any) -> None:
|
||||
"""Raise if ``value`` does not match this port's declared type."""
|
||||
where = self.name or self.port
|
||||
stray = _nonfinite(value)
|
||||
if stray is not None:
|
||||
raise TypeError(
|
||||
f"{where}: {stray} cannot travel as JSON. A subset with nothing "
|
||||
"in it, or a division that had no denominator, is what usually "
|
||||
"produces one — publish None, or a number that says so."
|
||||
)
|
||||
if self.dtype is DType.SERIES:
|
||||
ok = _is_series(value)
|
||||
elif self.dtype is DType.LIST:
|
||||
|
||||
@@ -287,7 +287,9 @@ class Node:
|
||||
|
||||
Only ``None`` means "nothing to publish". A falsy value of the wrong
|
||||
shape — ``0``, ``""``, an empty list — is a mistake worth naming rather
|
||||
than silence someone has to debug from an empty canvas.
|
||||
than silence someone has to debug from an empty canvas. So is a key no
|
||||
port declares: a mistyped metric name is how a training curve goes
|
||||
missing, and it costs nothing to say so at the first yield.
|
||||
"""
|
||||
if retval is None:
|
||||
return None
|
||||
@@ -302,8 +304,18 @@ class Node:
|
||||
for key, value in retval.items():
|
||||
spec = by_port.get(key) or self.provides.get(key)
|
||||
if spec is None:
|
||||
continue
|
||||
spec.check(value)
|
||||
declared = sorted(by_port) or ["none"]
|
||||
raise NodeOutputError(
|
||||
f"'{self.local_id}' produced '{key}', which no port "
|
||||
f"declares. Its ports are: {', '.join(declared)}."
|
||||
)
|
||||
try:
|
||||
spec.check(value)
|
||||
except TypeError as exc:
|
||||
# Same category as an undeclared key: what the node produced is
|
||||
# wrong. Naming it as one is what lets an emission fail the
|
||||
# call rather than being logged where nobody reads it.
|
||||
raise NodeOutputError(f"'{self.local_id}' {exc}") from exc
|
||||
outputs[spec.name] = value
|
||||
return outputs or None
|
||||
|
||||
|
||||
@@ -126,6 +126,15 @@ class WorkQueue(ABC):
|
||||
def ack(self, item: WorkItem) -> None:
|
||||
"""Mark an item done, so it is never redelivered."""
|
||||
|
||||
def touch(self, entry_ids: list[str]) -> None:
|
||||
"""Say these items are still being worked on, not abandoned.
|
||||
|
||||
A node with no timeout may run far longer than the reclaim window, and
|
||||
nothing else distinguishes that from an engine that died holding the
|
||||
item. Concrete rather than abstract: a queue with no redelivery has
|
||||
nothing to answer here.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def reclaim_stale(self, min_idle_ms: int) -> list[WorkItem]:
|
||||
"""Take back items claimed by a consumer that never acknowledged them."""
|
||||
@@ -361,6 +370,23 @@ class RedisWorkQueue(WorkQueue):
|
||||
if item.entry_id:
|
||||
self._redis.xack(self._stream, GROUP, item.entry_id)
|
||||
|
||||
def touch(self, entry_ids: list[str]) -> None:
|
||||
if not entry_ids:
|
||||
return
|
||||
# Claiming an entry we already hold resets how long it has been idle,
|
||||
# which is the only thing `reclaim_stale` reads. `justid` keeps the
|
||||
# delivery count where it is, so a long node neither redelivers nor
|
||||
# spends its way towards the dead-letter cap. An entry already
|
||||
# acknowledged is simply not there, and this is a no-op for it.
|
||||
self._redis.xclaim(
|
||||
self._stream,
|
||||
GROUP,
|
||||
self._consumer,
|
||||
min_idle_time=0,
|
||||
message_ids=entry_ids,
|
||||
justid=True,
|
||||
)
|
||||
|
||||
def reclaim_stale(self, min_idle_ms: int) -> list[WorkItem]:
|
||||
"""Take over entries a dead consumer never acknowledged."""
|
||||
_cursor, entries, _deleted = cast(
|
||||
|
||||
@@ -30,14 +30,17 @@ import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from fluksio.flow.nodes.base import NodeOutputError
|
||||
from fluksio.flow.workers import NodeTimeout, RemoteError, _remote_class
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: How long the loop is given to accept a frame we are handing it.
|
||||
SEND_TIMEOUT_S = 30.0
|
||||
#: A worker that has said nothing for this long is treated as gone. It sends a
|
||||
#: heartbeat while it is executing, so this only ever catches a dead socket.
|
||||
#: A worker that has said nothing at all for this long — not even a heartbeat —
|
||||
#: is treated as gone. It beats every ten seconds while it is executing, so this
|
||||
#: catches a dead socket rather than a slow node, and it is what bounds a call
|
||||
#: whose node has no timeout of its own.
|
||||
SILENCE_S = 90.0
|
||||
#: Protocol version this engine speaks. A worker announcing anything else is
|
||||
#: refused rather than half-understood.
|
||||
@@ -125,15 +128,27 @@ class RemoteWorker:
|
||||
future = asyncio.run_coroutine_threadsafe(self._send(payload), self._loop)
|
||||
future.result(timeout=SEND_TIMEOUT_S)
|
||||
|
||||
# The node's own deadline measures silence, so a node reporting its
|
||||
# progress is never mistaken for a hung one. A heartbeat is not
|
||||
# progress: it says the agent is alive, which is what SILENCE_S
|
||||
# asks, and says nothing about the node — so it feeds the liveness
|
||||
# bound below and never the node's own.
|
||||
deadline = time.monotonic() + timeout if timeout > 0 else None
|
||||
while True:
|
||||
wait = SILENCE_S
|
||||
if deadline is not None:
|
||||
wait = min(SILENCE_S, deadline - time.monotonic())
|
||||
try:
|
||||
# Reset per frame: the deadline measures silence, so a node
|
||||
# reporting its progress is never mistaken for a hung one.
|
||||
message = inbox.get(timeout=timeout)
|
||||
message = inbox.get(timeout=max(wait, 0.0))
|
||||
except queue.Empty:
|
||||
self.cancel(call_id)
|
||||
raise NodeTimeout(
|
||||
f"'{self.name}' was silent for {timeout}s"
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
raise NodeTimeout(
|
||||
f"'{self.name}' was silent for {timeout}s"
|
||||
) from None
|
||||
raise RemoteError(
|
||||
f"worker '{self.name}' sent nothing for "
|
||||
f"{SILENCE_S:.0f}s and is presumed gone"
|
||||
) from None
|
||||
if message is None:
|
||||
raise RemoteError(f"worker '{self.name}' went away mid-call")
|
||||
@@ -144,12 +159,19 @@ class RemoteWorker:
|
||||
if on_event is not None:
|
||||
try:
|
||||
on_event(message)
|
||||
except NodeOutputError:
|
||||
# A port the node never declared. Stop the call
|
||||
# rather than let the rest of its emissions arrive.
|
||||
self.cancel(call_id)
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Could not record a worker event")
|
||||
if deadline is not None:
|
||||
deadline = time.monotonic() + timeout
|
||||
continue
|
||||
return message
|
||||
except Exception as exc:
|
||||
if isinstance(exc, (NodeTimeout, RemoteError)):
|
||||
if isinstance(exc, (NodeTimeout, RemoteError, NodeOutputError)):
|
||||
raise
|
||||
raise RemoteError(f"worker '{self.name}': {exc}") from exc
|
||||
finally:
|
||||
|
||||
@@ -28,6 +28,7 @@ from __future__ import annotations
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import socket
|
||||
import threading
|
||||
@@ -43,9 +44,9 @@ from sqlalchemy.dialects.sqlite import insert as upsert
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from fluksio.core.db import engine as db_engine
|
||||
from fluksio.flow.artifacts import ArtifactStore, is_reference
|
||||
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
|
||||
from fluksio.flow.controller import FlowController, RunContext
|
||||
from fluksio.flow.messages import qualify
|
||||
from fluksio.flow.messages import DType, qualify
|
||||
from fluksio.flow.pipeline import NodeOutcome, Pipeline
|
||||
from fluksio.flow.queue import WorkItem, WorkQueue
|
||||
from fluksio.flow.schemas import FlowDef
|
||||
@@ -144,6 +145,112 @@ def required_labels(flow: FlowDef) -> list[str]:
|
||||
)
|
||||
|
||||
|
||||
#: What a run's output is called from outside it: ``@run:<id>.<output>``.
|
||||
RUN_REF_PREFIX = "@run:"
|
||||
|
||||
|
||||
def resolve_references(
|
||||
flow: FlowDef, params: dict[str, Any], artifacts: ArtifactStore | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Turn the text spellings of an artifact input into the reference itself.
|
||||
|
||||
A python caller hands one run's output straight to the next, because it has
|
||||
the reference in its hand. A shell does not, and pasting the whole object
|
||||
is not a command anyone wants to type — so an artifact input also takes
|
||||
``@run:<id>.<output>``, naming what a run produced, or a bare
|
||||
``sha256:...`` digest naming the bytes. Resolved here rather than in each
|
||||
client, so the CLI, the browser and a python caller all mean the same
|
||||
thing by the same string.
|
||||
"""
|
||||
wanted = {
|
||||
declared.spec.name
|
||||
for declared in flow.inputs
|
||||
if declared.spec.dtype is DType.ARTIFACT
|
||||
}
|
||||
pending = {
|
||||
key: value
|
||||
for key, value in params.items()
|
||||
if key in wanted and isinstance(value, str)
|
||||
}
|
||||
if not pending:
|
||||
return params
|
||||
|
||||
resolved = dict(params)
|
||||
with Session(db_engine) as session:
|
||||
for key, text in pending.items():
|
||||
if text.startswith(RUN_REF_PREFIX):
|
||||
resolved[key] = _from_run(session, key, text[len(RUN_REF_PREFIX) :])
|
||||
elif valid_digest(text):
|
||||
resolved[key] = _from_digest(session, key, text, artifacts)
|
||||
# Anything else is left alone: the type check names it better than
|
||||
# a guess about what was meant would.
|
||||
return resolved
|
||||
|
||||
|
||||
def _from_run(session: Session, key: str, spelling: str) -> dict[str, Any]:
|
||||
"""``<run id>.<output>`` as the reference that run produced."""
|
||||
run_id, _, output = spelling.partition(".")
|
||||
if not run_id or not output:
|
||||
raise RunRejected(
|
||||
f"Parameter '{key}': '{RUN_REF_PREFIX}{spelling}' names no output — "
|
||||
f"write {RUN_REF_PREFIX}<run id>.<output>"
|
||||
)
|
||||
run = session.get(Run, run_id)
|
||||
if run is None:
|
||||
raise RunRejected(f"Parameter '{key}': there is no run '{run_id}'")
|
||||
|
||||
# The run's own result first: that is the reference as its producer made
|
||||
# it, file name and all. The rows are the fallback, and they carry the
|
||||
# message name instead — which loads the same bytes either way.
|
||||
candidate = (run.result or {}).get(output)
|
||||
if is_reference(candidate):
|
||||
return dict(candidate)
|
||||
|
||||
rows = session.exec(
|
||||
select(RunArtifact).where(col(RunArtifact.run_id) == run_id)
|
||||
).all()
|
||||
for row in rows:
|
||||
if output in (row.name, row.name.rsplit(".", 1)[-1]):
|
||||
return {
|
||||
"digest": row.digest,
|
||||
"size": row.size,
|
||||
"media_type": row.media_type or "application/octet-stream",
|
||||
"name": row.name,
|
||||
}
|
||||
known = ", ".join(sorted(row.name for row in rows)) or "none"
|
||||
raise RunRejected(
|
||||
f"Parameter '{key}': run '{run_id}' has no artifact '{output}' "
|
||||
f"(it made: {known})"
|
||||
)
|
||||
|
||||
|
||||
def _from_digest(
|
||||
session: Session, key: str, digest: str, artifacts: ArtifactStore | None
|
||||
) -> dict[str, Any]:
|
||||
"""A bare digest as a reference, with the size the store needs."""
|
||||
row = session.exec(
|
||||
select(RunArtifact)
|
||||
.where(col(RunArtifact.digest) == digest)
|
||||
.order_by(col(RunArtifact.run_id).desc())
|
||||
).first()
|
||||
if row is None:
|
||||
raise RunRejected(
|
||||
f"Parameter '{key}': no run has produced '{digest}', so there is "
|
||||
"nothing here under that digest"
|
||||
)
|
||||
if artifacts is not None and artifacts.path(digest) is None:
|
||||
raise RunRejected(
|
||||
f"Parameter '{key}': '{digest}' is known but its bytes are gone "
|
||||
"from this installation's store"
|
||||
)
|
||||
return {
|
||||
"digest": row.digest,
|
||||
"size": row.size,
|
||||
"media_type": row.media_type or "application/octet-stream",
|
||||
"name": row.name,
|
||||
}
|
||||
|
||||
|
||||
def seed_values(
|
||||
flow: FlowDef, params: dict[str, Any], seed: int | None = None
|
||||
) -> dict[str, Any]:
|
||||
@@ -230,6 +337,11 @@ class MetricSink:
|
||||
# A checkpoint or a record is on the run some other way —
|
||||
# as an artifact, or as its result. Only numbers are series.
|
||||
continue
|
||||
if not math.isfinite(value):
|
||||
# The ports refuse these, so one here came from somewhere
|
||||
# that does not go through them. A column that cannot hold
|
||||
# it would take the whole batch down with it.
|
||||
continue
|
||||
step = self._steps.get(name, -1) + 1
|
||||
self._steps[name] = step
|
||||
row = RunMetric(
|
||||
@@ -368,6 +480,7 @@ class RunService:
|
||||
) -> None:
|
||||
self.controller = controller
|
||||
self.queue = queue
|
||||
self._artifacts = artifacts
|
||||
self._cache = RunCache(artifacts)
|
||||
# Without one, a run gets a private in-memory state — which is exactly
|
||||
# the isolation it wants, minus surviving the process.
|
||||
@@ -433,6 +546,10 @@ class RunService:
|
||||
if issues:
|
||||
raise RunRejected(" ".join(issues))
|
||||
params = params or {}
|
||||
# Resolved before it is stored, so what the run records is the same
|
||||
# reference a python caller would have passed and every later reader —
|
||||
# the digest, the cache, the run detail — sees one spelling.
|
||||
params = resolve_references(flow, params, self._artifacts)
|
||||
# Checked here rather than in the driver: a caller who mistyped a
|
||||
# parameter should be told now, not by a run that fails in a minute.
|
||||
seed_values(flow, params, seed)
|
||||
|
||||
@@ -46,13 +46,13 @@ class NodeDef(BaseModel):
|
||||
source_ref: str | None = None
|
||||
timeout: float | None = Field(
|
||||
default=None,
|
||||
gt=0,
|
||||
ge=0,
|
||||
description=(
|
||||
"Seconds this node's code may run before it is stopped. This "
|
||||
"covers the first call's imports, which can be much slower than "
|
||||
"the body. Above 60 the engine may deliver its work again while "
|
||||
"it is still running — in a batch run, which never redelivers, "
|
||||
"it is an idle timeout instead: silence this long is a kill."
|
||||
"Seconds this node's code may be silent before it is stopped. A "
|
||||
"yield or an emit resets the clock, and the first call's imports "
|
||||
"are not charged to it. 0 disables the limit: the node runs until "
|
||||
"it finishes, and only a dead worker fails the call. Empty "
|
||||
"inherits the engine default."
|
||||
),
|
||||
)
|
||||
device: str | None = Field(
|
||||
|
||||
@@ -31,6 +31,7 @@ from typing import Any
|
||||
from fluksio_worker import worker_main as _worker_main
|
||||
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.nodes.base import NodeOutputError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -42,6 +43,12 @@ WORKER_MAIN = Path(_worker_main.__file__)
|
||||
#: something a person is watching a spinner for.
|
||||
COMPILE_TIMEOUT = 60.0
|
||||
|
||||
#: How often to wake up while waiting on a node with no timeout. Nothing is
|
||||
#: checked on a schedule — a worker that dies closes its pipe and wakes the
|
||||
#: read immediately — so this is only here to notice a pipe held open by
|
||||
#: something that outlived the worker it belonged to.
|
||||
POLL_S = 30.0
|
||||
|
||||
#: Environment the worker is not given. ``SECRET_KEY`` decrypts every stored
|
||||
#: secret, not only the ones bound to the node asking.
|
||||
ENV_DENY_PREFIXES = ("DATABASE_URL", "FIRST_SUPERUSER", "SENTRY_DSN")
|
||||
@@ -402,7 +409,9 @@ class PythonWorkerPool:
|
||||
# been silent rather than how long it has been working. A node
|
||||
# that reports nothing is still held to it, which is what keeps
|
||||
# the deadline meaningful for the ones that never report.
|
||||
line = worker.read_line(time.monotonic() + timeout)
|
||||
line = worker.read_line(
|
||||
time.monotonic() + (timeout if timeout > 0 else POLL_S)
|
||||
)
|
||||
if line:
|
||||
try:
|
||||
message = dict(json.loads(line))
|
||||
@@ -418,6 +427,14 @@ class PythonWorkerPool:
|
||||
if on_event is not None:
|
||||
try:
|
||||
on_event(message)
|
||||
except NodeOutputError:
|
||||
# An emission on a port the node never declared.
|
||||
# The call is already wrong, and its generator has
|
||||
# more frames coming down this pipe — so retire the
|
||||
# worker and let the author see what they emitted.
|
||||
worker.cancelled = True
|
||||
worker.kill()
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Could not record a worker event")
|
||||
continue
|
||||
@@ -435,8 +452,18 @@ class PythonWorkerPool:
|
||||
if worker.cancelled:
|
||||
raise NodeCancelled("cancelled while it was running")
|
||||
if line is None:
|
||||
worker.kill()
|
||||
raise NodeTimeout(f"was silent for {timeout}s and was killed")
|
||||
if timeout > 0:
|
||||
worker.kill()
|
||||
raise NodeTimeout(f"was silent for {timeout}s and was killed")
|
||||
# No limit, so silence is the node working. What ends the call
|
||||
# is the worker dying, and that arrives as the pipe closing
|
||||
# rather than as a deadline — unless something else inherited
|
||||
# the write end and is holding it open, which is what this
|
||||
# asks about.
|
||||
if worker.alive():
|
||||
continue
|
||||
worker.cancelled = True
|
||||
raise RemoteError("worker died")
|
||||
worker.cancelled = True
|
||||
raise RemoteError("worker died")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user