Merge remote-tracking branch 'origin/main' into HEAD
Docs / docs (push) Successful in 36s
Playwright Tests / test-playwright (1, 2) (push) Successful in 4m11s
Playwright Tests / test-playwright (2, 2) (push) Successful in 2m34s
pre-commit / pre-commit (push) Successful in 3m35s
Test Backend / test-backend (3.12) (push) Failing after 6m25s
Test Backend / test-backend (3.14) (push) Failing after 6m2s
Compose Smoke Test / test-compose (push) Successful in 38s
Playwright Tests / merge-reports (push) Successful in 1m52s
Docs / docs (push) Successful in 36s
Playwright Tests / test-playwright (1, 2) (push) Successful in 4m11s
Playwright Tests / test-playwright (2, 2) (push) Successful in 2m34s
pre-commit / pre-commit (push) Successful in 3m35s
Test Backend / test-backend (3.12) (push) Failing after 6m25s
Test Backend / test-backend (3.14) (push) Failing after 6m2s
Compose Smoke Test / test-compose (push) Successful in 38s
Playwright Tests / merge-reports (push) Successful in 1m52s
This commit is contained in:
@@ -148,6 +148,17 @@ class RebuildBusy(RuntimeError):
|
||||
"""
|
||||
|
||||
|
||||
class NodesNotLoaded(RuntimeError):
|
||||
"""A run's flow has nodes that could not be built, so none of it runs."""
|
||||
|
||||
def __init__(self, errors: dict[str, str]) -> None:
|
||||
#: The load error of each node, by its qualified id.
|
||||
self.errors = errors
|
||||
node, error = next(iter(errors.items()))
|
||||
more = f" and {len(errors) - 1} more" if len(errors) > 1 else ""
|
||||
super().__init__(f"Node '{node}'{more} could not be loaded: {error}")
|
||||
|
||||
|
||||
class NodeStatus(str, Enum):
|
||||
ACTIVE = "active"
|
||||
ERROR = "error"
|
||||
@@ -730,8 +741,13 @@ class FlowController:
|
||||
try:
|
||||
# Work already claimed belongs to the pipeline it was claimed
|
||||
# against; let it finish there before swapping the graph out.
|
||||
if self.execution is not None:
|
||||
await run_in_threadpool(self.execution.pause_intake)
|
||||
# Work that does not finish in time keeps the old graph, which
|
||||
# nothing has touched yet.
|
||||
if self.execution is not None and not await run_in_threadpool(
|
||||
self.execution.pause_intake
|
||||
):
|
||||
self.execution.resume_intake()
|
||||
raise RebuildBusy("Claimed work is still running on the pipeline")
|
||||
await self._teardown()
|
||||
# A fresh supervisor per build, so a flow quarantined by the last
|
||||
# one gets another chance once its author has changed something.
|
||||
@@ -2010,10 +2026,21 @@ class FlowController:
|
||||
and webhooks would only be a second copy of what the live pipeline
|
||||
already holds. The state is the run's, which is what keeps two runs of
|
||||
one flow from overwriting each other's messages.
|
||||
|
||||
:raises NodesNotLoaded: When a node could not be built. The live engine
|
||||
leaves such a node out, but a run without it would finish as if
|
||||
the flow had done its work.
|
||||
"""
|
||||
nodes, _loaded, initial_values, _inputs = self._build_flows(
|
||||
nodes, loaded, initial_values, _inputs = self._build_flows(
|
||||
[(flow, draft)], run=run
|
||||
)
|
||||
errors = {
|
||||
entry.id: entry.error or "unknown error"
|
||||
for entry in loaded.values()
|
||||
if entry.status is NodeStatus.ERROR
|
||||
}
|
||||
if errors:
|
||||
raise NodesNotLoaded(errors)
|
||||
pipeline = Pipeline(
|
||||
nodes=nodes,
|
||||
state=state,
|
||||
|
||||
@@ -64,6 +64,10 @@ BAR_ROWS = 8
|
||||
#: client (``ui/core/counter.ts``, ``MAX_COUNTERS``).
|
||||
COUNTERS = 3
|
||||
|
||||
#: How many windows one querying chart offers, stacked down the tile's edge.
|
||||
#: Mirrored in the client (``ChartWidget.tsx``, ``MAX_RANGES``).
|
||||
CHART_RANGES = 8
|
||||
|
||||
#: Resolved out here on purpose: the store has a ``list`` method, which
|
||||
#: shadows the builtin for any annotation written inside the class.
|
||||
Bindings = list[dict[str, Any]]
|
||||
@@ -259,10 +263,15 @@ class WidgetDef(BaseModel):
|
||||
A chart comes in two kinds. The default reads what the engine kept for a
|
||||
message. One with ``source: "query"`` asks instead, and its config is
|
||||
``{source, request, request_dtype: "record", message, dtype: "series",
|
||||
refresh_s, range_s}``: it publishes ``{range_s, interval_s}`` to
|
||||
refresh_s, range_s, ranges}``: it publishes ``{range_s, interval_s}`` to
|
||||
``request`` exactly as a slider publishes a value, and draws the ``series``
|
||||
a flow answers with on ``message``.
|
||||
|
||||
``ranges`` lists the windows its rail offers, up to ``CHART_RANGES`` of
|
||||
``{label, range_s, interval_s}``, and ``range_s`` names the one it opens on.
|
||||
Without them it offers 1h, 6h, 24h and 7d. ``range_s: 0`` asks for the
|
||||
whole history, and the flow answering decides how far back that reaches.
|
||||
|
||||
A colour widget picks what it publishes with ``format``, because fixtures
|
||||
differ and a change node per tile is not the answer:
|
||||
|
||||
@@ -448,6 +457,28 @@ class WidgetDef(BaseModel):
|
||||
raise ValueError(
|
||||
f"a querying chart's {key} must be '{want}', not '{bound}'"
|
||||
)
|
||||
ranges = self.config.get("ranges") or []
|
||||
if not isinstance(ranges, list) or len(ranges) > CHART_RANGES:
|
||||
raise ValueError(
|
||||
f"a querying chart offers a list of at most {CHART_RANGES} ranges"
|
||||
)
|
||||
for entry in ranges:
|
||||
window = entry if isinstance(entry, dict) else {}
|
||||
label = window.get("label")
|
||||
span, step = window.get("range_s"), window.get("interval_s")
|
||||
# `type() is int`, because a bool is an int to isinstance.
|
||||
if not (
|
||||
isinstance(label, str)
|
||||
and label.strip()
|
||||
and type(span) is int
|
||||
and span >= 0
|
||||
and type(step) is int
|
||||
and step > 0
|
||||
):
|
||||
raise ValueError(
|
||||
"a querying chart's range needs a label, a whole range_s "
|
||||
f"of 0 or more and a whole interval_s above 0, not {entry!r}"
|
||||
)
|
||||
return self
|
||||
|
||||
if self.type == "bar" and len(self.bar_rows) > BAR_ROWS:
|
||||
@@ -961,21 +992,23 @@ class DashboardStore:
|
||||
self._dump(shown.model_copy(update={"version": top + 1}))
|
||||
)
|
||||
draft.unlink()
|
||||
self.flows._commit(f"Discard draft of dashboard '{name}'")
|
||||
return self.read(name)
|
||||
shown = self.read(name)
|
||||
self.flows._commit(f"Discard draft of dashboard {named(shown.name, name)}")
|
||||
return shown
|
||||
|
||||
def delete(self, name: str) -> None:
|
||||
if not self.exists(name):
|
||||
raise DashboardNotFound(name)
|
||||
path = self._file(name)
|
||||
with self.flows._write_lock:
|
||||
key = next((k for k, i in self.key_index().items() if i == name), name)
|
||||
path.unlink(missing_ok=True)
|
||||
self._draft_file(name).unlink(missing_ok=True)
|
||||
try:
|
||||
path.parent.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
self.flows._commit(f"Delete dashboard '{name}'")
|
||||
self.flows._commit(f"Delete dashboard {named(key, name)}")
|
||||
|
||||
def duplicate(self, name: str) -> DashboardDef:
|
||||
"""Copy the working copy into a new dashboard, as a draft only.
|
||||
|
||||
@@ -255,14 +255,26 @@ class ExecutionService:
|
||||
with self._inflight_lock:
|
||||
self._inflight_lock.notify_all()
|
||||
|
||||
def pause_intake(self) -> None:
|
||||
def pause_intake(self) -> bool:
|
||||
"""Stop claiming, and wait for what is already claimed to finish.
|
||||
|
||||
Called around a rebuild: items claimed against the old pipeline should
|
||||
finish there rather than half-run against the new one.
|
||||
finish there rather than half-run against the new one. That includes
|
||||
a claim still under way: the consumer holds its lock from the claim to
|
||||
the dispatch, so once that lock is free here the claim is counted, and
|
||||
every later pass finds the intake closed.
|
||||
|
||||
False when that took longer than `DRAIN_TIMEOUT_S`. The intake stays
|
||||
closed either way; the caller decides whether to go on.
|
||||
"""
|
||||
self._intake.clear()
|
||||
# So a consumer waiting for capacity sees the intake closed.
|
||||
self._capacity_changed()
|
||||
deadline = time.monotonic() + DRAIN_TIMEOUT_S
|
||||
if not self._consumer_lock.acquire(timeout=DRAIN_TIMEOUT_S):
|
||||
logger.warning("Rebuild did not wait out a claim")
|
||||
return False
|
||||
self._consumer_lock.release()
|
||||
with self._inflight_lock:
|
||||
while self._running:
|
||||
remaining = deadline - time.monotonic()
|
||||
@@ -270,8 +282,9 @@ class ExecutionService:
|
||||
logger.warning(
|
||||
"Rebuild did not wait out %d cascades", self._running.total()
|
||||
)
|
||||
return
|
||||
return False
|
||||
self._inflight_lock.wait(remaining)
|
||||
return True
|
||||
|
||||
def resume_intake(self) -> None:
|
||||
self._intake.set()
|
||||
@@ -284,8 +297,8 @@ class ExecutionService:
|
||||
"""Stop claiming, promoting, touching and reclaiming, once every
|
||||
claimed cascade has finished. For a rename, which rewrites the queue.
|
||||
|
||||
Unlike `pause_intake` this also waits out a claim already blocking
|
||||
when intake clears, and a timer pass. It waits without a limit and
|
||||
Unlike `pause_intake` this also waits out a timer pass, and keeps the
|
||||
consumer lock until `release`. It waits without a limit and
|
||||
tells ``report`` the nodes whose cascades are still running each time
|
||||
they change. Setting ``abort`` gives up: claiming resumes and it
|
||||
returns False. True means held until `release`.
|
||||
|
||||
@@ -148,6 +148,8 @@ class NodeOutcome(BaseModel):
|
||||
artifacts: dict[str, dict[str, Any]] = {}
|
||||
#: Restored from an earlier run rather than executed.
|
||||
cached: bool = False
|
||||
#: Not executed, because a node it waits on failed or was skipped itself.
|
||||
skipped: bool = False
|
||||
#: Which run the restored values came from — and, because a restored row
|
||||
#: holds no series of its own, which run this node's metrics live in.
|
||||
cached_from: str = ""
|
||||
@@ -1073,7 +1075,8 @@ class Pipeline:
|
||||
inputs: dict[str, Any] | None = None,
|
||||
arrived: list[str] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Run one node and record its outputs. Never raises.
|
||||
"""Run one node and record its outputs. Never raises: None means it
|
||||
failed, and a node that published nothing answers ``{}``.
|
||||
|
||||
``inputs`` is what the readiness check already read — the same keys, a
|
||||
moment earlier. Reading them again here was the second of two MGETs
|
||||
@@ -1108,7 +1111,7 @@ class Pipeline:
|
||||
if key:
|
||||
hit, restored = self._from_cache(node, key, state, entry_id)
|
||||
if hit:
|
||||
return restored
|
||||
return restored or {}
|
||||
|
||||
token = ARRIVED.set(arrived)
|
||||
try:
|
||||
@@ -1175,7 +1178,7 @@ class Pipeline:
|
||||
output_values=result,
|
||||
)
|
||||
)
|
||||
return result
|
||||
return result or {}
|
||||
except Exception as exc:
|
||||
# One failing node must not take the rest of the graph down.
|
||||
error = self.publish_error(node, exc, collected, entry_id)
|
||||
@@ -1437,6 +1440,11 @@ class Pipeline:
|
||||
|
||||
ready_nodes = deque(n for n, count in in_degree.items() if count == 0)
|
||||
blocked: set[Node] = set()
|
||||
# A batch run starts from an empty state, so a node whose producer
|
||||
# failed would be called without that input. It is skipped instead,
|
||||
# and so is everything waiting on it. A live wave still has the last
|
||||
# values to run on.
|
||||
failed: set[Node] | None = set() if self.run_id else None
|
||||
gate_revision = self._gate_revision
|
||||
node_futures: dict[Future[dict[str, Any] | None], Node] = {}
|
||||
finished: SimpleQueue[Future[dict[str, Any] | None]] = SimpleQueue()
|
||||
@@ -1475,10 +1483,26 @@ class Pipeline:
|
||||
if n.passive:
|
||||
complete(n)
|
||||
continue
|
||||
if failed is not None and not failed.isdisjoint(deps[n]):
|
||||
failed.add(n)
|
||||
self._observe(NodeOutcome(node=n.id, ok=False, skipped=True))
|
||||
complete(n)
|
||||
continue
|
||||
if fresh is not None and untouched(n):
|
||||
complete(n)
|
||||
continue
|
||||
ready_inputs: dict[str, Any] | None = None
|
||||
if failed is not None:
|
||||
# A live wave's readiness rule: a node runs once every
|
||||
# triggering input has a value. A run has nothing to wait
|
||||
# for, so the node fails instead.
|
||||
ready_inputs = state.get_present(list(n.requires))
|
||||
missing = self._no_value(n, ready_inputs)
|
||||
if missing:
|
||||
failed.add(n)
|
||||
self._observe(NodeOutcome(node=n.id, ok=False, error=missing))
|
||||
complete(n)
|
||||
continue
|
||||
if check_ready:
|
||||
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
|
||||
if not ready:
|
||||
@@ -1518,6 +1542,8 @@ class Pipeline:
|
||||
future = finished.get()
|
||||
n = node_futures.pop(future)
|
||||
result = future.result()
|
||||
if result is None and failed is not None:
|
||||
failed.add(n)
|
||||
if result is not None and fresh is not None:
|
||||
fresh.update(result)
|
||||
# Silent, failed and throttled producers still release their
|
||||
@@ -1536,6 +1562,18 @@ class Pipeline:
|
||||
|
||||
return state
|
||||
|
||||
def _no_value(self, node: Node, values: dict[str, Any]) -> str:
|
||||
"""The first triggering input with nothing in ``values``, as an error,
|
||||
or "" when there is none."""
|
||||
for name, spec in node.requires.items():
|
||||
if spec.trigger and name not in values:
|
||||
producers = ", ".join(
|
||||
f"'{p.local_id}'" for p in self.produces.get(name, ())
|
||||
)
|
||||
where = f" from {producers}" if producers else ""
|
||||
return f"No value on input '{spec.port or name}'{where}"
|
||||
return ""
|
||||
|
||||
def run(
|
||||
self, inputs: dict[str, Any] | None = None, nodes: set[Node] | None = None
|
||||
) -> StateBackend:
|
||||
|
||||
@@ -51,7 +51,12 @@ from sqlmodel import Session, col, select
|
||||
from fluksio.core.db import engine as db_engine
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
|
||||
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
|
||||
from fluksio.flow.controller import (
|
||||
NODE_TYPES,
|
||||
FlowController,
|
||||
NodesNotLoaded,
|
||||
RunContext,
|
||||
)
|
||||
from fluksio.flow.messages import qualify, requalify
|
||||
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
|
||||
from fluksio.flow.queue import WorkItem, WorkQueue
|
||||
@@ -137,8 +142,9 @@ def batch_issues(
|
||||
history keeps every value, and the interval only thins what is published.
|
||||
Anywhere else it is a message quietly going missing.
|
||||
|
||||
A delay node is fine; without a queue to defer into it simply sleeps,
|
||||
which in a run is what was asked for.
|
||||
A delay node's ``interval`` is the same limit, put on every output it
|
||||
has. Its ``delay`` is fine: without a queue to defer into it simply
|
||||
sleeps, which in a run is what was asked for.
|
||||
|
||||
The other is a node with no body. The store answers a new node's template
|
||||
when nothing was ever written for one, so such a node runs — and returns
|
||||
@@ -148,8 +154,11 @@ def batch_issues(
|
||||
"""
|
||||
issues: list[str] = []
|
||||
for node in flow.nodes:
|
||||
for spec in list(node.requires) + list(node.provides):
|
||||
if spec.interval > 0 and not spec.stream:
|
||||
held = float(node.params.get("interval") or 0) if node.type == "delay" else 0
|
||||
limits = [(spec, spec.interval) for spec in node.requires]
|
||||
limits += [(spec, max(spec.interval, held)) for spec in node.provides]
|
||||
for spec, interval in limits:
|
||||
if interval > 0 and not spec.stream:
|
||||
issues.append(
|
||||
f"Node '{node.id}' rate-limits '{spec.port or spec.name}'. "
|
||||
"A run has no timer to release what that holds back, so "
|
||||
@@ -1236,8 +1245,10 @@ class RunService:
|
||||
return True
|
||||
# The gate first, so nothing new is submitted while the running nodes
|
||||
# are being killed; a gated node is never handed to the executor, so
|
||||
# the graph drains instead of going further.
|
||||
pipeline.pause(self._key_of(run_id))
|
||||
# the graph drains instead of going further. Stopped rather than
|
||||
# paused: a pause announces itself on the engine's bus, under the key
|
||||
# the live flow shares, so every open editor showed that one paused.
|
||||
pipeline.set_disabled({self._key_of(run_id)})
|
||||
workers = self.controller.workers
|
||||
if workers is not None:
|
||||
# Keyed by run, so a sweep cancelling one config leaves the others
|
||||
@@ -1502,7 +1513,7 @@ class RunService:
|
||||
|
||||
def observe(outcome: NodeOutcome) -> None:
|
||||
nonlocal errors
|
||||
if not outcome.ok:
|
||||
if not outcome.ok and not outcome.skipped:
|
||||
errors += 1
|
||||
self._record_node(run, key, outcome)
|
||||
|
||||
@@ -1537,19 +1548,26 @@ class RunService:
|
||||
digests = node_digests(flow)
|
||||
run.code_digest = self._restamp(run, run_digest(flow))
|
||||
state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}")
|
||||
pipeline = self.controller.build_run_pipeline(
|
||||
flow,
|
||||
state=state,
|
||||
draft=run.draft,
|
||||
observer=observe,
|
||||
emission_observer=sink.handle,
|
||||
run=RunContext(
|
||||
run_id=run_id,
|
||||
code_digest=run.code_digest,
|
||||
node_digests=digests,
|
||||
),
|
||||
run_cache=None if run.no_cache else self._cache,
|
||||
)
|
||||
try:
|
||||
pipeline = self.controller.build_run_pipeline(
|
||||
flow,
|
||||
state=state,
|
||||
draft=run.draft,
|
||||
observer=observe,
|
||||
emission_observer=sink.handle,
|
||||
run=RunContext(
|
||||
run_id=run_id,
|
||||
code_digest=run.code_digest,
|
||||
node_digests=digests,
|
||||
),
|
||||
run_cache=None if run.no_cache else self._cache,
|
||||
)
|
||||
except NodesNotLoaded as exc:
|
||||
# Nothing of it runs; the finally below records it.
|
||||
for node, error in exc.errors.items():
|
||||
observe(NodeOutcome(node=node, ok=False, error=error))
|
||||
status, reason = "error", str(exc)[:1024]
|
||||
return
|
||||
with self._lock:
|
||||
if run_id in self._cancelled:
|
||||
# Cancelled while it was being built, so nothing of it
|
||||
@@ -1627,10 +1645,16 @@ class RunService:
|
||||
run_id = run.id
|
||||
node = requalify(outcome.node, key, run.flow_id)[:255]
|
||||
outputs = _cacheable(outcome)
|
||||
if outcome.cached:
|
||||
status = "cached"
|
||||
elif outcome.ok:
|
||||
status = "ok"
|
||||
else:
|
||||
status = "skipped" if outcome.skipped else "error"
|
||||
row = RunNode(
|
||||
run_id=run_id,
|
||||
node=node,
|
||||
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
|
||||
status=status,
|
||||
started_at=datetime.now(UTC),
|
||||
duration_ms=outcome.duration_ms,
|
||||
error=outcome.error[:ERROR_CAP],
|
||||
|
||||
@@ -639,6 +639,14 @@ class RedisState(StateBackend):
|
||||
names = [k for k in keys if not k.startswith("__")]
|
||||
if names:
|
||||
pipe.sadd(self._names_key(), *names)
|
||||
if self._ttl:
|
||||
pipe.expire(self._names_key(), self._ttl)
|
||||
|
||||
def _incr(self, pipe: Any, key: str) -> None:
|
||||
"""INCR a counter in the caller's pipeline, expiring with the values."""
|
||||
pipe.incr(self._key(key))
|
||||
if self._ttl:
|
||||
pipe.expire(self._key(key), self._ttl)
|
||||
|
||||
def set(self, key: str, value: Any) -> None:
|
||||
data = self._serialize(value)
|
||||
@@ -747,7 +755,9 @@ class RedisState(StateBackend):
|
||||
|
||||
def increment(self, key: str) -> int:
|
||||
"""Atomically increment a counter using Redis INCR."""
|
||||
return cast(int, self._client.incr(self._key(key)))
|
||||
pipe = self._client.pipeline()
|
||||
self._incr(pipe, key)
|
||||
return cast(int, pipe.execute()[0])
|
||||
|
||||
def increment_multi(self, keys: list[str]) -> None:
|
||||
"""Bump every counter in one round trip rather than one INCR each."""
|
||||
@@ -755,7 +765,7 @@ class RedisState(StateBackend):
|
||||
return
|
||||
pipe = self._client.pipeline()
|
||||
for key in keys:
|
||||
pipe.incr(self._key(key))
|
||||
self._incr(pipe, key)
|
||||
pipe.execute()
|
||||
|
||||
def get_multi(self, keys: list[str]) -> dict[str, Any]:
|
||||
@@ -824,34 +834,31 @@ class RedisState(StateBackend):
|
||||
|
||||
watch_keys = [self._key(k) for k in expected.keys()]
|
||||
|
||||
try:
|
||||
# Start watching the keys for changes
|
||||
self._client.watch(*watch_keys)
|
||||
# WATCH has to go through the pipeline: it pins one connection, and
|
||||
# the client-level call is a no-op in redis-py 7. Leaving the block
|
||||
# resets the pipeline, which unwatches.
|
||||
with self._client.pipeline() as pipe:
|
||||
try:
|
||||
pipe.watch(*watch_keys) # type: ignore[no-untyped-call]
|
||||
# A watching pipeline runs commands immediately.
|
||||
current = cast(list[bytes | None], pipe.mget(watch_keys))
|
||||
for raw, exp_value in zip(current, expected.values(), strict=True):
|
||||
if self._deserialize(raw) != exp_value:
|
||||
return False
|
||||
|
||||
# Check current values
|
||||
current = self.get_multi(list(expected.keys()))
|
||||
|
||||
for key, exp_value in expected.items():
|
||||
if current.get(key) != exp_value:
|
||||
self._client.unwatch()
|
||||
return False
|
||||
|
||||
# All values match - execute the transaction
|
||||
pipe = self._client.pipeline(True) # True = use MULTI/EXEC
|
||||
|
||||
for key, value in updates.items():
|
||||
data = self._serialize(value)
|
||||
if self._ttl:
|
||||
pipe.setex(self._key(key), self._ttl, data)
|
||||
else:
|
||||
pipe.set(self._key(key), data)
|
||||
|
||||
pipe.execute()
|
||||
return True
|
||||
|
||||
except redis.WatchError:
|
||||
# Another client modified one of the watched keys
|
||||
return False
|
||||
pipe.multi()
|
||||
for key, value in updates.items():
|
||||
data = self._serialize(value)
|
||||
if self._ttl:
|
||||
pipe.setex(self._key(key), self._ttl, data)
|
||||
else:
|
||||
pipe.set(self._key(key), data)
|
||||
self._note(pipe, updates)
|
||||
pipe.execute()
|
||||
return True
|
||||
except redis.WatchError:
|
||||
# Another client modified one of the watched keys
|
||||
return False
|
||||
|
||||
def close(self) -> None:
|
||||
"""Give the connection pool back."""
|
||||
@@ -916,7 +923,7 @@ class RedisState(StateBackend):
|
||||
if self._ttl:
|
||||
pipe.expire(history_key, self._ttl)
|
||||
for key in counters:
|
||||
pipe.incr(self._key(key))
|
||||
self._incr(pipe, key)
|
||||
self._note(pipe, values)
|
||||
pipe.execute()
|
||||
|
||||
|
||||
@@ -202,12 +202,14 @@ def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str:
|
||||
|
||||
|
||||
class FlowNotFound(KeyError):
|
||||
def __init__(self, name: str) -> None:
|
||||
def __init__(self, name: str, published: bool = False) -> None:
|
||||
super().__init__(name)
|
||||
self.name = name
|
||||
self.published = published
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"No flow named '{self.name}'"
|
||||
what = "published flow" if self.published else "flow"
|
||||
return f"No {what} with the id '{self.name}'"
|
||||
|
||||
|
||||
class FlowExists(ValueError):
|
||||
@@ -720,7 +722,7 @@ class FlowStore:
|
||||
return self._load(path)
|
||||
path = self._flow_file(name)
|
||||
if not path.exists():
|
||||
raise FlowNotFound(name)
|
||||
raise FlowNotFound(name, published=not draft)
|
||||
return self._load(path)
|
||||
|
||||
def key_index(self) -> dict[str, str]:
|
||||
@@ -782,7 +784,9 @@ class FlowStore:
|
||||
and returned as a copy so a caller sorting it cannot disturb the next.
|
||||
"""
|
||||
with self._read_all_lock:
|
||||
if self._read_all_at != self.revision:
|
||||
# Taken before scanning, so a commit landing meanwhile scans again.
|
||||
at = self.revision
|
||||
if self._read_all_at != at:
|
||||
flows = []
|
||||
for path in sorted(self.root.glob("*/flow.json")):
|
||||
name = path.parent.name
|
||||
@@ -791,7 +795,7 @@ class FlowStore:
|
||||
except Exception:
|
||||
logger.exception("Skipping unreadable flow '%s'", name)
|
||||
self._read_all = flows
|
||||
self._read_all_at = self.revision
|
||||
self._read_all_at = at
|
||||
return list(self._read_all)
|
||||
|
||||
def write_flow(self, flow: FlowDef) -> bool:
|
||||
|
||||
@@ -428,8 +428,10 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
app.state.device_store = devices
|
||||
controller.devices = devices
|
||||
# What the sockets want and the flows read, mapped onto the devices'
|
||||
# on-demand ports.
|
||||
# on-demand ports. It is process-wide, so it lets go of this
|
||||
# controller when the engine stops.
|
||||
demand.bind(controller)
|
||||
started.append(lambda: demand.bind(None))
|
||||
# Charts need a deeper series than the default; tell the engine
|
||||
# before it starts recording.
|
||||
controller.set_history_limits(dashboards.history_requirements())
|
||||
|
||||
@@ -877,12 +877,16 @@ def _shim(spec: NodeSpec) -> str:
|
||||
The import is aliased because ``process``'s parameters are the node's port
|
||||
names, and a port may be named after the very function it feeds — which
|
||||
would otherwise shadow the import inside the body.
|
||||
|
||||
Only the triggering ports are named. The engine leaves out an input that
|
||||
has no value yet, which only a port that does not trigger can lack, so
|
||||
those travel in ``**settings`` and the function's own default applies.
|
||||
"""
|
||||
_refuse_main(spec)
|
||||
fn = spec.fn
|
||||
where = inspect.getsourcefile(fn) or fn.__module__
|
||||
repo = import_root(fn)
|
||||
ports = [port.port for port in spec.requires]
|
||||
ports = [port.port for port in spec.requires if port.trigger]
|
||||
signature = ", ".join(ports + ["**settings"])
|
||||
arguments = ", ".join([f"{port}={port}" for port in ports] + ["**settings"])
|
||||
call = f"_impl({arguments})"
|
||||
|
||||
@@ -170,6 +170,11 @@ class Conflict(ApiError):
|
||||
self.current_version = int(detail.get("current_version") or 0)
|
||||
|
||||
|
||||
def engine_url(url: str) -> str:
|
||||
"""An engine's address, whether it was given with the API's path or not."""
|
||||
return url.rstrip("/").removesuffix(API).rstrip("/")
|
||||
|
||||
|
||||
class Client:
|
||||
"""An authenticated engine, addressed over its HTTP API."""
|
||||
|
||||
@@ -182,7 +187,9 @@ class Client:
|
||||
retries: int = RETRIES,
|
||||
) -> None:
|
||||
stored = _stored()
|
||||
self.url = url or os.environ.get("FLUKSIO_URL") or stored.get("url") or ""
|
||||
self.url = engine_url(
|
||||
url or os.environ.get("FLUKSIO_URL") or stored.get("url") or ""
|
||||
)
|
||||
self.token = (
|
||||
token or os.environ.get("FLUKSIO_TOKEN") or stored.get("token") or ""
|
||||
)
|
||||
@@ -766,8 +773,9 @@ def login(
|
||||
"""Exchange credentials for a token and remember the engine."""
|
||||
import httpx
|
||||
|
||||
url = engine_url(url)
|
||||
response = httpx.post(
|
||||
f"{url.rstrip('/')}{API}/login/access-token",
|
||||
f"{url}{API}/login/access-token",
|
||||
data={"username": email, "password": password},
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user