"""The `fluksio sync`, `run`, `runs`, `sweep`, `status` and `login` commands. Kept beside the SDK rather than in `fluksio.cli`: these are the client half of the tool, and none of them needs the engine to be importable — `--local`, which does, imports it inside the branch that asked for it. """ from __future__ import annotations import argparse import getpass import importlib import itertools import json import pkgutil import sys import time from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path from typing import Any import httpx from fluksio.sdk import FLOWS, Flow, SyncError from fluksio.sdk.client import ( GLOBAL_DATA_DIR, WAIT_TOLERANCE, ApiError, Client, RunHandle, data_dir, login, origin_of, repo_root, sync, ) __all__ = ["add_parsers", "discover"] #: Statuses worth a colour, and the SGR code each gets. _COLORS = { "ok": "32", "error": "31", "cached": "36", "cancelled": "33", "abandoned": "31", "running": "36", "queued": "33", } def _say(message: str = "") -> None: print(message) def _fail(message: str) -> int: print(f"fluksio: {message}", file=sys.stderr) return 1 def _unreachable(exc: Exception, note: str = "") -> int: """The engine did not answer. Say so as a sentence, not a traceback.""" return _fail( f"engine not answering ({type(exc).__name__}: {exc})" + (f". {note}" if note else "") ) def _status(text: str, width: int = 0) -> str: """A status, coloured when a terminal is reading it. Padded before it is coloured: the escape sequences are characters as far as `str.format` is concerned, and a column that lines up in a pipe would not line up on screen. """ body = f"{text:<{width}}" if width else text code = _COLORS.get(text) if not code or not sys.stdout.isatty(): return body return f"\033[{code}m{body}\033[0m" # --------------------------------------------------------------------------- # Discovery # --------------------------------------------------------------------------- def _package_of(directory: Path) -> tuple[str, str]: """The path root and dotted name of a package directory.""" parts = [directory.name] parent = directory.parent while (parent / "__init__.py").exists(): parts.append(parent.name) parent = parent.parent return str(parent), ".".join(reversed(parts)) def _module_of(path: Path) -> tuple[str, str]: """The path root and dotted name of a module file.""" parts = [path.stem] directory = path.parent while (directory / "__init__.py").exists(): parts.append(directory.name) directory = directory.parent return str(directory), ".".join(reversed(parts)) def _import(root: str, dotted: str) -> None: if root not in sys.path: sys.path.insert(0, root) importlib.import_module(dotted) def discover(targets: list[str]) -> list[Flow]: """Import what was named and hand back the flows it declared. Imported by dotted name with its root on the path, never from a file location: the generated node bodies import the same way, and a module loaded under a different name would generate an import that does not resolve. """ for target in targets: path = Path(target) if not path.exists(): _import(str(Path.cwd()), target) continue path = path.resolve() if path.is_file(): _import(*_module_of(path)) continue if (path / "__init__.py").exists(): root, dotted = _package_of(path) _import(root, dotted) package = sys.modules[dotted] for info in pkgutil.walk_packages(package.__path__, f"{dotted}."): importlib.import_module(info.name) continue # A plain directory — a repository root, usually. Its own modules, # and the packages inside it: `myresearch/` beside a `README` is the # ordinary shape, and naming it explicitly should not be the price of # keeping your code in a package. for module in sorted(path.glob("*.py")): _import(*_module_of(module)) for child in sorted(path.iterdir()): if child.name.startswith(".") or not (child / "__init__.py").exists(): continue root, dotted = _package_of(child) _import(root, dotted) for info in pkgutil.walk_packages( sys.modules[dotted].__path__, f"{dotted}." ): importlib.import_module(info.name) return list(FLOWS.values()) # --------------------------------------------------------------------------- # Which engine # --------------------------------------------------------------------------- @contextmanager def _engine_client() -> Iterator[Client]: """The engine itself, in this process, behind the ordinary client. Everything `fluksio serve` does apart from listening on a socket: the same data directory, the same database, the same admin, the same lifespan. The app is driven through its ASGI interface, so a run costs what it costs on a served engine and lands in the same history — which is what makes the stage cache carry across the two. The engine only exists for the length of the command, so nothing here is written back as a login: a stored token belongs to whichever engine `fluksio login` was pointed at. """ # Imported here, not at module scope: everything else in this file is the # client half and must keep working with no engine installed. import logging from fluksio.cli import _data_dir, _prepare, _print_new_admin directory = _data_dir(None) # Settings are read when the app is imported, so this comes first. _prepare(directory) # Nothing here goes over a network, so httpx logging each call as a # request to "testserver" is noise that also happens to be untrue. logging.getLogger("httpx").setLevel(logging.WARNING) from datetime import timedelta from fastapi.testclient import TestClient from sqlmodel import Session from fluksio.core import security from fluksio.core.bootstrap import ensure_superuser from fluksio.core.db import engine with Session(engine) as session: admin, generated = ensure_superuser(session) admin_id = admin.id if generated: _print_new_admin(admin.email, generated) token = security.create_access_token(admin_id, expires_delta=timedelta(hours=12)) from fluksio.main import app # Entering the client is what runs the lifespan: the worker pool, the # controller and the run service all start here and stop on the way out. with TestClient(app) as http: yield Client(http=http, token=token) @contextmanager def _client_for(args: argparse.Namespace) -> Iterator[Client]: """The engine this command talks to: one running somewhere, or this one.""" if getattr(args, "local", False): with _engine_client() as client: yield client else: yield Client(url=args.url, token=args.token) # --------------------------------------------------------------------------- # Commands # --------------------------------------------------------------------------- def cmd_login(args: argparse.Namespace) -> int: """Only needed for an engine somewhere else — `serve` signs you in here.""" email = args.email or input("Email: ") password = args.password or getpass.getpass("Password: ") directory = GLOBAL_DATA_DIR.expanduser() if args.shared else data_dir() try: path = login(args.url, email, password, directory=directory) except ApiError as exc: return _fail(f"could not log in: {exc.detail}") _say(f"Logged in to {args.url}; the token is in {path}.") return 0 def cmd_sync(args: argparse.Namespace) -> int: targets = args.targets or ["."] try: flows = discover(targets) except (ImportError, SyncError) as exc: return _fail(str(exc)) if not flows: return _fail( f"no flows declared in {', '.join(targets)} — a flow is a `Flow(...)` " "at module level. Name the package if it is somewhere else: " "`fluksio sync src/myresearch`." ) repo = repo_root(targets[0]) origin = origin_of(repo) if origin["dirty"]: _say(f"warning: {repo} has uncommitted changes, so the stamp says -dirty") if not origin["commit"]: _say(f"warning: {repo} is not a git repository, so runs cannot name a commit") if args.dry_run: for target in flows: _say(f"=== flow {target.name}") _say(json.dumps(target.document(origin), indent=2)) for node_id, code in target.shims().items(): _say(f"=== {target.name}.{node_id}") _say(code) return 0 try: client = Client(url=args.url, token=args.token) reports = sync( flows, client, origin=origin, publish=not args.no_publish, force=args.force, ) except (SyncError, ApiError) as exc: return _fail(str(exc)) except httpx.HTTPError as exc: return _unreachable(exc, "Nothing was published; sync again when it is back.") for report in reports: if report.unchanged: _say(f" {report.flow}: unchanged") continue what = "created" if report.created else "updated" detail = ", ".join(report.changed) state = "published" if report.published else "draft" _say(f" {report.flow}: {what} ({detail}) — {state}") stamp = origin["commit"][:7] + ("-dirty" if origin["dirty"] else "") _say(f"Stamped with {stamp or 'no commit'} from {repo}.") return 0 def _coerce(value: str, dtype: str) -> Any: if value.startswith("@run:") or value.startswith("sha256:"): # A run's output, named rather than typed out. Sent as it stands: the # engine turns it into the value itself, whatever type that is. Passing # the whole thing as JSON still works, and is what a script that # already has it in hand would do. return value if dtype == "int": return int(value) if dtype == "float": return float(value) if dtype == "bool": return value.lower() in ("true", "1", "yes", "on") if dtype == "str": return value return json.loads(value) def _input_types(definition: dict[str, Any]) -> dict[str, str]: """What each of a flow's inputs is declared to be.""" return { str(entry["spec"]["name"]): str(entry["spec"].get("dtype", "float")) for entry in definition.get("inputs") or [] } def _ask_params(definition: dict[str, Any]) -> dict[str, Any]: """The run dialog, in a terminal: one line per declared input. An empty answer leaves the input out, which is what keeps its declared value — the same thing the browser's dialog does with a field nobody filled in. So pressing Enter through the lot runs the defaults. """ params: dict[str, Any] = {} for entry in definition.get("inputs") or []: spec = entry.get("spec") or {} name = str(spec.get("name", "")) if not name: continue dtype = str(spec.get("dtype", "float")) initial = entry.get("initial") shown = "" if initial is None else json.dumps(initial) prompt = f"{name} ({dtype})" + (f" [{shown}]" if shown else "") + ": " answer = input(prompt).strip() if not answer: continue try: params[name] = _coerce(answer, dtype) except ValueError as exc: raise SyncError(f"'{name}' takes {dtype}: {exc}") from exc return params def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]: """Turn `--lr 0.05` into a typed parameter, using the flow's own inputs.""" types = _input_types(definition) params: dict[str, Any] = {} pending: str | None = None for token in rest: if token.startswith("--"): if pending is not None: # A flag with no value is a flag: `--resume` means true. params[pending] = True name, sep, value = token[2:].partition("=") # Only the name is spelled with dashes; a value may hold one, and # `--lr=1e-4` is the case that says so. pending = name.replace("-", "_") if sep: params[pending] = _coerce(value, types.get(pending, "json")) pending = None continue if pending is None: raise SyncError(f"unexpected argument '{token}'") params[pending] = _coerce(token, types.get(pending, "json")) pending = None if pending is not None: params[pending] = True unknown = sorted(set(params) - set(types)) if unknown: raise SyncError( f"'{unknown[0]}' is not an input of this flow (it takes " f"{', '.join(sorted(types)) or 'none'})" ) return params def _sync_first(client: Client) -> None: """Upload what the working directory declares, before running it. The reason a run exists is usually the edit that came before it, and remembering to sync is remembering to do the thing the computer could have done. So `run` syncs by default — including the worker refresh, which is what makes an edit to your own package take effect at all. A directory that declares nothing is not an error: a flow drawn on the canvas is run the same way, and has nothing to upload. """ try: flows = discover(["."]) except (ImportError, SyncError) as exc: # Do not fail a run for a module the run may not even need. _say(f"warning: nothing synced — {exc}") return if not flows: return repo = repo_root(".") reports = sync(flows, client, origin=origin_of(repo)) changed = [r for r in reports if not r.unchanged] if changed: _say(f"synced {', '.join(r.flow for r in changed)}") def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None: """Print a run's numbers as they arrive, until it is over. Polled rather than pushed: the engine writes a metric down when it is reported, so asking once a second draws the same curve a socket would have, without either side holding a connection open. The status is read before the numbers, so the last batch is never the one that gets missed. """ seen: set[tuple[str, int]] = set() failures = 0 while True: try: done = handle.refresh().done points = client.metrics(handle.id) except (httpx.HTTPError, ApiError) as exc: # Following an eight-hour run must not end because one poll of it # did. The run is still going; only this side lost sight of it. if isinstance(exc, ApiError) and exc.status < 500: raise failures += 1 if failures >= WAIT_TOLERANCE: raise time.sleep(poll) continue failures = 0 for point in points: mark = (str(point.get("name", "")), int(point.get("step", -1))) if mark in seen: continue seen.add(mark) _say(f" {mark[0]}[{mark[1]}] = {point['value']:g}") if done: return time.sleep(poll) def _cancel(client: Client, handle: RunHandle) -> int: """Ctrl-C means stop the run, not just stop watching it.""" try: client.cancel(handle.id) except (SyncError, ApiError, httpx.HTTPError) as exc: return _fail(f"could not cancel {handle.id}: {exc}") _say(f"{handle.id} {_status('cancelled')}") return 130 def _cached_note(client: Client, handle: RunHandle) -> str: """How much of the run earlier ones had already answered.""" try: nodes = client.run(handle.id).get("nodes") or [] except (SyncError, ApiError, httpx.HTTPError): return "" cached = sum(1 for node in nodes if node.get("status") == "cached") return f" ({cached}/{len(nodes)} {_status('cached')})" if cached else "" def cmd_run(args: argparse.Namespace, rest: list[str]) -> int: # An in-process engine lives exactly as long as this command, so a run # nobody waits for would be thrown away with the queue holding it. wait = args.wait or args.follow or args.local handle: RunHandle | None = None try: with _client_for(args) as client: if not args.no_sync: _sync_first(client) stored = client.get_flow(args.flow) if stored is None: return _fail(f"no flow '{args.flow}' on that engine") definition = stored.get("definition") or {} params = _params(definition, rest) # Nothing on the command line and somebody watching: ask, the way # pressing Run in the browser asks. A scripted run — piped, or # carrying parameters already — is left exactly as it was. if ( not params and not args.defaults and (definition.get("inputs") or []) and sys.stdin.isatty() ): params = _ask_params(definition) handle = client.submit( args.flow, params, seed=args.seed, no_cache=args.no_cache, cause="cli", ) _say(f"{handle.id} queued {json.dumps(params)}") if not wait: return 0 try: if args.follow: _follow(client, handle) else: handle.wait(timeout=args.timeout) except KeyboardInterrupt: return _cancel(client, handle) _say( f"{handle.id} {_status(handle.status)} " f"{json.dumps(handle.result)}{_cached_note(client, handle)}" ) return 0 if handle.status == "ok" else 1 except (SyncError, ApiError) as exc: return _fail(str(exc)) except httpx.HTTPError as exc: # The run is the engine's, not this command's: it carries on, and its # id is how to find it again. return _unreachable( exc, f"Run {handle.id} is still on the engine." if handle else "" ) # --------------------------------------------------------------------------- # Status # # The home screen's top half, in a terminal: how the engine is, what each flow # is doing, and what failed recently. Drawn with rich, which is already here # under fastapi's own CLI. # --------------------------------------------------------------------------- #: How often `--watch` asks again. The dashboard polls its summary every ten #: seconds; nothing here moves faster than that. WATCH_INTERVAL_S = 5.0 def _flow_state(flow: dict[str, Any]) -> str: if flow.get("quarantined"): return "quarantined" if not flow.get("enabled", True): return "stopped" if flow.get("paused"): return "paused" return "running" def _portal_phrase(portal: dict[str, Any]) -> tuple[str, str]: """Where this installation stands with its portal, and how to colour it. Three states worth telling apart: never paired, paired and linked, and paired but not reaching it — the last being the one somebody needs to know about, since the dashboard is served from the other end. """ if not portal.get("enrolled"): return "no portal", "dim" if portal.get("connected"): host = str(portal.get("portal_url") or "").split("//")[-1].rstrip("/") return f"portal {host}", "green" trouble = str(portal.get("last_error") or "").strip() return "portal unreachable" + (f" ({trouble[:60]})" if trouble else ""), "red" def _status_screen(client: Client) -> Any: """One frame: health, the flows, and the failures under them.""" from rich.console import Group from rich.table import Table from rich.text import Text summary = client.summary() flows = client.flows() # Both, because an installation is usually one or the other: a live flow # fails as an engine event, while a batch run fails on its own row. failures = client.events(kind="failure", limit=5) runs = client.runs(limit=5) try: portal = client.cloud_status() except (SyncError, ApiError): # Never enrolled, or an engine too old to answer. Neither is worth # failing a status screen over. portal = {} healthy = summary.get("status") == "ok" head = Text() head.append( "healthy" if healthy else str(summary.get("status", "unknown")), style="green" if healthy else "yellow", ) problems = summary.get("problems") or [] if problems: head.append(" " + " · ".join(str(p) for p in problems), style="yellow") phrase, style = _portal_phrase(portal) head.append(" " + phrase, style=style) counts = summary.get("flows") or {} nodes = summary.get("nodes") or {} queue = summary.get("queue") or {} lag = summary.get("loop_lag") or {} facts = Text( f"{counts.get('running', 0)}/{counts.get('total', 0)} flows running " f"{nodes.get('total', 0)} nodes" + (f", {nodes['error']} failed" if nodes.get("error") else "") + f" queue {queue.get('pending', 0)} pending" + (f", {queue['parked']} parked" if queue.get("parked") else "") + f" loop lag {lag.get('ewma', 0.0) * 1000:.0f}ms", style="dim", ) table = Table(box=None, pad_edge=False, header_style="dim") table.add_column("flow") table.add_column("state") table.add_column("nodes", justify="right") table.add_column("") for flow in sorted(flows, key=lambda one: str(one.get("name", ""))): state = _flow_state(flow) notes = [] if flow.get("error_count"): notes.append(f"{flow['error_count']} in error") if flow.get("has_draft"): notes.append("unpublished changes") table.add_row( str(flow.get("title") or flow.get("name", "")), Text( state, style={"running": "green", "paused": "yellow"}.get(state, "red"), ), str(flow.get("node_count", 0)), Text(" · ".join(notes), style="red" if flow.get("error_count") else "dim"), ) parts: list[Any] = [head, facts, ""] parts.append( table if flows else Text("No flows yet. `fluksio sync` uploads yours.", "dim") ) if runs: parts += ["", Text("recent runs", style="dim")] for row in runs: state = str(row.get("status", "")) parts.append( Text(f" {str(row.get('id', ''))[-8:]} ") + Text( f"{state:<9}", style={"ok": "green", "cached": "cyan", "running": "cyan"}.get( state, "red" if state == "error" else "yellow" ), ) + Text( f"{str(row.get('flow', '')):<16}" f"{(row.get('duration_ms') or 0) / 1000:7.1f}s", style="dim", ) ) if failures: parts += ["", Text("recent failures", style="dim")] for event in failures: where = " ".join( str(event.get(key, "")) for key in ("flow", "node") if event.get(key) ) parts.append( Text(f" {where} ", style="red") + Text(str(event.get("detail", ""))[:100], style="dim") ) return Group(*parts) def cmd_status(args: argparse.Namespace) -> int: """How the engine is doing, once or until Ctrl-C.""" from rich.console import Console from rich.live import Live console = Console() try: with _client_for(args) as client: if not args.watch: console.print(_status_screen(client)) return 0 if not sys.stdout.isatty(): return _fail("--watch needs a terminal; without one, drop it") if getattr(args, "local", False): # An in-process engine is this command and nothing else, so # nothing can change under it while it watches. return _fail("--watch wants an engine that outlives it; drop --local") with Live(_status_screen(client), console=console) as live: while True: time.sleep(WATCH_INTERVAL_S) live.update(_status_screen(client)) except KeyboardInterrupt: return 130 except (SyncError, ApiError) as exc: return _fail(str(exc)) except httpx.HTTPError as exc: return _unreachable(exc) #: How much of a run's inputs the list shows. A flow taking a few kB of JSON #: would otherwise make the table unreadable; `client.runs()` is where the #: whole value is read. PARAMS_WIDTH = 80 def cmd_runs(args: argparse.Namespace) -> int: try: with _client_for(args) as client: rows = client.runs(flow=args.flow, limit=args.limit) except (SyncError, ApiError) as exc: return _fail(str(exc)) except httpx.HTTPError as exc: return _unreachable(exc) for row in rows: commit = (row.get("origin_commit") or "")[:7] params = json.dumps(row["params"]) if len(params) > PARAMS_WIDTH: params = params[: PARAMS_WIDTH - 3] + "..." _say( f"{row['id']} {_status(row['status'], 9)} {row['flow']:<16} " f"{row['duration_ms'] / 1000:7.1f}s {commit:<8} {params}" ) return 0 def _grid( definition: dict[str, Any], values: list[str], seed: int | None ) -> list[dict[str, Any]]: """`--param lr=0.1,0.01 --param epochs=10,50` — every combination of them.""" types = _input_types(definition) names: list[str] = [] columns: list[list[Any]] = [] for raw in values: name, sep, listed = raw.partition("=") name = name.replace("-", "_") if not sep or not listed: raise SyncError(f"--param takes name=value,value — got '{raw}'") if name not in types: raise SyncError( f"'{name}' is not an input of this flow (it takes " f"{', '.join(sorted(types)) or 'none'})" ) names.append(name) columns.append([_coerce(item, types[name]) for item in listed.split(",")]) return [ {"params": dict(zip(names, combination, strict=True)), "seed": seed} for combination in itertools.product(*columns) ] def cmd_sweep(args: argparse.Namespace) -> int: wait = args.wait or args.local try: with _client_for(args) as client: if not args.no_sync: _sync_first(client) stored = client.get_flow(args.flow) if stored is None: return _fail(f"no flow '{args.flow}' on that engine") entries = _grid(stored.get("definition") or {}, args.param, args.seed) handles = client.sweep(args.flow, entries, no_cache=args.no_cache) for handle, entry in zip(handles, entries, strict=True): _say(f"{handle.id} queued {json.dumps(entry['params'])}") if not wait: return 0 failed = 0 try: for handle in handles: handle.wait(timeout=args.timeout) _say( f"{handle.id} {_status(handle.status)} " f"{json.dumps(handle.result)}" ) failed += handle.status != "ok" except KeyboardInterrupt: for handle in handles: if not handle.refresh().done: _cancel(client, handle) return 130 return 1 if failed else 0 except (SyncError, ApiError) as exc: return _fail(str(exc)) except httpx.HTTPError as exc: return _unreachable(exc, "The runs are still on the engine; `fluksio runs`.") # --------------------------------------------------------------------------- # Wiring # --------------------------------------------------------------------------- def add_parsers(subparsers: Any) -> None: """Register the client commands on `fluksio`'s parser.""" def with_engine(sub: argparse.ArgumentParser, local: bool = False) -> None: sub.add_argument( "--url", default="", help="the engine (default: the last login)" ) sub.add_argument("--token", default="", help="override the stored token") if local: sub.add_argument( "--local", action="store_true", help="boot the engine in this process instead of talking to one", ) parser = subparsers.add_parser( "login", help="store a token for an engine elsewhere" ) parser.add_argument("--url", default="http://localhost:8000") parser.add_argument("--email", default="") parser.add_argument("--password", default="") parser.add_argument( "--global", dest="shared", action="store_true", help="store it for the machine rather than this project", ) parser.set_defaults(func=cmd_login) parser = subparsers.add_parser( "sync", help="upload the flows declared in your own code" ) parser.add_argument( "targets", nargs="*", help="modules, packages or directories to import (default: .)", ) parser.add_argument( "--dry-run", action="store_true", help="print what would be uploaded" ) parser.add_argument( "--no-publish", action="store_true", help="leave the changes as a draft" ) parser.add_argument( "--force", action="store_true", help="overwrite work done on the canvas" ) with_engine(parser) parser.set_defaults(func=cmd_sync) parser = subparsers.add_parser( "run", help="sync this directory, then start a run of one of its flows" ) parser.add_argument("flow") parser.add_argument( "--seed", type=int, default=None, help=( "the run's seed: recorded on it, part of what tells two runs of one " "configuration apart, and passed to an input named 'seed' when the " "flow declares one" ), ) parser.add_argument("--wait", action="store_true", help="block until it finishes") parser.add_argument( "--follow", action="store_true", help="wait, printing the numbers it reports as they arrive", ) parser.add_argument( "--timeout", type=float, default=0.0, metavar="SECONDS", help=( "give up waiting after this long and leave the run going; 0, the " "default, waits as long as it takes. Not the node timeout" ), ) parser.add_argument( "--no-sync", action="store_true", help="run what is already on the engine, without uploading first", ) parser.add_argument( "--no-cache", action="store_true", help="execute every node, even one an earlier run already answered", ) parser.add_argument( "--defaults", action="store_true", help="take every input's declared value instead of asking for it", ) with_engine(parser, local=True) parser.set_defaults(func=cmd_run) parser = subparsers.add_parser( "status", help="how the engine is doing, and what each flow is up to" ) parser.add_argument( "--watch", action="store_true", help="keep it on screen, refreshed until Ctrl-C", ) # `--local` for the same reason `runs` has it: the flows and the history are # in this directory, and reading them should not need a server. with_engine(parser, local=True) parser.set_defaults(func=cmd_status) parser = subparsers.add_parser("runs", help="the runs an engine has recorded") parser.add_argument("--flow", default="") parser.add_argument("--limit", type=int, default=20) with_engine(parser, local=True) parser.set_defaults(func=cmd_runs) parser = subparsers.add_parser( "sweep", help="one flow, once per combination of the parameters given" ) parser.add_argument("flow") parser.add_argument( "--param", action="append", default=[], metavar="NAME=V1,V2", help="an input and the values to try; repeat for a grid", ) parser.add_argument( "--seed", type=int, default=None, help="the seed every run in the sweep gets; vary it with --param seed=1,2", ) parser.add_argument("--wait", action="store_true", help="block until all finish") parser.add_argument( "--timeout", type=float, default=0.0, metavar="SECONDS", help="give up waiting after this long; 0, the default, waits them out", ) parser.add_argument( "--no-sync", action="store_true", help="run what is already on the engine, without uploading first", ) parser.add_argument( "--no-cache", action="store_true", help="execute every node, even one an earlier run already answered", ) with_engine(parser, local=True) parser.set_defaults(func=cmd_sweep)