stroblmeandClaude Opus 5
8bd30db016
GPU count is not detected, so FLOW_GPUS was 0 on a fresh install and a node asking for one was silently clamped to zero and ran concurrently with every other. Setting the variable serialised them, but it was an environment variable only — `serve` had --max-runs and --max-workers and no --gpus. The clamp warning now names the flag when nothing here declares a card. The same flags are written into the environment before the settings are built, so a value they refused died in a pydantic import naming no flag. They are checked where they are typed instead. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019Hra4ndWMCLU5F3KjUuVAc
525 lines
19 KiB
Python
525 lines
19 KiB
Python
"""`fluksio serve`, `fluksio enroll`, `fluksio worker`, `sync`, `run`, `runs`.
|
|
|
|
The point of this module is a machine nobody can route to: a node on a cluster
|
|
where ports cannot be opened, or a laptop with no Docker. `fluksio serve`
|
|
starts the engine with no infrastructure and no configuration; `fluksio enroll`
|
|
hands it a claim code, and it dials the portal itself. What a browser would
|
|
have done locally is then done through the portal, which serves the dashboard
|
|
from its own side.
|
|
|
|
Nothing from the engine is imported at module level. `fluksio.core.config`
|
|
builds its settings when it is first imported, and the database engine and the
|
|
user venv's path are computed from those — so the environment has to be right
|
|
before any of that happens, which is what `_configure_environment` is for.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import copy
|
|
import os
|
|
import secrets
|
|
import socket
|
|
import sys
|
|
from collections.abc import Callable
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import fluksio
|
|
|
|
#: Where an installation keeps everything, unless it is told otherwise.
|
|
DEFAULT_HOME = Path("~/.fluksio")
|
|
|
|
#: The portal a claim code is redeemed at, unless another is named. Having a
|
|
#: default is the difference between one flag and two on the one command that
|
|
#: is run before anything works.
|
|
DEFAULT_PORTAL = "https://hub.fluksio.com"
|
|
|
|
#: WAL — what lets readers work while the engine writes — is not supported on
|
|
#: these. The database would be corrupt or locked, so it is worth saying.
|
|
NETWORK_FILESYSTEMS = ("nfs", "nfs4", "cifs", "smb", "smb3", "lustre", "fuse.sshfs")
|
|
|
|
|
|
def _say(message: str = "") -> None:
|
|
"""Print, and mean it.
|
|
|
|
Redirected to a file or a journal, stdout is block-buffered, and the admin
|
|
password below is shown exactly once — sitting in a buffer until the
|
|
process exits is the same as never printing it.
|
|
"""
|
|
print(message, flush=True)
|
|
|
|
|
|
def _data_dir(raw: str | None, shared: bool = False) -> Path:
|
|
"""Which installation this command is for.
|
|
|
|
A repository with its own venv wants its own engine too — its own flows,
|
|
its own run history, its own token — so the default is a `.fluksio` beside
|
|
the code, found the way `.git` is. `--global` asks for the shared one
|
|
instead, and `--data-dir` names any directory outright.
|
|
"""
|
|
from fluksio.sdk.client import find_data_dir, ignore_self
|
|
|
|
if raw:
|
|
path = Path(raw).expanduser()
|
|
elif shared:
|
|
path = DEFAULT_HOME.expanduser()
|
|
else:
|
|
found = find_data_dir()
|
|
path = found if found is not None else Path.cwd() / ".fluksio"
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
ignore_self(path)
|
|
return path.resolve()
|
|
|
|
|
|
def _mention_other_installation(data_dir: Path) -> None:
|
|
"""Say when there is a second engine, so neither goes looking lost."""
|
|
shared = DEFAULT_HOME.expanduser().resolve()
|
|
if data_dir == shared or not (shared / "fluksio.db").exists():
|
|
return
|
|
_say(f" Note {shared} holds another installation; this one is separate.")
|
|
_say(" `fluksio serve --global` runs that one instead.")
|
|
|
|
|
|
def _warn_if_networked(path: Path) -> None:
|
|
"""A cluster's $HOME is often NFS, and SQLite's WAL does not work there."""
|
|
try:
|
|
mounts = Path("/proc/mounts").read_text().splitlines()
|
|
except OSError: # pragma: no cover - not Linux
|
|
return
|
|
best, kind = "", ""
|
|
for line in mounts:
|
|
parts = line.split()
|
|
if len(parts) < 3:
|
|
continue
|
|
point, fstype = parts[1], parts[2]
|
|
if (path == Path(point) or point in map(str, path.parents)) and len(
|
|
point
|
|
) > len(best):
|
|
best, kind = point, fstype
|
|
if kind in NETWORK_FILESYSTEMS:
|
|
print(
|
|
f"warning: {path} is on {kind}, where SQLite's write-ahead log does "
|
|
"not work. Point --data-dir at local disk.",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
|
|
|
|
def load_or_create_secret_key(path: Path) -> str:
|
|
"""The key that signs sessions and encrypts the secrets store, kept once.
|
|
|
|
Regenerating it per process would sign out every session on restart and,
|
|
worse, leave `secrets.enc` unreadable.
|
|
"""
|
|
if path.exists():
|
|
return path.read_text().strip()
|
|
key = secrets.token_urlsafe(32)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(key)
|
|
path.chmod(0o600)
|
|
return key
|
|
|
|
|
|
def _configure_environment(data_dir: Path) -> None:
|
|
"""Everything the settings need, before anything reads them.
|
|
|
|
Imports nothing from the engine, deliberately: the first import of
|
|
`fluksio.core.config` builds the settings, and any engine module reaches it
|
|
within an import or two. Doing that here would fix `DATA_DIR` at whatever
|
|
the current directory happened to be.
|
|
|
|
`setdefault` throughout: an operator who exported one of these means it.
|
|
"""
|
|
os.environ.setdefault("DATA_DIR", str(data_dir))
|
|
# Without this the settings would read whatever `../.env` resolves to from
|
|
# the current directory — a checkout's development configuration, if the
|
|
# command happened to be run from inside one.
|
|
os.environ.setdefault("FLUKSIO_ENV_FILE", str(data_dir / "env"))
|
|
os.environ.setdefault(
|
|
"SECRET_KEY", load_or_create_secret_key(data_dir / "secret_key")
|
|
)
|
|
|
|
|
|
def _prepare(data_dir: Path) -> None:
|
|
_warn_if_networked(data_dir)
|
|
_configure_environment(data_dir)
|
|
from fluksio.core.db import engine, prepare
|
|
|
|
prepare(engine)
|
|
|
|
|
|
def _enroll(portal: str, code: str, as_email: str | None) -> int:
|
|
from sqlmodel import Session
|
|
|
|
from fluksio.cloud import enroll as enroll_mod
|
|
from fluksio.core.bootstrap import ensure_superuser, pick_superuser
|
|
from fluksio.core.config import settings
|
|
from fluksio.core.db import engine
|
|
|
|
with Session(engine) as session:
|
|
admin, generated = ensure_superuser(session, email=as_email)
|
|
if generated:
|
|
_print_new_admin(admin.email, generated)
|
|
try:
|
|
user = pick_superuser(session, as_email)
|
|
except LookupError as exc:
|
|
print(f"error: {exc}", file=sys.stderr, flush=True)
|
|
return 1
|
|
# Read before the session closes: the instance is detached after it,
|
|
# and touching an attribute then goes back to a database that is gone.
|
|
email = user.email
|
|
try:
|
|
config = enroll_mod.enroll(session, user, portal, code)
|
|
except enroll_mod.AlreadyEnrolled as exc:
|
|
print(
|
|
f"error: {exc.detail} (see {settings.CLOUD_CONFIG_FILE}).",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
return 1
|
|
except enroll_mod.EnrollError as exc:
|
|
print(f"error: {exc.detail}", file=sys.stderr, flush=True)
|
|
return 1
|
|
|
|
_say(
|
|
f"Connected to {config.portal_url} as {email} "
|
|
f"(installation {config.installation_id})."
|
|
)
|
|
return 0
|
|
|
|
|
|
def _print_new_admin(email: str, password: str) -> None:
|
|
_say(f"Created the admin account {email}")
|
|
_say(f" password: {password}")
|
|
_say(" Shown once. Change it from the dashboard.")
|
|
|
|
|
|
#: How long the token `serve` writes for its own machine is good for. Long,
|
|
#: because the thing it saves you from is logging in again, and it is rewritten
|
|
#: on every start anyway — an engine left running for a year is the only case
|
|
#: this has to cover.
|
|
LOCAL_TOKEN_DAYS = 365
|
|
|
|
|
|
def _sign_in(admin_id: Any, url: str, data_dir: Path) -> Path:
|
|
"""Write the credential for the engine this command is about to start.
|
|
|
|
Logging in to your own machine is a formality: the password was printed by
|
|
this same process, and the database it authenticates against is in the
|
|
directory the token goes into. So it costs nothing to be honest about it
|
|
and hand the client a token, rather than asking somebody to type back
|
|
something we just told them.
|
|
"""
|
|
from datetime import timedelta
|
|
|
|
from fluksio.core import security
|
|
from fluksio.sdk.client import write_config
|
|
|
|
token = security.create_access_token(
|
|
admin_id, expires_delta=timedelta(days=LOCAL_TOKEN_DAYS)
|
|
)
|
|
return write_config(url, token, data_dir)
|
|
|
|
|
|
#: The `serve` flags that are settings under another name, and the setting each
|
|
#: one writes. A flag is a real environment variable, which outranks the env
|
|
#: file the settings read — so the order is flag, environment, `<data-dir>/env`.
|
|
CONCURRENCY_FLAGS = {
|
|
"max_workers": "FLOW_MAX_WORKERS",
|
|
"max_cascades": "FLOW_MAX_CASCADES",
|
|
"max_runs": "FLOW_MAX_RUNS",
|
|
"gpus": "FLOW_GPUS",
|
|
}
|
|
|
|
|
|
def _at_least(minimum: int) -> Callable[[str], int]:
|
|
"""A flag's value, checked here rather than by the settings.
|
|
|
|
These are written into the environment before the settings are built, so a
|
|
number they refuse dies inside a pydantic import with no flag named in it.
|
|
"""
|
|
|
|
def parse(text: str) -> int:
|
|
value = int(text)
|
|
if value < minimum:
|
|
raise argparse.ArgumentTypeError(f"is {value}, needs at least {minimum}")
|
|
return value
|
|
|
|
return parse
|
|
|
|
|
|
#: What `serve` listens on when nobody says. Taken often enough — another
|
|
#: engine, another framework's dev server — that dying on it is the first
|
|
#: thing a zero-config start would hit.
|
|
DEFAULT_PORT = 8000
|
|
|
|
#: How far up from it to look before giving up and letting the bind fail.
|
|
PORT_TRIES = 20
|
|
|
|
|
|
def _free_port(host: str, start: int) -> int:
|
|
"""The first port from ``start`` that nothing is listening on.
|
|
|
|
Probed with the same address and options uvicorn will bind with, so this
|
|
answers the question uvicorn is about to ask rather than a similar one.
|
|
"""
|
|
for port in range(start, start + PORT_TRIES):
|
|
with socket.socket() as probe:
|
|
probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
try:
|
|
probe.bind((host, port))
|
|
except OSError:
|
|
continue
|
|
return port
|
|
return start
|
|
|
|
|
|
def cmd_serve(args: argparse.Namespace) -> int:
|
|
data_dir = _data_dir(args.data_dir, args.shared)
|
|
for flag, name in CONCURRENCY_FLAGS.items():
|
|
value = getattr(args, flag, None)
|
|
if value is not None:
|
|
os.environ[name] = str(value)
|
|
_prepare(data_dir)
|
|
|
|
from sqlmodel import Session
|
|
|
|
from fluksio.cloud import config as cloud_config
|
|
from fluksio.core.bootstrap import ensure_superuser
|
|
from fluksio.core.db import engine
|
|
|
|
with Session(engine) as session:
|
|
admin, generated = ensure_superuser(
|
|
session, email=args.admin_email, password=args.admin_password
|
|
)
|
|
admin_id, admin_email = admin.id, admin.email
|
|
if generated:
|
|
_print_new_admin(admin_email, generated)
|
|
|
|
if args.enroll:
|
|
if not cloud_config.exists():
|
|
# Before the engine starts, so the connector finds the config and
|
|
# dials out as part of coming up rather than needing a restart.
|
|
failed = _enroll(
|
|
args.portal or DEFAULT_PORTAL, args.enroll, args.admin_email
|
|
)
|
|
if failed:
|
|
return failed
|
|
|
|
import uvicorn
|
|
|
|
from fluksio.flow import modules
|
|
from fluksio.main import app
|
|
|
|
port = args.port
|
|
if port is None:
|
|
port = _free_port(args.host, DEFAULT_PORT)
|
|
if port != DEFAULT_PORT:
|
|
_say(f"Port {DEFAULT_PORT} is in use; serving on {port} instead.")
|
|
|
|
# The client talks to this engine, and 0.0.0.0 is not an address to talk
|
|
# to — it is a statement about which interfaces to listen on.
|
|
reachable = "127.0.0.1" if args.host in ("0.0.0.0", "::", "") else args.host
|
|
url = f"http://{reachable}:{port}"
|
|
token_path = _sign_in(admin_id, url, data_dir)
|
|
|
|
config = cloud_config.load()
|
|
_say(f"Fluksio {fluksio.__version__} — data in {data_dir}")
|
|
_say(f" API {url}/api/v1")
|
|
# Which environment node code imports from is the thing a data scientist
|
|
# most needs to know at this moment, and the answer differs depending on
|
|
# how Fluksio was installed. Saying it costs one line.
|
|
if modules.adopted() is not None:
|
|
_say(f" Nodes {modules.venv_python()}")
|
|
_say(" your environment, adopted. Add packages with pip.")
|
|
else:
|
|
# Not `venv_python()`: the managed venv is built when the engine comes
|
|
# up, which is after this prints, and until then that would answer with
|
|
# whatever interpreter happens to be running this.
|
|
_say(f" Nodes {modules.venv_dir() / 'bin' / 'python'}")
|
|
_say(" a venv of its own; the Modules screen installs into it.")
|
|
if config is not None:
|
|
_say(f" Portal {config.portal_url}, installation {config.installation_id}")
|
|
_say(" The dashboard is served by the portal; nothing is served here.")
|
|
else:
|
|
_say(" No portal. Pair this installation with:")
|
|
_say(" fluksio enroll <code>")
|
|
_say(f" Signed in as {admin_email}")
|
|
_say(f" token in {token_path}")
|
|
_mention_other_installation(data_dir)
|
|
|
|
# One process: it holds the flow engine, and a second worker would be a
|
|
# second engine — duplicated subscriptions, cron ticks and webhooks.
|
|
uvicorn.run(
|
|
app,
|
|
host=args.host,
|
|
port=port,
|
|
log_level=args.log_level,
|
|
log_config=_log_config(args.log_level),
|
|
)
|
|
return 0
|
|
|
|
|
|
def _log_config(level: str) -> dict[str, Any]:
|
|
"""Uvicorn's logging, with the engine's own loggers drawn the same way.
|
|
|
|
Without this the engine's lines go to the root logger, which has no handler
|
|
configured and falls back to `INFO:fluksio.cloud.connector:...` — beside
|
|
uvicorn's own aligned, coloured output it reads like something went wrong.
|
|
"""
|
|
from uvicorn.config import LOGGING_CONFIG
|
|
|
|
config = copy.deepcopy(LOGGING_CONFIG)
|
|
# Named rather than configuring the root: everything else that logs — httpx
|
|
# on every portal call, for one — is at INFO too, and today none of it is
|
|
# printed at all. Styling the root would turn all of it on.
|
|
for name in ("fluksio", "alembic"):
|
|
config["loggers"][name] = {
|
|
"handlers": ["default"],
|
|
"level": level.upper(),
|
|
# It has a handler of its own now; propagating would print each
|
|
# line twice the moment anything configures the root.
|
|
"propagate": False,
|
|
}
|
|
return config
|
|
|
|
|
|
def cmd_enroll(args: argparse.Namespace) -> int:
|
|
data_dir = _data_dir(args.data_dir, args.shared)
|
|
_prepare(data_dir)
|
|
result = _enroll(args.portal, args.code, args.as_email)
|
|
if result == 0:
|
|
# True either way round: one already serving notices within seconds,
|
|
# and one not yet started dials as it comes up.
|
|
_say("An engine already running picks this up; otherwise `fluksio serve`.")
|
|
return result
|
|
|
|
|
|
def cmd_worker(rest: list[str]) -> int:
|
|
from fluksio_worker.agent import main as worker_main
|
|
|
|
return worker_main(rest)
|
|
|
|
|
|
def _parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(
|
|
prog="fluksio", description="Node-based automation: flows, dashboards, runs."
|
|
)
|
|
parser.add_argument("--version", action="version", version=fluksio.__version__)
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
def with_data_dir(sub: argparse.ArgumentParser) -> None:
|
|
sub.add_argument(
|
|
"--data-dir",
|
|
default=os.environ.get("FLUKSIO_HOME"),
|
|
help="where this installation keeps everything (default: ./.fluksio)",
|
|
)
|
|
sub.add_argument(
|
|
"--global",
|
|
dest="shared",
|
|
action="store_true",
|
|
help=f"use the shared installation at {DEFAULT_HOME} instead",
|
|
)
|
|
|
|
serve = subparsers.add_parser("serve", help="run the engine")
|
|
with_data_dir(serve)
|
|
serve.add_argument("--host", default="127.0.0.1")
|
|
# No default: a port nobody asked for may move when it is taken, and one
|
|
# that was asked for may not.
|
|
serve.add_argument(
|
|
"--port",
|
|
type=int,
|
|
default=None,
|
|
help=f"default {DEFAULT_PORT}, or the next free port when it is in use",
|
|
)
|
|
serve.add_argument("--log-level", default="info")
|
|
serve.add_argument("--admin-email", default=None)
|
|
serve.add_argument("--admin-password", default=None)
|
|
serve.add_argument("--enroll", metavar="CODE", help="claim code, if not yet paired")
|
|
serve.add_argument(
|
|
"--portal",
|
|
metavar="URL",
|
|
help=f"the portal --enroll redeems at (default {DEFAULT_PORTAL})",
|
|
)
|
|
serve.add_argument(
|
|
"--max-runs",
|
|
type=_at_least(1),
|
|
default=None,
|
|
metavar="N",
|
|
help="batch runs driven at once (default 4, FLOW_MAX_RUNS)",
|
|
)
|
|
serve.add_argument(
|
|
"--max-cascades",
|
|
type=_at_least(1),
|
|
default=None,
|
|
metavar="N",
|
|
help="cascades in flight at once (default 4, FLOW_MAX_CASCADES)",
|
|
)
|
|
serve.add_argument(
|
|
"--max-workers",
|
|
type=_at_least(1),
|
|
default=None,
|
|
metavar="N",
|
|
help="python worker processes (default 4, FLOW_MAX_WORKERS)",
|
|
)
|
|
serve.add_argument(
|
|
"--gpus",
|
|
type=_at_least(0),
|
|
default=None,
|
|
metavar="N",
|
|
help="GPUs on this machine a node may be given (default 0, FLOW_GPUS)",
|
|
)
|
|
serve.set_defaults(func=cmd_serve)
|
|
|
|
enroll = subparsers.add_parser(
|
|
"enroll", help="pair this installation with a portal"
|
|
)
|
|
enroll.add_argument("code", help="the claim code minted on the portal")
|
|
enroll.add_argument(
|
|
"--portal",
|
|
default=DEFAULT_PORTAL,
|
|
metavar="URL",
|
|
help=f"a portal of your own, instead of {DEFAULT_PORTAL}",
|
|
)
|
|
enroll.add_argument(
|
|
"--as",
|
|
dest="as_email",
|
|
default=None,
|
|
help="the local account a portal session arrives as",
|
|
)
|
|
with_data_dir(enroll)
|
|
enroll.set_defaults(func=cmd_enroll)
|
|
|
|
subparsers.add_parser(
|
|
"worker",
|
|
help="run nodes for an engine elsewhere (fluksio-worker)",
|
|
add_help=False,
|
|
)
|
|
|
|
# The client half: talking to an engine rather than being one.
|
|
from fluksio.sdk.cli import add_parsers
|
|
|
|
add_parsers(subparsers)
|
|
return parser
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
argv = list(sys.argv[1:] if argv is None else argv)
|
|
# Everything after `worker` belongs to the agent's own parser.
|
|
if argv and argv[0] == "worker":
|
|
return cmd_worker(argv[1:])
|
|
parser = _parser()
|
|
if argv and argv[0] == "run":
|
|
# A flow's inputs are its own, so `--lr 0.05` cannot be declared here:
|
|
# whatever this parser does not know is typed against the flow.
|
|
args, rest = parser.parse_known_args(argv)
|
|
return int(args.func(args, rest))
|
|
args = parser.parse_args(argv)
|
|
result: int = args.func(args)
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|