Add the fluksio CLI: serve, enroll, worker
`pip install fluksio && fluksio serve` on a machine with no Docker, no database and no configuration — which is the case this is for: a node on a cluster where ports cannot be opened. It makes its data directory, its key and an admin account, prints the password once, and serves. Pairing is `fluksio enroll <code> --portal …`, doing what the Settings screen does through the same function, before the engine starts and without one running — a machine nobody can route to has no browser pointed at it either. The portal serves the dashboard, so nothing is served here. Two things had to give way. `fastapi[standard]` pulls a cloud CLI that wants sentry-sdk 2.x while we pinned below it — no pip resolution existed, so the pin is lifted, which the comment beside it had been waiting for and which also lets the Python cap go. And `uv` is now a dependency rather than something to find on PATH: the Modules screen is how a data scientist installs torch, and it was quietly falling back to the engine's own interpreter. The CLI imports nothing from the engine before it has set DATA_DIR — the settings are built on the first import of core.config, and reaching it early put the database in the working directory. There is a test for that now, because the failure is silent. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,290 @@
|
||||
"""`fluksio serve`, `fluksio enroll`, `fluksio worker`.
|
||||
|
||||
The point of this module is a machine nobody can route to: a node on a cluster
|
||||
where ports cannot be opened, or a laptop with no Docker. `fluksio serve`
|
||||
starts the engine with no infrastructure and no configuration; `fluksio enroll`
|
||||
hands it a claim code, and it dials the portal itself. What a browser would
|
||||
have done locally is then done through the portal, which serves the dashboard
|
||||
from its own side.
|
||||
|
||||
Nothing from the engine is imported at module level. `fluksio.core.config`
|
||||
builds its settings when it is first imported, and the database engine and the
|
||||
user venv's path are computed from those — so the environment has to be right
|
||||
before any of that happens, which is what `_configure_environment` is for.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import secrets
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import fluksio
|
||||
|
||||
#: Where an installation keeps everything, unless it is told otherwise.
|
||||
DEFAULT_HOME = Path("~/.fluksio")
|
||||
|
||||
#: WAL — what lets readers work while the engine writes — is not supported on
|
||||
#: these. The database would be corrupt or locked, so it is worth saying.
|
||||
NETWORK_FILESYSTEMS = ("nfs", "nfs4", "cifs", "smb", "smb3", "lustre", "fuse.sshfs")
|
||||
|
||||
|
||||
def _say(message: str = "") -> None:
|
||||
"""Print, and mean it.
|
||||
|
||||
Redirected to a file or a journal, stdout is block-buffered, and the admin
|
||||
password below is shown exactly once — sitting in a buffer until the
|
||||
process exits is the same as never printing it.
|
||||
"""
|
||||
print(message, flush=True)
|
||||
|
||||
|
||||
def _data_dir(raw: str | None) -> Path:
|
||||
path = Path(raw).expanduser() if raw else DEFAULT_HOME.expanduser()
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path.resolve()
|
||||
|
||||
|
||||
def _warn_if_networked(path: Path) -> None:
|
||||
"""A cluster's $HOME is often NFS, and SQLite's WAL does not work there."""
|
||||
try:
|
||||
mounts = Path("/proc/mounts").read_text().splitlines()
|
||||
except OSError: # pragma: no cover - not Linux
|
||||
return
|
||||
best, kind = "", ""
|
||||
for line in mounts:
|
||||
parts = line.split()
|
||||
if len(parts) < 3:
|
||||
continue
|
||||
point, fstype = parts[1], parts[2]
|
||||
if (path == Path(point) or point in map(str, path.parents)) and len(
|
||||
point
|
||||
) > len(best):
|
||||
best, kind = point, fstype
|
||||
if kind in NETWORK_FILESYSTEMS:
|
||||
print(
|
||||
f"warning: {path} is on {kind}, where SQLite's write-ahead log does "
|
||||
"not work. Point --data-dir at local disk.",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def load_or_create_secret_key(path: Path) -> str:
|
||||
"""The key that signs sessions and encrypts the secrets store, kept once.
|
||||
|
||||
Regenerating it per process would sign out every session on restart and,
|
||||
worse, leave `secrets.enc` unreadable.
|
||||
"""
|
||||
if path.exists():
|
||||
return path.read_text().strip()
|
||||
key = secrets.token_urlsafe(32)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(key)
|
||||
path.chmod(0o600)
|
||||
return key
|
||||
|
||||
|
||||
def _configure_environment(data_dir: Path) -> None:
|
||||
"""Everything the settings need, before anything reads them.
|
||||
|
||||
Imports nothing from the engine, deliberately: the first import of
|
||||
`fluksio.core.config` builds the settings, and any engine module reaches it
|
||||
within an import or two. Doing that here would fix `DATA_DIR` at whatever
|
||||
the current directory happened to be.
|
||||
|
||||
`setdefault` throughout: an operator who exported one of these means it.
|
||||
"""
|
||||
os.environ.setdefault("DATA_DIR", str(data_dir))
|
||||
# Without this the settings would read whatever `../.env` resolves to from
|
||||
# the current directory — a checkout's development configuration, if the
|
||||
# command happened to be run from inside one.
|
||||
os.environ.setdefault("FLUKSIO_ENV_FILE", str(data_dir / "env"))
|
||||
os.environ.setdefault(
|
||||
"SECRET_KEY", load_or_create_secret_key(data_dir / "secret_key")
|
||||
)
|
||||
|
||||
|
||||
def _prepare(data_dir: Path) -> None:
|
||||
_warn_if_networked(data_dir)
|
||||
_configure_environment(data_dir)
|
||||
from fluksio.core.db import engine, prepare
|
||||
|
||||
prepare(engine)
|
||||
|
||||
|
||||
def _enroll(portal: str, code: str, as_email: str | None) -> int:
|
||||
from sqlmodel import Session
|
||||
|
||||
from fluksio.cloud import enroll as enroll_mod
|
||||
from fluksio.core.bootstrap import ensure_superuser, pick_superuser
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.core.db import engine
|
||||
|
||||
with Session(engine) as session:
|
||||
_, generated = ensure_superuser(session, email=as_email)
|
||||
if generated:
|
||||
_print_new_admin(session, generated)
|
||||
try:
|
||||
user = pick_superuser(session, as_email)
|
||||
except LookupError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr, flush=True)
|
||||
return 1
|
||||
try:
|
||||
config = enroll_mod.enroll(session, user, portal, code)
|
||||
except enroll_mod.AlreadyEnrolled as exc:
|
||||
print(
|
||||
f"error: {exc.detail} (see {settings.CLOUD_CONFIG_FILE}).",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
return 1
|
||||
except enroll_mod.EnrollError as exc:
|
||||
print(f"error: {exc.detail}", file=sys.stderr, flush=True)
|
||||
return 1
|
||||
|
||||
_say(
|
||||
f"Connected to {config.portal_url} as {user.email} "
|
||||
f"(installation {config.installation_id})."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _print_new_admin(session: object, password: str) -> None:
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from fluksio.models import User
|
||||
|
||||
assert isinstance(session, Session)
|
||||
user = session.exec(
|
||||
select(User).where(User.is_superuser == True) # noqa: E712
|
||||
).first()
|
||||
_say(f"Created the admin account {user.email if user else ''}")
|
||||
_say(f" password: {password}")
|
||||
_say(" Shown once. Change it from the dashboard.")
|
||||
|
||||
|
||||
def cmd_serve(args: argparse.Namespace) -> int:
|
||||
data_dir = _data_dir(args.data_dir)
|
||||
_prepare(data_dir)
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from fluksio.cloud import config as cloud_config
|
||||
from fluksio.core.bootstrap import ensure_superuser
|
||||
from fluksio.core.db import engine
|
||||
|
||||
with Session(engine) as session:
|
||||
_, generated = ensure_superuser(
|
||||
session, email=args.admin_email, password=args.admin_password
|
||||
)
|
||||
if generated:
|
||||
_print_new_admin(session, generated)
|
||||
|
||||
if args.enroll:
|
||||
if not args.portal:
|
||||
print("error: --enroll needs --portal", file=sys.stderr, flush=True)
|
||||
return 1
|
||||
if not cloud_config.exists():
|
||||
# Before the engine starts, so the connector finds the config and
|
||||
# dials out as part of coming up rather than needing a restart.
|
||||
failed = _enroll(args.portal, args.enroll, args.admin_email)
|
||||
if failed:
|
||||
return failed
|
||||
|
||||
import uvicorn
|
||||
|
||||
from fluksio.main import app
|
||||
|
||||
config = cloud_config.load()
|
||||
_say(f"Fluksio {fluksio.__version__} — data in {data_dir}")
|
||||
_say(f" API http://{args.host}:{args.port}{'/api/v1'}")
|
||||
if config is not None:
|
||||
_say(f" Portal {config.portal_url}, installation {config.installation_id}")
|
||||
_say(" The dashboard is served by the portal; nothing is served here.")
|
||||
else:
|
||||
_say(" No portal. Pair this installation with:")
|
||||
_say(" fluksio enroll <code> --portal https://hub.example.com")
|
||||
|
||||
# One process: it holds the flow engine, and a second worker would be a
|
||||
# second engine — duplicated subscriptions, cron ticks and webhooks.
|
||||
uvicorn.run(app, host=args.host, port=args.port, log_level=args.log_level)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_enroll(args: argparse.Namespace) -> int:
|
||||
data_dir = _data_dir(args.data_dir)
|
||||
_prepare(data_dir)
|
||||
result = _enroll(args.portal, args.code, args.as_email)
|
||||
if result == 0:
|
||||
_say("Start it with `fluksio serve`; it dials the portal as it comes up.")
|
||||
return result
|
||||
|
||||
|
||||
def cmd_worker(rest: list[str]) -> int:
|
||||
from fluksio_worker.agent import main as worker_main
|
||||
|
||||
return worker_main(rest)
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="fluksio", description="Node-based automation: flows, dashboards, runs."
|
||||
)
|
||||
parser.add_argument("--version", action="version", version=fluksio.__version__)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
def with_data_dir(sub: argparse.ArgumentParser) -> None:
|
||||
sub.add_argument(
|
||||
"--data-dir",
|
||||
default=os.environ.get("FLUKSIO_HOME"),
|
||||
help=f"where this installation keeps everything (default {DEFAULT_HOME})",
|
||||
)
|
||||
|
||||
serve = subparsers.add_parser("serve", help="run the engine")
|
||||
with_data_dir(serve)
|
||||
serve.add_argument("--host", default="127.0.0.1")
|
||||
serve.add_argument("--port", type=int, default=8000)
|
||||
serve.add_argument("--log-level", default="info")
|
||||
serve.add_argument("--admin-email", default=None)
|
||||
serve.add_argument("--admin-password", default=None)
|
||||
serve.add_argument("--enroll", metavar="CODE", help="claim code, if not yet paired")
|
||||
serve.add_argument("--portal", metavar="URL", help="the portal --enroll redeems at")
|
||||
serve.set_defaults(func=cmd_serve)
|
||||
|
||||
enroll = subparsers.add_parser(
|
||||
"enroll", help="pair this installation with a portal"
|
||||
)
|
||||
enroll.add_argument("code", help="the claim code minted on the portal")
|
||||
enroll.add_argument("--portal", required=True, metavar="URL")
|
||||
enroll.add_argument(
|
||||
"--as",
|
||||
dest="as_email",
|
||||
default=None,
|
||||
help="the local account a portal session arrives as",
|
||||
)
|
||||
with_data_dir(enroll)
|
||||
enroll.set_defaults(func=cmd_enroll)
|
||||
|
||||
subparsers.add_parser(
|
||||
"worker",
|
||||
help="run nodes for an engine elsewhere (fluksio-worker)",
|
||||
add_help=False,
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
argv = list(sys.argv[1:] if argv is None else argv)
|
||||
# Everything after `worker` belongs to the agent's own parser.
|
||||
if argv and argv[0] == "worker":
|
||||
return cmd_worker(argv[1:])
|
||||
args = _parser().parse_args(argv)
|
||||
result: int = args.func(args)
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user