A `pip install` on a locked-down host — the case the CLI exists for — may have no git, and the store shelled out to it while building the flow repository, so the engine refused to start at all. The store is files; git is their history. Missing it is now one warning and no commits rather than a stack trace, which is the difference between a machine that runs your experiments and one that does not. Found by installing the wheels into a bare python:3.12-slim and pairing it with the portal: `fluksio enroll` took the code, `fluksio serve` dialled out, and the hub was proxying requests through the tunnel. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
286 lines
10 KiB
Python
286 lines
10 KiB
Python
"""`fluksio serve`, `fluksio enroll`, `fluksio worker`.
|
|
|
|
The point of this module is a machine nobody can route to: a node on a cluster
|
|
where ports cannot be opened, or a laptop with no Docker. `fluksio serve`
|
|
starts the engine with no infrastructure and no configuration; `fluksio enroll`
|
|
hands it a claim code, and it dials the portal itself. What a browser would
|
|
have done locally is then done through the portal, which serves the dashboard
|
|
from its own side.
|
|
|
|
Nothing from the engine is imported at module level. `fluksio.core.config`
|
|
builds its settings when it is first imported, and the database engine and the
|
|
user venv's path are computed from those — so the environment has to be right
|
|
before any of that happens, which is what `_configure_environment` is for.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import secrets
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import fluksio
|
|
|
|
#: Where an installation keeps everything, unless it is told otherwise.
|
|
DEFAULT_HOME = Path("~/.fluksio")
|
|
|
|
#: WAL — what lets readers work while the engine writes — is not supported on
|
|
#: these. The database would be corrupt or locked, so it is worth saying.
|
|
NETWORK_FILESYSTEMS = ("nfs", "nfs4", "cifs", "smb", "smb3", "lustre", "fuse.sshfs")
|
|
|
|
|
|
def _say(message: str = "") -> None:
|
|
"""Print, and mean it.
|
|
|
|
Redirected to a file or a journal, stdout is block-buffered, and the admin
|
|
password below is shown exactly once — sitting in a buffer until the
|
|
process exits is the same as never printing it.
|
|
"""
|
|
print(message, flush=True)
|
|
|
|
|
|
def _data_dir(raw: str | None) -> Path:
|
|
path = Path(raw).expanduser() if raw else DEFAULT_HOME.expanduser()
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
return path.resolve()
|
|
|
|
|
|
def _warn_if_networked(path: Path) -> None:
|
|
"""A cluster's $HOME is often NFS, and SQLite's WAL does not work there."""
|
|
try:
|
|
mounts = Path("/proc/mounts").read_text().splitlines()
|
|
except OSError: # pragma: no cover - not Linux
|
|
return
|
|
best, kind = "", ""
|
|
for line in mounts:
|
|
parts = line.split()
|
|
if len(parts) < 3:
|
|
continue
|
|
point, fstype = parts[1], parts[2]
|
|
if (path == Path(point) or point in map(str, path.parents)) and len(
|
|
point
|
|
) > len(best):
|
|
best, kind = point, fstype
|
|
if kind in NETWORK_FILESYSTEMS:
|
|
print(
|
|
f"warning: {path} is on {kind}, where SQLite's write-ahead log does "
|
|
"not work. Point --data-dir at local disk.",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
|
|
|
|
def load_or_create_secret_key(path: Path) -> str:
|
|
"""The key that signs sessions and encrypts the secrets store, kept once.
|
|
|
|
Regenerating it per process would sign out every session on restart and,
|
|
worse, leave `secrets.enc` unreadable.
|
|
"""
|
|
if path.exists():
|
|
return path.read_text().strip()
|
|
key = secrets.token_urlsafe(32)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(key)
|
|
path.chmod(0o600)
|
|
return key
|
|
|
|
|
|
def _configure_environment(data_dir: Path) -> None:
|
|
"""Everything the settings need, before anything reads them.
|
|
|
|
Imports nothing from the engine, deliberately: the first import of
|
|
`fluksio.core.config` builds the settings, and any engine module reaches it
|
|
within an import or two. Doing that here would fix `DATA_DIR` at whatever
|
|
the current directory happened to be.
|
|
|
|
`setdefault` throughout: an operator who exported one of these means it.
|
|
"""
|
|
os.environ.setdefault("DATA_DIR", str(data_dir))
|
|
# Without this the settings would read whatever `../.env` resolves to from
|
|
# the current directory — a checkout's development configuration, if the
|
|
# command happened to be run from inside one.
|
|
os.environ.setdefault("FLUKSIO_ENV_FILE", str(data_dir / "env"))
|
|
os.environ.setdefault(
|
|
"SECRET_KEY", load_or_create_secret_key(data_dir / "secret_key")
|
|
)
|
|
|
|
|
|
def _prepare(data_dir: Path) -> None:
|
|
_warn_if_networked(data_dir)
|
|
_configure_environment(data_dir)
|
|
from fluksio.core.db import engine, prepare
|
|
|
|
prepare(engine)
|
|
|
|
|
|
def _enroll(portal: str, code: str, as_email: str | None) -> int:
|
|
from sqlmodel import Session
|
|
|
|
from fluksio.cloud import enroll as enroll_mod
|
|
from fluksio.core.bootstrap import ensure_superuser, pick_superuser
|
|
from fluksio.core.config import settings
|
|
from fluksio.core.db import engine
|
|
|
|
with Session(engine) as session:
|
|
admin, generated = ensure_superuser(session, email=as_email)
|
|
if generated:
|
|
_print_new_admin(admin.email, generated)
|
|
try:
|
|
user = pick_superuser(session, as_email)
|
|
except LookupError as exc:
|
|
print(f"error: {exc}", file=sys.stderr, flush=True)
|
|
return 1
|
|
# Read before the session closes: the instance is detached after it,
|
|
# and touching an attribute then goes back to a database that is gone.
|
|
email = user.email
|
|
try:
|
|
config = enroll_mod.enroll(session, user, portal, code)
|
|
except enroll_mod.AlreadyEnrolled as exc:
|
|
print(
|
|
f"error: {exc.detail} (see {settings.CLOUD_CONFIG_FILE}).",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
return 1
|
|
except enroll_mod.EnrollError as exc:
|
|
print(f"error: {exc.detail}", file=sys.stderr, flush=True)
|
|
return 1
|
|
|
|
_say(
|
|
f"Connected to {config.portal_url} as {email} "
|
|
f"(installation {config.installation_id})."
|
|
)
|
|
return 0
|
|
|
|
|
|
def _print_new_admin(email: str, password: str) -> None:
|
|
_say(f"Created the admin account {email}")
|
|
_say(f" password: {password}")
|
|
_say(" Shown once. Change it from the dashboard.")
|
|
|
|
|
|
def cmd_serve(args: argparse.Namespace) -> int:
|
|
data_dir = _data_dir(args.data_dir)
|
|
_prepare(data_dir)
|
|
|
|
from sqlmodel import Session
|
|
|
|
from fluksio.cloud import config as cloud_config
|
|
from fluksio.core.bootstrap import ensure_superuser
|
|
from fluksio.core.db import engine
|
|
|
|
with Session(engine) as session:
|
|
admin, generated = ensure_superuser(
|
|
session, email=args.admin_email, password=args.admin_password
|
|
)
|
|
if generated:
|
|
_print_new_admin(admin.email, generated)
|
|
|
|
if args.enroll:
|
|
if not args.portal:
|
|
print("error: --enroll needs --portal", file=sys.stderr, flush=True)
|
|
return 1
|
|
if not cloud_config.exists():
|
|
# Before the engine starts, so the connector finds the config and
|
|
# dials out as part of coming up rather than needing a restart.
|
|
failed = _enroll(args.portal, args.enroll, args.admin_email)
|
|
if failed:
|
|
return failed
|
|
|
|
import uvicorn
|
|
|
|
from fluksio.main import app
|
|
|
|
config = cloud_config.load()
|
|
_say(f"Fluksio {fluksio.__version__} — data in {data_dir}")
|
|
_say(f" API http://{args.host}:{args.port}{'/api/v1'}")
|
|
if config is not None:
|
|
_say(f" Portal {config.portal_url}, installation {config.installation_id}")
|
|
_say(" The dashboard is served by the portal; nothing is served here.")
|
|
else:
|
|
_say(" No portal. Pair this installation with:")
|
|
_say(" fluksio enroll <code> --portal https://hub.example.com")
|
|
|
|
# One process: it holds the flow engine, and a second worker would be a
|
|
# second engine — duplicated subscriptions, cron ticks and webhooks.
|
|
uvicorn.run(app, host=args.host, port=args.port, log_level=args.log_level)
|
|
return 0
|
|
|
|
|
|
def cmd_enroll(args: argparse.Namespace) -> int:
|
|
data_dir = _data_dir(args.data_dir)
|
|
_prepare(data_dir)
|
|
result = _enroll(args.portal, args.code, args.as_email)
|
|
if result == 0:
|
|
_say("Start it with `fluksio serve`; it dials the portal as it comes up.")
|
|
return result
|
|
|
|
|
|
def cmd_worker(rest: list[str]) -> int:
|
|
from fluksio_worker.agent import main as worker_main
|
|
|
|
return worker_main(rest)
|
|
|
|
|
|
def _parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(
|
|
prog="fluksio", description="Node-based automation: flows, dashboards, runs."
|
|
)
|
|
parser.add_argument("--version", action="version", version=fluksio.__version__)
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
def with_data_dir(sub: argparse.ArgumentParser) -> None:
|
|
sub.add_argument(
|
|
"--data-dir",
|
|
default=os.environ.get("FLUKSIO_HOME"),
|
|
help=f"where this installation keeps everything (default {DEFAULT_HOME})",
|
|
)
|
|
|
|
serve = subparsers.add_parser("serve", help="run the engine")
|
|
with_data_dir(serve)
|
|
serve.add_argument("--host", default="127.0.0.1")
|
|
serve.add_argument("--port", type=int, default=8000)
|
|
serve.add_argument("--log-level", default="info")
|
|
serve.add_argument("--admin-email", default=None)
|
|
serve.add_argument("--admin-password", default=None)
|
|
serve.add_argument("--enroll", metavar="CODE", help="claim code, if not yet paired")
|
|
serve.add_argument("--portal", metavar="URL", help="the portal --enroll redeems at")
|
|
serve.set_defaults(func=cmd_serve)
|
|
|
|
enroll = subparsers.add_parser(
|
|
"enroll", help="pair this installation with a portal"
|
|
)
|
|
enroll.add_argument("code", help="the claim code minted on the portal")
|
|
enroll.add_argument("--portal", required=True, metavar="URL")
|
|
enroll.add_argument(
|
|
"--as",
|
|
dest="as_email",
|
|
default=None,
|
|
help="the local account a portal session arrives as",
|
|
)
|
|
with_data_dir(enroll)
|
|
enroll.set_defaults(func=cmd_enroll)
|
|
|
|
subparsers.add_parser(
|
|
"worker",
|
|
help="run nodes for an engine elsewhere (fluksio-worker)",
|
|
add_help=False,
|
|
)
|
|
return parser
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
argv = list(sys.argv[1:] if argv is None else argv)
|
|
# Everything after `worker` belongs to the agent's own parser.
|
|
if argv and argv[0] == "worker":
|
|
return cmd_worker(argv[1:])
|
|
args = _parser().parse_args(argv)
|
|
result: int = args.func(args)
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|