"""`fluksio serve`, `fluksio enroll`, `fluksio worker`, `sync`, `run`, `runs`. The point of this module is a machine nobody can route to: a node on a cluster where ports cannot be opened, or a laptop with no Docker. `fluksio serve` starts the engine with no infrastructure and no configuration; `fluksio enroll` hands it a claim code, and it dials the portal itself. What a browser would have done locally is then done through the portal, which serves the dashboard from its own side. Nothing from the engine is imported at module level. `fluksio.core.config` builds its settings when it is first imported, and the database engine and the user venv's path are computed from those — so the environment has to be right before any of that happens, which is what `_configure_environment` is for. """ from __future__ import annotations import argparse import os import secrets import sys from pathlib import Path from typing import Any import fluksio #: Where an installation keeps everything, unless it is told otherwise. DEFAULT_HOME = Path("~/.fluksio") #: WAL — what lets readers work while the engine writes — is not supported on #: these. The database would be corrupt or locked, so it is worth saying. NETWORK_FILESYSTEMS = ("nfs", "nfs4", "cifs", "smb", "smb3", "lustre", "fuse.sshfs") def _say(message: str = "") -> None: """Print, and mean it. Redirected to a file or a journal, stdout is block-buffered, and the admin password below is shown exactly once — sitting in a buffer until the process exits is the same as never printing it. """ print(message, flush=True) def _data_dir(raw: str | None, shared: bool = False) -> Path: """Which installation this command is for. A repository with its own venv wants its own engine too — its own flows, its own run history, its own token — so the default is a `.fluksio` beside the code, found the way `.git` is. `--global` asks for the shared one instead, and `--data-dir` names any directory outright. """ from fluksio.sdk.client import find_data_dir, ignore_self if raw: path = Path(raw).expanduser() elif shared: path = DEFAULT_HOME.expanduser() else: found = find_data_dir() path = found if found is not None else Path.cwd() / ".fluksio" path.mkdir(parents=True, exist_ok=True) ignore_self(path) return path.resolve() def _mention_other_installation(data_dir: Path) -> None: """Say when there is a second engine, so neither goes looking lost.""" shared = DEFAULT_HOME.expanduser().resolve() if data_dir == shared or not (shared / "fluksio.db").exists(): return _say(f" Note {shared} holds another installation; this one is separate.") _say(" `fluksio serve --global` runs that one instead.") def _warn_if_networked(path: Path) -> None: """A cluster's $HOME is often NFS, and SQLite's WAL does not work there.""" try: mounts = Path("/proc/mounts").read_text().splitlines() except OSError: # pragma: no cover - not Linux return best, kind = "", "" for line in mounts: parts = line.split() if len(parts) < 3: continue point, fstype = parts[1], parts[2] if (path == Path(point) or point in map(str, path.parents)) and len( point ) > len(best): best, kind = point, fstype if kind in NETWORK_FILESYSTEMS: print( f"warning: {path} is on {kind}, where SQLite's write-ahead log does " "not work. Point --data-dir at local disk.", file=sys.stderr, flush=True, ) def load_or_create_secret_key(path: Path) -> str: """The key that signs sessions and encrypts the secrets store, kept once. Regenerating it per process would sign out every session on restart and, worse, leave `secrets.enc` unreadable. """ if path.exists(): return path.read_text().strip() key = secrets.token_urlsafe(32) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(key) path.chmod(0o600) return key def _configure_environment(data_dir: Path) -> None: """Everything the settings need, before anything reads them. Imports nothing from the engine, deliberately: the first import of `fluksio.core.config` builds the settings, and any engine module reaches it within an import or two. Doing that here would fix `DATA_DIR` at whatever the current directory happened to be. `setdefault` throughout: an operator who exported one of these means it. """ os.environ.setdefault("DATA_DIR", str(data_dir)) # Without this the settings would read whatever `../.env` resolves to from # the current directory — a checkout's development configuration, if the # command happened to be run from inside one. os.environ.setdefault("FLUKSIO_ENV_FILE", str(data_dir / "env")) os.environ.setdefault( "SECRET_KEY", load_or_create_secret_key(data_dir / "secret_key") ) def _prepare(data_dir: Path) -> None: _warn_if_networked(data_dir) _configure_environment(data_dir) from fluksio.core.db import engine, prepare prepare(engine) def _enroll(portal: str, code: str, as_email: str | None) -> int: from sqlmodel import Session from fluksio.cloud import enroll as enroll_mod from fluksio.core.bootstrap import ensure_superuser, pick_superuser from fluksio.core.config import settings from fluksio.core.db import engine with Session(engine) as session: admin, generated = ensure_superuser(session, email=as_email) if generated: _print_new_admin(admin.email, generated) try: user = pick_superuser(session, as_email) except LookupError as exc: print(f"error: {exc}", file=sys.stderr, flush=True) return 1 # Read before the session closes: the instance is detached after it, # and touching an attribute then goes back to a database that is gone. email = user.email try: config = enroll_mod.enroll(session, user, portal, code) except enroll_mod.AlreadyEnrolled as exc: print( f"error: {exc.detail} (see {settings.CLOUD_CONFIG_FILE}).", file=sys.stderr, flush=True, ) return 1 except enroll_mod.EnrollError as exc: print(f"error: {exc.detail}", file=sys.stderr, flush=True) return 1 _say( f"Connected to {config.portal_url} as {email} " f"(installation {config.installation_id})." ) return 0 def _print_new_admin(email: str, password: str) -> None: _say(f"Created the admin account {email}") _say(f" password: {password}") _say(" Shown once. Change it from the dashboard.") #: How long the token `serve` writes for its own machine is good for. Long, #: because the thing it saves you from is logging in again, and it is rewritten #: on every start anyway — an engine left running for a year is the only case #: this has to cover. LOCAL_TOKEN_DAYS = 365 def _sign_in(admin_id: Any, url: str, data_dir: Path) -> Path: """Write the credential for the engine this command is about to start. Logging in to your own machine is a formality: the password was printed by this same process, and the database it authenticates against is in the directory the token goes into. So it costs nothing to be honest about it and hand the client a token, rather than asking somebody to type back something we just told them. """ from datetime import timedelta from fluksio.core import security from fluksio.sdk.client import write_config token = security.create_access_token( admin_id, expires_delta=timedelta(days=LOCAL_TOKEN_DAYS) ) return write_config(url, token, data_dir) def cmd_serve(args: argparse.Namespace) -> int: data_dir = _data_dir(args.data_dir, args.shared) _prepare(data_dir) from sqlmodel import Session from fluksio.cloud import config as cloud_config from fluksio.core.bootstrap import ensure_superuser from fluksio.core.db import engine with Session(engine) as session: admin, generated = ensure_superuser( session, email=args.admin_email, password=args.admin_password ) admin_id, admin_email = admin.id, admin.email if generated: _print_new_admin(admin_email, generated) if args.enroll: if not args.portal: print("error: --enroll needs --portal", file=sys.stderr, flush=True) return 1 if not cloud_config.exists(): # Before the engine starts, so the connector finds the config and # dials out as part of coming up rather than needing a restart. failed = _enroll(args.portal, args.enroll, args.admin_email) if failed: return failed import uvicorn from fluksio.flow import modules from fluksio.main import app # The client talks to this engine, and 0.0.0.0 is not an address to talk # to — it is a statement about which interfaces to listen on. reachable = "127.0.0.1" if args.host in ("0.0.0.0", "::", "") else args.host url = f"http://{reachable}:{args.port}" token_path = _sign_in(admin_id, url, data_dir) config = cloud_config.load() _say(f"Fluksio {fluksio.__version__} — data in {data_dir}") _say(f" API {url}/api/v1") # Which environment node code imports from is the thing a data scientist # most needs to know at this moment, and the answer differs depending on # how Fluksio was installed. Saying it costs one line. if modules.adopted() is not None: _say(f" Nodes {modules.venv_python()}") _say(" your environment, adopted. Add packages with pip.") else: # Not `venv_python()`: the managed venv is built when the engine comes # up, which is after this prints, and until then that would answer with # whatever interpreter happens to be running this. _say(f" Nodes {modules.venv_dir() / 'bin' / 'python'}") _say(" a venv of its own; the Modules screen installs into it.") if config is not None: _say(f" Portal {config.portal_url}, installation {config.installation_id}") _say(" The dashboard is served by the portal; nothing is served here.") else: _say(" No portal. Pair this installation with:") _say(" fluksio enroll --portal https://hub.example.com") _say(f" Signed in as {admin_email}") _say(f" token in {token_path}") _mention_other_installation(data_dir) # One process: it holds the flow engine, and a second worker would be a # second engine — duplicated subscriptions, cron ticks and webhooks. uvicorn.run(app, host=args.host, port=args.port, log_level=args.log_level) return 0 def cmd_enroll(args: argparse.Namespace) -> int: data_dir = _data_dir(args.data_dir, args.shared) _prepare(data_dir) result = _enroll(args.portal, args.code, args.as_email) if result == 0: _say("Start it with `fluksio serve`; it dials the portal as it comes up.") return result def cmd_worker(rest: list[str]) -> int: from fluksio_worker.agent import main as worker_main return worker_main(rest) def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="fluksio", description="Node-based automation: flows, dashboards, runs." ) parser.add_argument("--version", action="version", version=fluksio.__version__) subparsers = parser.add_subparsers(dest="command", required=True) def with_data_dir(sub: argparse.ArgumentParser) -> None: sub.add_argument( "--data-dir", default=os.environ.get("FLUKSIO_HOME"), help="where this installation keeps everything (default: ./.fluksio)", ) sub.add_argument( "--global", dest="shared", action="store_true", help=f"use the shared installation at {DEFAULT_HOME} instead", ) serve = subparsers.add_parser("serve", help="run the engine") with_data_dir(serve) serve.add_argument("--host", default="127.0.0.1") serve.add_argument("--port", type=int, default=8000) serve.add_argument("--log-level", default="info") serve.add_argument("--admin-email", default=None) serve.add_argument("--admin-password", default=None) serve.add_argument("--enroll", metavar="CODE", help="claim code, if not yet paired") serve.add_argument("--portal", metavar="URL", help="the portal --enroll redeems at") serve.set_defaults(func=cmd_serve) enroll = subparsers.add_parser( "enroll", help="pair this installation with a portal" ) enroll.add_argument("code", help="the claim code minted on the portal") enroll.add_argument("--portal", required=True, metavar="URL") enroll.add_argument( "--as", dest="as_email", default=None, help="the local account a portal session arrives as", ) with_data_dir(enroll) enroll.set_defaults(func=cmd_enroll) subparsers.add_parser( "worker", help="run nodes for an engine elsewhere (fluksio-worker)", add_help=False, ) # The client half: talking to an engine rather than being one. from fluksio.sdk.cli import add_parsers add_parsers(subparsers) return parser def main(argv: list[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) # Everything after `worker` belongs to the agent's own parser. if argv and argv[0] == "worker": return cmd_worker(argv[1:]) parser = _parser() if argv and argv[0] == "run": # A flow's inputs are its own, so `--lr 0.05` cannot be declared here: # whatever this parser does not know is typed against the flow. args, rest = parser.parse_known_args(argv) return int(args.func(args, rest)) args = parser.parse_args(argv) result: int = args.func(args) return result if __name__ == "__main__": raise SystemExit(main())