Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Three fewer things to remember

Browse Source 
**The portal link puts itself back up.** It already retried a connection that
raised, but a session that ended *cleanly* — a portal restarting, a proxy
closing an idle socket — returned normally and went straight back round the
loop with no wait at all, so an engine could spin against a portal that was
merely saying goodbye politely. Every ending now reconnects on a delay, and
the delay turns on whether the attempt got as far as attaching: one that stood
up and dropped is a network event and retries at once, one that never stood up
waits longer each time. Jittered, so a portal coming back is not met by every
installation it serves in the same instant. Ping timeouts are named rather
than defaulted, since they are what bounds how long a suspended laptop's dead
socket looks alive, and the keepalive task is awaited so the reason a link
went reaches the log instead of the garbage collector.

**`fluksio enroll <code>`** is the whole command now; hub.fluksio.com is the
default and `--portal` names another. The one command run before anything
works should not need two flags.

**`fluksio run` syncs first.** The reason a run exists is usually the edit
before it, so remembering to sync was remembering to do something the computer
could do — including the worker refresh, which is what makes an edit to your
own package take effect at all. `--no-sync` opts out for a tight loop.

That last one needed discovery fixed: it only ever looked at top-level `*.py`,
so a repository whose code is in a package — the ordinary shape — found
nothing from its own root. It now descends into the packages it holds.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ue1tkFWB1bcGy3aWhCKpU
This commit is contained in:
stroblme
2026-08-24 18:17:16 +02:00
co-authored by Claude Fable 5
parent 99f6530698
commit 7e4f03369b
8 changed files with 294 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/fluksio/cli.py
+20 -7
View File
@@ -27,6 +27,11 @@ import fluksio
#: Where an installation keeps everything, unless it is told otherwise.
DEFAULT_HOME = Path("~/.fluksio")
#: The portal a claim code is redeemed at, unless another is named. Having a
#: default is the difference between one flag and two on the one command that
#: is run before anything works.
DEFAULT_PORTAL = "https://hub.fluksio.com"
#: WAL — what lets readers work while the engine writes — is not supported on
#: these. The database would be corrupt or locked, so it is worth saying.
NETWORK_FILESYSTEMS = ("nfs", "nfs4", "cifs", "smb", "smb3", "lustre", "fuse.sshfs")
@@ -233,13 +238,12 @@ def cmd_serve(args: argparse.Namespace) -> int:
_print_new_admin(admin_email, generated)
if args.enroll:
if not args.portal:
print("error: --enroll needs --portal", file=sys.stderr, flush=True)
return 1
if not cloud_config.exists():
# Before the engine starts, so the connector finds the config and
# dials out as part of coming up rather than needing a restart.
failed = _enroll(args.portal, args.enroll, args.admin_email)
failed = _enroll(
args.portal or DEFAULT_PORTAL, args.enroll, args.admin_email
)
if failed:
return failed
@@ -274,7 +278,7 @@ def cmd_serve(args: argparse.Namespace) -> int:
_say(" The dashboard is served by the portal; nothing is served here.")
else:
_say(" No portal. Pair this installation with:")
_say(" fluksio enroll <code> --portal https://hub.example.com")
_say(" fluksio enroll <code>")
_say(f" Signed in as {admin_email}")
_say(f" token in {token_path}")
_mention_other_installation(data_dir)
@@ -328,14 +332,23 @@ def _parser() -> argparse.ArgumentParser:
serve.add_argument("--admin-email", default=None)
serve.add_argument("--admin-password", default=None)
serve.add_argument("--enroll", metavar="CODE", help="claim code, if not yet paired")
serve.add_argument("--portal", metavar="URL", help="the portal --enroll redeems at")
serve.add_argument(
"--portal",
metavar="URL",
help=f"the portal --enroll redeems at (default {DEFAULT_PORTAL})",
)
serve.set_defaults(func=cmd_serve)
enroll = subparsers.add_parser(
"enroll", help="pair this installation with a portal"
)
enroll.add_argument("code", help="the claim code minted on the portal")
enroll.add_argument("--portal", required=True, metavar="URL")
enroll.add_argument(
"--portal",
default=DEFAULT_PORTAL,
metavar="URL",
help=f"a portal of your own, instead of {DEFAULT_PORTAL}",
)
enroll.add_argument(
"--as",
dest="as_email",
backend/fluksio/cloud/connector.py
+46 -9
View File
@@ -19,6 +19,7 @@ import asyncio
import base64
import contextlib
import logging
import random
import time
import uuid
from dataclasses import replace
@@ -36,7 +37,14 @@ logger = logging.getLogger(__name__)
PROTOCOL = 1
HEARTBEAT_S = 20.0
STATUS_S = 60.0
BASE_BACKOFF_S = 1.0
MAX_BACKOFF_S = 30.0
#: How long a silent socket is given before it is treated as gone. A laptop
#: that suspended, or a wifi that changed underneath us, leaves a connection
#: the operating system still believes in — nothing arrives and nothing fails.
#: Only an unanswered ping tells us, so this is the worst case for noticing.
PING_INTERVAL_S = 20.0
PING_TIMEOUT_S = 20.0
CHUNK_BYTES = 256 * 1024
MAX_WS_MESSAGE = 1024 * 1024
#: Only the versioned API is served over the tunnel. The MCP mount and the
@@ -79,31 +87,53 @@ class CloudConnector:
# -------------------------------------------------------------------------
async def serve_forever(self) -> None:
backoff = 1.0
"""Hold the link up, and put it back up when it goes down.
Every ending is the same ending here: a portal that closed the socket,
a network that changed under it, a laptop that woke up somewhere else.
The link is dialled again in all of them — the only question is how
soon, and that turns on whether this attempt got as far as being
attached. One that did and then dropped is a network event, so it
retries at once; one that never stood up is met with a longer wait
each time, because whatever is refusing is unlikely to stop within a
second.
"""
delay = BASE_BACKOFF_S
while True:
config = cloud_config.load()
if config is None:
# Disconnected locally while we were running.
return
attached = False
try:
await self._session(config)
backoff = 1.0
attached = await self._session(config)
except asyncio.CancelledError:
raise
except Exception as exc:
self._last_error = str(exc)
logger.warning("Portal link down: %s", exc)
finally:
self._connected = False
self._connected_since = None
self._last_error = str(exc)
logger.warning("Portal link down: %s — retrying in %.0fs", exc, backoff)
await asyncio.sleep(backoff)
backoff = min(MAX_BACKOFF_S, backoff * 2)
async def _session(self, config: cloud_config.CloudConfig) -> None:
delay = BASE_BACKOFF_S if attached else min(MAX_BACKOFF_S, delay * 2)
# Jittered, so a portal coming back up is not met by every
# installation it serves in the same instant.
wait = delay * (0.75 + random.random() * 0.5)
logger.info("Reconnecting to the portal in %.0fs", wait)
await asyncio.sleep(wait)
async def _session(self, config: cloud_config.CloudConfig) -> bool:
"""One connection, from dial to close. True if it ever attached."""
import websockets
attached = False
url = f"{config.ws_url}?token={config.token}"
async with websockets.connect(
url, max_size=MAX_WS_MESSAGE, ping_interval=20
url,
max_size=MAX_WS_MESSAGE,
ping_interval=PING_INTERVAL_S,
ping_timeout=PING_TIMEOUT_S,
) as socket:
await socket.send(
_dump({"op": "hello", "protocol": PROTOCOL, "app_version": _version()})
@@ -116,6 +146,7 @@ class CloudConnector:
self._adopt_owner(config, welcome.get("owner"))
attached = True
self._connected = True
self._connected_since = time.time()
self._last_error = None
@@ -138,12 +169,18 @@ class CloudConnector:
self._cancel_stream(str(frame.get("id") or ""))
finally:
keepalive.cancel()
# Awaited, not just cancelled: a keepalive that died of its own
# accord holds the reason the link went, and an un-awaited task
# takes it to the garbage collector instead of the log.
with contextlib.suppress(asyncio.CancelledError, Exception):
await keepalive
self._connected = False
self._connected_since = None
for task in list(self._calls.values()) + list(self._streams.values()):
task.cancel()
self._calls.clear()
self._streams.clear()
return attached
@staticmethod
def _adopt_owner(config: cloud_config.CloudConfig, owner: Any) -> None:
backend/fluksio/sdk/cli.py
+50 -3
View File
@@ -94,8 +94,21 @@ def discover(targets: list[str]) -> list[Flow]:
for info in pkgutil.walk_packages(package.__path__, f"{dotted}."):
importlib.import_module(info.name)
continue
# A plain directory — a repository root, usually. Its own modules,
# and the packages inside it: `myresearch/` beside a `README` is the
# ordinary shape, and naming it explicitly should not be the price of
# keeping your code in a package.
for module in sorted(path.glob("*.py")):
_import(*_module_of(module))
for child in sorted(path.iterdir()):
if child.name.startswith(".") or not (child / "__init__.py").exists():
continue
root, dotted = _package_of(child)
_import(root, dotted)
for info in pkgutil.walk_packages(
sys.modules[dotted].__path__, f"{dotted}."
):
importlib.import_module(info.name)
return list(FLOWS.values())
@@ -125,8 +138,9 @@ def cmd_sync(args: argparse.Namespace) -> int:
return _fail(str(exc))
if not flows:
return _fail(
f"no flows declared in {', '.join(targets)} — a flow is a `flow(...)` "
"call at module level"
f"no flows declared in {', '.join(targets)} — a flow is a `Flow(...)` "
"at module level. Name the package if it is somewhere else: "
"`fluksio sync src/myresearch`."
)
repo = repo_root(targets[0])
@@ -218,9 +232,37 @@ def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
return params
def _sync_first(client: Client) -> None:
"""Upload what the working directory declares, before running it.
The reason a run exists is usually the edit that came before it, and
remembering to sync is remembering to do the thing the computer could have
done. So `run` syncs by default — including the worker refresh, which is
what makes an edit to your own package take effect at all.
A directory that declares nothing is not an error: a flow drawn on the
canvas is run the same way, and has nothing to upload.
"""
try:
flows = discover(["."])
except (ImportError, SyncError) as exc:
# Do not fail a run for a module the run may not even need.
_say(f"warning: nothing synced — {exc}")
return
if not flows:
return
repo = repo_root(".")
reports = sync(flows, client, origin=origin_of(repo))
changed = [r for r in reports if not r.unchanged]
if changed:
_say(f"synced {', '.join(r.flow for r in changed)}")
def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
try:
client = Client(url=args.url, token=args.token)
if not args.no_sync:
_sync_first(client)
stored = client.get_flow(args.flow)
if stored is None:
return _fail(f"no flow '{args.flow}' on that engine")
@@ -301,12 +343,17 @@ def add_parsers(subparsers: Any) -> None:
parser.set_defaults(func=cmd_sync)
parser = subparsers.add_parser(
"run", help="start a run, passing the flow's inputs as --name value"
"run", help="sync this directory, then start a run of one of its flows"
)
parser.add_argument("flow")
parser.add_argument("--seed", type=int, default=None)
parser.add_argument("--wait", action="store_true", help="block until it finishes")
parser.add_argument("--timeout", type=float, default=0.0)
parser.add_argument(
"--no-sync",
action="store_true",
help="run what is already on the engine, without uploading first",
)
with_engine(parser)
parser.set_defaults(func=cmd_run)