Close the nine open SDK tasks: one engine per directory, a tabbed dashboard, re-pairing, run recovery

serve: refuse a second engine for one data directory whatever port it was
asked for, using the pidfile and a token this directory signed. The check
runs before the database is touched and before the credential is written,
which is what left every later CLI call pointing at a dead port.

The terminal dashboard is three tabs (Overview, Runs, Logs) with the toolbar
following the focused pane, the engine's output goes to serve.log rather than
down a pipe, and closing the screen stops both reader threads so the prompt
comes back. It adopts a running engine on every start, so stop/start and
restart work on one it did not start, and a stop waits for the process to be
gone before the next start. Enrolment reports itself in the modal.

enroll: a new claim code replaces the pairing instead of being refused. The
code is redeemed before anything is written, mappings to a portal being left
are cleared, and a running engine redials when the stored enrolment changes.

runs: an engine re-queues the runs left `queued` by the one before it, and
`fluksio retry <id>` / `retry --group <sweep>` submits an interrupted run
again with the same inputs and group, recorded through Run.parent_id.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U9BoNGq6V9MdRWAte7JBuC

backend/fluksio/cloud/connector.py

@@ -60,6 +60,11 @@ MAX_IN_FLIGHT = 256
 ENROL_POLL_S = 3.0
 
 
+def _identity(config: cloud_config.CloudConfig | None) -> tuple[str, str] | None:
+    """Which enrolment a link is running on, so a replaced one is noticed."""
+    return (config.instance_id, config.token) if config is not None else None
+
+
 def start(app: FastAPI) -> None:
     """Dial the portal, replacing any link already up."""
     existing = getattr(app.state, "cloud_task", None)
@@ -67,6 +72,7 @@ def start(app: FastAPI) -> None:
         existing.cancel()
     connector = CloudConnector(app)
     app.state.cloud_connector = connector
+    app.state.cloud_identity = _identity(cloud_config.load())
     app.state.cloud_task = asyncio.create_task(
         connector.serve_forever(), name="cloud-connector"
     )
@@ -85,7 +91,7 @@ async def watch_enrolment(app: FastAPI) -> None:
         task = getattr(app.state, "cloud_task", None)
         if task is not None and task.done():
             # It returns of its own accord when the config goes away, which is
-            # what `fluksio disconnect` and the portal's own Disconnect do.
+            # what Disconnect, here or on the portal, does.
             app.state.cloud_task = None
             app.state.cloud_connector = None
             task = None
@@ -94,7 +100,17 @@ async def watch_enrolment(app: FastAPI) -> None:
         # started — which, started from here, is a restart every few seconds.
         # A config that is fine but unreachable keeps its task, and the
         # retrying belongs to the connector rather than to this.
-        if task is None and cloud_config.load() is not None:
+        config = cloud_config.load()
+        if task is not None and _identity(config) != getattr(
+            app.state, "cloud_identity", None
+        ):
+            # Enrolled again, at this portal or another one. `fluksio enroll`
+            # is its own process and cannot cancel this task, so the link would
+            # otherwise stay up on the credential that was replaced.
+            logger.info("Enrolment replaced while running; redialling")
+            task.cancel()
+            task = None
+        if task is None and config is not None:
             logger.info("Enrolled while running; dialling the portal")
             start(app)
 

backend/fluksio/cloud/enroll.py

@@ -14,7 +14,7 @@ from typing import Any
 from urllib.parse import urlsplit
 
 import httpx
-from sqlmodel import Session, select
+from sqlmodel import Session, col, select
 
 import fluksio
 from fluksio.cloud import config as cloud_config
@@ -30,11 +30,6 @@ class EnrollError(Exception):
         self.detail = detail
 
 
-class AlreadyEnrolled(EnrollError):
-    def __init__(self) -> None:
-        super().__init__(409, "This instance is already connected to a portal")
-
-
 def _is_local(url: str) -> bool:
     """Whether the address is this machine or a compose-internal service."""
     host = urlsplit(url).hostname or ""
@@ -83,10 +78,15 @@ def redeem_claim(
 def enroll(
     session: Session, user: User, portal_url: str, claim_code: str
 ) -> cloud_config.CloudConfig:
-    """Redeem the code and write the config the connector dials with."""
-    if cloud_config.exists():
-        raise AlreadyEnrolled()
+    """Redeem the code and write the config the connector dials with.
 
+    An instance that is already paired is re-paired rather than refused: a new
+    claim code is somebody asking for this, and refusing left deleting
+    `cloud.json` by hand as the only way through. The code is redeemed before
+    anything is written, so a code the portal rejects leaves a working
+    connection working.
+    """
+    previous = cloud_config.load()
     data = redeem_claim(portal_url, claim_code)
     config = cloud_config.CloudConfig(
         portal_url=portal_url.rstrip("/"),
@@ -104,6 +104,17 @@ def enroll(
     cloud_config.save(config)
 
     owner_id = str(data["owner_id"])
+    if previous is not None and previous.issuer != config.issuer:
+        # A mapping is a subject of the portal that issued it, and nothing in
+        # the row says which portal that was — so against a different portal
+        # the old rows would both block the new owner from being adopted and
+        # resolve a stranger's subject onto a local account.
+        for stale in session.exec(
+            select(User).where(col(User.portal_sub).is_not(None))
+        ):
+            stale.portal_sub = None
+            session.add(stale)
+        session.flush()
     # Re-enrolling from a different local account moves the mapping rather than
     # leaving two accounts claiming the same portal identity, which the unique
     # index would refuse and the lookup could not choose between anyway.