Stand down when the portal revokes this instance
The connector treats close code 4410 from the hub as final: it stops dialling, keeps cloud.json so a new claim code replaces it, and reports revoked in /cloud/status, which Settings → Remote access shows as Removed at the portal. Enrolling again, or a restart, brings the link back. An older hub still answers 403 and the link keeps retrying as before.
This commit is contained in:
@@ -60,6 +60,7 @@ def read_status(request: Request) -> dict[str, Any]:
|
|||||||
"instance_id": config.instance_id if config else None,
|
"instance_id": config.instance_id if config else None,
|
||||||
"last_error": None,
|
"last_error": None,
|
||||||
"connected_since": None,
|
"connected_since": None,
|
||||||
|
"revoked": False,
|
||||||
}
|
}
|
||||||
status: dict[str, Any] = connector.status()
|
status: dict[str, Any] = connector.status()
|
||||||
return status
|
return status
|
||||||
|
|||||||
@@ -35,6 +35,10 @@ from fluksio.cloud import config as cloud_config
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
PROTOCOL = 1
|
PROTOCOL = 1
|
||||||
|
#: The hub closes a credential it will never accept again with this code (see
|
||||||
|
#: the hub's ``tunnel/protocol.py``). The link then stands down until enrolled
|
||||||
|
#: again or restarted, rather than dialling a deleted instance forever.
|
||||||
|
REVOKED_CLOSE = 4410
|
||||||
HEARTBEAT_S = 20.0
|
HEARTBEAT_S = 20.0
|
||||||
STATUS_S = 60.0
|
STATUS_S = 60.0
|
||||||
BASE_BACKOFF_S = 1.0
|
BASE_BACKOFF_S = 1.0
|
||||||
@@ -89,18 +93,27 @@ async def watch_enrolment(app: FastAPI) -> None:
|
|||||||
while True:
|
while True:
|
||||||
await asyncio.sleep(ENROL_POLL_S)
|
await asyncio.sleep(ENROL_POLL_S)
|
||||||
task = getattr(app.state, "cloud_task", None)
|
task = getattr(app.state, "cloud_task", None)
|
||||||
if task is not None and task.done():
|
|
||||||
# It returns of its own accord when the config goes away, which is
|
|
||||||
# what Disconnect, here or on the portal, does.
|
|
||||||
app.state.cloud_task = None
|
|
||||||
app.state.cloud_connector = None
|
|
||||||
task = None
|
|
||||||
# `load()`, not `exists()`: a file that cannot be read is not an
|
# `load()`, not `exists()`: a file that cannot be read is not an
|
||||||
# enrolment, and the connector would give up on it the instant it
|
# enrolment, and the connector would give up on it the instant it
|
||||||
# started — which, started from here, is a restart every few seconds.
|
# started — which, started from here, is a restart every few seconds.
|
||||||
# A config that is fine but unreachable keeps its task, and the
|
# A config that is fine but unreachable keeps its task, and the
|
||||||
# retrying belongs to the connector rather than to this.
|
# retrying belongs to the connector rather than to this.
|
||||||
config = cloud_config.load()
|
config = cloud_config.load()
|
||||||
|
if task is not None and task.done():
|
||||||
|
connector = getattr(app.state, "cloud_connector", None)
|
||||||
|
if (
|
||||||
|
connector is not None
|
||||||
|
and connector.revoked
|
||||||
|
and _identity(config) == getattr(app.state, "cloud_identity", None)
|
||||||
|
):
|
||||||
|
# Stood down on the portal's word, and kept for its status.
|
||||||
|
# Only a new enrolment brings the link back.
|
||||||
|
continue
|
||||||
|
# It returns of its own accord when the config goes away, which is
|
||||||
|
# what Disconnect, here or on the portal, does.
|
||||||
|
app.state.cloud_task = None
|
||||||
|
app.state.cloud_connector = None
|
||||||
|
task = None
|
||||||
if task is not None and _identity(config) != getattr(
|
if task is not None and _identity(config) != getattr(
|
||||||
app.state, "cloud_identity", None
|
app.state, "cloud_identity", None
|
||||||
):
|
):
|
||||||
@@ -125,6 +138,11 @@ class CloudConnector:
|
|||||||
self._connected = False
|
self._connected = False
|
||||||
self._connected_since: float | None = None
|
self._connected_since: float | None = None
|
||||||
self._last_error: str | None = None
|
self._last_error: str | None = None
|
||||||
|
self._revoked = False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def revoked(self) -> bool:
|
||||||
|
return self._revoked
|
||||||
|
|
||||||
# -------------------------------------------------------------------------
|
# -------------------------------------------------------------------------
|
||||||
# Status, for the local settings screen
|
# Status, for the local settings screen
|
||||||
@@ -143,6 +161,7 @@ class CloudConnector:
|
|||||||
"instance_id": config.instance_id if config else None,
|
"instance_id": config.instance_id if config else None,
|
||||||
"last_error": self._last_error,
|
"last_error": self._last_error,
|
||||||
"connected_since": self._connected_since,
|
"connected_since": self._connected_since,
|
||||||
|
"revoked": self._revoked,
|
||||||
}
|
}
|
||||||
|
|
||||||
# -------------------------------------------------------------------------
|
# -------------------------------------------------------------------------
|
||||||
@@ -174,6 +193,12 @@ class CloudConnector:
|
|||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._last_error = str(exc)
|
self._last_error = str(exc)
|
||||||
|
if _is_revoked(exc):
|
||||||
|
# Dialling again would be refused the same way forever.
|
||||||
|
# The config stays, so enrolling again replaces it.
|
||||||
|
self._revoked = True
|
||||||
|
logger.warning("The portal revoked this instance; standing down")
|
||||||
|
return
|
||||||
logger.warning("Portal link down: %s", exc)
|
logger.warning("Portal link down: %s", exc)
|
||||||
finally:
|
finally:
|
||||||
self._connected = False
|
self._connected = False
|
||||||
@@ -606,6 +631,17 @@ class CloudConnector:
|
|||||||
task.cancel()
|
task.cancel()
|
||||||
|
|
||||||
|
|
||||||
|
def _is_revoked(exc: Exception) -> bool:
|
||||||
|
"""Whether the hub closed the link as revoked, at the handshake or later."""
|
||||||
|
from websockets.exceptions import ConnectionClosed
|
||||||
|
|
||||||
|
return (
|
||||||
|
isinstance(exc, ConnectionClosed)
|
||||||
|
and exc.rcvd is not None
|
||||||
|
and exc.rcvd.code == REVOKED_CLOSE
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _dump(payload: Any) -> str:
|
def _dump(payload: Any) -> str:
|
||||||
import json
|
import json
|
||||||
|
|
||||||
|
|||||||
@@ -641,3 +641,53 @@ async def test_the_link_sends_its_credential_as_a_header() -> None:
|
|||||||
await CloudConnector(Mock())._session(config) # type: ignore[arg-type]
|
await CloudConnector(Mock())._session(config) # type: ignore[arg-type]
|
||||||
|
|
||||||
assert seen == {"path": "/api/v1/tunnel/attach", "auth": "Bearer t0k"}
|
assert seen == {"path": "/api/v1/tunnel/attach", "auth": "Bearer t0k"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_a_revoked_link_stands_down(enrolled: User) -> None: # noqa: ARG001
|
||||||
|
"""A deleted instance is told so once, rather than refused forever.
|
||||||
|
|
||||||
|
The config stays, so enrolling again is all it takes to come back.
|
||||||
|
"""
|
||||||
|
from websockets.asyncio.server import ServerConnection, serve
|
||||||
|
|
||||||
|
from fluksio.cloud import connector
|
||||||
|
|
||||||
|
dials = 0
|
||||||
|
|
||||||
|
async def hub(socket: ServerConnection) -> None:
|
||||||
|
nonlocal dials
|
||||||
|
dials += 1
|
||||||
|
await socket.recv()
|
||||||
|
await socket.close(code=connector.REVOKED_CLOSE, reason="revoked")
|
||||||
|
|
||||||
|
async with serve(hub, "127.0.0.1", 0) as server:
|
||||||
|
port = next(iter(server.sockets)).getsockname()[1]
|
||||||
|
config = cloud_config.load()
|
||||||
|
assert config is not None
|
||||||
|
cloud_config.save(replace(config, ws_url=f"ws://127.0.0.1:{port}/attach"))
|
||||||
|
link = connector.CloudConnector(Mock())
|
||||||
|
await asyncio.wait_for(link.serve_forever(), timeout=5)
|
||||||
|
|
||||||
|
assert dials == 1
|
||||||
|
assert link.status()["revoked"] is True
|
||||||
|
assert cloud_config.load() is not None
|
||||||
|
|
||||||
|
# And the watcher leaves it down until the enrolment changes.
|
||||||
|
app = Mock()
|
||||||
|
done = asyncio.get_running_loop().create_future()
|
||||||
|
done.set_result(None)
|
||||||
|
app.state = Mock(
|
||||||
|
cloud_task=done,
|
||||||
|
cloud_connector=link,
|
||||||
|
cloud_identity=connector._identity(cloud_config.load()),
|
||||||
|
)
|
||||||
|
started: list[object] = []
|
||||||
|
with (
|
||||||
|
patch.object(connector, "ENROL_POLL_S", 0.01),
|
||||||
|
patch.object(connector, "start", lambda one: started.append(one)),
|
||||||
|
):
|
||||||
|
watcher = asyncio.create_task(connector.watch_enrolment(app))
|
||||||
|
await asyncio.sleep(0.05)
|
||||||
|
watcher.cancel()
|
||||||
|
assert not started
|
||||||
|
|||||||
@@ -66,6 +66,10 @@ proxy on that route must support WebSockets.
|
|||||||
- To replace the pairing, use **Connect with a new code**. Remote users and
|
- To replace the pairing, use **Connect with a new code**. Remote users and
|
||||||
panels may need to pair again.
|
panels may need to pair again.
|
||||||
|
|
||||||
|
If the instance is removed at the portal, or its code is renewed there, the
|
||||||
|
status reads **Removed at the portal** and the instance stops dialling. Connect
|
||||||
|
with a new code to restore remote access.
|
||||||
|
|
||||||
Local sign-in and flows continue after disconnecting. Portal requests follow
|
Local sign-in and flows continue after disconnecting. Portal requests follow
|
||||||
the instance's normal account permissions, and the instance verifies portal
|
the instance's normal account permissions, and the instance verifies portal
|
||||||
tokens against the public signing key pinned during enrollment.
|
tokens against the public signing key pinned during enrollment.
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ type CloudStatus = {
|
|||||||
instance_id: string | null
|
instance_id: string | null
|
||||||
last_error: string | null
|
last_error: string | null
|
||||||
connected_since: number | null
|
connected_since: number | null
|
||||||
|
revoked: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -167,11 +168,20 @@ export function RemoteAccess() {
|
|||||||
<div className="flex flex-col gap-4">
|
<div className="flex flex-col gap-4">
|
||||||
<dl className="grid gap-3 sm:grid-cols-2">
|
<dl className="grid gap-3 sm:grid-cols-2">
|
||||||
<Field label="Status">
|
<Field label="Status">
|
||||||
{status.connected
|
{status.connected ? (
|
||||||
? "Connected"
|
"Connected"
|
||||||
: status.last_error
|
) : status.revoked ? (
|
||||||
? `Reconnecting — ${status.last_error}`
|
<>
|
||||||
: "Reconnecting…"}
|
Removed at the portal
|
||||||
|
<span className="mt-1 block text-xs text-muted-foreground">
|
||||||
|
Connect with a new code to restore remote access.
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
) : status.last_error ? (
|
||||||
|
`Reconnecting — ${status.last_error}`
|
||||||
|
) : (
|
||||||
|
"Reconnecting…"
|
||||||
|
)}
|
||||||
</Field>
|
</Field>
|
||||||
<Field label="Portal">{status.portal_url ?? "—"}</Field>
|
<Field label="Portal">{status.portal_url ?? "—"}</Field>
|
||||||
<Field label="Acting as">
|
<Field label="Acting as">
|
||||||
|
|||||||
Reference in New Issue
Block a user