Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(connect): hold a live outbound connection to the relay

Juan Denis232c6978c509
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index 7f78604a..eef51398 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -443,6 +443,11 @@ def create_app(config_name=None):
from app.services.linked_panel_service import LinkedPanelService
LinkedPanelService.start_client_if_linked(app)
+ # Hold the outbound ServerKit Cloud relay connection when this
+ # panel is paired.
+ from app.services import connect_client
+ connect_client.start_client_if_paired(app)
+
# Request body size limit
app.config['MAX_CONTENT_LENGTH'] = 100 * 1024 * 1024 # 100MB limit
diff --git a/backend/app/services/connect_client.py b/backend/app/services/connect_client.py
index 22d16e06..019fe350 100644
--- a/backend/app/services/connect_client.py
+++ b/backend/app/services/connect_client.py
@@ -1,23 +1,36 @@
-"""ServerKit Cloud connect pairing — panel side.
+"""ServerKit Cloud connect — panel side.
-M1 scope is pairing ONLY: enroll with ServerKit Cloud, print the pairing code +
-fingerprint, poll with an Ed25519 proof-of-possession until the user approves
-in the browser, then write ``connect.json`` next to the panel config. No
-WebSocket transport, no reconnect loop — those are later milestones.
+Pairing: enroll with ServerKit Cloud, print the pairing code + fingerprint, poll with an
+Ed25519 proof-of-possession until the user approves in the browser, then
+write ``connect.json`` next to the panel config. The enrollment_secret is
+kept in memory for the pairing session only — never written to disk or
+logged.
+
+Transport: once ``connect.json`` exists, a managed background thread (``RelayClient``,
+started from app init via ``start_client_if_paired`` — same pattern as
+linked_panel_agent) holds an outbound WebSocket to the relay: signed hello
+(``device_id|ts|nonce``), a ``{"t":"ping"}`` heartbeat every 20 s, jittered
+exponential backoff on drops, flap detection, and a long-poll fallback for
+networks that break WebSockets. Runtime state is persisted to
+``connect-state.json`` so `serverkit connect status` and the API show it
+without the thread running. Stream opens this client does not implement are answered with close +
+``unsupported``.
ServerKit Cloud endpoints (no session auth on the panel side):
POST /api/pair/enroll -> enrollment_id, enrollment_secret (shown once), code
POST /api/pair/poll -> 202 pending / 200 claimed (signed proof-of-possession)
POST /api/pair/rotate -> new code + expires_at
-
-The enrollment_secret is kept in memory for the pairing session only — it is
-never written to disk or logged.
"""
+import collections
import json
import logging
import os
+import random
+import secrets
import socket
+import threading
import time
+import urllib.parse
from datetime import datetime, timezone
import requests
@@ -301,7 +314,12 @@ def _past(iso_ts: str) -> bool:
def status() -> dict:
- """Connection state as the UI/CLI should show it (reads connect.json + key)."""
+ """Connection state as the UI/CLI should show it.
+
+ Reads connect.json (pairing), connect-state.json (runtime transport
+ state, written by RelayClient) and checks the device key — no thread
+ needs to be running in this process.
+ """
data = _read_connect_file()
key_path = data.get(KEY_PATH_FIELD) or connect_keys.default_key_path()
key_present = os.path.exists(key_path)
@@ -314,11 +332,16 @@ def status() -> dict:
'key_present': key_present,
'key_path': key_path,
}
- state = 'paired_offline' if key_present else 'degraded'
+ saved = _read_state_file()
+ if saved.get('state'):
+ state = saved['state']
+ reason = saved.get('state_reason')
+ else:
+ state = 'paired_offline' if key_present else 'degraded'
+ reason = 'never_connected' if key_present else 'device_key_missing'
return {
'state': state,
- # No transport yet in M1 — a paired panel has never been online.
- 'state_reason': 'never_connected' if key_present else 'device_key_missing',
+ 'state_reason': reason,
'paired': True,
'cloud_url': data.get('cloud_url') or resolve_cloud_url(),
'device_id': data.get('device_id'),
@@ -326,6 +349,9 @@ def status() -> dict:
'name': data.get('name'),
'relay_url': data.get('relay_url'),
'scopes': data.get('scopes') or [],
+ 'transport': saved.get('transport'),
+ 'relay_instance': saved.get('relay_instance'),
+ 'last_connected_at': saved.get('last_connected_at'),
'fingerprint': data.get('fingerprint'),
'fingerprint_grouped': connect_keys.format_fingerprint(data['fingerprint'])
if data.get('fingerprint') else None,
@@ -337,14 +363,667 @@ def status() -> dict:
def disconnect(remove_key: bool = False) -> dict:
"""Forget the pairing locally. Does NOT revoke anything on ServerKit Cloud."""
+ stop_relay_client() # no-op unless the client runs in this process
removed = []
- path = connect_file_path()
- if os.path.exists(path):
- os.unlink(path)
- removed.append(path)
+ for path in (connect_file_path(), state_file_path()):
+ if os.path.exists(path):
+ os.unlink(path)
+ removed.append(path)
if remove_key:
key_path = connect_keys.default_key_path()
if os.path.exists(key_path):
os.unlink(key_path)
removed.append(key_path)
return {'success': True, 'removed': removed, 'state': 'unpaired'}
+
+
+# ==================== Relay transport ====================
+
+DEFAULT_RELAY_URL = 'wss://relay.serverkit.ai/v1/device'
+STATE_FILENAME = 'connect-state.json'
+
+PROTO_VERSION = 1
+HELLO_TIMEOUT_S = 10 # the relay closes devices that take longer to hello
+PING_INTERVAL_S = 20 # panel -> relay {"t":"ping"} heartbeat
+MAX_FRAME_BYTES = 256 * 1024
+
+BACKOFF_BASE_S = 1.0
+BACKOFF_CAP_S = 60.0
+BACKOFF_JITTER = 0.25 # ±25%
+STABLE_AFTER_S = 120 # a connection stable this long resets the backoff
+WS_RETRY_AFTER_S = 15 * 60 # while on the poll fallback, re-probe WS this often
+POLL_TIMEOUT_S = 35 # slightly above the relay's 25 s hold
+FLAP_WINDOW_S = 600
+FLAP_LIMIT = 6 # >6 reconnects in 10 minutes = flapping
+
+# Relay close codes -> state_reason enum (mirrors the relay's protocol).
+CLOSE_REASON_BY_CODE = {
+ 4001: 'auth_failed_signature',
+ 4002: 'auth_failed_clock_skew',
+ 4003: 'version_unsupported',
+ 4004: 'auth_failed_signature', # nonce replay
+ 4009: 'revoked',
+}
+
+
+class RelayRevoked(Exception):
+ """The relay closed us with 4009 — terminal until re-paired on ServerKit Cloud."""
+
+
+class _HandshakeRefused(Exception):
+ """WS handshake/hello failed with a classified state_reason."""
+
+
+# ---------- pure helpers (unit-tested) ----------
+
+
+def close_reason_for_code(code) -> str:
+ """Relay close code -> state_reason; unknown codes are a dropped heartbeat."""
+ return CLOSE_REASON_BY_CODE.get(code, 'heartbeat_timeout')
+
+
+def backoff_delay(attempt: int, rand=random.uniform) -> float:
+ """Jittered exponential backoff: 1 s doubling to a 60 s cap, ±25% jitter."""
+ base = min(BACKOFF_CAP_S, BACKOFF_BASE_S * (2 ** max(0, attempt)))
+ return base * (1 + rand(-BACKOFF_JITTER, BACKOFF_JITTER))
+
+
+def is_flapping(attempts_at, now: float = None,
+ window: float = FLAP_WINDOW_S, limit: int = FLAP_LIMIT) -> bool:
+ """True when more than `limit` reconnect attempts happened within `window`."""
+ now = time.time() if now is None else now
+ return sum(1 for t in attempts_at if now - t <= window) > limit
+
+
+def relay_http_base(relay_url: str) -> str:
+ """wss://host:port/v1/device -> https://host:port (ws -> http)."""
+ parsed = urllib.parse.urlparse(relay_url)
+ scheme = {'wss': 'https', 'ws': 'http'}.get(parsed.scheme, parsed.scheme)
+ return urllib.parse.urlunparse((scheme, parsed.netloc, '', '', '', ''))
+
+
+def poll_url_for(relay_url: str) -> str:
+ """The relay's long-poll endpoint for a device WS URL."""
+ return relay_http_base(relay_url) + '/v1/device/poll'
+
+
+def build_hello(private_key, device_id: str, client_version: str) -> dict:
+ """The signed first frame. sig covers the exact string `device_id|ts|nonce`."""
+ ts = int(time.time())
+ nonce = secrets.token_hex(16)
+ sig = private_key.sign(f'{device_id}|{ts}|{nonce}'.encode()).hex()
+ return {
+ 't': 'hello',
+ 'device_id': device_id,
+ 'ts': ts,
+ 'nonce': nonce,
+ 'sig': sig,
+ 'proto': PROTO_VERSION,
+ 'client_version': client_version,
+ }
+
+
+HELLO_HEADERS = {
+ 'device_id': 'X-ServerKit-Device',
+ 'ts': 'X-ServerKit-Ts',
+ 'nonce': 'X-ServerKit-Nonce',
+ 'sig': 'X-ServerKit-Sig',
+ 'proto': 'X-ServerKit-Proto',
+ 'client_version': 'X-ServerKit-Client-Version',
+}
+
+
+def hello_headers(hello: dict) -> dict:
+ """The same hello carried in X-ServerKit-* headers for the poll fallback."""
+ return {header: str(hello[field]) for field, header in HELLO_HEADERS.items()}
+
+
+# ---------- runtime state file ----------
+
+
+def state_file_path() -> str:
+ return os.path.join(paths.SERVERKIT_CONFIG_DIR, STATE_FILENAME)
+
+
+def _read_state_file() -> dict:
+ path = state_file_path()
+ if not os.path.exists(path):
+ return {}
+ try:
+ with open(path, 'r', encoding='utf-8') as f:
+ data = json.load(f)
+ return data if isinstance(data, dict) else {}
+ except (OSError, ValueError) as exc:
+ logger.warning('Could not read %s: %s', path, exc)
+ return {}
+
+
+def _write_state_file(state: str, reason: str = None, transport: str = None,
+ relay_instance: str = None, last_connected_at: str = None):
+ """Persist a state transition, preserving fields not being overwritten."""
+ saved = _read_state_file()
+ saved.update({
+ 'state': state,
+ 'state_reason': reason,
+ 'transport': transport,
+ 'relay_instance': relay_instance if relay_instance is not None
+ else saved.get('relay_instance'),
+ 'last_connected_at': last_connected_at or saved.get('last_connected_at'),
+ 'updated_at': datetime.now(timezone.utc).isoformat(),
+ })
+ path = state_file_path()
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, 'w', encoding='utf-8') as f:
+ json.dump(saved, f, indent=2)
+ f.write('\n')
+
+
+def _classify_ws_failure(exc) -> str:
+ """Map a transport exception to a state_reason (pure; unit-tested)."""
+ import ssl
+
+ from websockets.exceptions import ConnectionClosed, InvalidHandshake
+
+ if isinstance(exc, ConnectionClosed):
+ code = exc.rcvd.code if exc.rcvd else None
+ return close_reason_for_code(code)
+ if isinstance(exc, InvalidHandshake):
+ # HTTP 4xx/5xx or a non-101 answer on the upgrade: a proxy breaking WS.
+ return 'relay_unreachable'
+ if isinstance(exc, ssl.SSLError):
+ return 'tls_error'
+ if isinstance(exc, socket.gaierror):
+ return 'dns_error'
+ # connect refused/reset, timeouts, anything else network-shaped.
+ return 'relay_unreachable'
+
+
+def _relay_config() -> dict:
+ """What the transport needs from connect.json + the device key.
+
+ Returns None when the panel is not paired (anymore) or the key is gone —
+ the client loop treats that as "exit", not "retry".
+ """
+ data = _read_connect_file()
+ if not data.get('device_id') or not data.get('relay_url'):
+ return None
+ key_path = data.get(KEY_PATH_FIELD) or connect_keys.default_key_path()
+ try:
+ private_key, _pub, _fpr = connect_keys.load_keypair(key_path)
+ except Exception as exc:
+ logger.warning('Connect relay: cannot load device key %s: %s', key_path, exc)
+ return None
+ return {
+ 'device_id': data['device_id'],
+ 'relay_url': data['relay_url'],
+ 'private_key': private_key,
+ 'key_path': key_path,
+ }
+
+
+# ---------- the client ----------
+
+
+class RelayClient:
+ """Holds one outbound connection to the ServerKit relay.
+
+ WebSocket first (signed hello, 20 s pings); when the upgrade smells like
+ a proxy block (non-101 answer, timeout) the client falls back to the
+ long-poll loop and reports `degraded`, re-probing WS every
+ WS_RETRY_AFTER_S. All state transitions are persisted to
+ connect-state.json so the CLI/API read them without this thread.
+ """
+
+ def __init__(self, app=None):
+ self.app = app
+ self.running = False
+ self.transport = None
+ self.relay_instance = None
+ self.last_error = None
+ self._thread = None
+ self._stop = threading.Event()
+ self._ws = None
+ self._attempts_at = collections.deque()
+ self._flap_logged = False
+ self._last_written = None # (state, reason, transport) transition dedup
+
+ # -- lifecycle -----------------------------------------------------
+
+ def start(self):
+ if self.running:
+ return
+ self.running = True
+ self._stop.clear()
+ self._thread = threading.Thread(
+ target=self._run, daemon=True, name='connect-relay')
+ self._thread.start()
+ logger.info('Connect relay client started')
+
+ def stop(self):
+ self.running = False
+ self._stop.set()
+ ws, self._ws = self._ws, None
+ if ws is not None:
+ try:
+ ws.close()
+ except Exception:
+ pass
+ # A terminal 'revoked' survives a stop: re-pairing on ServerKit Cloud, not a
+ # local restart, is what clears it.
+ if _read_state_file().get('state') != 'revoked':
+ self._set_state('paired_offline', 'client_stopped', transport=None)
+
+ # -- state persistence ---------------------------------------------
+
+ def _set_state(self, state, reason, transport=None):
+ key = (state, reason, transport)
+ if key == self._last_written:
+ return
+ self._last_written = key
+ connected = state in ('online', 'degraded')
+ _write_state_file(
+ state, reason, transport=transport, relay_instance=self.relay_instance,
+ last_connected_at=datetime.now(timezone.utc).isoformat()
+ if connected else None)
+
+ def _sleep(self, seconds):
+ self._stop.wait(max(0.0, seconds))
+
+ # -- main loop ------------------------------------------------------
+
+ def _run(self):
+ attempt = 0
+ ws_blocked_until = 0.0
+ while self.running:
+ cfg = _relay_config()
+ if cfg is None:
+ logger.info('Connect relay: not paired (connect.json/key gone); exiting')
+ return
+
+ if time.monotonic() < ws_blocked_until:
+ if self._poll_session(cfg, ws_blocked_until) == 'stopped':
+ return
+ ws_blocked_until = 0.0 # time to re-probe WS
+ continue
+
+ started = time.monotonic()
+ outcome = self._ws_session(cfg)
+ if outcome == 'stopped':
+ return
+ kind, _, reason = outcome.partition(':')
+
+ if kind == 'revoked':
+ self._set_state('revoked', 'revoked', transport=None)
+ logger.warning('Connect relay: device revoked on ServerKit Cloud; stopping')
+ return
+
+ if time.monotonic() - started >= STABLE_AFTER_S:
+ attempt = 0
+ else:
+ attempt += 1
+ self._note_attempt()
+
+ if kind == 'refused' and reason == 'relay_unreachable':
+ # Proxy-blocked or unanswerable upgrade: limited mode.
+ ws_blocked_until = time.monotonic() + WS_RETRY_AFTER_S
+ self._set_state('degraded', 'relay_unreachable', transport='poll')
+ continue
+
+ self._set_state('paired_offline', reason, transport=None)
+ self._sleep(backoff_delay(attempt))
+
+ def _note_attempt(self):
+ now = time.time()
+ self._attempts_at.append(now)
+ while self._attempts_at and now - self._attempts_at[0] > FLAP_WINDOW_S:
+ self._attempts_at.popleft()
+ flapping = is_flapping(self._attempts_at, now=now)
+ if flapping and not self._flap_logged:
+ self._flap_logged = True
+ logger.warning(
+ 'Connect relay flapping: %d reconnects in the last %d s',
+ len(self._attempts_at), FLAP_WINDOW_S)
+ elif not flapping:
+ self._flap_logged = False
+
+ # -- websocket ------------------------------------------------------
+
+ def _ws_session(self, cfg) -> str:
+ """One WS connection: handshake, hello, heartbeat until drop.
+
+ Returns 'stopped', 'revoked', 'refused:<reason>' (never got ready —
+ auth/handshake failure) or 'drop:<reason>' (was ready, then lost).
+ """
+ try:
+ ws = self._ws_connect(cfg)
+ except RelayRevoked:
+ return 'revoked'
+ except _HandshakeRefused as exc:
+ return f'refused:{exc}'
+ except Exception as exc:
+ return f'refused:{_classify_ws_failure(exc)}'
+
+ self._ws = ws
+ try:
+ self.transport = 'ws'
+ self._set_state('online', None, transport='ws')
+ logger.info('Connect relay: online via ws (instance %s)',
+ self.relay_instance)
+ while self.running:
+ try:
+ raw = ws.recv(timeout=PING_INTERVAL_S)
+ except TimeoutError:
+ ws.send('{"t":"ping"}')
+ continue
+ self._handle_frame(ws, raw)
+ return 'stopped'
+ except Exception as exc:
+ from websockets.exceptions import ConnectionClosed
+ if isinstance(exc, ConnectionClosed):
+ code = exc.rcvd.code if exc.rcvd else None
+ if code == 4009:
+ return 'revoked'
+ self.last_error = f'connection closed (code {code})'
+ return f'drop:{close_reason_for_code(code)}'
+ self.last_error = str(exc)
+ return f'drop:{_classify_ws_failure(exc)}'
+ finally:
+ self._ws = None
+ try:
+ ws.close()
+ except Exception:
+ pass
+
+ def _ws_connect(self, cfg):
+ """Handshake + signed hello + wait for `ready` (10 s, relay-enforced)."""
+ from app.utils.version import get_panel_version
+ from websockets.exceptions import ConnectionClosed
+ from websockets.sync.client import connect as ws_connect
+
+ ws = ws_connect(
+ cfg['relay_url'],
+ open_timeout=HELLO_TIMEOUT_S,
+ close_timeout=5,
+ ping_interval=None, # heartbeats are app-level {"t":"ping"} frames
+ max_size=MAX_FRAME_BYTES,
+ )
+ try:
+ hello = build_hello(cfg['private_key'], cfg['device_id'],
+ get_panel_version())
+ ws.send(json.dumps(hello))
+ frame = json.loads(ws.recv(timeout=HELLO_TIMEOUT_S))
+ if frame.get('t') != 'ready':
+ raise _HandshakeRefused('relay_unreachable')
+ self.relay_instance = frame.get('instance')
+ return ws
+ except ConnectionClosed as exc:
+ code = exc.rcvd.code if exc.rcvd else None
+ if code == 4009:
+ raise RelayRevoked()
+ raise _HandshakeRefused(close_reason_for_code(code))
+ except Exception:
+ try:
+ ws.close()
+ except Exception:
+ pass
+ raise
+
+ def _handle_frame(self, ws, raw):
+ try:
+ frame = json.loads(raw)
+ except ValueError:
+ return
+ if frame.get('t') == 'open':
+ # M2 has no streams yet: refuse honestly (mirrors the relay).
+ ws.send(json.dumps({
+ 's': frame.get('s'),
+ 't': 'close',
+ 'reason': 'unsupported',
+ 'detail': 'streams arrive with the next Connect release',
+ }))
+
+ # -- long-poll fallback ----------------------------------------------
+
+ def _poll_session(self, cfg, ws_retry_at) -> str:
+ """Long-poll loop ('limited mode'): each held GET is the heartbeat.
+
+ Returns 'stopped', 'revoked', or 'retry_ws' when it's time to try the
+ WebSocket again.
+ """
+ self.transport = 'poll'
+ self._set_state('degraded', 'relay_unreachable', transport='poll')
+ logger.info('Connect relay: using long-poll fallback (degraded)')
+ fail_attempt = 0
+ while self.running:
+ if time.monotonic() >= ws_retry_at:
+ return 'retry_ws'
+ try:
+ self._poll_once(cfg)
+ fail_attempt = 0
+ except RelayRevoked:
+ return 'revoked'
+ except Exception as exc:
+ fail_attempt += 1
+ self.last_error = str(exc)
+ logger.warning('Connect relay poll failed: %s', exc)
+ self._sleep(min(30.0, backoff_delay(fail_attempt)))
+ return 'stopped'
+
+ def _poll_once(self, cfg):
+ from app.utils.version import get_panel_version
+
+ hello = build_hello(cfg['private_key'], cfg['device_id'],
+ get_panel_version())
+ resp = requests.get(
+ poll_url_for(cfg['relay_url']),
+ headers=hello_headers(hello),
+ timeout=POLL_TIMEOUT_S,
+ )
+ if resp.status_code == 200:
+ return
+ detail = _error_detail(resp)
+ if resp.status_code == 403 and detail == 'revoked':
+ raise RelayRevoked()
+ raise ConnectError(f'poll failed ({resp.status_code}): {detail}')
+
+
+# ---------- singleton management (mirrors linked_panel_agent) ----------
+
+_relay_client = None
+
+
+def get_client():
+ return _relay_client
+
+
+def start_relay_client(app=None):
+ global _relay_client
+ if _relay_client is not None:
+ return _relay_client
+ client = RelayClient(app)
+ client.start()
+ _relay_client = client
+ return client
+
+
+def stop_relay_client():
+ global _relay_client
+ if _relay_client is not None:
+ _relay_client.stop()
+ _relay_client = None
+
+
+def start_client_if_paired(app):
+ """App-startup hook: resume the relay connection when this panel is paired."""
+ try:
+ if app.config.get('ENV') == 'testing' or app.config.get('TESTING'):
+ return None
+ if not _read_connect_file().get('device_id'):
+ return None
+ return start_relay_client(app)
+ except Exception as exc: # never block boot on the relay
+ logger.warning('Could not start connect relay client: %s', exc)
+ return None
+
+
+# ==================== Doctor ====================
+
+# Hard checks fail the doctor run (exit 1); soft ones are warnings with a
+# working fallback. (name, ok, hard, copy-on-failure, note-on-pass)
+
+
+def _doctor_check(name, ok, hard, fail_copy, note=None):
+ return {
+ 'name': name,
+ 'ok': bool(ok),
+ 'hard': bool(hard),
+ 'error': None if ok else fail_copy,
+ 'note': note if ok else None,
+ }
+
+
+def _key_file_ok(key_path: str):
+ if not os.path.exists(key_path):
+ return False
+ if os.name != 'nt':
+ import stat
+ mode = stat.S_IMODE(os.stat(key_path).st_mode)
+ return mode & 0o077 == 0
+ return True
+
+
+def run_doctor() -> list:
+ """The connect check table: pairing, key, DNS, TCP, TLS, clock, panel, WS."""
+ data = _read_connect_file()
+ relay_url = data.get('relay_url') or DEFAULT_RELAY_URL
+ parsed = urllib.parse.urlparse(relay_url)
+ host = parsed.hostname or ''
+ port = parsed.port or (443 if parsed.scheme == 'wss' else 80)
+ use_tls = parsed.scheme == 'wss'
+
+ checks = []
+
+ # Pairing state (informational: doctor is also useful pre-pair).
+ paired = bool(data.get('device_id'))
+ checks.append(_doctor_check(
+ 'Paired with ServerKit Cloud', paired, hard=False,
+ fail_copy='Not paired yet. Run `serverkit connect` first.',
+ note=f'device {data.get("device_id")}' if paired else None))
+
+ # Device key present and mode 0600.
+ key_path = data.get(KEY_PATH_FIELD) or connect_keys.default_key_path()
+ checks.append(_doctor_check(
+ 'Device key', _key_file_ok(key_path), hard=True,
+ fail_copy=f'Device key missing or readable by others: {key_path}.',
+ note=key_path))
+
+ # DNS resolution of the relay host.
+ try:
+ socket.getaddrinfo(host, port)
+ dns_ok = True
+ checks.append(_doctor_check(f'DNS {host}', True, hard=True,
+ fail_copy=None, note=host))
+ except socket.gaierror:
+ dns_ok = False
+ checks.append(_doctor_check(
+ f'DNS {host}', False, hard=True,
+ fail_copy=f'Cannot resolve {host}. Check DNS or outbound filtering.'))
+
+ # TCP egress to the relay port (443 for wss).
+ if dns_ok:
+ try:
+ with socket.create_connection((host, port), timeout=5):
+ pass
+ checks.append(_doctor_check(f'TCP {host}:{port}', True, hard=True,
+ fail_copy=None))
+ except OSError:
+ checks.append(_doctor_check(
+ f'TCP {host}:{port}', False, hard=True,
+ fail_copy='Outbound HTTPS to the relay is blocked.'))
+ else:
+ checks.append(_doctor_check(f'TCP {host}:{port}', False, hard=True,
+ fail_copy='Skipped: DNS failed.'))
+
+ # TLS chain (plain ws:// relays have none to check).
+ if not use_tls:
+ checks.append(_doctor_check(f'TLS {host}', True, hard=True,
+ fail_copy=None,
+ note='not applicable (plain ws:// relay)'))
+ elif dns_ok:
+ import ssl
+ try:
+ ctx = ssl.create_default_context()
+ with socket.create_connection((host, port), timeout=5) as sock:
+ with ctx.wrap_socket(sock, server_hostname=host):
+ pass
+ checks.append(_doctor_check(f'TLS {host}', True, hard=True,
+ fail_copy=None))
+ except Exception as exc:
+ checks.append(_doctor_check(
+ f'TLS {host}', False, hard=True,
+ fail_copy=f'TLS to the relay failed: {exc}. '
+ 'Corporate proxies that inspect TLS are not supported.'))
+ else:
+ checks.append(_doctor_check(f'TLS {host}', False, hard=True,
+ fail_copy='Skipped: DNS failed.'))
+
+ # Clock skew: the relay/ServerKit Cloud Date header vs local time (limit 60 s).
+ if dns_ok:
+ import email.utils
+ try:
+ resp = requests.get(relay_http_base(relay_url) + '/', timeout=10)
+ remote = email.utils.parsedate_to_datetime(resp.headers['Date'])
+ skew = abs(time.time() - remote.timestamp())
+ checks.append(_doctor_check(
+ 'Clock skew', skew <= 60, hard=True,
+ fail_copy=f'System clock is {int(skew)}s off. Enable NTP; the '
+ 'relay rejects signatures more than 60 s off.',
+ note=f'{skew:.1f}s off UTC'))
+ except Exception as exc:
+ checks.append(_doctor_check(
+ 'Clock skew', False, hard=True,
+ fail_copy=f'Could not read the relay clock: {exc}.'))
+ else:
+ checks.append(_doctor_check('Clock skew', False, hard=True,
+ fail_copy='Skipped: DNS failed.'))
+
+ # Loopback panel reachable (the relay will proxy browser traffic to it).
+ from app.services.cli_api_client import resolve_port
+ panel_port = resolve_port()
+ try:
+ with socket.create_connection(('127.0.0.1', panel_port), timeout=3):
+ pass
+ checks.append(_doctor_check('Panel loopback', True, hard=True,
+ fail_copy=None,
+ note=f'127.0.0.1:{panel_port}'))
+ except OSError:
+ checks.append(_doctor_check(
+ 'Panel loopback', False, hard=True,
+ fail_copy=f'Panel not reachable at 127.0.0.1:{panel_port}.'))
+
+ # WS upgrade attempt (soft: the long-poll fallback keeps limited mode).
+ if dns_ok:
+ try:
+ from websockets.sync.client import connect as ws_connect
+ ws = ws_connect(relay_url, open_timeout=10, close_timeout=2,
+ ping_interval=None)
+ try:
+ ws.close()
+ finally:
+ pass
+ checks.append(_doctor_check('WebSocket upgrade', True, hard=False,
+ fail_copy=None))
+ except Exception:
+ checks.append(_doctor_check(
+ 'WebSocket upgrade', False, hard=False,
+ fail_copy='WebSocket upgrade refused; will use limited mode.'))
+ else:
+ checks.append(_doctor_check('WebSocket upgrade', False, hard=False,
+ fail_copy='Skipped: DNS failed.'))
+
+ return checks
+
+
+def doctor_ok(checks) -> bool:
+ """True when no HARD check failed (soft checks are warnings)."""
+ return not any(c['hard'] and not c['ok'] for c in checks)
diff --git a/backend/app/services/connect_keys.py b/backend/app/services/connect_keys.py
index 9b7b3388..6d979f44 100644
--- a/backend/app/services/connect_keys.py
+++ b/backend/app/services/connect_keys.py
@@ -43,6 +43,29 @@ def _public_bytes(private_key: Ed25519PrivateKey) -> bytes:
)
+def _load_existing(path: str) -> Ed25519PrivateKey:
+ with open(path, 'rb') as f:
+ private_key = serialization.load_pem_private_key(f.read(), password=None)
+ if not isinstance(private_key, Ed25519PrivateKey):
+ raise ValueError(f'{path} is not an Ed25519 private key')
+ if os.name != 'nt':
+ os.chmod(path, 0o600)
+ return private_key
+
+
+def load_keypair(path: str = None):
+ """Strictly load the device keypair; raises FileNotFoundError if absent.
+
+ Returns (private_key, pubkey_hex, fingerprint). Used by the relay
+ transport: generating a fresh key here would silently change the device
+ identity and fail every hello signature.
+ """
+ path = path or default_key_path()
+ private_key = _load_existing(path)
+ pubkey_bytes = _public_bytes(private_key)
+ return private_key, pubkey_bytes.hex(), fingerprint_of(pubkey_bytes)
+
+
def load_or_create_keypair(path: str = None):
"""Load the device keypair from ``path``, generating + persisting one if absent.
@@ -51,12 +74,7 @@ def load_or_create_keypair(path: str = None):
"""
path = path or default_key_path()
if os.path.exists(path):
- with open(path, 'rb') as f:
- private_key = serialization.load_pem_private_key(f.read(), password=None)
- if not isinstance(private_key, Ed25519PrivateKey):
- raise ValueError(f'{path} is not an Ed25519 private key')
- if os.name != 'nt':
- os.chmod(path, 0o600)
+ private_key = _load_existing(path)
else:
private_key = Ed25519PrivateKey.generate()
os.makedirs(os.path.dirname(path), exist_ok=True)
diff --git a/backend/cli.py b/backend/cli.py
index b4561b1e..d3793c0d 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -1609,6 +1609,9 @@ def connect_status(as_json):
rows = [
('State', state.get('state')),
('Reason', state.get('state_reason') or '-'),
+ ('Transport', state.get('transport') or '-'),
+ ('Last seen', state.get('last_connected_at') or '-'),
+ ('Relay instance', state.get('relay_instance') or '-'),
('ServerKit Cloud', state.get('cloud_url')),
('Organization', state.get('org_slug') or '-'),
('Name', state.get('name') or '-'),
@@ -1622,6 +1625,48 @@ def connect_status(as_json):
_echo_table(['Field', 'Value'], rows)
[email protected]('--json', 'as_json', is_flag=True, help='Output machine-readable JSON')
+def connect_doctor(as_json):
+ """Diagnose the path from this panel to the ServerKit relay.
+
+ Exits non-zero when any hard check fails (DNS, TCP, TLS, clock skew,
+ device key, panel loopback). A refused WebSocket upgrade is a warning —
+ the client falls back to limited (long-poll) mode.
+ """
+ from app.services import connect_client
+
+ checks = connect_client.run_doctor()
+ healthy = connect_client.doctor_ok(checks)
+
+ if as_json:
+ _echo_json({'ok': healthy, 'checks': checks})
+ else:
+ click.echo('ServerKit Cloud connect doctor')
+ click.echo()
+ for check in checks:
+ if check['ok']:
+ mark = click.style('OK ', fg='green')
+ elif check['hard']:
+ mark = click.style('FAIL', fg='red')
+ else:
+ mark = click.style('WARN', fg='yellow')
+ line = f' {mark} {check["name"]}'
+ if check.get('note'):
+ line += f' ({check["note"]})'
+ click.echo(line)
+ if check.get('error'):
+ click.echo(f' {check["error"]}')
+ click.echo()
+ if healthy:
+ click.echo(click.style('All hard checks passed.', fg='green'))
+ else:
+ click.echo(click.style('Some checks failed — see above.', fg='red'))
+
+ if not healthy:
+ sys.exit(1)
+
+
@connect.command('disconnect')
@click.option('--remove-key', is_flag=True,
help='Also delete the device keypair (a new identity on next pair)')
diff --git a/backend/requirements.txt b/backend/requirements.txt
index aa2d4eea..5d58e034 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -24,6 +24,9 @@ Flask-Cors==6.0.2
Flask-SocketIO==5.6.1
python-socketio==5.16.3
python-engineio==4.13.3
+# WebSocket CLIENT for the ServerKit Cloud relay (see
+# app/services/connect_client.py). The server-side stack above cannot dial out.
+websockets==15.0.1
# System monitoring
psutil==5.9.7
diff --git a/backend/tests/test_connect_relay.py b/backend/tests/test_connect_relay.py
new file mode 100644
index 00000000..81721d08
--- /dev/null
+++ b/backend/tests/test_connect_relay.py
@@ -0,0 +1,285 @@
+"""Unit tests for the relay transport helpers (no network)."""
+import json
+import socket
+import ssl
+
+import pytest
+from websockets.exceptions import ConnectionClosedOK
+from websockets.frames import Close
+
+from app.services import connect_client, connect_keys
+
+
+def config_dir(tmp_path, monkeypatch):
+ """Point the panel config dir at a temp dir for connect*.json files."""
+ monkeypatch.setattr(connect_client.paths, 'SERVERKIT_CONFIG_DIR',
+ str(tmp_path))
+ return tmp_path
+
+
+# ---------- close-code / failure classification ----------
+
+
+def test_close_reason_for_code_maps_relay_codes():
+ assert connect_client.close_reason_for_code(4001) == 'auth_failed_signature'
+ assert connect_client.close_reason_for_code(4002) == 'auth_failed_clock_skew'
+ assert connect_client.close_reason_for_code(4003) == 'version_unsupported'
+ assert connect_client.close_reason_for_code(4004) == 'auth_failed_signature'
+ assert connect_client.close_reason_for_code(4009) == 'revoked'
+
+
+def test_close_reason_for_unknown_code_is_heartbeat_timeout():
+ assert connect_client.close_reason_for_code(1006) == 'heartbeat_timeout'
+ assert connect_client.close_reason_for_code(None) == 'heartbeat_timeout'
+
+
+def test_classify_ws_failure():
+ closed = ConnectionClosedOK(Close(4002, 'auth_failed_clock_skew'), None)
+ assert connect_client._classify_ws_failure(closed) == 'auth_failed_clock_skew'
+ assert connect_client._classify_ws_failure(ssl.SSLError('x')) == 'tls_error'
+ assert connect_client._classify_ws_failure(
+ socket.gaierror('x')) == 'dns_error'
+ assert connect_client._classify_ws_failure(
+ ValueError('x')) == 'relay_unreachable'
+
+
+# ---------- backoff / flap detection ----------
+
+
+def test_backoff_doubles_to_cap():
+ no_jitter = lambda a, b: 0.0
+ assert connect_client.backoff_delay(0, rand=no_jitter) == 1.0
+ assert connect_client.backoff_delay(1, rand=no_jitter) == 2.0
+ assert connect_client.backoff_delay(5, rand=no_jitter) == 32.0
+ assert connect_client.backoff_delay(10, rand=no_jitter) == 60.0
+ assert connect_client.backoff_delay(100, rand=no_jitter) == 60.0
+
+
+def test_backoff_jitter_stays_within_bounds():
+ for attempt in (0, 3, 10):
+ for _ in range(50):
+ delay = connect_client.backoff_delay(attempt)
+ base = min(60.0, 2 ** attempt)
+ assert base * 0.75 <= delay <= base * 1.25
+
+
+def test_is_flapping():
+ now = 1000.0
+ recent = [now - i for i in range(7)]
+ assert connect_client.is_flapping(recent, now=now)
+ assert not connect_client.is_flapping(recent[:6], now=now)
+ # Attempts outside the window don't count.
+ stale = [now - 700 - i for i in range(10)]
+ assert not connect_client.is_flapping(stale, now=now)
+ assert connect_client.is_flapping(recent + stale, now=now)
+
+
+# ---------- URL / hello helpers ----------
+
+
+def test_relay_http_base_and_poll_url():
+ assert connect_client.relay_http_base(
+ 'wss://relay.serverkit.ai/v1/device') == 'https://relay.serverkit.ai'
+ assert connect_client.relay_http_base(
+ 'ws://127.0.0.1:8791/v1/device') == 'http://127.0.0.1:8791'
+ assert connect_client.poll_url_for('wss://relay.serverkit.ai/v1/device') == \
+ 'https://relay.serverkit.ai/v1/device/poll'
+
+
+def test_build_hello_signature_verifies(config_dir):
+ private_key, pubkey_hex, _fpr = connect_keys.load_or_create_keypair()
+ hello = connect_client.build_hello(private_key, 'dev_123', '1.9.22')
+
+ assert hello['t'] == 'hello'
+ assert hello['device_id'] == 'dev_123'
+ assert hello['proto'] == 1
+ assert hello['client_version'] == '1.9.22'
+ assert isinstance(hello['ts'], int)
+
+ from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+ Ed25519PublicKey,
+ )
+ pub = Ed25519PublicKey.from_public_bytes(bytes.fromhex(pubkey_hex))
+ message = f"dev_123|{hello['ts']}|{hello['nonce']}".encode()
+ pub.verify(bytes.fromhex(hello['sig']), message) # raises if invalid
+
+
+def test_hello_headers_use_relay_names(config_dir):
+ private_key, _pub, _fpr = connect_keys.load_or_create_keypair()
+ hello = connect_client.build_hello(private_key, 'dev_123', '1.9.22')
+ headers = connect_client.hello_headers(hello)
+ assert headers['X-ServerKit-Device'] == 'dev_123'
+ assert headers['X-ServerKit-Sig'] == hello['sig']
+ assert headers['X-ServerKit-Ts'] == str(hello['ts'])
+ assert headers['X-ServerKit-Proto'] == '1'
+ assert headers['X-ServerKit-Client-Version'] == '1.9.22'
+
+
+# ---------- state file / status merge ----------
+
+
+def test_state_file_roundtrip_preserves_last_connected(config_dir):
+ connect_client._write_state_file('online', None, transport='ws',
+ relay_instance='i-1',
+ last_connected_at='2026-01-01T00:00:00+00:00')
+ connect_client._write_state_file('paired_offline', 'heartbeat_timeout',
+ transport=None)
+ saved = connect_client._read_state_file()
+ assert saved['state'] == 'paired_offline'
+ assert saved['state_reason'] == 'heartbeat_timeout'
+ assert saved['transport'] is None
+ # Not overwritten by the later, offline transition.
+ assert saved['last_connected_at'] == '2026-01-01T00:00:00+00:00'
+ assert saved['relay_instance'] == 'i-1'
+ assert saved['updated_at']
+
+
+def test_status_merges_state_file(config_dir):
+ connect_client._write_connect_file({
+ 'device_id': 'dev_1', 'org_slug': 'acme', 'name': 'web-1',
+ 'relay_url': 'wss://relay.example.com/v1/device', 'scopes': [],
+ 'fingerprint': 'a1b2c3d4e5f67890',
+ 'key_path': str(config_dir / 'connect_device_key.pem'),
+ 'cloud_url': 'https://app.serverkit.ai',
+ 'paired_at': '2026-01-01T00:00:00+00:00',
+ })
+ connect_client._write_state_file('online', None, transport='ws',
+ relay_instance='i-1',
+ last_connected_at='2026-01-02T00:00:00+00:00')
+
+ state = connect_client.status()
+ assert state['state'] == 'online'
+ assert state['transport'] == 'ws'
+ assert state['relay_instance'] == 'i-1'
+ assert state['last_connected_at'] == '2026-01-02T00:00:00+00:00'
+ assert state['key_present'] is False # no key written in this test
+
+
+def test_status_without_state_file_is_never_connected(config_dir):
+ connect_keys.load_or_create_keypair()
+ connect_client._write_connect_file({
+ 'device_id': 'dev_1', 'org_slug': 'acme', 'name': 'web-1',
+ 'relay_url': 'wss://relay.example.com/v1/device', 'scopes': [],
+ 'fingerprint': 'a1b2c3d4e5f67890',
+ 'key_path': connect_keys.default_key_path(),
+ 'cloud_url': 'https://app.serverkit.ai',
+ 'paired_at': '2026-01-01T00:00:00+00:00',
+ })
+ state = connect_client.status()
+ assert state['state'] == 'paired_offline'
+ assert state['state_reason'] == 'never_connected'
+
+
+def test_disconnect_removes_state_file(config_dir):
+ connect_client._write_connect_file({'device_id': 'dev_1'})
+ connect_client._write_state_file('online', None, transport='ws')
+ result = connect_client.disconnect()
+ assert result['state'] == 'unpaired'
+ assert not (config_dir / 'connect.json').exists()
+ assert not (config_dir / 'connect-state.json').exists()
+ assert connect_client.status()['state'] == 'unpaired'
+
+
+def test_state_file_is_valid_json_on_disk(config_dir):
+ connect_client._write_state_file('degraded', 'relay_unreachable',
+ transport='poll')
+ with open(connect_client.state_file_path()) as f:
+ saved = json.load(f)
+ assert saved['state'] == 'degraded'
+
+
+# ---------- client loop state transitions (no network: sessions stubbed) ----------
+
+
+def _wait_state(want, timeout=5.0):
+ import time
+ deadline = time.time() + timeout
+ while time.time() < deadline:
+ saved = connect_client._read_state_file()
+ if saved.get('state') == want:
+ return saved
+ time.sleep(0.05)
+ raise AssertionError(f'timed out waiting for {want}; '
+ f'have {connect_client._read_state_file()}')
+
+
+def _fake_config(monkeypatch):
+ monkeypatch.setattr(connect_client, '_relay_config', lambda: {
+ 'device_id': 'dev_1', 'relay_url': 'ws://127.0.0.1:9/v1/device',
+ 'private_key': None, 'key_path': '/nope',
+ })
+ monkeypatch.setattr(connect_client, 'backoff_delay',
+ lambda attempt, **kw: 0.01)
+
+
+def test_ws_refused_falls_back_to_poll_degraded(config_dir, monkeypatch):
+ """A proxy-smelling WS failure switches to the poll loop as degraded."""
+ _fake_config(monkeypatch)
+ monkeypatch.setattr(connect_client.RelayClient, '_ws_session',
+ lambda self, cfg: 'refused:relay_unreachable')
+
+ def fake_poll(self, cfg, ws_retry_at):
+ # The real _poll_session sets transport='poll' and writes the degraded
+ # state; the loop has already written it by the time we're called.
+ assert connect_client._read_state_file()['state'] == 'degraded'
+ assert connect_client._read_state_file()['transport'] == 'poll'
+ return 'stopped'
+
+ monkeypatch.setattr(connect_client.RelayClient, '_poll_session', fake_poll)
+
+ client = connect_client.RelayClient()
+ client.start()
+ client._thread.join(timeout=5)
+ assert not client._thread.is_alive()
+
+
+def test_drop_after_ready_reports_heartbeat_timeout(config_dir, monkeypatch):
+ """An established connection that drops un cleanly -> paired_offline."""
+ _fake_config(monkeypatch)
+ calls = []
+
+ def fake_ws(self, cfg):
+ calls.append(1)
+ return 'drop:heartbeat_timeout' if len(calls) == 1 else 'stopped'
+
+ monkeypatch.setattr(connect_client.RelayClient, '_ws_session', fake_ws)
+
+ client = connect_client.RelayClient()
+ client.start()
+ saved = _wait_state('paired_offline')
+ assert saved['state_reason'] == 'heartbeat_timeout'
+ client._thread.join(timeout=5)
+ assert not client._thread.is_alive()
+
+
+def test_revoked_is_terminal(config_dir, monkeypatch):
+ """Close 4009 stops the loop for good, and stop() must not overwrite it."""
+ _fake_config(monkeypatch)
+ monkeypatch.setattr(connect_client.RelayClient, '_ws_session',
+ lambda self, cfg: 'revoked')
+
+ client = connect_client.RelayClient()
+ client.start()
+ saved = _wait_state('revoked')
+ client._thread.join(timeout=5)
+ assert not client._thread.is_alive()
+ assert saved['state_reason'] == 'revoked'
+
+ client.stop()
+ assert connect_client._read_state_file()['state'] == 'revoked'
+
+
+def test_flap_logging_fires_after_limit(config_dir, monkeypatch, caplog):
+ _fake_config(monkeypatch)
+ # FLAP_LIMIT is 6: 8 quick drops exceed it inside the window.
+ outcomes = iter(['drop:heartbeat_timeout'] * 8 + ['stopped'])
+ monkeypatch.setattr(
+ connect_client.RelayClient, '_ws_session',
+ lambda self, cfg: next(outcomes, 'stopped'))
+
+ client = connect_client.RelayClient()
+ with caplog.at_level('WARNING', logger='app.services.connect_client'):
+ client.start()
+ client._thread.join(timeout=10)
+ assert any('flapping' in r.message for r in caplog.records)