ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat(connect): hold a live outbound connection to the relay
Juan Denis232c6978c509
diff --git a/backend/app/__init__.py b/backend/app/__init__.pyindex 7f78604a..eef51398 100644--- a/backend/app/__init__.py+++ b/backend/app/__init__.py@@ -443,6 +443,11 @@ def create_app(config_name=None):from app.services.linked_panel_service import LinkedPanelServiceLinkedPanelService.start_client_if_linked(app)+ # Hold the outbound ServerKit Cloud relay connection when this+ # panel is paired.+ from app.services import connect_client+ connect_client.start_client_if_paired(app)+# Request body size limitapp.config['MAX_CONTENT_LENGTH'] = 100 * 1024 * 1024 # 100MB limitdiff --git a/backend/app/services/connect_client.py b/backend/app/services/connect_client.pyindex 22d16e06..019fe350 100644--- a/backend/app/services/connect_client.py+++ b/backend/app/services/connect_client.py@@ -1,23 +1,36 @@-"""ServerKit Cloud connect pairing — panel side.+"""ServerKit Cloud connect — panel side.-M1 scope is pairing ONLY: enroll with ServerKit Cloud, print the pairing code +-fingerprint, poll with an Ed25519 proof-of-possession until the user approves-in the browser, then write ``connect.json`` next to the panel config. No-WebSocket transport, no reconnect loop — those are later milestones.+Pairing: enroll with ServerKit Cloud, print the pairing code + fingerprint, poll with an+Ed25519 proof-of-possession until the user approves in the browser, then+write ``connect.json`` next to the panel config. The enrollment_secret is+kept in memory for the pairing session only — never written to disk or+logged.++Transport: once ``connect.json`` exists, a managed background thread (``RelayClient``,+started from app init via ``start_client_if_paired`` — same pattern as+linked_panel_agent) holds an outbound WebSocket to the relay: signed hello+(``device_id|ts|nonce``), a ``{"t":"ping"}`` heartbeat every 20 s, jittered+exponential backoff on drops, flap detection, and a long-poll fallback for+networks that break WebSockets. Runtime state is persisted to+``connect-state.json`` so `serverkit connect status` and the API show it+without the thread running. Stream opens this client does not implement are answered with close ++``unsupported``.ServerKit Cloud endpoints (no session auth on the panel side):POST /api/pair/enroll -> enrollment_id, enrollment_secret (shown once), codePOST /api/pair/poll -> 202 pending / 200 claimed (signed proof-of-possession)POST /api/pair/rotate -> new code + expires_at--The enrollment_secret is kept in memory for the pairing session only — it is-never written to disk or logged."""+import collectionsimport jsonimport loggingimport os+import random+import secretsimport socket+import threadingimport time+import urllib.parsefrom datetime import datetime, timezoneimport requests@@ -301,7 +314,12 @@ def _past(iso_ts: str) -> bool:def status() -> dict:- """Connection state as the UI/CLI should show it (reads connect.json + key)."""+ """Connection state as the UI/CLI should show it.++ Reads connect.json (pairing), connect-state.json (runtime transport+ state, written by RelayClient) and checks the device key — no thread+ needs to be running in this process.+ """data = _read_connect_file()key_path = data.get(KEY_PATH_FIELD) or connect_keys.default_key_path()key_present = os.path.exists(key_path)@@ -314,11 +332,16 @@ def status() -> dict:'key_present': key_present,'key_path': key_path,}- state = 'paired_offline' if key_present else 'degraded'+ saved = _read_state_file()+ if saved.get('state'):+ state = saved['state']+ reason = saved.get('state_reason')+ else:+ state = 'paired_offline' if key_present else 'degraded'+ reason = 'never_connected' if key_present else 'device_key_missing'return {'state': state,- # No transport yet in M1 — a paired panel has never been online.- 'state_reason': 'never_connected' if key_present else 'device_key_missing',+ 'state_reason': reason,'paired': True,'cloud_url': data.get('cloud_url') or resolve_cloud_url(),'device_id': data.get('device_id'),@@ -326,6 +349,9 @@ def status() -> dict:'name': data.get('name'),'relay_url': data.get('relay_url'),'scopes': data.get('scopes') or [],+ 'transport': saved.get('transport'),+ 'relay_instance': saved.get('relay_instance'),+ 'last_connected_at': saved.get('last_connected_at'),'fingerprint': data.get('fingerprint'),'fingerprint_grouped': connect_keys.format_fingerprint(data['fingerprint'])if data.get('fingerprint') else None,@@ -337,14 +363,667 @@ def status() -> dict:def disconnect(remove_key: bool = False) -> dict:"""Forget the pairing locally. Does NOT revoke anything on ServerKit Cloud."""+ stop_relay_client() # no-op unless the client runs in this processremoved = []- path = connect_file_path()- if os.path.exists(path):- os.unlink(path)- removed.append(path)+ for path in (connect_file_path(), state_file_path()):+ if os.path.exists(path):+ os.unlink(path)+ removed.append(path)if remove_key:key_path = connect_keys.default_key_path()if os.path.exists(key_path):os.unlink(key_path)removed.append(key_path)return {'success': True, 'removed': removed, 'state': 'unpaired'}+++# ==================== Relay transport ====================++DEFAULT_RELAY_URL = 'wss://relay.serverkit.ai/v1/device'+STATE_FILENAME = 'connect-state.json'++PROTO_VERSION = 1+HELLO_TIMEOUT_S = 10 # the relay closes devices that take longer to hello+PING_INTERVAL_S = 20 # panel -> relay {"t":"ping"} heartbeat+MAX_FRAME_BYTES = 256 * 1024++BACKOFF_BASE_S = 1.0+BACKOFF_CAP_S = 60.0+BACKOFF_JITTER = 0.25 # ±25%+STABLE_AFTER_S = 120 # a connection stable this long resets the backoff+WS_RETRY_AFTER_S = 15 * 60 # while on the poll fallback, re-probe WS this often+POLL_TIMEOUT_S = 35 # slightly above the relay's 25 s hold+FLAP_WINDOW_S = 600+FLAP_LIMIT = 6 # >6 reconnects in 10 minutes = flapping++# Relay close codes -> state_reason enum (mirrors the relay's protocol).+CLOSE_REASON_BY_CODE = {+ 4001: 'auth_failed_signature',+ 4002: 'auth_failed_clock_skew',+ 4003: 'version_unsupported',+ 4004: 'auth_failed_signature', # nonce replay+ 4009: 'revoked',+}+++class RelayRevoked(Exception):+ """The relay closed us with 4009 — terminal until re-paired on ServerKit Cloud."""+++class _HandshakeRefused(Exception):+ """WS handshake/hello failed with a classified state_reason."""+++# ---------- pure helpers (unit-tested) ----------+++def close_reason_for_code(code) -> str:+ """Relay close code -> state_reason; unknown codes are a dropped heartbeat."""+ return CLOSE_REASON_BY_CODE.get(code, 'heartbeat_timeout')+++def backoff_delay(attempt: int, rand=random.uniform) -> float:+ """Jittered exponential backoff: 1 s doubling to a 60 s cap, ±25% jitter."""+ base = min(BACKOFF_CAP_S, BACKOFF_BASE_S * (2 ** max(0, attempt)))+ return base * (1 + rand(-BACKOFF_JITTER, BACKOFF_JITTER))+++def is_flapping(attempts_at, now: float = None,+ window: float = FLAP_WINDOW_S, limit: int = FLAP_LIMIT) -> bool:+ """True when more than `limit` reconnect attempts happened within `window`."""+ now = time.time() if now is None else now+ return sum(1 for t in attempts_at if now - t <= window) > limit+++def relay_http_base(relay_url: str) -> str:+ """wss://host:port/v1/device -> https://host:port (ws -> http)."""+ parsed = urllib.parse.urlparse(relay_url)+ scheme = {'wss': 'https', 'ws': 'http'}.get(parsed.scheme, parsed.scheme)+ return urllib.parse.urlunparse((scheme, parsed.netloc, '', '', '', ''))+++def poll_url_for(relay_url: str) -> str:+ """The relay's long-poll endpoint for a device WS URL."""+ return relay_http_base(relay_url) + '/v1/device/poll'+++def build_hello(private_key, device_id: str, client_version: str) -> dict:+ """The signed first frame. sig covers the exact string `device_id|ts|nonce`."""+ ts = int(time.time())+ nonce = secrets.token_hex(16)+ sig = private_key.sign(f'{device_id}|{ts}|{nonce}'.encode()).hex()+ return {+ 't': 'hello',+ 'device_id': device_id,+ 'ts': ts,+ 'nonce': nonce,+ 'sig': sig,+ 'proto': PROTO_VERSION,+ 'client_version': client_version,+ }+++HELLO_HEADERS = {+ 'device_id': 'X-ServerKit-Device',+ 'ts': 'X-ServerKit-Ts',+ 'nonce': 'X-ServerKit-Nonce',+ 'sig': 'X-ServerKit-Sig',+ 'proto': 'X-ServerKit-Proto',+ 'client_version': 'X-ServerKit-Client-Version',+}+++def hello_headers(hello: dict) -> dict:+ """The same hello carried in X-ServerKit-* headers for the poll fallback."""+ return {header: str(hello[field]) for field, header in HELLO_HEADERS.items()}+++# ---------- runtime state file ----------+++def state_file_path() -> str:+ return os.path.join(paths.SERVERKIT_CONFIG_DIR, STATE_FILENAME)+++def _read_state_file() -> dict:+ path = state_file_path()+ if not os.path.exists(path):+ return {}+ try:+ with open(path, 'r', encoding='utf-8') as f:+ data = json.load(f)+ return data if isinstance(data, dict) else {}+ except (OSError, ValueError) as exc:+ logger.warning('Could not read %s: %s', path, exc)+ return {}+++def _write_state_file(state: str, reason: str = None, transport: str = None,+ relay_instance: str = None, last_connected_at: str = None):+ """Persist a state transition, preserving fields not being overwritten."""+ saved = _read_state_file()+ saved.update({+ 'state': state,+ 'state_reason': reason,+ 'transport': transport,+ 'relay_instance': relay_instance if relay_instance is not None+ else saved.get('relay_instance'),+ 'last_connected_at': last_connected_at or saved.get('last_connected_at'),+ 'updated_at': datetime.now(timezone.utc).isoformat(),+ })+ path = state_file_path()+ os.makedirs(os.path.dirname(path), exist_ok=True)+ with open(path, 'w', encoding='utf-8') as f:+ json.dump(saved, f, indent=2)+ f.write('\n')+++def _classify_ws_failure(exc) -> str:+ """Map a transport exception to a state_reason (pure; unit-tested)."""+ import ssl++ from websockets.exceptions import ConnectionClosed, InvalidHandshake++ if isinstance(exc, ConnectionClosed):+ code = exc.rcvd.code if exc.rcvd else None+ return close_reason_for_code(code)+ if isinstance(exc, InvalidHandshake):+ # HTTP 4xx/5xx or a non-101 answer on the upgrade: a proxy breaking WS.+ return 'relay_unreachable'+ if isinstance(exc, ssl.SSLError):+ return 'tls_error'+ if isinstance(exc, socket.gaierror):+ return 'dns_error'+ # connect refused/reset, timeouts, anything else network-shaped.+ return 'relay_unreachable'+++def _relay_config() -> dict:+ """What the transport needs from connect.json + the device key.++ Returns None when the panel is not paired (anymore) or the key is gone —+ the client loop treats that as "exit", not "retry".+ """+ data = _read_connect_file()+ if not data.get('device_id') or not data.get('relay_url'):+ return None+ key_path = data.get(KEY_PATH_FIELD) or connect_keys.default_key_path()+ try:+ private_key, _pub, _fpr = connect_keys.load_keypair(key_path)+ except Exception as exc:+ logger.warning('Connect relay: cannot load device key %s: %s', key_path, exc)+ return None+ return {+ 'device_id': data['device_id'],+ 'relay_url': data['relay_url'],+ 'private_key': private_key,+ 'key_path': key_path,+ }+++# ---------- the client ----------+++class RelayClient:+ """Holds one outbound connection to the ServerKit relay.++ WebSocket first (signed hello, 20 s pings); when the upgrade smells like+ a proxy block (non-101 answer, timeout) the client falls back to the+ long-poll loop and reports `degraded`, re-probing WS every+ WS_RETRY_AFTER_S. All state transitions are persisted to+ connect-state.json so the CLI/API read them without this thread.+ """++ def __init__(self, app=None):+ self.app = app+ self.running = False+ self.transport = None+ self.relay_instance = None+ self.last_error = None+ self._thread = None+ self._stop = threading.Event()+ self._ws = None+ self._attempts_at = collections.deque()+ self._flap_logged = False+ self._last_written = None # (state, reason, transport) transition dedup++ # -- lifecycle -----------------------------------------------------++ def start(self):+ if self.running:+ return+ self.running = True+ self._stop.clear()+ self._thread = threading.Thread(+ target=self._run, daemon=True, name='connect-relay')+ self._thread.start()+ logger.info('Connect relay client started')++ def stop(self):+ self.running = False+ self._stop.set()+ ws, self._ws = self._ws, None+ if ws is not None:+ try:+ ws.close()+ except Exception:+ pass+ # A terminal 'revoked' survives a stop: re-pairing on ServerKit Cloud, not a+ # local restart, is what clears it.+ if _read_state_file().get('state') != 'revoked':+ self._set_state('paired_offline', 'client_stopped', transport=None)++ # -- state persistence ---------------------------------------------++ def _set_state(self, state, reason, transport=None):+ key = (state, reason, transport)+ if key == self._last_written:+ return+ self._last_written = key+ connected = state in ('online', 'degraded')+ _write_state_file(+ state, reason, transport=transport, relay_instance=self.relay_instance,+ last_connected_at=datetime.now(timezone.utc).isoformat()+ if connected else None)++ def _sleep(self, seconds):+ self._stop.wait(max(0.0, seconds))++ # -- main loop ------------------------------------------------------++ def _run(self):+ attempt = 0+ ws_blocked_until = 0.0+ while self.running:+ cfg = _relay_config()+ if cfg is None:+ logger.info('Connect relay: not paired (connect.json/key gone); exiting')+ return++ if time.monotonic() < ws_blocked_until:+ if self._poll_session(cfg, ws_blocked_until) == 'stopped':+ return+ ws_blocked_until = 0.0 # time to re-probe WS+ continue++ started = time.monotonic()+ outcome = self._ws_session(cfg)+ if outcome == 'stopped':+ return+ kind, _, reason = outcome.partition(':')++ if kind == 'revoked':+ self._set_state('revoked', 'revoked', transport=None)+ logger.warning('Connect relay: device revoked on ServerKit Cloud; stopping')+ return++ if time.monotonic() - started >= STABLE_AFTER_S:+ attempt = 0+ else:+ attempt += 1+ self._note_attempt()++ if kind == 'refused' and reason == 'relay_unreachable':+ # Proxy-blocked or unanswerable upgrade: limited mode.+ ws_blocked_until = time.monotonic() + WS_RETRY_AFTER_S+ self._set_state('degraded', 'relay_unreachable', transport='poll')+ continue++ self._set_state('paired_offline', reason, transport=None)+ self._sleep(backoff_delay(attempt))++ def _note_attempt(self):+ now = time.time()+ self._attempts_at.append(now)+ while self._attempts_at and now - self._attempts_at[0] > FLAP_WINDOW_S:+ self._attempts_at.popleft()+ flapping = is_flapping(self._attempts_at, now=now)+ if flapping and not self._flap_logged:+ self._flap_logged = True+ logger.warning(+ 'Connect relay flapping: %d reconnects in the last %d s',+ len(self._attempts_at), FLAP_WINDOW_S)+ elif not flapping:+ self._flap_logged = False++ # -- websocket ------------------------------------------------------++ def _ws_session(self, cfg) -> str:+ """One WS connection: handshake, hello, heartbeat until drop.++ Returns 'stopped', 'revoked', 'refused:<reason>' (never got ready —+ auth/handshake failure) or 'drop:<reason>' (was ready, then lost).+ """+ try:+ ws = self._ws_connect(cfg)+ except RelayRevoked:+ return 'revoked'+ except _HandshakeRefused as exc:+ return f'refused:{exc}'+ except Exception as exc:+ return f'refused:{_classify_ws_failure(exc)}'++ self._ws = ws+ try:+ self.transport = 'ws'+ self._set_state('online', None, transport='ws')+ logger.info('Connect relay: online via ws (instance %s)',+ self.relay_instance)+ while self.running:+ try:+ raw = ws.recv(timeout=PING_INTERVAL_S)+ except TimeoutError:+ ws.send('{"t":"ping"}')+ continue+ self._handle_frame(ws, raw)+ return 'stopped'+ except Exception as exc:+ from websockets.exceptions import ConnectionClosed+ if isinstance(exc, ConnectionClosed):+ code = exc.rcvd.code if exc.rcvd else None+ if code == 4009:+ return 'revoked'+ self.last_error = f'connection closed (code {code})'+ return f'drop:{close_reason_for_code(code)}'+ self.last_error = str(exc)+ return f'drop:{_classify_ws_failure(exc)}'+ finally:+ self._ws = None+ try:+ ws.close()+ except Exception:+ pass++ def _ws_connect(self, cfg):+ """Handshake + signed hello + wait for `ready` (10 s, relay-enforced)."""+ from app.utils.version import get_panel_version+ from websockets.exceptions import ConnectionClosed+ from websockets.sync.client import connect as ws_connect++ ws = ws_connect(+ cfg['relay_url'],+ open_timeout=HELLO_TIMEOUT_S,+ close_timeout=5,+ ping_interval=None, # heartbeats are app-level {"t":"ping"} frames+ max_size=MAX_FRAME_BYTES,+ )+ try:+ hello = build_hello(cfg['private_key'], cfg['device_id'],+ get_panel_version())+ ws.send(json.dumps(hello))+ frame = json.loads(ws.recv(timeout=HELLO_TIMEOUT_S))+ if frame.get('t') != 'ready':+ raise _HandshakeRefused('relay_unreachable')+ self.relay_instance = frame.get('instance')+ return ws+ except ConnectionClosed as exc:+ code = exc.rcvd.code if exc.rcvd else None+ if code == 4009:+ raise RelayRevoked()+ raise _HandshakeRefused(close_reason_for_code(code))+ except Exception:+ try:+ ws.close()+ except Exception:+ pass+ raise++ def _handle_frame(self, ws, raw):+ try:+ frame = json.loads(raw)+ except ValueError:+ return+ if frame.get('t') == 'open':+ # M2 has no streams yet: refuse honestly (mirrors the relay).+ ws.send(json.dumps({+ 's': frame.get('s'),+ 't': 'close',+ 'reason': 'unsupported',+ 'detail': 'streams arrive with the next Connect release',+ }))++ # -- long-poll fallback ----------------------------------------------++ def _poll_session(self, cfg, ws_retry_at) -> str:+ """Long-poll loop ('limited mode'): each held GET is the heartbeat.++ Returns 'stopped', 'revoked', or 'retry_ws' when it's time to try the+ WebSocket again.+ """+ self.transport = 'poll'+ self._set_state('degraded', 'relay_unreachable', transport='poll')+ logger.info('Connect relay: using long-poll fallback (degraded)')+ fail_attempt = 0+ while self.running:+ if time.monotonic() >= ws_retry_at:+ return 'retry_ws'+ try:+ self._poll_once(cfg)+ fail_attempt = 0+ except RelayRevoked:+ return 'revoked'+ except Exception as exc:+ fail_attempt += 1+ self.last_error = str(exc)+ logger.warning('Connect relay poll failed: %s', exc)+ self._sleep(min(30.0, backoff_delay(fail_attempt)))+ return 'stopped'++ def _poll_once(self, cfg):+ from app.utils.version import get_panel_version++ hello = build_hello(cfg['private_key'], cfg['device_id'],+ get_panel_version())+ resp = requests.get(+ poll_url_for(cfg['relay_url']),+ headers=hello_headers(hello),+ timeout=POLL_TIMEOUT_S,+ )+ if resp.status_code == 200:+ return+ detail = _error_detail(resp)+ if resp.status_code == 403 and detail == 'revoked':+ raise RelayRevoked()+ raise ConnectError(f'poll failed ({resp.status_code}): {detail}')+++# ---------- singleton management (mirrors linked_panel_agent) ----------++_relay_client = None+++def get_client():+ return _relay_client+++def start_relay_client(app=None):+ global _relay_client+ if _relay_client is not None:+ return _relay_client+ client = RelayClient(app)+ client.start()+ _relay_client = client+ return client+++def stop_relay_client():+ global _relay_client+ if _relay_client is not None:+ _relay_client.stop()+ _relay_client = None+++def start_client_if_paired(app):+ """App-startup hook: resume the relay connection when this panel is paired."""+ try:+ if app.config.get('ENV') == 'testing' or app.config.get('TESTING'):+ return None+ if not _read_connect_file().get('device_id'):+ return None+ return start_relay_client(app)+ except Exception as exc: # never block boot on the relay+ logger.warning('Could not start connect relay client: %s', exc)+ return None+++# ==================== Doctor ====================++# Hard checks fail the doctor run (exit 1); soft ones are warnings with a+# working fallback. (name, ok, hard, copy-on-failure, note-on-pass)+++def _doctor_check(name, ok, hard, fail_copy, note=None):+ return {+ 'name': name,+ 'ok': bool(ok),+ 'hard': bool(hard),+ 'error': None if ok else fail_copy,+ 'note': note if ok else None,+ }+++def _key_file_ok(key_path: str):+ if not os.path.exists(key_path):+ return False+ if os.name != 'nt':+ import stat+ mode = stat.S_IMODE(os.stat(key_path).st_mode)+ return mode & 0o077 == 0+ return True+++def run_doctor() -> list:+ """The connect check table: pairing, key, DNS, TCP, TLS, clock, panel, WS."""+ data = _read_connect_file()+ relay_url = data.get('relay_url') or DEFAULT_RELAY_URL+ parsed = urllib.parse.urlparse(relay_url)+ host = parsed.hostname or ''+ port = parsed.port or (443 if parsed.scheme == 'wss' else 80)+ use_tls = parsed.scheme == 'wss'++ checks = []++ # Pairing state (informational: doctor is also useful pre-pair).+ paired = bool(data.get('device_id'))+ checks.append(_doctor_check(+ 'Paired with ServerKit Cloud', paired, hard=False,+ fail_copy='Not paired yet. Run `serverkit connect` first.',+ note=f'device {data.get("device_id")}' if paired else None))++ # Device key present and mode 0600.+ key_path = data.get(KEY_PATH_FIELD) or connect_keys.default_key_path()+ checks.append(_doctor_check(+ 'Device key', _key_file_ok(key_path), hard=True,+ fail_copy=f'Device key missing or readable by others: {key_path}.',+ note=key_path))++ # DNS resolution of the relay host.+ try:+ socket.getaddrinfo(host, port)+ dns_ok = True+ checks.append(_doctor_check(f'DNS {host}', True, hard=True,+ fail_copy=None, note=host))+ except socket.gaierror:+ dns_ok = False+ checks.append(_doctor_check(+ f'DNS {host}', False, hard=True,+ fail_copy=f'Cannot resolve {host}. Check DNS or outbound filtering.'))++ # TCP egress to the relay port (443 for wss).+ if dns_ok:+ try:+ with socket.create_connection((host, port), timeout=5):+ pass+ checks.append(_doctor_check(f'TCP {host}:{port}', True, hard=True,+ fail_copy=None))+ except OSError:+ checks.append(_doctor_check(+ f'TCP {host}:{port}', False, hard=True,+ fail_copy='Outbound HTTPS to the relay is blocked.'))+ else:+ checks.append(_doctor_check(f'TCP {host}:{port}', False, hard=True,+ fail_copy='Skipped: DNS failed.'))++ # TLS chain (plain ws:// relays have none to check).+ if not use_tls:+ checks.append(_doctor_check(f'TLS {host}', True, hard=True,+ fail_copy=None,+ note='not applicable (plain ws:// relay)'))+ elif dns_ok:+ import ssl+ try:+ ctx = ssl.create_default_context()+ with socket.create_connection((host, port), timeout=5) as sock:+ with ctx.wrap_socket(sock, server_hostname=host):+ pass+ checks.append(_doctor_check(f'TLS {host}', True, hard=True,+ fail_copy=None))+ except Exception as exc:+ checks.append(_doctor_check(+ f'TLS {host}', False, hard=True,+ fail_copy=f'TLS to the relay failed: {exc}. '+ 'Corporate proxies that inspect TLS are not supported.'))+ else:+ checks.append(_doctor_check(f'TLS {host}', False, hard=True,+ fail_copy='Skipped: DNS failed.'))++ # Clock skew: the relay/ServerKit Cloud Date header vs local time (limit 60 s).+ if dns_ok:+ import email.utils+ try:+ resp = requests.get(relay_http_base(relay_url) + '/', timeout=10)+ remote = email.utils.parsedate_to_datetime(resp.headers['Date'])+ skew = abs(time.time() - remote.timestamp())+ checks.append(_doctor_check(+ 'Clock skew', skew <= 60, hard=True,+ fail_copy=f'System clock is {int(skew)}s off. Enable NTP; the '+ 'relay rejects signatures more than 60 s off.',+ note=f'{skew:.1f}s off UTC'))+ except Exception as exc:+ checks.append(_doctor_check(+ 'Clock skew', False, hard=True,+ fail_copy=f'Could not read the relay clock: {exc}.'))+ else:+ checks.append(_doctor_check('Clock skew', False, hard=True,+ fail_copy='Skipped: DNS failed.'))++ # Loopback panel reachable (the relay will proxy browser traffic to it).+ from app.services.cli_api_client import resolve_port+ panel_port = resolve_port()+ try:+ with socket.create_connection(('127.0.0.1', panel_port), timeout=3):+ pass+ checks.append(_doctor_check('Panel loopback', True, hard=True,+ fail_copy=None,+ note=f'127.0.0.1:{panel_port}'))+ except OSError:+ checks.append(_doctor_check(+ 'Panel loopback', False, hard=True,+ fail_copy=f'Panel not reachable at 127.0.0.1:{panel_port}.'))++ # WS upgrade attempt (soft: the long-poll fallback keeps limited mode).+ if dns_ok:+ try:+ from websockets.sync.client import connect as ws_connect+ ws = ws_connect(relay_url, open_timeout=10, close_timeout=2,+ ping_interval=None)+ try:+ ws.close()+ finally:+ pass+ checks.append(_doctor_check('WebSocket upgrade', True, hard=False,+ fail_copy=None))+ except Exception:+ checks.append(_doctor_check(+ 'WebSocket upgrade', False, hard=False,+ fail_copy='WebSocket upgrade refused; will use limited mode.'))+ else:+ checks.append(_doctor_check('WebSocket upgrade', False, hard=False,+ fail_copy='Skipped: DNS failed.'))++ return checks+++def doctor_ok(checks) -> bool:+ """True when no HARD check failed (soft checks are warnings)."""+ return not any(c['hard'] and not c['ok'] for c in checks)diff --git a/backend/app/services/connect_keys.py b/backend/app/services/connect_keys.pyindex 9b7b3388..6d979f44 100644--- a/backend/app/services/connect_keys.py+++ b/backend/app/services/connect_keys.py@@ -43,6 +43,29 @@ def _public_bytes(private_key: Ed25519PrivateKey) -> bytes:)+def _load_existing(path: str) -> Ed25519PrivateKey:+ with open(path, 'rb') as f:+ private_key = serialization.load_pem_private_key(f.read(), password=None)+ if not isinstance(private_key, Ed25519PrivateKey):+ raise ValueError(f'{path} is not an Ed25519 private key')+ if os.name != 'nt':+ os.chmod(path, 0o600)+ return private_key+++def load_keypair(path: str = None):+ """Strictly load the device keypair; raises FileNotFoundError if absent.++ Returns (private_key, pubkey_hex, fingerprint). Used by the relay+ transport: generating a fresh key here would silently change the device+ identity and fail every hello signature.+ """+ path = path or default_key_path()+ private_key = _load_existing(path)+ pubkey_bytes = _public_bytes(private_key)+ return private_key, pubkey_bytes.hex(), fingerprint_of(pubkey_bytes)++def load_or_create_keypair(path: str = None):"""Load the device keypair from ``path``, generating + persisting one if absent.@@ -51,12 +74,7 @@ def load_or_create_keypair(path: str = None):"""path = path or default_key_path()if os.path.exists(path):- with open(path, 'rb') as f:- private_key = serialization.load_pem_private_key(f.read(), password=None)- if not isinstance(private_key, Ed25519PrivateKey):- raise ValueError(f'{path} is not an Ed25519 private key')- if os.name != 'nt':- os.chmod(path, 0o600)+ private_key = _load_existing(path)else:private_key = Ed25519PrivateKey.generate()os.makedirs(os.path.dirname(path), exist_ok=True)diff --git a/backend/cli.py b/backend/cli.pyindex b4561b1e..d3793c0d 100644--- a/backend/cli.py+++ b/backend/cli.py@@ -1609,6 +1609,9 @@ def connect_status(as_json):rows = [('State', state.get('state')),('Reason', state.get('state_reason') or '-'),+ ('Transport', state.get('transport') or '-'),+ ('Last seen', state.get('last_connected_at') or '-'),+ ('Relay instance', state.get('relay_instance') or '-'),('ServerKit Cloud', state.get('cloud_url')),('Organization', state.get('org_slug') or '-'),('Name', state.get('name') or '-'),@@ -1622,6 +1625,48 @@ def connect_status(as_json):_echo_table(['Field', 'Value'], rows)[email protected]('doctor')[email protected]('--json', 'as_json', is_flag=True, help='Output machine-readable JSON')+def connect_doctor(as_json):+ """Diagnose the path from this panel to the ServerKit relay.++ Exits non-zero when any hard check fails (DNS, TCP, TLS, clock skew,+ device key, panel loopback). A refused WebSocket upgrade is a warning —+ the client falls back to limited (long-poll) mode.+ """+ from app.services import connect_client++ checks = connect_client.run_doctor()+ healthy = connect_client.doctor_ok(checks)++ if as_json:+ _echo_json({'ok': healthy, 'checks': checks})+ else:+ click.echo('ServerKit Cloud connect doctor')+ click.echo()+ for check in checks:+ if check['ok']:+ mark = click.style('OK ', fg='green')+ elif check['hard']:+ mark = click.style('FAIL', fg='red')+ else:+ mark = click.style('WARN', fg='yellow')+ line = f' {mark} {check["name"]}'+ if check.get('note'):+ line += f' ({check["note"]})'+ click.echo(line)+ if check.get('error'):+ click.echo(f' {check["error"]}')+ click.echo()+ if healthy:+ click.echo(click.style('All hard checks passed.', fg='green'))+ else:+ click.echo(click.style('Some checks failed — see above.', fg='red'))++ if not healthy:+ sys.exit(1)++@connect.command('disconnect')@click.option('--remove-key', is_flag=True,help='Also delete the device keypair (a new identity on next pair)')diff --git a/backend/requirements.txt b/backend/requirements.txtindex aa2d4eea..5d58e034 100644--- a/backend/requirements.txt+++ b/backend/requirements.txt@@ -24,6 +24,9 @@ Flask-Cors==6.0.2Flask-SocketIO==5.6.1python-socketio==5.16.3python-engineio==4.13.3+# WebSocket CLIENT for the ServerKit Cloud relay (see+# app/services/connect_client.py). The server-side stack above cannot dial out.+websockets==15.0.1# System monitoringpsutil==5.9.7diff --git a/backend/tests/test_connect_relay.py b/backend/tests/test_connect_relay.pynew file mode 100644index 00000000..81721d08--- /dev/null+++ b/backend/tests/test_connect_relay.py@@ -0,0 +1,285 @@+"""Unit tests for the relay transport helpers (no network)."""+import json+import socket+import ssl++import pytest+from websockets.exceptions import ConnectionClosedOK+from websockets.frames import Close++from app.services import connect_client, connect_keys+++def config_dir(tmp_path, monkeypatch):+ """Point the panel config dir at a temp dir for connect*.json files."""+ monkeypatch.setattr(connect_client.paths, 'SERVERKIT_CONFIG_DIR',+ str(tmp_path))+ return tmp_path+++# ---------- close-code / failure classification ----------+++def test_close_reason_for_code_maps_relay_codes():+ assert connect_client.close_reason_for_code(4001) == 'auth_failed_signature'+ assert connect_client.close_reason_for_code(4002) == 'auth_failed_clock_skew'+ assert connect_client.close_reason_for_code(4003) == 'version_unsupported'+ assert connect_client.close_reason_for_code(4004) == 'auth_failed_signature'+ assert connect_client.close_reason_for_code(4009) == 'revoked'+++def test_close_reason_for_unknown_code_is_heartbeat_timeout():+ assert connect_client.close_reason_for_code(1006) == 'heartbeat_timeout'+ assert connect_client.close_reason_for_code(None) == 'heartbeat_timeout'+++def test_classify_ws_failure():+ closed = ConnectionClosedOK(Close(4002, 'auth_failed_clock_skew'), None)+ assert connect_client._classify_ws_failure(closed) == 'auth_failed_clock_skew'+ assert connect_client._classify_ws_failure(ssl.SSLError('x')) == 'tls_error'+ assert connect_client._classify_ws_failure(+ socket.gaierror('x')) == 'dns_error'+ assert connect_client._classify_ws_failure(+ ValueError('x')) == 'relay_unreachable'+++# ---------- backoff / flap detection ----------+++def test_backoff_doubles_to_cap():+ no_jitter = lambda a, b: 0.0+ assert connect_client.backoff_delay(0, rand=no_jitter) == 1.0+ assert connect_client.backoff_delay(1, rand=no_jitter) == 2.0+ assert connect_client.backoff_delay(5, rand=no_jitter) == 32.0+ assert connect_client.backoff_delay(10, rand=no_jitter) == 60.0+ assert connect_client.backoff_delay(100, rand=no_jitter) == 60.0+++def test_backoff_jitter_stays_within_bounds():+ for attempt in (0, 3, 10):+ for _ in range(50):+ delay = connect_client.backoff_delay(attempt)+ base = min(60.0, 2 ** attempt)+ assert base * 0.75 <= delay <= base * 1.25+++def test_is_flapping():+ now = 1000.0+ recent = [now - i for i in range(7)]+ assert connect_client.is_flapping(recent, now=now)+ assert not connect_client.is_flapping(recent[:6], now=now)+ # Attempts outside the window don't count.+ stale = [now - 700 - i for i in range(10)]+ assert not connect_client.is_flapping(stale, now=now)+ assert connect_client.is_flapping(recent + stale, now=now)+++# ---------- URL / hello helpers ----------+++def test_relay_http_base_and_poll_url():+ assert connect_client.relay_http_base(+ 'wss://relay.serverkit.ai/v1/device') == 'https://relay.serverkit.ai'+ assert connect_client.relay_http_base(+ 'ws://127.0.0.1:8791/v1/device') == 'http://127.0.0.1:8791'+ assert connect_client.poll_url_for('wss://relay.serverkit.ai/v1/device') == \+ 'https://relay.serverkit.ai/v1/device/poll'+++def test_build_hello_signature_verifies(config_dir):+ private_key, pubkey_hex, _fpr = connect_keys.load_or_create_keypair()+ hello = connect_client.build_hello(private_key, 'dev_123', '1.9.22')++ assert hello['t'] == 'hello'+ assert hello['device_id'] == 'dev_123'+ assert hello['proto'] == 1+ assert hello['client_version'] == '1.9.22'+ assert isinstance(hello['ts'], int)++ from cryptography.hazmat.primitives.asymmetric.ed25519 import (+ Ed25519PublicKey,+ )+ pub = Ed25519PublicKey.from_public_bytes(bytes.fromhex(pubkey_hex))+ message = f"dev_123|{hello['ts']}|{hello['nonce']}".encode()+ pub.verify(bytes.fromhex(hello['sig']), message) # raises if invalid+++def test_hello_headers_use_relay_names(config_dir):+ private_key, _pub, _fpr = connect_keys.load_or_create_keypair()+ hello = connect_client.build_hello(private_key, 'dev_123', '1.9.22')+ headers = connect_client.hello_headers(hello)+ assert headers['X-ServerKit-Device'] == 'dev_123'+ assert headers['X-ServerKit-Sig'] == hello['sig']+ assert headers['X-ServerKit-Ts'] == str(hello['ts'])+ assert headers['X-ServerKit-Proto'] == '1'+ assert headers['X-ServerKit-Client-Version'] == '1.9.22'+++# ---------- state file / status merge ----------+++def test_state_file_roundtrip_preserves_last_connected(config_dir):+ connect_client._write_state_file('online', None, transport='ws',+ relay_instance='i-1',+ last_connected_at='2026-01-01T00:00:00+00:00')+ connect_client._write_state_file('paired_offline', 'heartbeat_timeout',+ transport=None)+ saved = connect_client._read_state_file()+ assert saved['state'] == 'paired_offline'+ assert saved['state_reason'] == 'heartbeat_timeout'+ assert saved['transport'] is None+ # Not overwritten by the later, offline transition.+ assert saved['last_connected_at'] == '2026-01-01T00:00:00+00:00'+ assert saved['relay_instance'] == 'i-1'+ assert saved['updated_at']+++def test_status_merges_state_file(config_dir):+ connect_client._write_connect_file({+ 'device_id': 'dev_1', 'org_slug': 'acme', 'name': 'web-1',+ 'relay_url': 'wss://relay.example.com/v1/device', 'scopes': [],+ 'fingerprint': 'a1b2c3d4e5f67890',+ 'key_path': str(config_dir / 'connect_device_key.pem'),+ 'cloud_url': 'https://app.serverkit.ai',+ 'paired_at': '2026-01-01T00:00:00+00:00',+ })+ connect_client._write_state_file('online', None, transport='ws',+ relay_instance='i-1',+ last_connected_at='2026-01-02T00:00:00+00:00')++ state = connect_client.status()+ assert state['state'] == 'online'+ assert state['transport'] == 'ws'+ assert state['relay_instance'] == 'i-1'+ assert state['last_connected_at'] == '2026-01-02T00:00:00+00:00'+ assert state['key_present'] is False # no key written in this test+++def test_status_without_state_file_is_never_connected(config_dir):+ connect_keys.load_or_create_keypair()+ connect_client._write_connect_file({+ 'device_id': 'dev_1', 'org_slug': 'acme', 'name': 'web-1',+ 'relay_url': 'wss://relay.example.com/v1/device', 'scopes': [],+ 'fingerprint': 'a1b2c3d4e5f67890',+ 'key_path': connect_keys.default_key_path(),+ 'cloud_url': 'https://app.serverkit.ai',+ 'paired_at': '2026-01-01T00:00:00+00:00',+ })+ state = connect_client.status()+ assert state['state'] == 'paired_offline'+ assert state['state_reason'] == 'never_connected'+++def test_disconnect_removes_state_file(config_dir):+ connect_client._write_connect_file({'device_id': 'dev_1'})+ connect_client._write_state_file('online', None, transport='ws')+ result = connect_client.disconnect()+ assert result['state'] == 'unpaired'+ assert not (config_dir / 'connect.json').exists()+ assert not (config_dir / 'connect-state.json').exists()+ assert connect_client.status()['state'] == 'unpaired'+++def test_state_file_is_valid_json_on_disk(config_dir):+ connect_client._write_state_file('degraded', 'relay_unreachable',+ transport='poll')+ with open(connect_client.state_file_path()) as f:+ saved = json.load(f)+ assert saved['state'] == 'degraded'+++# ---------- client loop state transitions (no network: sessions stubbed) ----------+++def _wait_state(want, timeout=5.0):+ import time+ deadline = time.time() + timeout+ while time.time() < deadline:+ saved = connect_client._read_state_file()+ if saved.get('state') == want:+ return saved+ time.sleep(0.05)+ raise AssertionError(f'timed out waiting for {want}; '+ f'have {connect_client._read_state_file()}')+++def _fake_config(monkeypatch):+ monkeypatch.setattr(connect_client, '_relay_config', lambda: {+ 'device_id': 'dev_1', 'relay_url': 'ws://127.0.0.1:9/v1/device',+ 'private_key': None, 'key_path': '/nope',+ })+ monkeypatch.setattr(connect_client, 'backoff_delay',+ lambda attempt, **kw: 0.01)+++def test_ws_refused_falls_back_to_poll_degraded(config_dir, monkeypatch):+ """A proxy-smelling WS failure switches to the poll loop as degraded."""+ _fake_config(monkeypatch)+ monkeypatch.setattr(connect_client.RelayClient, '_ws_session',+ lambda self, cfg: 'refused:relay_unreachable')++ def fake_poll(self, cfg, ws_retry_at):+ # The real _poll_session sets transport='poll' and writes the degraded+ # state; the loop has already written it by the time we're called.+ assert connect_client._read_state_file()['state'] == 'degraded'+ assert connect_client._read_state_file()['transport'] == 'poll'+ return 'stopped'++ monkeypatch.setattr(connect_client.RelayClient, '_poll_session', fake_poll)++ client = connect_client.RelayClient()+ client.start()+ client._thread.join(timeout=5)+ assert not client._thread.is_alive()+++def test_drop_after_ready_reports_heartbeat_timeout(config_dir, monkeypatch):+ """An established connection that drops un cleanly -> paired_offline."""+ _fake_config(monkeypatch)+ calls = []++ def fake_ws(self, cfg):+ calls.append(1)+ return 'drop:heartbeat_timeout' if len(calls) == 1 else 'stopped'++ monkeypatch.setattr(connect_client.RelayClient, '_ws_session', fake_ws)++ client = connect_client.RelayClient()+ client.start()+ saved = _wait_state('paired_offline')+ assert saved['state_reason'] == 'heartbeat_timeout'+ client._thread.join(timeout=5)+ assert not client._thread.is_alive()+++def test_revoked_is_terminal(config_dir, monkeypatch):+ """Close 4009 stops the loop for good, and stop() must not overwrite it."""+ _fake_config(monkeypatch)+ monkeypatch.setattr(connect_client.RelayClient, '_ws_session',+ lambda self, cfg: 'revoked')++ client = connect_client.RelayClient()+ client.start()+ saved = _wait_state('revoked')+ client._thread.join(timeout=5)+ assert not client._thread.is_alive()+ assert saved['state_reason'] == 'revoked'++ client.stop()+ assert connect_client._read_state_file()['state'] == 'revoked'+++def test_flap_logging_fires_after_limit(config_dir, monkeypatch, caplog):+ _fake_config(monkeypatch)+ # FLAP_LIMIT is 6: 8 quick drops exceed it inside the window.+ outcomes = iter(['drop:heartbeat_timeout'] * 8 + ['stopped'])+ monkeypatch.setattr(+ connect_client.RelayClient, '_ws_session',+ lambda self, cfg: next(outcomes, 'stopped'))++ client = connect_client.RelayClient()+ with caplog.at_level('WARNING', logger='app.services.connect_client'):+ client.start()+ client._thread.join(timeout=10)+ assert any('flapping' in r.message for r in caplog.records)