ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
fix(agent): fail-closed auth + heartbeat reaper reconnect race
Juan Denis749152c09eff
diff --git a/backend/app/models/server.py b/backend/app/models/server.pyindex 4ee520f9..25995cf1 100644--- a/backend/app/models/server.py+++ b/backend/app/models/server.py@@ -208,6 +208,23 @@ class Server(db.Model):print(f"Error decrypting API secret: {e}")return None+ def get_pending_api_secret(self):+ """Decrypt and return the pending API secret used during key rotation.++ Mirrors get_api_secret() for the api_secret_pending_encrypted column so+ an agent that authenticates with the pending key during a rotation+ window can have its signature verified against the right secret.+ Returns None when there is no pending secret or it can't be decrypted.+ """+ if not self.api_secret_pending_encrypted:+ return None+ try:+ from app.utils.crypto import decrypt_secret+ return decrypt_secret(self.api_secret_pending_encrypted)+ except Exception as e:+ print(f"Error decrypting pending API secret: {e}")+ return None+def start_key_rotation(self):"""Start API key rotation process.diff --git a/backend/app/services/agent_registry.py b/backend/app/services/agent_registry.pyindex 7daed8a0..78da565f 100644--- a/backend/app/services/agent_registry.py+++ b/backend/app/services/agent_registry.py@@ -90,6 +90,11 @@ class AgentRegistry:_instance = None_lock = threading.Lock()+ # How long an agent can go without a heartbeat before the reaper+ # considers it dead (3 missed 30s heartbeats). Shared between the scan+ # and the re-validation done just before eviction so the two can't drift.+ HEARTBEAT_TIMEOUT = timedelta(seconds=90)+def __new__(cls):if cls._instance is None:with cls._lock:@@ -156,64 +161,90 @@ class AgentRegistry:while not self._stop_heartbeat.is_set():try:now = datetime.utcnow()- timeout = timedelta(seconds=90) # 3 missed heartbeats+ timeout = self.HEARTBEAT_TIMEOUT+ # Snapshot (server_id, socket_id) for stale agents. The+ # socket_id is part of the snapshot so the eviction step can+ # tell whether the agent it's about to drop is still the same+ # connection we flagged, or a fresh reconnect that landed+ # after we released the lock.with self._lock:- dead_agents = []- for server_id, agent in self._agents.items():- if now - agent.last_heartbeat > timeout:- dead_agents.append(server_id)+ dead_agents = [+ (server_id, agent.socket_id)+ for server_id, agent in self._agents.items()+ if now - agent.last_heartbeat > timeout+ ]# Wrap DB access in an app context — the timeout# handler queries Server / AgentSession and commits.# In a thread there's no implicit context.if dead_agents and self._app is not None:with self._app.app_context():- for server_id in dead_agents:- self._handle_agent_timeout(server_id)+ for server_id, socket_id in dead_agents:+ self._handle_agent_timeout(server_id, socket_id)elif dead_agents:- # No app captured (test environment); just remove- # from in-memory registry without touching DB.- for server_id in dead_agents:+ # No app captured (test environment); just remove from the+ # in-memory registry without touching DB — but still+ # re-validate so a reconnect isn't clobbered.+ now2 = datetime.utcnow()+ for server_id, socket_id in dead_agents:with self._lock:- agent = self._agents.pop(server_id, None)- if agent:- self._socket_to_server.pop(agent.socket_id, None)+ agent = self._agents.get(server_id)+ if (agent and agent.socket_id == socket_id+ and now2 - agent.last_heartbeat > timeout):+ self._agents.pop(server_id, None)+ self._socket_to_server.pop(socket_id, None)except Exception as e:logger.error("Error in heartbeat checker: %s", e)self._stop_heartbeat.wait(30) # Check every 30 seconds- def _handle_agent_timeout(self, server_id: str):- """Handle agent that hasn't sent heartbeats"""+ def _handle_agent_timeout(self, server_id: str, socket_id: str):+ """Handle agent that hasn't sent heartbeats.++ Re-validates under the lock before evicting. The reaper snapshots+ stale agents in _check_heartbeats and releases the lock before calling+ this, so an agent can reconnect in between — register_agent installs a+ fresh ConnectedAgent (new socket_id) and marks the server online.+ Without re-checking, we'd pop that healthy new connection and flip the+ server offline right after it reconnected — the reconnection-stability+ bug. Only evict when the entry still refers to the same socket we+ flagged AND it is still stale.+ """with self._lock:- agent = self._agents.pop(server_id, None)- if agent:- self._socket_to_server.pop(agent.socket_id, None)+ agent = self._agents.get(server_id)+ if not agent or agent.socket_id != socket_id:+ # Already gone, or replaced by a reconnect — leave it alone.+ return+ if datetime.utcnow() - agent.last_heartbeat <= self.HEARTBEAT_TIMEOUT:+ # A heartbeat landed after the snapshot — no longer stale.+ return+ self._agents.pop(server_id, None)+ self._socket_to_server.pop(socket_id, None)- if agent:- # Update server status in database- try:- server = Server.query.get(server_id)- if server:- server.status = 'offline'- server.last_error = 'Agent heartbeat timeout'- db.session.commit()+ # Update server status in database+ try:+ server = Server.query.get(server_id)+ if server:+ server.status = 'offline'+ server.last_error = 'Agent heartbeat timeout'+ db.session.commit()- # Mark session as inactive- session = AgentSession.query.filter_by(- server_id=server_id,- socket_id=agent.socket_id,- is_active=True- ).first()- if session:- session.is_active = False- session.disconnected_at = datetime.utcnow()- session.disconnect_reason = 'heartbeat_timeout'- db.session.commit()- except Exception as e:- logger.exception("Error updating server status")+ # Mark session as inactive+ session = AgentSession.query.filter_by(+ server_id=server_id,+ socket_id=socket_id,+ is_active=True+ ).first()+ if session:+ session.is_active = False+ session.disconnected_at = datetime.utcnow()+ session.disconnect_reason = 'heartbeat_timeout'+ db.session.commit()+ except Exception as e:+ logger.exception("Error updating server status")+ db.session.rollback()# ==================== Connection Management ====================@@ -702,26 +733,45 @@ class AgentRegistry:anomaly_detection_service.track_replay_attack(server.id, ip_address, nonce)return None- # Verify HMAC signature if we have the encrypted secret- api_secret = server.get_api_secret()- if api_secret:- # Construct the message that was signed- # Format: agent_id:timestamp:nonce (nonce is optional for backward compatibility)- if nonce:- message = f"{agent_id}:{timestamp}:{nonce}"- else:- message = f"{agent_id}:{timestamp}"-- # Calculate expected signature- expected_signature = hmac.new(- api_secret.encode(),- message.encode(),- hashlib.sha256- ).hexdigest()+ # Verify HMAC signature. This is MANDATORY — a server whose shared+ # secret is missing or no longer decryptable must FAIL authentication,+ # never fall through to "authenticated". Skipping it would reduce agent+ # auth to agent_id + api_key_prefix, both of which are non-secret,+ # observable values (an attacker who has seen one connect could forge+ # auth). Select the secret matching whichever prefix matched: the+ # pending secret during a key-rotation window, otherwise the active+ # one. get_*_api_secret() returns None on missing/undecryptable+ # ciphertext, which fails closed below.+ if pending_prefix_matches and not prefix_matches:+ api_secret = server.get_pending_api_secret()+ else:+ api_secret = server.get_api_secret()++ if not api_secret:+ logger.error(+ "Agent auth rejected for server %s: no decryptable API secret "+ "on file (re-registration or re-pairing required)", server.id+ )+ anomaly_detection_service.track_auth_attempt(server.id, False, ip_address)+ return None- if not hmac.compare_digest(signature, expected_signature):- anomaly_detection_service.track_auth_attempt(server.id, False, ip_address)- return None+ # Construct the message that was signed.+ # Format: agent_id:timestamp:nonce (nonce omitted only by legacy agents)+ if nonce:+ message = f"{agent_id}:{timestamp}:{nonce}"+ else:+ message = f"{agent_id}:{timestamp}"++ # Calculate expected signature+ expected_signature = hmac.new(+ api_secret.encode(),+ message.encode(),+ hashlib.sha256+ ).hexdigest()++ if not hmac.compare_digest(signature, expected_signature):+ anomaly_detection_service.track_auth_attempt(server.id, False, ip_address)+ return None# Authentication successfulif ip_address:diff --git a/backend/tests/test_agent_registry_security.py b/backend/tests/test_agent_registry_security.pynew file mode 100644index 00000000..58e6de97--- /dev/null+++ b/backend/tests/test_agent_registry_security.py@@ -0,0 +1,172 @@+"""+Regression tests for two agent_registry defects found in the deep-dive review:++1. verify_agent_auth() must FAIL CLOSED when no decryptable API secret is on+ file. Previously the HMAC check was wrapped in `if api_secret:`, so a server+ with a missing/undecryptable secret authenticated on agent_id + api_key_prefix+ alone (both non-secret, observable values).++2. The heartbeat reaper must NOT evict an agent that reconnected between the+ stale-scan snapshot and eviction. _check_heartbeats snapshots under the lock+ then releases it; _handle_agent_timeout used to pop unconditionally, so a+ fresh reconnect (new socket_id) got clobbered and flipped offline.+"""+import time+import hmac+import hashlib+from datetime import datetime, timedelta++import pytest++from app import db as _db+from app.models.server import Server+from app.services.agent_registry import agent_registry++[email protected](autouse=True)+def _silence_side_effects(monkeypatch):+ """No-op the anomaly/nonce side-effects so these tests exercise only the+ auth decision logic, not unrelated subsystems."""+ import app.services.anomaly_detection_service as ad+ monkeypatch.setattr(ad.anomaly_detection_service, "track_auth_attempt",+ lambda *a, **k: None, raising=False)+ monkeypatch.setattr(ad.anomaly_detection_service, "track_replay_attack",+ lambda *a, **k: None, raising=False)+++# --------------------------------------------------------------------------+# Bug #1 — mandatory signature verification+# --------------------------------------------------------------------------++def _signed(agent_id, secret, ts, nonce=None):+ msg = f"{agent_id}:{ts}:{nonce}" if nonce else f"{agent_id}:{ts}"+ return hmac.new(secret.encode(), msg.encode(), hashlib.sha256).hexdigest()+++def test_auth_fails_closed_when_no_decryptable_secret(app, monkeypatch):+ """The critical fix: no secret on file => auth rejected, never bypassed."""+ monkeypatch.setattr(Server, "get_api_secret", lambda self: None)+ monkeypatch.setattr(Server, "get_pending_api_secret", lambda self: None)++ s = Server(name="t", agent_id="agent-nosecret", api_key_prefix="sk_test12345")+ _db.session.add(s)+ _db.session.commit()++ ts = int(time.time() * 1000)+ # Attacker knows the (non-secret) agent_id + prefix and sends any signature.+ result = agent_registry.verify_agent_auth(+ agent_id="agent-nosecret",+ api_key_prefix="sk_test12345",+ signature="deadbeef" * 8,+ timestamp=ts,+ nonce=None,+ ip_address="203.0.113.7",+ )+ assert result is None+++def test_auth_succeeds_with_valid_signature(app, monkeypatch):+ secret = "shared-secret-value-123"+ monkeypatch.setattr(Server, "get_api_secret", lambda self: secret)++ s = Server(name="t", agent_id="agent-ok", api_key_prefix="sk_okokokok12")+ _db.session.add(s)+ _db.session.commit()++ ts = int(time.time() * 1000)+ sig = _signed("agent-ok", secret, ts)+ result = agent_registry.verify_agent_auth(+ agent_id="agent-ok",+ api_key_prefix="sk_okokokok12",+ signature=sig,+ timestamp=ts,+ nonce=None,+ ip_address="203.0.113.7",+ )+ assert result is not None+ assert result.agent_id == "agent-ok"+++def test_auth_fails_with_wrong_signature(app, monkeypatch):+ monkeypatch.setattr(Server, "get_api_secret", lambda self: "the-real-secret")++ s = Server(name="t", agent_id="agent-wrong", api_key_prefix="sk_wrongwrong")+ _db.session.add(s)+ _db.session.commit()++ ts = int(time.time() * 1000)+ bad_sig = _signed("agent-wrong", "a-different-secret", ts)+ result = agent_registry.verify_agent_auth(+ agent_id="agent-wrong",+ api_key_prefix="sk_wrongwrong",+ signature=bad_sig,+ timestamp=ts,+ nonce=None,+ ip_address="203.0.113.7",+ )+ assert result is None+++# --------------------------------------------------------------------------+# Bug #2 — reaper must not clobber a reconnected agent+# --------------------------------------------------------------------------++def clean_registry():+ """Isolate the in-memory singleton registry per test."""+ with agent_registry._lock:+ agent_registry._agents.clear()+ agent_registry._socket_to_server.clear()+ yield+ with agent_registry._lock:+ agent_registry._agents.clear()+ agent_registry._socket_to_server.clear()+++def _make_stale(server_id):+ with agent_registry._lock:+ agent_registry._agents[server_id].last_heartbeat = (+ datetime.utcnow() - timedelta(seconds=120)+ )+++def test_reaper_does_not_evict_reconnected_agent(app, clean_registry):+ agent_registry._app = app++ s = Server(name="t", agent_id="agent-recon")+ _db.session.add(s)+ _db.session.commit()+ sid = s.id++ # Original connection, then it goes stale.+ agent_registry.register_agent(sid, "socket-OLD", "203.0.113.7", "1.0.0")+ _make_stale(sid)++ # Agent reconnects on a NEW socket (fresh heartbeat, server online again).+ agent_registry.register_agent(sid, "socket-NEW", "203.0.113.7", "1.0.0")++ # Reaper fires for the OLD snapshot it took before the reconnect.+ agent_registry._handle_agent_timeout(sid, "socket-OLD")++ # The fresh connection must survive and the server must stay online.+ assert agent_registry.is_agent_connected(sid) is True+ assert agent_registry.get_agent(sid).socket_id == "socket-NEW"+ assert Server.query.get(sid).status != "offline"+++def test_reaper_evicts_genuinely_stale_agent(app, clean_registry):+ agent_registry._app = app++ s = Server(name="t", agent_id="agent-stale")+ _db.session.add(s)+ _db.session.commit()+ sid = s.id++ agent_registry.register_agent(sid, "socket-1", "203.0.113.7", "1.0.0")+ _make_stale(sid)++ # Same socket, genuinely stale -> evicted and marked offline.+ agent_registry._handle_agent_timeout(sid, "socket-1")++ assert agent_registry.is_agent_connected(sid) is False+ assert Server.query.get(sid).status == "offline"