Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(agent): fail-closed auth + heartbeat reaper reconnect race

Juan Denis749152c09eff
diff --git a/backend/app/models/server.py b/backend/app/models/server.py
index 4ee520f9..25995cf1 100644
--- a/backend/app/models/server.py
+++ b/backend/app/models/server.py
@@ -208,6 +208,23 @@ class Server(db.Model):
print(f"Error decrypting API secret: {e}")
return None
+ def get_pending_api_secret(self):
+ """Decrypt and return the pending API secret used during key rotation.
+
+ Mirrors get_api_secret() for the api_secret_pending_encrypted column so
+ an agent that authenticates with the pending key during a rotation
+ window can have its signature verified against the right secret.
+ Returns None when there is no pending secret or it can't be decrypted.
+ """
+ if not self.api_secret_pending_encrypted:
+ return None
+ try:
+ from app.utils.crypto import decrypt_secret
+ return decrypt_secret(self.api_secret_pending_encrypted)
+ except Exception as e:
+ print(f"Error decrypting pending API secret: {e}")
+ return None
+
def start_key_rotation(self):
"""
Start API key rotation process.
diff --git a/backend/app/services/agent_registry.py b/backend/app/services/agent_registry.py
index 7daed8a0..78da565f 100644
--- a/backend/app/services/agent_registry.py
+++ b/backend/app/services/agent_registry.py
@@ -90,6 +90,11 @@ class AgentRegistry:
_instance = None
_lock = threading.Lock()
+ # How long an agent can go without a heartbeat before the reaper
+ # considers it dead (3 missed 30s heartbeats). Shared between the scan
+ # and the re-validation done just before eviction so the two can't drift.
+ HEARTBEAT_TIMEOUT = timedelta(seconds=90)
+
def __new__(cls):
if cls._instance is None:
with cls._lock:
@@ -156,64 +161,90 @@ class AgentRegistry:
while not self._stop_heartbeat.is_set():
try:
now = datetime.utcnow()
- timeout = timedelta(seconds=90) # 3 missed heartbeats
+ timeout = self.HEARTBEAT_TIMEOUT
+ # Snapshot (server_id, socket_id) for stale agents. The
+ # socket_id is part of the snapshot so the eviction step can
+ # tell whether the agent it's about to drop is still the same
+ # connection we flagged, or a fresh reconnect that landed
+ # after we released the lock.
with self._lock:
- dead_agents = []
- for server_id, agent in self._agents.items():
- if now - agent.last_heartbeat > timeout:
- dead_agents.append(server_id)
+ dead_agents = [
+ (server_id, agent.socket_id)
+ for server_id, agent in self._agents.items()
+ if now - agent.last_heartbeat > timeout
+ ]
# Wrap DB access in an app context — the timeout
# handler queries Server / AgentSession and commits.
# In a thread there's no implicit context.
if dead_agents and self._app is not None:
with self._app.app_context():
- for server_id in dead_agents:
- self._handle_agent_timeout(server_id)
+ for server_id, socket_id in dead_agents:
+ self._handle_agent_timeout(server_id, socket_id)
elif dead_agents:
- # No app captured (test environment); just remove
- # from in-memory registry without touching DB.
- for server_id in dead_agents:
+ # No app captured (test environment); just remove from the
+ # in-memory registry without touching DB — but still
+ # re-validate so a reconnect isn't clobbered.
+ now2 = datetime.utcnow()
+ for server_id, socket_id in dead_agents:
with self._lock:
- agent = self._agents.pop(server_id, None)
- if agent:
- self._socket_to_server.pop(agent.socket_id, None)
+ agent = self._agents.get(server_id)
+ if (agent and agent.socket_id == socket_id
+ and now2 - agent.last_heartbeat > timeout):
+ self._agents.pop(server_id, None)
+ self._socket_to_server.pop(socket_id, None)
except Exception as e:
logger.error("Error in heartbeat checker: %s", e)
self._stop_heartbeat.wait(30) # Check every 30 seconds
- def _handle_agent_timeout(self, server_id: str):
- """Handle agent that hasn't sent heartbeats"""
+ def _handle_agent_timeout(self, server_id: str, socket_id: str):
+ """Handle agent that hasn't sent heartbeats.
+
+ Re-validates under the lock before evicting. The reaper snapshots
+ stale agents in _check_heartbeats and releases the lock before calling
+ this, so an agent can reconnect in between — register_agent installs a
+ fresh ConnectedAgent (new socket_id) and marks the server online.
+ Without re-checking, we'd pop that healthy new connection and flip the
+ server offline right after it reconnected — the reconnection-stability
+ bug. Only evict when the entry still refers to the same socket we
+ flagged AND it is still stale.
+ """
with self._lock:
- agent = self._agents.pop(server_id, None)
- if agent:
- self._socket_to_server.pop(agent.socket_id, None)
+ agent = self._agents.get(server_id)
+ if not agent or agent.socket_id != socket_id:
+ # Already gone, or replaced by a reconnect — leave it alone.
+ return
+ if datetime.utcnow() - agent.last_heartbeat <= self.HEARTBEAT_TIMEOUT:
+ # A heartbeat landed after the snapshot — no longer stale.
+ return
+ self._agents.pop(server_id, None)
+ self._socket_to_server.pop(socket_id, None)
- if agent:
- # Update server status in database
- try:
- server = Server.query.get(server_id)
- if server:
- server.status = 'offline'
- server.last_error = 'Agent heartbeat timeout'
- db.session.commit()
+ # Update server status in database
+ try:
+ server = Server.query.get(server_id)
+ if server:
+ server.status = 'offline'
+ server.last_error = 'Agent heartbeat timeout'
+ db.session.commit()
- # Mark session as inactive
- session = AgentSession.query.filter_by(
- server_id=server_id,
- socket_id=agent.socket_id,
- is_active=True
- ).first()
- if session:
- session.is_active = False
- session.disconnected_at = datetime.utcnow()
- session.disconnect_reason = 'heartbeat_timeout'
- db.session.commit()
- except Exception as e:
- logger.exception("Error updating server status")
+ # Mark session as inactive
+ session = AgentSession.query.filter_by(
+ server_id=server_id,
+ socket_id=socket_id,
+ is_active=True
+ ).first()
+ if session:
+ session.is_active = False
+ session.disconnected_at = datetime.utcnow()
+ session.disconnect_reason = 'heartbeat_timeout'
+ db.session.commit()
+ except Exception as e:
+ logger.exception("Error updating server status")
+ db.session.rollback()
# ==================== Connection Management ====================
@@ -702,26 +733,45 @@ class AgentRegistry:
anomaly_detection_service.track_replay_attack(server.id, ip_address, nonce)
return None
- # Verify HMAC signature if we have the encrypted secret
- api_secret = server.get_api_secret()
- if api_secret:
- # Construct the message that was signed
- # Format: agent_id:timestamp:nonce (nonce is optional for backward compatibility)
- if nonce:
- message = f"{agent_id}:{timestamp}:{nonce}"
- else:
- message = f"{agent_id}:{timestamp}"
-
- # Calculate expected signature
- expected_signature = hmac.new(
- api_secret.encode(),
- message.encode(),
- hashlib.sha256
- ).hexdigest()
+ # Verify HMAC signature. This is MANDATORY — a server whose shared
+ # secret is missing or no longer decryptable must FAIL authentication,
+ # never fall through to "authenticated". Skipping it would reduce agent
+ # auth to agent_id + api_key_prefix, both of which are non-secret,
+ # observable values (an attacker who has seen one connect could forge
+ # auth). Select the secret matching whichever prefix matched: the
+ # pending secret during a key-rotation window, otherwise the active
+ # one. get_*_api_secret() returns None on missing/undecryptable
+ # ciphertext, which fails closed below.
+ if pending_prefix_matches and not prefix_matches:
+ api_secret = server.get_pending_api_secret()
+ else:
+ api_secret = server.get_api_secret()
+
+ if not api_secret:
+ logger.error(
+ "Agent auth rejected for server %s: no decryptable API secret "
+ "on file (re-registration or re-pairing required)", server.id
+ )
+ anomaly_detection_service.track_auth_attempt(server.id, False, ip_address)
+ return None
- if not hmac.compare_digest(signature, expected_signature):
- anomaly_detection_service.track_auth_attempt(server.id, False, ip_address)
- return None
+ # Construct the message that was signed.
+ # Format: agent_id:timestamp:nonce (nonce omitted only by legacy agents)
+ if nonce:
+ message = f"{agent_id}:{timestamp}:{nonce}"
+ else:
+ message = f"{agent_id}:{timestamp}"
+
+ # Calculate expected signature
+ expected_signature = hmac.new(
+ api_secret.encode(),
+ message.encode(),
+ hashlib.sha256
+ ).hexdigest()
+
+ if not hmac.compare_digest(signature, expected_signature):
+ anomaly_detection_service.track_auth_attempt(server.id, False, ip_address)
+ return None
# Authentication successful
if ip_address:
diff --git a/backend/tests/test_agent_registry_security.py b/backend/tests/test_agent_registry_security.py
new file mode 100644
index 00000000..58e6de97
--- /dev/null
+++ b/backend/tests/test_agent_registry_security.py
@@ -0,0 +1,172 @@
+"""
+Regression tests for two agent_registry defects found in the deep-dive review:
+
+1. verify_agent_auth() must FAIL CLOSED when no decryptable API secret is on
+ file. Previously the HMAC check was wrapped in `if api_secret:`, so a server
+ with a missing/undecryptable secret authenticated on agent_id + api_key_prefix
+ alone (both non-secret, observable values).
+
+2. The heartbeat reaper must NOT evict an agent that reconnected between the
+ stale-scan snapshot and eviction. _check_heartbeats snapshots under the lock
+ then releases it; _handle_agent_timeout used to pop unconditionally, so a
+ fresh reconnect (new socket_id) got clobbered and flipped offline.
+"""
+import time
+import hmac
+import hashlib
+from datetime import datetime, timedelta
+
+import pytest
+
+from app import db as _db
+from app.models.server import Server
+from app.services.agent_registry import agent_registry
+
+
[email protected](autouse=True)
+def _silence_side_effects(monkeypatch):
+ """No-op the anomaly/nonce side-effects so these tests exercise only the
+ auth decision logic, not unrelated subsystems."""
+ import app.services.anomaly_detection_service as ad
+ monkeypatch.setattr(ad.anomaly_detection_service, "track_auth_attempt",
+ lambda *a, **k: None, raising=False)
+ monkeypatch.setattr(ad.anomaly_detection_service, "track_replay_attack",
+ lambda *a, **k: None, raising=False)
+
+
+# --------------------------------------------------------------------------
+# Bug #1 — mandatory signature verification
+# --------------------------------------------------------------------------
+
+def _signed(agent_id, secret, ts, nonce=None):
+ msg = f"{agent_id}:{ts}:{nonce}" if nonce else f"{agent_id}:{ts}"
+ return hmac.new(secret.encode(), msg.encode(), hashlib.sha256).hexdigest()
+
+
+def test_auth_fails_closed_when_no_decryptable_secret(app, monkeypatch):
+ """The critical fix: no secret on file => auth rejected, never bypassed."""
+ monkeypatch.setattr(Server, "get_api_secret", lambda self: None)
+ monkeypatch.setattr(Server, "get_pending_api_secret", lambda self: None)
+
+ s = Server(name="t", agent_id="agent-nosecret", api_key_prefix="sk_test12345")
+ _db.session.add(s)
+ _db.session.commit()
+
+ ts = int(time.time() * 1000)
+ # Attacker knows the (non-secret) agent_id + prefix and sends any signature.
+ result = agent_registry.verify_agent_auth(
+ agent_id="agent-nosecret",
+ api_key_prefix="sk_test12345",
+ signature="deadbeef" * 8,
+ timestamp=ts,
+ nonce=None,
+ ip_address="203.0.113.7",
+ )
+ assert result is None
+
+
+def test_auth_succeeds_with_valid_signature(app, monkeypatch):
+ secret = "shared-secret-value-123"
+ monkeypatch.setattr(Server, "get_api_secret", lambda self: secret)
+
+ s = Server(name="t", agent_id="agent-ok", api_key_prefix="sk_okokokok12")
+ _db.session.add(s)
+ _db.session.commit()
+
+ ts = int(time.time() * 1000)
+ sig = _signed("agent-ok", secret, ts)
+ result = agent_registry.verify_agent_auth(
+ agent_id="agent-ok",
+ api_key_prefix="sk_okokokok12",
+ signature=sig,
+ timestamp=ts,
+ nonce=None,
+ ip_address="203.0.113.7",
+ )
+ assert result is not None
+ assert result.agent_id == "agent-ok"
+
+
+def test_auth_fails_with_wrong_signature(app, monkeypatch):
+ monkeypatch.setattr(Server, "get_api_secret", lambda self: "the-real-secret")
+
+ s = Server(name="t", agent_id="agent-wrong", api_key_prefix="sk_wrongwrong")
+ _db.session.add(s)
+ _db.session.commit()
+
+ ts = int(time.time() * 1000)
+ bad_sig = _signed("agent-wrong", "a-different-secret", ts)
+ result = agent_registry.verify_agent_auth(
+ agent_id="agent-wrong",
+ api_key_prefix="sk_wrongwrong",
+ signature=bad_sig,
+ timestamp=ts,
+ nonce=None,
+ ip_address="203.0.113.7",
+ )
+ assert result is None
+
+
+# --------------------------------------------------------------------------
+# Bug #2 — reaper must not clobber a reconnected agent
+# --------------------------------------------------------------------------
+
+def clean_registry():
+ """Isolate the in-memory singleton registry per test."""
+ with agent_registry._lock:
+ agent_registry._agents.clear()
+ agent_registry._socket_to_server.clear()
+ yield
+ with agent_registry._lock:
+ agent_registry._agents.clear()
+ agent_registry._socket_to_server.clear()
+
+
+def _make_stale(server_id):
+ with agent_registry._lock:
+ agent_registry._agents[server_id].last_heartbeat = (
+ datetime.utcnow() - timedelta(seconds=120)
+ )
+
+
+def test_reaper_does_not_evict_reconnected_agent(app, clean_registry):
+ agent_registry._app = app
+
+ s = Server(name="t", agent_id="agent-recon")
+ _db.session.add(s)
+ _db.session.commit()
+ sid = s.id
+
+ # Original connection, then it goes stale.
+ agent_registry.register_agent(sid, "socket-OLD", "203.0.113.7", "1.0.0")
+ _make_stale(sid)
+
+ # Agent reconnects on a NEW socket (fresh heartbeat, server online again).
+ agent_registry.register_agent(sid, "socket-NEW", "203.0.113.7", "1.0.0")
+
+ # Reaper fires for the OLD snapshot it took before the reconnect.
+ agent_registry._handle_agent_timeout(sid, "socket-OLD")
+
+ # The fresh connection must survive and the server must stay online.
+ assert agent_registry.is_agent_connected(sid) is True
+ assert agent_registry.get_agent(sid).socket_id == "socket-NEW"
+ assert Server.query.get(sid).status != "offline"
+
+
+def test_reaper_evicts_genuinely_stale_agent(app, clean_registry):
+ agent_registry._app = app
+
+ s = Server(name="t", agent_id="agent-stale")
+ _db.session.add(s)
+ _db.session.commit()
+ sid = s.id
+
+ agent_registry.register_agent(sid, "socket-1", "203.0.113.7", "1.0.0")
+ _make_stale(sid)
+
+ # Same socket, genuinely stale -> evicted and marked offline.
+ agent_registry._handle_agent_timeout(sid, "socket-1")
+
+ assert agent_registry.is_agent_connected(sid) is False
+ assert Server.query.get(sid).status == "offline"