Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Beat every 10 seconds instead of 2, and spread the auth cache expiries (CHOO-3013) (#643)

Abel Dantasc33b46980f6c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ac9c02ea0..11559e993 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -111,6 +111,14 @@ version of their own to them without also giving them a release of their own.
compares it only when someone signs in.
#### Changed
+- **Agents and controllers are pinged every 10 seconds instead of every 2.**
+ A connection is now declared dead after 30 seconds without a beat, up from
+ 6, so a client that goes away holds its room slot and shows online for up to
+ 30 seconds. The interval is advertised on every open, so clients follow it
+ without an update. The agent auth cache now gives each entry a random 80 to
+ 120% of `AGENT_AUTH_CACHE_TTL_SECONDS`, so entries filled together after a
+ restart do not all expire together, and that setting must stay below 25
+ seconds (the 30 second heartbeat TTL, less the jitter) or startup refuses it.
- **Following the conversation is described the same way everywhere, and the
chart can turn it on.** The feature added in 0.31.0 is off unless
`follow_the_conversation` is among the enabled feature flags
diff --git a/core/switch_core/bridges/agent/api/session_reporter.py b/core/switch_core/bridges/agent/api/session_reporter.py
index 861feb959..ddb3feffc 100644
--- a/core/switch_core/bridges/agent/api/session_reporter.py
+++ b/core/switch_core/bridges/agent/api/session_reporter.py
@@ -2,8 +2,8 @@
Two ordinary things make an agent hold a succession of short connections while
being continuously present: a stream rejected at the room claim and retried,
-and a stream whose heartbeat lapsed (six seconds) closing itself so the client
-reopens. Per connection, both read as a session every few seconds.
+and a stream whose heartbeat lapsed (`HEARTBEAT_TTL_SECONDS`) closing itself so
+the client reopens. Per connection, both read as a session every few seconds.
So this reports on the **agent**: a session begins when one that had no live
connection acquires its first, and ends only once it has had none for
diff --git a/core/switch_core/bridges/agent/api_key_cache.py b/core/switch_core/bridges/agent/api_key_cache.py
index 4cc5efc35..80e776e98 100644
--- a/core/switch_core/bridges/agent/api_key_cache.py
+++ b/core/switch_core/bridges/agent/api_key_cache.py
@@ -1,19 +1,25 @@
from __future__ import annotations
+import random
import time
from collections import OrderedDict
from switch_core.bridges.agent.protocol.agent_connections import HEARTBEAT_TTL_SECONDS
from switch_core.db.models import Agent, ApiKey
+# Each entry lives a random 80–120% of the TTL. A restart empties the cache
+# for every agent at once, and with a fixed TTL every entry then expires in
+# the same instant too, so the whole fleet hits the database together again on
+# every cycle. Spreading the expiries breaks that herd up after the first one.
+EXPIRY_JITTER = 0.2
+
class ApiKeyCache:
"""A short-lived memo of resolved agent bearer tokens.
Every authenticated request resolves its token against Postgres before the
- handler runs, and an agent connection beats every couple of seconds, so
- that one lookup is the bulk of the traffic through the connection pool —
- including the heartbeats whose deadline the pool is capable of missing.
+ handler runs, so that one lookup is the bulk of the traffic an agent's
+ requests put through the connection pool.
What it deliberately does not do:
@@ -35,10 +41,11 @@ class ApiKeyCache:
def __init__(self, *, ttl_seconds: float, max_entries: int) -> None:
if ttl_seconds < 0:
raise ValueError(f"ttl_seconds must not be negative, got {ttl_seconds!r}")
- if ttl_seconds >= HEARTBEAT_TTL_SECONDS:
+ if ttl_seconds * (1 + EXPIRY_JITTER) >= HEARTBEAT_TTL_SECONDS:
raise ValueError(
- f"ttl_seconds must stay below the agent heartbeat TTL of "
- f"{HEARTBEAT_TTL_SECONDS}s, got {ttl_seconds!r}"
+ f"ttl_seconds, with its {EXPIRY_JITTER:.0%} jitter, must stay "
+ f"below the agent heartbeat TTL of {HEARTBEAT_TTL_SECONDS}s, "
+ f"got {ttl_seconds!r}"
)
if max_entries < 1:
raise ValueError(f"max_entries must be at least 1, got {max_entries!r}")
@@ -64,7 +71,8 @@ class ApiKeyCache:
def put(self, token_hash: str, api_key: ApiKey, agent: Agent) -> None:
if not self.enabled:
return
- self._entries[token_hash] = (time.monotonic() + self._ttl, api_key, agent)
+ lifetime = self._ttl * random.uniform(1 - EXPIRY_JITTER, 1 + EXPIRY_JITTER)
+ self._entries[token_hash] = (time.monotonic() + lifetime, api_key, agent)
self._entries.move_to_end(token_hash)
while len(self._entries) > self._max_entries:
self._entries.popitem(last=False)
diff --git a/core/switch_core/bridges/agent/controller_auth_cache.py b/core/switch_core/bridges/agent/controller_auth_cache.py
index 27d828c8c..00960ce59 100644
--- a/core/switch_core/bridges/agent/controller_auth_cache.py
+++ b/core/switch_core/bridges/agent/controller_auth_cache.py
@@ -17,9 +17,9 @@ class ControllerAuthCache:
A controller token is a signed JWT, so verifying it needs no database. But
every request it authenticates also re-reads its controller row, to refuse
a revoked controller while its token is unexpired, and a request acting as
- one of its agents reads that agent's row too. A controller beats every two
- seconds and relays every call its agents' sessions make, so those reads are
- most of what it costs the connection pool.
+ one of its agents reads that agent's row too. A controller relays every
+ call its agents' sessions make, so those reads are most of what it costs
+ the connection pool.
Kept apart from :class:`ApiKeyCache`, which maps one bearer token to one
agent: a controller token stands for many agents, and its authorization is
diff --git a/core/switch_core/bridges/agent/protocol/connection_rows.py b/core/switch_core/bridges/agent/protocol/connection_rows.py
index 96201f529..c978ca245 100644
--- a/core/switch_core/bridges/agent/protocol/connection_rows.py
+++ b/core/switch_core/bridges/agent/protocol/connection_rows.py
@@ -99,7 +99,8 @@ WARM_SIGNAL = "agent_connections.warm"
INCARNATION_BATCH = 32
# How often beats forwarded to their holder go out, at most. A client beats
-# every two seconds and lapses after six, so this leaves the holder plenty.
+# every HEARTBEAT_INTERVAL_SECONDS and lapses after HEARTBEAT_TTL_SECONDS, so
+# this leaves the holder plenty.
BEAT_FORWARD_SECONDS = 0.5
# How often every lease is checked, so a process that died stops counting as
diff --git a/core/switch_core/bridges/agent/protocol/liveness.py b/core/switch_core/bridges/agent/protocol/liveness.py
index e5e59d2c0..fde2d916b 100644
--- a/core/switch_core/bridges/agent/protocol/liveness.py
+++ b/core/switch_core/bridges/agent/protocol/liveness.py
@@ -13,8 +13,17 @@ from typing import Literal
# Clients tick every HEARTBEAT_INTERVAL_SECONDS; a connection is declared dead
# once nothing has arrived for HEARTBEAT_TTL_SECONDS. One mechanism replaces
# /connection/renew, /watch/heartbeat and /leases/renew.
-HEARTBEAT_INTERVAL_SECONDS = 2.0
-HEARTBEAT_TTL_SECONDS = 6.0
+#
+# The server pings each agent's socket, and each controller's
+# (`controller_routes.py`), every HEARTBEAT_INTERVAL_SECONDS, and the pong is
+# the beat. The interval is also advertised on every open (`stream.py`). Ten
+# seconds rather than two: a beat on the socket costs no database query, so a
+# slower ping is fewer wakeups and fewer forwarded beats between processes for
+# the same answer. An old event-stream client that still beats over
+# `POST /connection/beat` keeps its own pace. The TTL stays three intervals,
+# so one lost tick never costs a connection.
+HEARTBEAT_INTERVAL_SECONDS = 10.0
+HEARTBEAT_TTL_SECONDS = 30.0
CloseCode = Literal["taken_over", "heartbeat_lapsed", "closed", "launch_superseded"]
diff --git a/core/switch_core/bridges/agent/protocol/stream.py b/core/switch_core/bridges/agent/protocol/stream.py
index 66bbec376..aeacaf312 100644
--- a/core/switch_core/bridges/agent/protocol/stream.py
+++ b/core/switch_core/bridges/agent/protocol/stream.py
@@ -31,6 +31,7 @@ from typing import TYPE_CHECKING, Any
from switch_core.bridges.agent.protocol.agent_connections import (
APPROVAL_OUTCOME_PROTOCOL_REVISION,
+ HEARTBEAT_INTERVAL_SECONDS,
HEARTBEAT_LAPSED,
PROTOCOL_VERSION,
TAKEN_OVER,
@@ -154,7 +155,7 @@ def _connection_state(conn: AgentConnection) -> dict[str, Any]:
"cursor": conn.cursor,
"position": dict(conn.position),
"protocol": PROTOCOL_VERSION,
- "heartbeat_interval_seconds": 2.0,
+ "heartbeat_interval_seconds": HEARTBEAT_INTERVAL_SECONDS,
"server": server_declaration("agent-protocol"),
# Echoed back so a client can see what the server understood it to
# have said — a declaration that silently failed to parse is worse
diff --git a/core/switch_core/config.py b/core/switch_core/config.py
index b263c3400..69dd9c02e 100644
--- a/core/switch_core/config.py
+++ b/core/switch_core/config.py
@@ -722,11 +722,12 @@ class SwitchConfig(BaseSettings):
template_max_bytes: int = 1024 * 1024
# Every authenticated agent request resolves its bearer token against the
- # database before the handler runs, and each live agent connection beats
- # every 2s, so the pool is sized against connection count rather than
- # human traffic: a fleet of N connections costs roughly N/2 checkouts per
- # second. Exceeding the pool does not shed load, it queues, and a queued
- # heartbeat that misses HEARTBEAT_TTL_SECONDS costs the connection.
+ # database before the handler runs, so the pool is sized against connection
+ # count rather than human traffic. Agent and controller heartbeats ride
+ # their sockets, authenticated once, but an old event-stream client still
+ # beats over HTTP, and every tool call an agent makes is a request. Exceeding
+ # the pool does not shed load, it queues, and a queued request that the
+ # connection's liveness depends on can still cost it the connection.
db_pool_size: int = 30
db_max_overflow: int = 10
db_pool_recycle: int = 1800
@@ -852,11 +853,26 @@ class SwitchConfig(BaseSettings):
@model_validator(mode="after")
def _validate_agent_auth_cache(self) -> "SwitchConfig":
+ # Imported here so loading config does not pull in the database models.
+ from switch_core.bridges.agent.api_key_cache import EXPIRY_JITTER
+ from switch_core.bridges.agent.protocol.agent_connections import (
+ HEARTBEAT_TTL_SECONDS,
+ )
+
if self.agent_auth_cache_ttl_seconds < 0:
raise ValueError(
"AGENT_AUTH_CACHE_TTL_SECONDS must not be negative (0 disables "
f"the cache), got {self.agent_auth_cache_ttl_seconds!r}."
)
+ if (
+ self.agent_auth_cache_ttl_seconds * (1 + EXPIRY_JITTER)
+ >= HEARTBEAT_TTL_SECONDS
+ ):
+ raise ValueError(
+ f"AGENT_AUTH_CACHE_TTL_SECONDS, with its {EXPIRY_JITTER:.0%} "
+ "jitter, must stay below the agent heartbeat TTL of "
+ f"{HEARTBEAT_TTL_SECONDS}s, got {self.agent_auth_cache_ttl_seconds!r}."
+ )
if self.agent_auth_cache_max_entries < 1:
raise ValueError(
"AGENT_AUTH_CACHE_MAX_ENTRIES must be at least 1, got "
diff --git a/core/switch_core/management/controller_routes.py b/core/switch_core/management/controller_routes.py
index ebd5e93f6..aa5da5d8e 100644
--- a/core/switch_core/management/controller_routes.py
+++ b/core/switch_core/management/controller_routes.py
@@ -468,8 +468,8 @@ async def record_beat(
) -> list[str]:
"""Keep the controller, and so every agent on it, live; confirm cursors.
- A pong on the socket, every heartbeat interval; six seconds without one
- and its agents are not live. The cursors confirmed here are where a
+ A pong on the socket, every heartbeat interval; `HEARTBEAT_TTL_SECONDS`
+ without one and its agents are not live. The cursors confirmed here are where a
stream reattached to this connection resumes each agent. Raises the
presence's refusal: `taken_over` once another connection has replaced
this one, terminal for the client, and `no_stream` or
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_connections.py b/core/tests/switch_core/bridges/agent/protocol/test_connections.py
index 53a779657..0dc09a2c1 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_connections.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_connections.py
@@ -8,6 +8,7 @@ import pytest
from switch_core.bridges.agent.protocol.agent_connections import (
FENCED_PROTOCOL_REVISION,
+ HEARTBEAT_INTERVAL_SECONDS,
HEARTBEAT_LAPSED,
HEARTBEAT_TTL_SECONDS,
MAX_CONNECTIONS_PER_AGENT,
@@ -634,3 +635,8 @@ def test_the_one_number_a_client_is_told_is_a_ticket_only_where_it_has_to_be() -
assert holds_position.cursor == 7
assert one_number.cursor == 2**48 + 5
assert own_events_only.cursor == 7
+
+
+def test_the_ttl_survives_two_lost_beats() -> None:
+ """One dropped or late beat must never cost a connection."""
+ assert HEARTBEAT_TTL_SECONDS >= 3 * HEARTBEAT_INTERVAL_SECONDS
diff --git a/core/tests/switch_core/bridges/agent/test_api_key_cache.py b/core/tests/switch_core/bridges/agent/test_api_key_cache.py
index 61b6487ff..cb42e560d 100644
--- a/core/tests/switch_core/bridges/agent/test_api_key_cache.py
+++ b/core/tests/switch_core/bridges/agent/test_api_key_cache.py
@@ -15,7 +15,7 @@ from typing import Any
import pytest
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
-from switch_core.bridges.agent.api_key_cache import ApiKeyCache
+from switch_core.bridges.agent.api_key_cache import EXPIRY_JITTER, ApiKeyCache
from switch_core.bridges.agent.auth import BearerAuthMiddleware
from switch_core.bridges.agent.protocol.agent_connections import (
HEARTBEAT_TTL_SECONDS,
@@ -189,6 +189,9 @@ class TestCacheUnit:
monkeypatch.setattr(
"switch_core.bridges.agent.api_key_cache.time.monotonic", lambda: now[0]
)
+ monkeypatch.setattr(
+ "switch_core.bridges.agent.api_key_cache.random.uniform", lambda a, b: 1.0
+ )
cache = ApiKeyCache(ttl_seconds=5, max_entries=8)
cache.put("h", SimpleNamespace(), SimpleNamespace(id="a")) # type: ignore[arg-type]
@@ -197,6 +200,30 @@ class TestCacheUnit:
now[0] = 1005.0
assert cache.get("h") is None
+ def test_expiries_are_spread_so_a_restart_does_not_repeat_its_herd(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ now = [1000.0]
+ monkeypatch.setattr(
+ "switch_core.bridges.agent.api_key_cache.time.monotonic", lambda: now[0]
+ )
+ cache = ApiKeyCache(ttl_seconds=5, max_entries=500)
+ for index in range(200):
+ cache.put(f"h{index}", SimpleNamespace(), SimpleNamespace(id="a")) # type: ignore[arg-type]
+
+ expiries = [expires for expires, _, _ in cache._entries.values()]
+ assert min(expiries) >= 1000.0 + 5 * (1 - EXPIRY_JITTER)
+ assert max(expiries) <= 1000.0 + 5 * (1 + EXPIRY_JITTER)
+ # Spread, not stacked: with a fixed lifetime every entry shares one.
+ assert len(set(expiries)) > 100
+
+ def test_the_jitter_counts_against_the_heartbeat_ttl(self) -> None:
+ """The longest an entry can live is what must stay below the TTL."""
+ just_too_long = HEARTBEAT_TTL_SECONDS / (1 + EXPIRY_JITTER)
+ with pytest.raises(ValueError, match="heartbeat TTL"):
+ ApiKeyCache(ttl_seconds=just_too_long, max_entries=8)
+ ApiKeyCache(ttl_seconds=just_too_long * 0.99, max_entries=8)
+
def test_invalidate_agent_drops_every_token_for_that_agent(self) -> None:
cache = ApiKeyCache(ttl_seconds=5, max_entries=8)
cache.put("old", SimpleNamespace(), SimpleNamespace(id="a1")) # type: ignore[arg-type]
@@ -467,6 +494,14 @@ class TestAuthCacheConfig:
with pytest.raises(ValueError, match="AGENT_AUTH_CACHE_TTL_SECONDS"):
_config(agent_auth_cache_ttl_seconds=-1)
+ def test_a_ttl_whose_jitter_reaches_the_heartbeat_ttl_fails_at_startup(
+ self,
+ ) -> None:
+ longest = HEARTBEAT_TTL_SECONDS / (1 + EXPIRY_JITTER)
+ assert _config(agent_auth_cache_ttl_seconds=longest - 0.1)
+ with pytest.raises(ValueError, match="AGENT_AUTH_CACHE_TTL_SECONDS"):
+ _config(agent_auth_cache_ttl_seconds=longest)
+
def test_an_unbounded_cache_fails_at_startup(self) -> None:
with pytest.raises(ValueError, match="AGENT_AUTH_CACHE_MAX_ENTRIES"):
_config(agent_auth_cache_max_entries=0)
diff --git a/core/tests/switch_core/management/test_stream.py b/core/tests/switch_core/management/test_stream.py
index 0abd54270..7c3471caa 100644
--- a/core/tests/switch_core/management/test_stream.py
+++ b/core/tests/switch_core/management/test_stream.py
@@ -15,7 +15,10 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.bridges.agent.commands import room_control_frame
from switch_core.bridges.agent.protocol.agent_detail import assemble_agent_detail
-from switch_core.bridges.agent.protocol.liveness import HEARTBEAT_TTL_SECONDS
+from switch_core.bridges.agent.protocol.liveness import (
+ HEARTBEAT_INTERVAL_SECONDS,
+ HEARTBEAT_TTL_SECONDS,
+)
from switch_core.bridges.agent.protocol.presence import (
agents_present_in,
rooms_occupied,
@@ -140,7 +143,7 @@ class TestOpening:
room_id = await add_room(harness.session_factory, agent_id)
opened = await open_connection(client, controller)
assert opened["agents"] == [agent_id]
- assert opened["heartbeat_interval_s"] == 2.0
+ assert opened["heartbeat_interval_s"] == HEARTBEAT_INTERVAL_SECONDS
stream = await open_stream(harness, controller, opened)
(state, attached) = await take(stream, 2)
@@ -154,7 +157,7 @@ class TestOpening:
"report_within_s": 60,
"connection_id": opened["connection_id"],
"generation": opened["generation"],
- "heartbeat_interval_s": 2.0,
+ "heartbeat_interval_s": HEARTBEAT_INTERVAL_SECONDS,
},
)
assert attached == (