Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Beat every 10 seconds instead of 2, and spread the auth cache expiries (CHOO-3013) (#643)
Abel Dantasc33b46980f6c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ac9c02ea0..11559e993 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -111,6 +111,14 @@ version of their own to them without also giving them a release of their own.compares it only when someone signs in.#### Changed+- **Agents and controllers are pinged every 10 seconds instead of every 2.**+ A connection is now declared dead after 30 seconds without a beat, up from+ 6, so a client that goes away holds its room slot and shows online for up to+ 30 seconds. The interval is advertised on every open, so clients follow it+ without an update. The agent auth cache now gives each entry a random 80 to+ 120% of `AGENT_AUTH_CACHE_TTL_SECONDS`, so entries filled together after a+ restart do not all expire together, and that setting must stay below 25+ seconds (the 30 second heartbeat TTL, less the jitter) or startup refuses it.- **Following the conversation is described the same way everywhere, and thechart can turn it on.** The feature added in 0.31.0 is off unless`follow_the_conversation` is among the enabled feature flagsdiff --git a/core/switch_core/bridges/agent/api/session_reporter.py b/core/switch_core/bridges/agent/api/session_reporter.pyindex 861feb959..ddb3feffc 100644--- a/core/switch_core/bridges/agent/api/session_reporter.py+++ b/core/switch_core/bridges/agent/api/session_reporter.py@@ -2,8 +2,8 @@Two ordinary things make an agent hold a succession of short connections whilebeing continuously present: a stream rejected at the room claim and retried,-and a stream whose heartbeat lapsed (six seconds) closing itself so the client-reopens. Per connection, both read as a session every few seconds.+and a stream whose heartbeat lapsed (`HEARTBEAT_TTL_SECONDS`) closing itself so+the client reopens. Per connection, both read as a session every few seconds.So this reports on the **agent**: a session begins when one that had no liveconnection acquires its first, and ends only once it has had none fordiff --git a/core/switch_core/bridges/agent/api_key_cache.py b/core/switch_core/bridges/agent/api_key_cache.pyindex 4cc5efc35..80e776e98 100644--- a/core/switch_core/bridges/agent/api_key_cache.py+++ b/core/switch_core/bridges/agent/api_key_cache.py@@ -1,19 +1,25 @@from __future__ import annotations+import randomimport timefrom collections import OrderedDictfrom switch_core.bridges.agent.protocol.agent_connections import HEARTBEAT_TTL_SECONDSfrom switch_core.db.models import Agent, ApiKey+# Each entry lives a random 80–120% of the TTL. A restart empties the cache+# for every agent at once, and with a fixed TTL every entry then expires in+# the same instant too, so the whole fleet hits the database together again on+# every cycle. Spreading the expiries breaks that herd up after the first one.+EXPIRY_JITTER = 0.2+class ApiKeyCache:"""A short-lived memo of resolved agent bearer tokens.Every authenticated request resolves its token against Postgres before the- handler runs, and an agent connection beats every couple of seconds, so- that one lookup is the bulk of the traffic through the connection pool —- including the heartbeats whose deadline the pool is capable of missing.+ handler runs, so that one lookup is the bulk of the traffic an agent's+ requests put through the connection pool.What it deliberately does not do:@@ -35,10 +41,11 @@ class ApiKeyCache:def __init__(self, *, ttl_seconds: float, max_entries: int) -> None:if ttl_seconds < 0:raise ValueError(f"ttl_seconds must not be negative, got {ttl_seconds!r}")- if ttl_seconds >= HEARTBEAT_TTL_SECONDS:+ if ttl_seconds * (1 + EXPIRY_JITTER) >= HEARTBEAT_TTL_SECONDS:raise ValueError(- f"ttl_seconds must stay below the agent heartbeat TTL of "- f"{HEARTBEAT_TTL_SECONDS}s, got {ttl_seconds!r}"+ f"ttl_seconds, with its {EXPIRY_JITTER:.0%} jitter, must stay "+ f"below the agent heartbeat TTL of {HEARTBEAT_TTL_SECONDS}s, "+ f"got {ttl_seconds!r}")if max_entries < 1:raise ValueError(f"max_entries must be at least 1, got {max_entries!r}")@@ -64,7 +71,8 @@ class ApiKeyCache:def put(self, token_hash: str, api_key: ApiKey, agent: Agent) -> None:if not self.enabled:return- self._entries[token_hash] = (time.monotonic() + self._ttl, api_key, agent)+ lifetime = self._ttl * random.uniform(1 - EXPIRY_JITTER, 1 + EXPIRY_JITTER)+ self._entries[token_hash] = (time.monotonic() + lifetime, api_key, agent)self._entries.move_to_end(token_hash)while len(self._entries) > self._max_entries:self._entries.popitem(last=False)diff --git a/core/switch_core/bridges/agent/controller_auth_cache.py b/core/switch_core/bridges/agent/controller_auth_cache.pyindex 27d828c8c..00960ce59 100644--- a/core/switch_core/bridges/agent/controller_auth_cache.py+++ b/core/switch_core/bridges/agent/controller_auth_cache.py@@ -17,9 +17,9 @@ class ControllerAuthCache:A controller token is a signed JWT, so verifying it needs no database. Butevery request it authenticates also re-reads its controller row, to refusea revoked controller while its token is unexpired, and a request acting as- one of its agents reads that agent's row too. A controller beats every two- seconds and relays every call its agents' sessions make, so those reads are- most of what it costs the connection pool.+ one of its agents reads that agent's row too. A controller relays every+ call its agents' sessions make, so those reads are most of what it costs+ the connection pool.Kept apart from :class:`ApiKeyCache`, which maps one bearer token to oneagent: a controller token stands for many agents, and its authorization isdiff --git a/core/switch_core/bridges/agent/protocol/connection_rows.py b/core/switch_core/bridges/agent/protocol/connection_rows.pyindex 96201f529..c978ca245 100644--- a/core/switch_core/bridges/agent/protocol/connection_rows.py+++ b/core/switch_core/bridges/agent/protocol/connection_rows.py@@ -99,7 +99,8 @@ WARM_SIGNAL = "agent_connections.warm"INCARNATION_BATCH = 32# How often beats forwarded to their holder go out, at most. A client beats-# every two seconds and lapses after six, so this leaves the holder plenty.+# every HEARTBEAT_INTERVAL_SECONDS and lapses after HEARTBEAT_TTL_SECONDS, so+# this leaves the holder plenty.BEAT_FORWARD_SECONDS = 0.5# How often every lease is checked, so a process that died stops counting asdiff --git a/core/switch_core/bridges/agent/protocol/liveness.py b/core/switch_core/bridges/agent/protocol/liveness.pyindex e5e59d2c0..fde2d916b 100644--- a/core/switch_core/bridges/agent/protocol/liveness.py+++ b/core/switch_core/bridges/agent/protocol/liveness.py@@ -13,8 +13,17 @@ from typing import Literal# Clients tick every HEARTBEAT_INTERVAL_SECONDS; a connection is declared dead# once nothing has arrived for HEARTBEAT_TTL_SECONDS. One mechanism replaces# /connection/renew, /watch/heartbeat and /leases/renew.-HEARTBEAT_INTERVAL_SECONDS = 2.0-HEARTBEAT_TTL_SECONDS = 6.0+#+# The server pings each agent's socket, and each controller's+# (`controller_routes.py`), every HEARTBEAT_INTERVAL_SECONDS, and the pong is+# the beat. The interval is also advertised on every open (`stream.py`). Ten+# seconds rather than two: a beat on the socket costs no database query, so a+# slower ping is fewer wakeups and fewer forwarded beats between processes for+# the same answer. An old event-stream client that still beats over+# `POST /connection/beat` keeps its own pace. The TTL stays three intervals,+# so one lost tick never costs a connection.+HEARTBEAT_INTERVAL_SECONDS = 10.0+HEARTBEAT_TTL_SECONDS = 30.0CloseCode = Literal["taken_over", "heartbeat_lapsed", "closed", "launch_superseded"]diff --git a/core/switch_core/bridges/agent/protocol/stream.py b/core/switch_core/bridges/agent/protocol/stream.pyindex 66bbec376..aeacaf312 100644--- a/core/switch_core/bridges/agent/protocol/stream.py+++ b/core/switch_core/bridges/agent/protocol/stream.py@@ -31,6 +31,7 @@ from typing import TYPE_CHECKING, Anyfrom switch_core.bridges.agent.protocol.agent_connections import (APPROVAL_OUTCOME_PROTOCOL_REVISION,+ HEARTBEAT_INTERVAL_SECONDS,HEARTBEAT_LAPSED,PROTOCOL_VERSION,TAKEN_OVER,@@ -154,7 +155,7 @@ def _connection_state(conn: AgentConnection) -> dict[str, Any]:"cursor": conn.cursor,"position": dict(conn.position),"protocol": PROTOCOL_VERSION,- "heartbeat_interval_seconds": 2.0,+ "heartbeat_interval_seconds": HEARTBEAT_INTERVAL_SECONDS,"server": server_declaration("agent-protocol"),# Echoed back so a client can see what the server understood it to# have said — a declaration that silently failed to parse is worsediff --git a/core/switch_core/config.py b/core/switch_core/config.pyindex b263c3400..69dd9c02e 100644--- a/core/switch_core/config.py+++ b/core/switch_core/config.py@@ -722,11 +722,12 @@ class SwitchConfig(BaseSettings):template_max_bytes: int = 1024 * 1024# Every authenticated agent request resolves its bearer token against the- # database before the handler runs, and each live agent connection beats- # every 2s, so the pool is sized against connection count rather than- # human traffic: a fleet of N connections costs roughly N/2 checkouts per- # second. Exceeding the pool does not shed load, it queues, and a queued- # heartbeat that misses HEARTBEAT_TTL_SECONDS costs the connection.+ # database before the handler runs, so the pool is sized against connection+ # count rather than human traffic. Agent and controller heartbeats ride+ # their sockets, authenticated once, but an old event-stream client still+ # beats over HTTP, and every tool call an agent makes is a request. Exceeding+ # the pool does not shed load, it queues, and a queued request that the+ # connection's liveness depends on can still cost it the connection.db_pool_size: int = 30db_max_overflow: int = 10db_pool_recycle: int = 1800@@ -852,11 +853,26 @@ class SwitchConfig(BaseSettings):@model_validator(mode="after")def _validate_agent_auth_cache(self) -> "SwitchConfig":+ # Imported here so loading config does not pull in the database models.+ from switch_core.bridges.agent.api_key_cache import EXPIRY_JITTER+ from switch_core.bridges.agent.protocol.agent_connections import (+ HEARTBEAT_TTL_SECONDS,+ )+if self.agent_auth_cache_ttl_seconds < 0:raise ValueError("AGENT_AUTH_CACHE_TTL_SECONDS must not be negative (0 disables "f"the cache), got {self.agent_auth_cache_ttl_seconds!r}.")+ if (+ self.agent_auth_cache_ttl_seconds * (1 + EXPIRY_JITTER)+ >= HEARTBEAT_TTL_SECONDS+ ):+ raise ValueError(+ f"AGENT_AUTH_CACHE_TTL_SECONDS, with its {EXPIRY_JITTER:.0%} "+ "jitter, must stay below the agent heartbeat TTL of "+ f"{HEARTBEAT_TTL_SECONDS}s, got {self.agent_auth_cache_ttl_seconds!r}."+ )if self.agent_auth_cache_max_entries < 1:raise ValueError("AGENT_AUTH_CACHE_MAX_ENTRIES must be at least 1, got "diff --git a/core/switch_core/management/controller_routes.py b/core/switch_core/management/controller_routes.pyindex ebd5e93f6..aa5da5d8e 100644--- a/core/switch_core/management/controller_routes.py+++ b/core/switch_core/management/controller_routes.py@@ -468,8 +468,8 @@ async def record_beat() -> list[str]:"""Keep the controller, and so every agent on it, live; confirm cursors.- A pong on the socket, every heartbeat interval; six seconds without one- and its agents are not live. The cursors confirmed here are where a+ A pong on the socket, every heartbeat interval; `HEARTBEAT_TTL_SECONDS`+ without one and its agents are not live. The cursors confirmed here are where astream reattached to this connection resumes each agent. Raises thepresence's refusal: `taken_over` once another connection has replacedthis one, terminal for the client, and `no_stream` ordiff --git a/core/tests/switch_core/bridges/agent/protocol/test_connections.py b/core/tests/switch_core/bridges/agent/protocol/test_connections.pyindex 53a779657..0dc09a2c1 100644--- a/core/tests/switch_core/bridges/agent/protocol/test_connections.py+++ b/core/tests/switch_core/bridges/agent/protocol/test_connections.py@@ -8,6 +8,7 @@ import pytestfrom switch_core.bridges.agent.protocol.agent_connections import (FENCED_PROTOCOL_REVISION,+ HEARTBEAT_INTERVAL_SECONDS,HEARTBEAT_LAPSED,HEARTBEAT_TTL_SECONDS,MAX_CONNECTIONS_PER_AGENT,@@ -634,3 +635,8 @@ def test_the_one_number_a_client_is_told_is_a_ticket_only_where_it_has_to_be() -assert holds_position.cursor == 7assert one_number.cursor == 2**48 + 5assert own_events_only.cursor == 7+++def test_the_ttl_survives_two_lost_beats() -> None:+ """One dropped or late beat must never cost a connection."""+ assert HEARTBEAT_TTL_SECONDS >= 3 * HEARTBEAT_INTERVAL_SECONDSdiff --git a/core/tests/switch_core/bridges/agent/test_api_key_cache.py b/core/tests/switch_core/bridges/agent/test_api_key_cache.pyindex 61b6487ff..cb42e560d 100644--- a/core/tests/switch_core/bridges/agent/test_api_key_cache.py+++ b/core/tests/switch_core/bridges/agent/test_api_key_cache.py@@ -15,7 +15,7 @@ from typing import Anyimport pytestfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker-from switch_core.bridges.agent.api_key_cache import ApiKeyCache+from switch_core.bridges.agent.api_key_cache import EXPIRY_JITTER, ApiKeyCachefrom switch_core.bridges.agent.auth import BearerAuthMiddlewarefrom switch_core.bridges.agent.protocol.agent_connections import (HEARTBEAT_TTL_SECONDS,@@ -189,6 +189,9 @@ class TestCacheUnit:monkeypatch.setattr("switch_core.bridges.agent.api_key_cache.time.monotonic", lambda: now[0])+ monkeypatch.setattr(+ "switch_core.bridges.agent.api_key_cache.random.uniform", lambda a, b: 1.0+ )cache = ApiKeyCache(ttl_seconds=5, max_entries=8)cache.put("h", SimpleNamespace(), SimpleNamespace(id="a")) # type: ignore[arg-type]@@ -197,6 +200,30 @@ class TestCacheUnit:now[0] = 1005.0assert cache.get("h") is None+ def test_expiries_are_spread_so_a_restart_does_not_repeat_its_herd(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ now = [1000.0]+ monkeypatch.setattr(+ "switch_core.bridges.agent.api_key_cache.time.monotonic", lambda: now[0]+ )+ cache = ApiKeyCache(ttl_seconds=5, max_entries=500)+ for index in range(200):+ cache.put(f"h{index}", SimpleNamespace(), SimpleNamespace(id="a")) # type: ignore[arg-type]++ expiries = [expires for expires, _, _ in cache._entries.values()]+ assert min(expiries) >= 1000.0 + 5 * (1 - EXPIRY_JITTER)+ assert max(expiries) <= 1000.0 + 5 * (1 + EXPIRY_JITTER)+ # Spread, not stacked: with a fixed lifetime every entry shares one.+ assert len(set(expiries)) > 100++ def test_the_jitter_counts_against_the_heartbeat_ttl(self) -> None:+ """The longest an entry can live is what must stay below the TTL."""+ just_too_long = HEARTBEAT_TTL_SECONDS / (1 + EXPIRY_JITTER)+ with pytest.raises(ValueError, match="heartbeat TTL"):+ ApiKeyCache(ttl_seconds=just_too_long, max_entries=8)+ ApiKeyCache(ttl_seconds=just_too_long * 0.99, max_entries=8)+def test_invalidate_agent_drops_every_token_for_that_agent(self) -> None:cache = ApiKeyCache(ttl_seconds=5, max_entries=8)cache.put("old", SimpleNamespace(), SimpleNamespace(id="a1")) # type: ignore[arg-type]@@ -467,6 +494,14 @@ class TestAuthCacheConfig:with pytest.raises(ValueError, match="AGENT_AUTH_CACHE_TTL_SECONDS"):_config(agent_auth_cache_ttl_seconds=-1)+ def test_a_ttl_whose_jitter_reaches_the_heartbeat_ttl_fails_at_startup(+ self,+ ) -> None:+ longest = HEARTBEAT_TTL_SECONDS / (1 + EXPIRY_JITTER)+ assert _config(agent_auth_cache_ttl_seconds=longest - 0.1)+ with pytest.raises(ValueError, match="AGENT_AUTH_CACHE_TTL_SECONDS"):+ _config(agent_auth_cache_ttl_seconds=longest)+def test_an_unbounded_cache_fails_at_startup(self) -> None:with pytest.raises(ValueError, match="AGENT_AUTH_CACHE_MAX_ENTRIES"):_config(agent_auth_cache_max_entries=0)diff --git a/core/tests/switch_core/management/test_stream.py b/core/tests/switch_core/management/test_stream.pyindex 0abd54270..7c3471caa 100644--- a/core/tests/switch_core/management/test_stream.py+++ b/core/tests/switch_core/management/test_stream.py@@ -15,7 +15,10 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.bridges.agent.commands import room_control_framefrom switch_core.bridges.agent.protocol.agent_detail import assemble_agent_detail-from switch_core.bridges.agent.protocol.liveness import HEARTBEAT_TTL_SECONDS+from switch_core.bridges.agent.protocol.liveness import (+ HEARTBEAT_INTERVAL_SECONDS,+ HEARTBEAT_TTL_SECONDS,+)from switch_core.bridges.agent.protocol.presence import (agents_present_in,rooms_occupied,@@ -140,7 +143,7 @@ class TestOpening:room_id = await add_room(harness.session_factory, agent_id)opened = await open_connection(client, controller)assert opened["agents"] == [agent_id]- assert opened["heartbeat_interval_s"] == 2.0+ assert opened["heartbeat_interval_s"] == HEARTBEAT_INTERVAL_SECONDSstream = await open_stream(harness, controller, opened)(state, attached) = await take(stream, 2)@@ -154,7 +157,7 @@ class TestOpening:"report_within_s": 60,"connection_id": opened["connection_id"],"generation": opened["generation"],- "heartbeat_interval_s": 2.0,+ "heartbeat_interval_s": HEARTBEAT_INTERVAL_SECONDS,},)assert attached == (