Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): invalidate the copilot integration-token cache across processes (#14320)

Reinier van der Leerc35bb102bbca
diff --git a/autogpt_platform/backend/backend/copilot/integration_creds.py b/autogpt_platform/backend/backend/copilot/integration_creds.py
index bd0c04a94b..0db4b617d9 100644
--- a/autogpt_platform/backend/backend/copilot/integration_creds.py
+++ b/autogpt_platform/backend/backend/copilot/integration_creds.py
@@ -5,7 +5,7 @@ Provides token retrieval for connected integrations so that copilot tools
hitting the database on every command.
Cache semantics (handled automatically by TTLCache):
-- Token found → cached for _TOKEN_CACHE_TTL (5 min). Avoids repeated DB hits
+- Token found → cached for _TOKEN_CACHE_TTL (60 s). Avoids repeated DB hits
for users who have credentials and are running many bash commands.
- No credentials found → cached for _NULL_CACHE_TTL (60 s). Avoids a DB hit
on every E2B command for users who haven't connected an account yet, while
@@ -14,24 +14,27 @@ Cache semantics (handled automatically by TTLCache):
Both caches are bounded to _CACHE_MAX_SIZE entries; cachetools evicts the
least-recently-used entry when the limit is reached.
-Multi-worker note: both caches are in-process only. Each worker/replica
-maintains its own independent cache, so a credential fetch may be duplicated
-across processes. This is acceptable for the current goal (reduce DB hits per
-session per-process), but if cache efficiency across replicas becomes important
-a shared cache (e.g. Redis) should be used instead.
+Multi-worker note: the cached values are per-process, but invalidation is not.
+The API server and the copilot executor each hold their own copy of these
+caches, so a write in one cannot evict the other's; a subscription to the Redis
+creds-changed bus does. See ``_ensure_cache_invalidation_listener``.
"""
+import asyncio
import logging
+import threading
from typing import cast
from cachetools import TTLCache
from backend.copilot.providers import SUPPORTED_PROVIDERS
from backend.data.model import APIKeyCredentials, OAuth2Credentials
+from backend.integrations.creds_events import listen_creds_changed
from backend.integrations.creds_manager import (
IntegrationCredentialsManager,
register_creds_changed_hook,
)
+from backend.util.retry import continuous_retry
logger = logging.getLogger(__name__)
@@ -40,32 +43,70 @@ PROVIDER_ENV_VARS: dict[str, list[str]] = {
slug: entry["env_vars"] for slug, entry in SUPPORTED_PROVIDERS.items()
}
-_TOKEN_CACHE_TTL = 300.0 # seconds — for found tokens
+# 60 s, not the original 300 s: the pub/sub invalidation below is best-effort
+# (a Redis blip drops the message), so the TTL is the floor on how long a stale
+# token can survive when it fails. Five minutes was long enough for AutoPilot
+# to verify a re-authorization against the provider and report it as failed.
+_TOKEN_CACHE_TTL = 60.0 # seconds — for found tokens
_NULL_CACHE_TTL = 60.0 # seconds — for "not connected" results
_CACHE_MAX_SIZE = 10_000
+
+# Sentinel so ``pop`` keeps ``Cache.pop``'s "raise without a default" contract.
+_MISSING = object()
+
+
+class _LockedTTLCache(TTLCache):
+ """TTLCache with a lock: the invalidation listener evicts entries from its
+ own thread while copilot workers read them from theirs."""
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._lock = threading.RLock()
+
+ def __getitem__(self, key):
+ with self._lock:
+ return super().__getitem__(key)
+
+ def __setitem__(self, key, value):
+ with self._lock:
+ super().__setitem__(key, value)
+
+ def __delitem__(self, key):
+ with self._lock:
+ super().__delitem__(key)
+
+ def __contains__(self, key):
+ with self._lock:
+ return super().__contains__(key)
+
+ def pop(self, key, default=_MISSING):
+ with self._lock:
+ if default is _MISSING:
+ return super().pop(key)
+ return super().pop(key, default)
+
+ def popitem(self):
+ with self._lock:
+ return super().popitem()
+
+
# (user_id, provider) → token string. TTLCache handles expiry + eviction.
-# Thread-safety note: TTLCache is NOT thread-safe, but that is acceptable here
-# because all callers (get_provider_token, invalidate_user_provider_cache) run
-# exclusively on the asyncio event loop. There are no await points between a
-# cache read and its corresponding write within any function, so no concurrent
-# coroutine can interleave. If ThreadPoolExecutor workers are ever added to
-# this path, a threading.RLock should be wrapped around these caches.
-_token_cache: TTLCache[tuple[str, str], str] = TTLCache(
+_token_cache: TTLCache[tuple[str, str], str] = _LockedTTLCache(
maxsize=_CACHE_MAX_SIZE, ttl=_TOKEN_CACHE_TTL
)
# Separate cache for "no credentials" results with a shorter TTL.
-_null_cache: TTLCache[tuple[str, str], bool] = TTLCache(
+_null_cache: TTLCache[tuple[str, str], bool] = _LockedTTLCache(
maxsize=_CACHE_MAX_SIZE, ttl=_NULL_CACHE_TTL
)
# GitHub user identity caches (keyed by user_id only, not provider tuple).
# Declared here so invalidate_user_provider_cache() can reference them.
_GH_IDENTITY_CACHE_TTL = 600.0 # 10 min — profile data rarely changes
-_gh_identity_cache: TTLCache[str, dict[str, str]] = TTLCache(
+_gh_identity_cache: TTLCache[str, dict[str, str]] = _LockedTTLCache(
maxsize=_CACHE_MAX_SIZE, ttl=_GH_IDENTITY_CACHE_TTL
)
-_gh_identity_null_cache: TTLCache[str, bool] = TTLCache(
+_gh_identity_null_cache: TTLCache[str, bool] = _LockedTTLCache(
maxsize=_CACHE_MAX_SIZE, ttl=_NULL_CACHE_TTL
)
@@ -90,10 +131,9 @@ def invalidate_user_provider_cache(user_id: str, provider: str) -> None:
_gh_identity_null_cache.pop(user_id, None)
-# Register this module's cache-bust function with the credentials manager so
-# that any create/update/delete operation immediately evicts stale cache
-# entries. This avoids a lazy import inside creds_manager and eliminates the
-# circular-import risk.
+# Same-process writes (a token refresh performed by this process) invalidate
+# through the hook, without a Redis round trip. Writes in other processes
+# arrive over the bus instead.
try:
register_creds_changed_hook(invalidate_user_provider_cache)
except RuntimeError:
@@ -109,11 +149,10 @@ async def get_provider_token(user_id: str, provider: str) -> str | None:
"""Return the user's access token for *provider*, or ``None`` if not connected.
OAuth2 tokens are preferred (refreshed if needed); API keys are the fallback.
- Found tokens are cached for _TOKEN_CACHE_TTL (5 min). "Not connected" results
- are cached for _NULL_CACHE_TTL (60 s) to avoid a DB hit on every bash_exec
- command for users who haven't connected yet, while still picking up a
- newly-connected account within one minute.
+ Both found tokens and "not connected" results are cached for 60 s, and a
+ credential write in any process evicts the entry before that lapses.
"""
+ _ensure_cache_invalidation_listener()
cache_key = (user_id, provider)
if cache_key in _null_cache:
@@ -162,6 +201,8 @@ async def get_provider_token(user_id: str, provider: str) -> str | None:
# preventing the LLM from receiving a non-functional token.
refresh_failed = True
continue
+ # A refresh here publishes, and this process's own listener may evict
+ # the entry just written; the cost is one extra lookup, not a leak.
_token_cache[cache_key] = token
return token
@@ -181,6 +222,36 @@ async def get_provider_token(user_id: str, provider: str) -> str | None:
return None
+def _ensure_cache_invalidation_listener() -> None:
+ """Subscribe this process to credential changes, once.
+
+ Started from the cache read path so that every process holding a cache
+ subscribes, and only those do — the in-process hook covers a write served
+ by this process, and this covers the ones served elsewhere.
+ """
+ global _listener_thread
+ with _listener_start_lock:
+ if _listener_thread is not None:
+ return
+ _listener_thread = threading.Thread(
+ target=lambda: asyncio.run(_consume_creds_changed_events()),
+ name="creds-cache-invalidation",
+ daemon=True,
+ )
+ _listener_thread.start()
+
+
+@continuous_retry(retry_delay=5.0)
+async def _consume_creds_changed_events() -> None:
+ async for event in listen_creds_changed():
+ invalidate_user_provider_cache(event.user_id, event.provider)
+ raise RuntimeError("creds-changed subscription ended; resubscribing")
+
+
+_listener_start_lock = threading.Lock()
+_listener_thread: threading.Thread | None = None
+
+
async def get_integration_env_vars(user_id: str) -> dict[str, str]:
"""Return env vars for all providers the user has connected.
diff --git a/autogpt_platform/backend/backend/copilot/integration_creds_test.py b/autogpt_platform/backend/backend/copilot/integration_creds_test.py
index 8f442b4e20..220be870e3 100644
--- a/autogpt_platform/backend/backend/copilot/integration_creds_test.py
+++ b/autogpt_platform/backend/backend/copilot/integration_creds_test.py
@@ -1,5 +1,7 @@
"""Tests for integration_creds — TTL cache and token lookup paths."""
+import asyncio
+import contextlib
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -9,6 +11,7 @@ from backend.copilot.integration_creds import (
_NULL_CACHE_TTL,
_TOKEN_CACHE_TTL,
PROVIDER_ENV_VARS,
+ _consume_creds_changed_events,
_gh_identity_cache,
_gh_identity_null_cache,
_null_cache,
@@ -18,6 +21,12 @@ from backend.copilot.integration_creds import (
invalidate_user_provider_cache,
)
from backend.data.model import APIKeyCredentials, OAuth2Credentials
+from backend.integrations.creds_events import CredentialsChangedEvent
+from backend.integrations.creds_manager import (
+ IntegrationCredentialsManager,
+ register_creds_changed_hook,
+ unregister_creds_changed_hook,
+)
_USER = "user-integration-creds-test"
_PROVIDER = "github"
@@ -46,6 +55,13 @@ def _make_oauth2_creds(token: str = "test-oauth-token") -> OAuth2Credentials:
)
[email protected](autouse=True)
+def no_listener_thread():
+ """Keep the lazy subscription from spawning a Redis-connecting thread."""
+ with patch("backend.copilot.integration_creds._ensure_cache_invalidation_listener"):
+ yield
+
+
@pytest.fixture(autouse=True)
def clear_caches():
"""Ensure clean caches before and after every test."""
@@ -212,11 +228,11 @@ class TestGetProviderToken:
assert (_USER, _PROVIDER) not in _null_cache
@pytest.mark.asyncio(loop_scope="session")
- async def test_null_cache_has_shorter_ttl_than_token_cache(self):
- """Verify the TTL constants are set correctly for each cache."""
+ async def test_token_cache_ttl_is_the_stale_token_ceiling(self):
+ """The TTL is the fallback when a pub/sub invalidation is lost."""
assert _null_cache.ttl == _NULL_CACHE_TTL
assert _token_cache.ttl == _TOKEN_CACHE_TTL
- assert _NULL_CACHE_TTL < _TOKEN_CACHE_TTL
+ assert _TOKEN_CACHE_TTL <= 60.0
class TestThreadSafetyLocks:
@@ -327,3 +343,97 @@ class TestGetIntegrationEnvVars:
result = await get_integration_env_vars(_USER)
assert result == {}
+
+
+class TestCacheInvalidationListener:
+ """The cross-process half: a write elsewhere reaches this process's cache."""
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_drops_exactly_the_affected_entry(self):
+ _token_cache[(_USER, "github")] = "stale"
+ _token_cache[(_USER, "notion")] = "untouched"
+ _token_cache[("other-user", "github")] = "untouched"
+
+ await _consume(CredentialsChangedEvent(user_id=_USER, provider="github"))
+
+ assert (_USER, "github") not in _token_cache
+ assert _token_cache[(_USER, "notion")] == "untouched"
+ assert _token_cache[("other-user", "github")] == "untouched"
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_resubscribes_when_the_stream_ends(self):
+ """A closed subscription must not leave the cache unwatched."""
+ streams = 0
+
+ async def stream():
+ nonlocal streams
+ streams += 1
+ if streams > 1:
+ raise asyncio.CancelledError
+ return
+ yield # pragma: no cover - makes this an async generator
+
+ with patch("backend.copilot.integration_creds.listen_creds_changed", stream):
+ with pytest.raises(asyncio.CancelledError):
+ await _consume_creds_changed_events()
+
+ assert streams == 2
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_upgrade_after_fetch_is_visible_without_waiting_for_the_ttl(self):
+ """OPEN-3472: a token fetched at T, upgraded at T+60s, was still served
+ from cache until the 300 s TTL lapsed at T+300s."""
+ manager = MagicMock()
+ manager.store.get_creds_by_provider = AsyncMock(
+ return_value=[_make_api_key_creds("old-scopes-token")]
+ )
+ with patch("backend.copilot.integration_creds._manager", manager):
+ assert await get_provider_token(_USER, _PROVIDER) == "old-scopes-token"
+
+ published = await _write_credential_as_another_process()
+ assert _token_cache[(_USER, _PROVIDER)] == "old-scopes-token"
+
+ for event in published:
+ await _consume(event)
+
+ manager.store.get_creds_by_provider = AsyncMock(
+ return_value=[_make_api_key_creds("new-scopes-token")]
+ )
+ with patch("backend.copilot.integration_creds._manager", manager):
+ assert await get_provider_token(_USER, _PROVIDER) == "new-scopes-token"
+
+
+async def _consume(*events: CredentialsChangedEvent) -> None:
+ """Run the listener over *events*, then stop it."""
+
+ async def stream():
+ for event in events:
+ yield event
+ raise asyncio.CancelledError
+
+ with patch("backend.copilot.integration_creds.listen_creds_changed", stream):
+ with pytest.raises(asyncio.CancelledError):
+ await _consume_creds_changed_events()
+
+
+async def _write_credential_as_another_process() -> list[CredentialsChangedEvent]:
+ """Perform a credential write with no in-process hook — the API process's
+ shape — and return what it broadcast."""
+ published: list[CredentialsChangedEvent] = []
+
+ async def capture(user_id: str, provider: str) -> None:
+ published.append(CredentialsChangedEvent(user_id=user_id, provider=provider))
+
+ manager = IntegrationCredentialsManager()
+ manager.store = MagicMock()
+ manager.store.update_creds = AsyncMock()
+ manager._locked = lambda *args, **kwargs: contextlib.nullcontext()
+
+ unregister_creds_changed_hook()
+ try:
+ with patch("backend.integrations.creds_manager.publish_creds_changed", capture):
+ await manager.update(_USER, _make_oauth2_creds("new-scopes-token"))
+ finally:
+ register_creds_changed_hook(invalidate_user_provider_cache)
+
+ return published
diff --git a/autogpt_platform/backend/backend/integrations/creds_events.py b/autogpt_platform/backend/backend/integrations/creds_events.py
new file mode 100644
index 0000000000..6701205fb2
--- /dev/null
+++ b/autogpt_platform/backend/backend/integrations/creds_events.py
@@ -0,0 +1,47 @@
+"""Cross-process broadcast of credential changes.
+
+The API server and the copilot executor are separate processes and each holds
+its own credential caches. A write serves one request in one of them, so an
+in-process callback only ever reaches that process's copy; every write is also
+published here so the other processes drop theirs.
+"""
+
+from collections.abc import AsyncGenerator
+
+from pydantic import BaseModel
+
+from backend.data.event_bus import AsyncRedisEventBus
+
+# Sharded pub/sub has no pattern-subscribe, and a subscriber cannot know which
+# users it holds tokens for — so everything goes on one channel.
+CREDS_CHANGED_CHANNEL = "all"
+
+
+class CredentialsChangedEvent(BaseModel):
+ user_id: str
+ provider: str
+
+
+async def publish_creds_changed(user_id: str, provider: str) -> None:
+ """Announce that *user_id*'s *provider* credentials were written."""
+ await _bus.publish_event(
+ CredentialsChangedEvent(user_id=user_id, provider=provider),
+ CREDS_CHANGED_CHANNEL,
+ )
+
+
+async def listen_creds_changed() -> AsyncGenerator[CredentialsChangedEvent, None]:
+ """Yield credential-change events published by any process."""
+ async for event in _bus.listen_events(CREDS_CHANGED_CHANNEL):
+ yield event
+
+
+class CredentialsChangedEventBus(AsyncRedisEventBus[CredentialsChangedEvent]):
+ Model = CredentialsChangedEvent
+
+ @property
+ def event_bus_name(self) -> str:
+ return "creds_changed"
+
+
+_bus = CredentialsChangedEventBus()
diff --git a/autogpt_platform/backend/backend/integrations/creds_events_test.py b/autogpt_platform/backend/backend/integrations/creds_events_test.py
new file mode 100644
index 0000000000..ef1999d2d5
--- /dev/null
+++ b/autogpt_platform/backend/backend/integrations/creds_events_test.py
@@ -0,0 +1,176 @@
+"""Tests for the credentials-changed bus — the cross-process half of invalidation."""
+
+import contextlib
+import logging
+import socket
+import subprocess
+import sys
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from pydantic import SecretStr
+
+from backend.data.model import OAuth2Credentials
+from backend.integrations.creds_events import (
+ CREDS_CHANGED_CHANNEL,
+ CredentialsChangedEvent,
+ listen_creds_changed,
+ publish_creds_changed,
+)
+from backend.integrations.creds_manager import IntegrationCredentialsManager
+
+_USER = "user-creds-events-test"
+
+
+async def test_publish_spublishes_on_the_broadcast_channel():
+ cluster = MagicMock()
+ cluster.execute_command = AsyncMock()
+
+ with patch("backend.data.event_bus.redis.get_redis_async", return_value=cluster):
+ await publish_creds_changed(_USER, "github")
+
+ command, channel, message = cluster.execute_command.await_args[0]
+ assert command == "SPUBLISH"
+ assert channel == f"creds_changed/{CREDS_CHANGED_CHANNEL}"
+ assert _USER in message and "github" in message
+
+
+async def test_a_dead_redis_does_not_fail_the_credential_write():
+ """A write is durable before it is announced, so a broadcast failure must
+ stay non-fatal — loudly, since the fallback is then the 60 s TTL."""
+ manager = IntegrationCredentialsManager()
+ manager.store = MagicMock()
+ manager.store.update_creds = AsyncMock()
+ manager._locked = lambda *args, **kwargs: contextlib.nullcontext()
+
+ records: list[logging.LogRecord] = []
+ handler = logging.Handler()
+ handler.emit = records.append
+ bus_logger = logging.getLogger("backend.data.event_bus")
+ bus_logger.addHandler(handler)
+ try:
+ with patch(
+ "backend.data.event_bus.redis.get_redis_async",
+ side_effect=ConnectionError("redis down"),
+ ):
+ await manager.update(_USER, _oauth_creds())
+ finally:
+ bus_logger.removeHandler(handler)
+
+ manager.store.update_creds.assert_awaited_once()
+ assert [r for r in records if r.levelno >= logging.ERROR and r.exc_info]
+
+
+async def test_credential_write_publishes_creds_changed():
+ """Every write through the manager announces itself, hook or no hook."""
+ manager = IntegrationCredentialsManager()
+ manager.store = MagicMock()
+ manager.store.update_creds = AsyncMock()
+ manager._locked = lambda *args, **kwargs: contextlib.nullcontext()
+
+ with patch(
+ "backend.integrations.creds_manager.publish_creds_changed",
+ new_callable=AsyncMock,
+ ) as publish:
+ await manager.update(_USER, _oauth_creds())
+
+ publish.assert_awaited_once_with(_USER, "github")
+
+
+ "module", ["backend.api.rest_api", "backend.copilot.executor.manager"]
+)
+def test_every_process_holding_the_cache_broadcasts_its_writes(module: str):
+ """The regression: the API server and the copilot executor each hold their
+ own copy of the token cache, so the in-process hook only ever evicts the
+ writer's copy and the OAuth callback could not reach the executor's."""
+ result = subprocess.run(
+ [sys.executable, "-c", _PROCESS_PROBE.replace("__MODULE__", module)],
+ capture_output=True,
+ text=True,
+ timeout=300,
+ )
+ assert result.returncode == 0, result.stderr
+ assert "cache=True hook=True published=('user-1', 'github')" in result.stdout
+
+
+def _has_live_cluster() -> bool:
+ """Probe the socket, not ``redis_client.connect()``: this runs at collection
+ time and that helper retries a dead host for ~45 minutes before giving up."""
+ from backend.data import redis_client
+
+ try:
+ socket.create_connection((redis_client.HOST, redis_client.PORT), 1).close()
+ except OSError:
+ return False
+ return True
+
+
+ not _has_live_cluster(),
+ reason="local redis cluster not reachable; skip creds-changed round trip",
+)
+async def test_publish_reaches_a_subscriber_in_another_event_loop():
+ import asyncio
+
+ from backend.data import redis_client
+
+ redis_client.get_redis.cache_clear()
+ redis_client._async_clients.clear()
+
+ received: list[CredentialsChangedEvent] = []
+
+ async def consume() -> None:
+ async for event in listen_creds_changed():
+ received.append(event)
+ return
+
+ task = asyncio.create_task(consume())
+ # Let SSUBSCRIBE settle; races drop the publish otherwise.
+ await asyncio.sleep(0.3)
+ try:
+ await publish_creds_changed(_USER, "github")
+ await asyncio.wait_for(task, timeout=5.0)
+ finally:
+ if not task.done():
+ task.cancel()
+ await redis_client.disconnect_async()
+
+ assert received and received[0].user_id == _USER
+
+
+def _oauth_creds(token: str = "tok") -> OAuth2Credentials:
+ return OAuth2Credentials(
+ id="creds-oauth2",
+ provider="github",
+ title="Test OAuth",
+ access_token=SecretStr(token),
+ refresh_token=SecretStr("test-refresh"),
+ access_token_expires_at=None,
+ refresh_token_expires_at=None,
+ scopes=[],
+ )
+
+
+_PROCESS_PROBE = """
+import asyncio
+import sys
+from unittest.mock import AsyncMock, patch
+
+import __MODULE__ # noqa: F401 - the process under test loads this
+import backend.integrations.creds_manager as cm
+
+with patch.object(cm, "publish_creds_changed", new_callable=AsyncMock) as publish:
+ asyncio.run(cm._invoke_creds_changed_hook("user-1", "github"))
+
+cached = "backend.copilot.integration_creds" in sys.modules
+print(
+ "cache=" + str(cached)
+ + " hook=" + str(cm._on_creds_changed is not None)
+ + " published=" + str(publish.await_args[0])
+)
+"""
diff --git a/autogpt_platform/backend/backend/integrations/creds_manager.py b/autogpt_platform/backend/backend/integrations/creds_manager.py
index e7bbe71531..351c77e6d5 100644
--- a/autogpt_platform/backend/backend/integrations/creds_manager.py
+++ b/autogpt_platform/backend/backend/integrations/creds_manager.py
@@ -15,6 +15,7 @@ from backend.integrations.credentials_store import (
IntegrationCredentialsStore,
provider_matches,
)
+from backend.integrations.creds_events import publish_creds_changed
from backend.integrations.oauth import (
CREDENTIALS_BY_PROVIDER,
DEVICE_HANDLERS_BY_NAME,
@@ -43,6 +44,10 @@ def register_creds_changed_hook(hook: Callable[[str, str], None]) -> None:
application startup (e.g. by the copilot module) without creating an
import cycle.
+ In-process only. Cross-process invalidation rides on
+ :func:`~backend.integrations.creds_events.publish_creds_changed`, which
+ :func:`_invoke_creds_changed_hook` calls alongside this hook.
+
Raises:
RuntimeError: If a hook is already registered. Call
:func:`unregister_creds_changed_hook` first if replacement is needed.
@@ -65,8 +70,12 @@ def unregister_creds_changed_hook() -> None:
_on_creds_changed = None
-def _invoke_creds_changed_hook(user_id: str, provider: str) -> None:
- """Invoke the registered creds-changed hook (if any)."""
+async def _invoke_creds_changed_hook(user_id: str, provider: str) -> None:
+ """Notify this process's hook, then every other process over Redis.
+
+ The hook alone is not enough: it evicts only the writing process's caches,
+ and the copilot executor holds its own in a different process.
+ """
if _on_creds_changed is not None:
try:
_on_creds_changed(user_id, provider)
@@ -77,6 +86,7 @@ def _invoke_creds_changed_hook(user_id: str, provider: str) -> None:
provider,
exc_info=True,
)
+ await publish_creds_changed(user_id, provider)
class IntegrationCredentialsManager:
@@ -124,7 +134,7 @@ class IntegrationCredentialsManager:
async def create(self, user_id: str, credentials: Credentials) -> None:
result = await self.store.add_creds(user_id, credentials)
# Notify listeners so downstream caches are invalidated immediately.
- _invoke_creds_changed_hook(user_id, credentials.provider)
+ await _invoke_creds_changed_hook(user_id, credentials.provider)
return result
async def exists(self, user_id: str, credentials_id: str) -> bool:
@@ -310,7 +320,9 @@ class IntegrationCredentialsManager:
try:
fresh_credentials = await oauth_handler.refresh_tokens(credentials)
await self.store.update_creds(user_id, fresh_credentials)
- _invoke_creds_changed_hook(user_id, fresh_credentials.provider)
+ await _invoke_creds_changed_hook(
+ user_id, fresh_credentials.provider
+ )
credentials = fresh_credentials
finally:
if (await _lock.locked()) and (await _lock.owned()):
@@ -346,7 +358,7 @@ class IntegrationCredentialsManager:
)
fresh_credentials = await oauth_handler.refresh_tokens(credentials)
await self.store.update_creds(user_id, fresh_credentials)
- _invoke_creds_changed_hook(user_id, fresh_credentials.provider)
+ await _invoke_creds_changed_hook(user_id, fresh_credentials.provider)
credentials = fresh_credentials
return credentials
@@ -354,7 +366,7 @@ class IntegrationCredentialsManager:
async with self._locked(user_id, updated.id):
await self.store.update_creds(user_id, updated)
# Notify listeners so the updated credential is picked up immediately.
- _invoke_creds_changed_hook(user_id, updated.provider)
+ await _invoke_creds_changed_hook(user_id, updated.provider)
async def upsert_single_provider(
self,
@@ -386,7 +398,7 @@ class IntegrationCredentialsManager:
credentials: Credentials,
) -> Credentials:
stored = await self.store.upsert_single_provider_creds(user_id, credentials)
- _invoke_creds_changed_hook(user_id, stored.provider)
+ await _invoke_creds_changed_hook(user_id, stored.provider)
return stored
async def update_acquired(
@@ -426,7 +438,7 @@ class IntegrationCredentialsManager:
):
raise RuntimeError("Provider-runtime credential identity changed")
await self.store.update_creds(user_id, updated)
- _invoke_creds_changed_hook(user_id, updated.provider)
+ await _invoke_creds_changed_hook(user_id, updated.provider)
async def delete_acquired(
self,
@@ -453,7 +465,7 @@ class IntegrationCredentialsManager:
):
raise RuntimeError("Credential identity changed before deletion")
await self.store.delete_creds_by_id(user_id, credentials.id)
- _invoke_creds_changed_hook(user_id, credentials.provider)
+ await _invoke_creds_changed_hook(user_id, credentials.provider)
async def delete(self, user_id: str, credentials_id: str) -> None:
async with self._locked(user_id, credentials_id):
@@ -462,7 +474,7 @@ class IntegrationCredentialsManager:
creds = await self.store.get_creds_by_id(user_id, credentials_id)
await self.store.delete_creds_by_id(user_id, credentials_id)
if creds:
- _invoke_creds_changed_hook(user_id, creds.provider)
+ await _invoke_creds_changed_hook(user_id, creds.provider)
# -- Locking utilities -- #
diff --git a/autogpt_platform/backend/backend/integrations/creds_manager_test.py b/autogpt_platform/backend/backend/integrations/creds_manager_test.py
index 0833fe3d8e..a8a4730822 100644
--- a/autogpt_platform/backend/backend/integrations/creds_manager_test.py
+++ b/autogpt_platform/backend/backend/integrations/creds_manager_test.py
@@ -1,6 +1,6 @@
"""Tests for credential hooks and provider-runtime credential leases."""
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import SecretStr
@@ -18,16 +18,22 @@ from backend.integrations.creds_manager import (
def _reset_hook():
"""Ensure global hook state is clean before and after every test."""
unregister_creds_changed_hook()
- yield
+ # The broadcast half is covered in creds_events_test; keep these tests off Redis.
+ with patch(
+ "backend.integrations.creds_manager.publish_creds_changed",
+ new_callable=AsyncMock,
+ ):
+ yield
unregister_creds_changed_hook()
class TestRegisterCredsChangedHook:
- def test_register_and_invoke(self):
+ @pytest.mark.asyncio
+ async def test_register_and_invoke(self):
calls: list[tuple[str, str]] = []
register_creds_changed_hook(lambda u, p: calls.append((u, p)))
- _invoke_creds_changed_hook("user-1", "github")
+ await _invoke_creds_changed_hook("user-1", "github")
assert calls == [("user-1", "github")]
def test_double_register_raises(self):
@@ -43,24 +49,39 @@ class TestRegisterCredsChangedHook:
class TestInvokeCredsChangedHook:
- def test_noop_when_no_hook_registered(self):
+ @pytest.mark.asyncio
+ async def test_noop_when_no_hook_registered(self):
# Must not raise even when no hook is registered.
- _invoke_creds_changed_hook("user-1", "github")
+ await _invoke_creds_changed_hook("user-1", "github")
- def test_hook_exception_is_swallowed(self):
+ @pytest.mark.asyncio
+ async def test_hook_exception_is_swallowed(self):
def bad_hook(user_id: str, provider: str) -> None:
raise ValueError("boom")
register_creds_changed_hook(bad_hook)
# Must not propagate the exception.
- _invoke_creds_changed_hook("user-1", "github")
+ await _invoke_creds_changed_hook("user-1", "github")
+
+ @pytest.mark.asyncio
+ async def test_broadcast_happens_even_when_the_hook_raises(self):
+ register_creds_changed_hook(MagicMock(side_effect=ValueError("boom")))
+
+ with patch(
+ "backend.integrations.creds_manager.publish_creds_changed",
+ new_callable=AsyncMock,
+ ) as publish:
+ await _invoke_creds_changed_hook("user-1", "github")
+
+ publish.assert_awaited_once_with("user-1", "github")
- def test_hook_receives_correct_args(self):
+ @pytest.mark.asyncio
+ async def test_hook_receives_correct_args(self):
calls: list[tuple[str, str]] = []
register_creds_changed_hook(lambda u, p: calls.append((u, p)))
- _invoke_creds_changed_hook("user-a", "github")
- _invoke_creds_changed_hook("user-b", "slack")
+ await _invoke_creds_changed_hook("user-a", "github")
+ await _invoke_creds_changed_hook("user-b", "slack")
assert calls == [("user-a", "github"), ("user-b", "slack")]