AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): replace a user's credential list in one transaction (#15413)
Nicholas Tindlefd1a3ffed4a9
diff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.pyindex b658843579..b0ddd127d7 100644--- a/autogpt_platform/backend/backend/data/user.py+++ b/autogpt_platform/backend/backend/data/user.py@@ -11,6 +11,7 @@ from urllib.parse import quote_plusfrom autogpt_libs.auth.models import DEFAULT_USER_IDfrom fastapi import HTTPException+from prisma import Prismafrom prisma.enums import BriefingFrequency, SubscriptionTierfrom prisma.errors import RecordNotFoundError, UniqueViolationErrorfrom prisma.models import AuthAccount, AuthUser@@ -562,61 +563,86 @@ async def set_user_credentials(user_id: str, credentials: list[Credentials]) ->around: rows missing from ``credentials`` are revoked (soft delete),new ids are created, existing ids get their payload refreshed(OAuth token rotation runs through here constantly).++ Runs as one transaction. Credential reads take no lock, so a reader must+ see the old list or the new one: committed row by row, it could see a+ replacement next to the credential it supersedes, and offer or pick the+ one about to be revoked."""cryptor = JSONCryptor()- existing_rows = await prisma.integrationcredential.find_many(- where={"ownerType": "USER", "ownerId": user_id},- )- existing_by_id = {row.id: row for row in existing_rows}incoming_ids = {c.id for c in credentials}-- org_id: str | None = None- for cred in credentials:- row = existing_by_id.get(cred.id)- encrypted = cryptor.encrypt(cred.model_dump())- if row is not None:- await prisma.integrationcredential.update(- where={"id": cred.id},+ async with transaction() as tx:+ await _lock_user_credentials(tx, user_id)+ existing_rows = await tx.integrationcredential.find_many(+ where={"ownerType": "USER", "ownerId": user_id},+ )+ existing_ids = {row.id for row in existing_rows}++ org_id: str | None = None+ for cred in credentials:+ encrypted = cryptor.encrypt(cred.model_dump())+ if cred.id in existing_ids:+ await tx.integrationcredential.update(+ where={"id": cred.id},+ data={+ "encryptedPayload": encrypted,+ "displayName": cred.title or cred.provider,+ "status": "active",+ },+ )+ continue+ if org_id is None:+ org_id = await _personal_org_id(tx, user_id)+ await tx.integrationcredential.create(data={- "encryptedPayload": encrypted,+ "id": cred.id,+ "organizationId": org_id,+ "ownerType": "USER",+ "ownerId": user_id,+ "provider": cred.provider,+ "credentialType": cred.type,"displayName": cred.title or cred.provider,- "status": "active",- },- )- continue- if org_id is None:- org_row = await prisma.organization.find_first(- where={- "isPersonal": True,- "Members": {"some": {"userId": user_id, "isOwner": True}},+ "encryptedPayload": encrypted,+ "createdByUserId": user_id,})- if org_row is None:- raise DatabaseError(- f"Cannot store credentials for user {user_id}: "- "personal org not bootstrapped"++ for row in existing_rows:+ if row.id not in incoming_ids and row.status == "active":+ await tx.integrationcredential.update(+ where={"id": row.id},+ data={"status": "revoked"},)- org_id = org_row.id- await prisma.integrationcredential.create(- data={- "id": cred.id,- "organizationId": org_id,- "ownerType": "USER",- "ownerId": user_id,- "provider": cred.provider,- "credentialType": cred.type,- "displayName": cred.title or cred.provider,- "encryptedPayload": encrypted,- "createdByUserId": user_id,- }- )- for row in existing_rows:- if row.id not in incoming_ids and row.status == "active":- await prisma.integrationcredential.update(- where={"id": row.id},- data={"status": "revoked"},- )++async def _lock_user_credentials(tx: Prisma, user_id: str) -> None:+ """Serialize full-list replaces for one user until the transaction ends.++ The credential store's Redis lock already does this; this keeps two+ replaces from interleaving row locks, and deadlocking, if that lock lapses.++ execute_raw, not query_raw: pg_advisory_xact_lock returns void, which+ Prisma cannot deserialize as a result column.+ """+ await tx.execute_raw(+ "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))",+ f"integration-credentials:{user_id}",+ )+++async def _personal_org_id(tx: Prisma, user_id: str) -> str:+ org_row = await tx.organization.find_first(+ where={+ "isPersonal": True,+ "Members": {"some": {"userId": user_id, "isOwner": True}},+ }+ )+ if org_row is None:+ raise DatabaseError(+ f"Cannot store credentials for user {user_id}: "+ "personal org not bootstrapped"+ )+ return org_row.idasync def migrate_and_encrypt_user_integrations():diff --git a/autogpt_platform/backend/backend/data/user_test.py b/autogpt_platform/backend/backend/data/user_test.pyindex b7bff49666..78381c9da3 100644--- a/autogpt_platform/backend/backend/data/user_test.py+++ b/autogpt_platform/backend/backend/data/user_test.py@@ -1007,6 +1007,19 @@ class TestGetBillingEmailRecipient:)+def _credentials_tx(mocker) -> MagicMock:+ """Stand in for the transaction ``set_user_credentials`` writes through."""+ tx = MagicMock()+ tx.execute_raw = AsyncMock()++ @asynccontextmanager+ async def fake_transaction():+ yield tx++ mocker.patch("backend.data.user.transaction", fake_transaction)+ return tx++class TestTableBackedCredentials:"""get/set_user_credentials — the IntegrationCredential-backed seam thecredential store runs on after the blob→table migration."""@@ -1063,26 +1076,25 @@ class TestTableBackedCredentials:row_gone.id = "cred-gone"row_gone.status = "active"- mock_prisma = MagicMock()- mock_prisma.integrationcredential.find_many = AsyncMock(+ mock_tx = _credentials_tx(mocker)+ mock_tx.integrationcredential.find_many = AsyncMock(return_value=[row_kept, row_gone])- mock_prisma.integrationcredential.update = AsyncMock()- mock_prisma.integrationcredential.create = AsyncMock()- mock_prisma.organization.find_first = AsyncMock(+ mock_tx.integrationcredential.update = AsyncMock()+ mock_tx.integrationcredential.create = AsyncMock()+ mock_tx.organization.find_first = AsyncMock(return_value=MagicMock(id="org-personal"))- mocker.patch("backend.data.user.prisma", mock_prisma)await set_user_credentials("u1", [kept, new])- create_data = mock_prisma.integrationcredential.create.call_args.kwargs["data"]+ create_data = mock_tx.integrationcredential.create.call_args.kwargs["data"]assert create_data["id"] == "cred-new"assert create_data["organizationId"] == "org-personal"update_calls = {c.kwargs["where"]["id"]: c.kwargs["data"]- for c in mock_prisma.integrationcredential.update.call_args_list+ for c in mock_tx.integrationcredential.update.call_args_list}# kept: payload refresh; gone: revokedassert "encryptedPayload" in update_calls["cred-kept"]@@ -1090,7 +1102,7 @@ class TestTableBackedCredentials:@pytest.mark.asyncioasync def test_set_raises_without_personal_org_for_new_cred(self, mocker):- from unittest.mock import AsyncMock, MagicMock+ from unittest.mock import AsyncMockfrom pydantic import SecretStr@@ -1101,10 +1113,9 @@ class TestTableBackedCredentials:new = APIKeyCredentials(id="cred-new", provider="notion", api_key=SecretStr("sk-3"), title="N")- mock_prisma = MagicMock()- mock_prisma.integrationcredential.find_many = AsyncMock(return_value=[])- mock_prisma.organization.find_first = AsyncMock(return_value=None)- mocker.patch("backend.data.user.prisma", mock_prisma)+ mock_tx = _credentials_tx(mocker)+ mock_tx.integrationcredential.find_many = AsyncMock(return_value=[])+ mock_tx.organization.find_first = AsyncMock(return_value=None)with pytest.raises(DatabaseError):await set_user_credentials("u1", [new])diff --git a/autogpt_platform/backend/backend/integrations/credentials_store.py b/autogpt_platform/backend/backend/integrations/credentials_store.pyindex 255acd2cdf..dad4544486 100644--- a/autogpt_platform/backend/backend/integrations/credentials_store.py+++ b/autogpt_platform/backend/backend/integrations/credentials_store.py@@ -358,8 +358,8 @@ class IntegrationCredentialsStore:async def get_all_creds(self, user_id: str) -> list[Credentials]:"""Return all credentials for *user_id*, without taking the lock.- Each row is read atomically, and a full-list replace creates new rows- before revoking old ones, so a read never misses a credential. Taking+ A full-list replace commits in one transaction, so a read sees the+ list from before it or after it, never a mix of the two. Takingthe lock here would queue a page load's concurrent reads behind eachother and behind every write or token refresh for the user."""diff --git a/autogpt_platform/backend/backend/integrations/credentials_store_replace_test.py b/autogpt_platform/backend/backend/integrations/credentials_store_replace_test.pynew file mode 100644index 0000000000..4309c572d8--- /dev/null+++ b/autogpt_platform/backend/backend/integrations/credentials_store_replace_test.py@@ -0,0 +1,136 @@+"""A full-list credential replace, seen by a concurrent reader, on the real database.++Credential reads take no lock (#15389), so a reader can run in the middle of a+replace. Committed row by row, the replace showed that reader the new+credential next to the one it supersedes: discovery offered both, and token+selection could pick the one about to be revoked. The reader must see the old+list or the new one, never a mix.+"""++import asyncio+import uuid++import pytest+from prisma.actions import IntegrationCredentialActions+from pydantic import SecretStr++from backend.data.model import APIKeyCredentials+from backend.data.user import get_or_create_user+from backend.integrations.credentials_store import IntegrationCredentialsStore+from backend.util.test import SpinTestServer++_STALL = 5.0+++async def _user() -> str:+ suffix = uuid.uuid4().hex[:8]+ user = await get_or_create_user(+ {+ "sub": str(uuid.uuid4()),+ "email": f"creds-replace-{suffix}@example.com",+ "name": "Credentials Owner",+ }+ )+ return user.id+++def _api_key(provider: str, *, managed: bool = False) -> APIKeyCredentials:+ return APIKeyCredentials(+ id=str(uuid.uuid4()),+ provider=provider,+ api_key=SecretStr(f"sk-{uuid.uuid4().hex}"),+ title=provider,+ is_managed=managed,+ )+++async def _persisted_ids(+ store: IntegrationCredentialsStore, user_id: str, ours: set[str]+) -> set[str]:+ """The ids *user_id* is offered, out of the ones this test created."""+ return {c.id for c in await store.get_all_creds(user_id)} & ours+++async def _wait_for(event: asyncio.Event, what: str) -> None:+ try:+ await asyncio.wait_for(event.wait(), timeout=_STALL)+ except TimeoutError:+ pytest.fail(f"timed out after {_STALL}s waiting for {what}")++[email protected](loop_scope="session")+async def test_readers_during_a_replace_see_the_old_list_or_the_new_one(+ server: SpinTestServer, monkeypatch: pytest.MonkeyPatch+):+ store = IntegrationCredentialsStore()+ user_id = await _user()+ unrelated = _api_key("github")+ superseded = _api_key("openai", managed=True)+ replacement = _api_key("openai", managed=True)+ await store.add_creds(user_id, unrelated)+ await store.add_managed_credential(user_id, superseded)++ ours = {unrelated.id, superseded.id, replacement.id}+ before = {unrelated.id, superseded.id}+ after = {unrelated.id, replacement.id}+ assert await _persisted_ids(store, user_id, ours) == before++ # Hold the replace between creating the new row and revoking the old one.+ created = asyncio.Event()+ release = asyncio.Event()+ create = IntegrationCredentialActions.create++ async def create_then_hold(self, *args, **kwargs):+ row = await create(self, *args, **kwargs)+ created.set()+ await release.wait()+ return row++ monkeypatch.setattr(IntegrationCredentialActions, "create", create_then_hold)++ writer = asyncio.create_task(+ store.add_managed_credential(user_id, replacement),+ )+ try:+ await _wait_for(created, "the replace to create its new row")+ seen = await asyncio.wait_for(+ asyncio.gather(*(_persisted_ids(store, user_id, ours) for _ in range(5))),+ timeout=_STALL,+ )+ finally:+ release.set()+ await asyncio.wait_for(writer, timeout=_STALL)++ assert all(ids == before for ids in seen), (+ f"a reader saw a mixed list mid-replace: {seen}; "+ f"expected only {before} until the replace commits"+ )+ assert await _persisted_ids(store, user_id, ours) == after++[email protected](loop_scope="session")+async def test_a_replace_that_fails_partway_leaves_the_old_list(+ server: SpinTestServer, monkeypatch: pytest.MonkeyPatch+):+ store = IntegrationCredentialsStore()+ user_id = await _user()+ unrelated = _api_key("github")+ superseded = _api_key("openai", managed=True)+ replacement = _api_key("openai", managed=True)+ await store.add_creds(user_id, unrelated)+ await store.add_managed_credential(user_id, superseded)+ ours = {unrelated.id, superseded.id, replacement.id}++ update = IntegrationCredentialActions.update++ async def fail_to_revoke(self, *args, **kwargs):+ if kwargs.get("data") == {"status": "revoked"}:+ raise RuntimeError("revocation failed")+ return await update(self, *args, **kwargs)++ monkeypatch.setattr(IntegrationCredentialActions, "update", fail_to_revoke)++ with pytest.raises(RuntimeError, match="revocation failed"):+ await store.add_managed_credential(user_id, replacement)++ assert await _persisted_ids(store, user_id, ours) == {unrelated.id, superseded.id}