Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/notifications): atomic upsert + drop eager include (#12919)

Zamil Majdy10ea46663f2d
diff --git a/autogpt_platform/backend/backend/data/notifications.py b/autogpt_platform/backend/backend/data/notifications.py
index 88c8b2f361..921077897f 100644
--- a/autogpt_platform/backend/backend/data/notifications.py
+++ b/autogpt_platform/backend/backend/data/notifications.py
@@ -5,6 +5,7 @@ from typing import Annotated, Any, Generic, Optional, TypeVar, Union
from prisma import Json
from prisma.enums import NotificationType
+from prisma.errors import UniqueViolationError
from prisma.models import NotificationEvent, UserNotificationBatch
from prisma.types import (
NotificationEventCreateInput,
@@ -459,57 +460,56 @@ async def create_or_add_to_user_notification_batch(
if not notification_data.data:
raise ValueError("Notification data must be provided")
- # Serialize the data
json_data: Json = SafeJson(notification_data.data.model_dump())
- # First try to find existing batch
- existing_batch = await UserNotificationBatch.prisma().find_unique(
- where={
- "userId_type": {
- "userId": user_id,
- "type": notification_type,
- }
- },
- include={"Notifications": True},
- )
-
- if not existing_batch:
- resp = await UserNotificationBatch.prisma().create(
- data=UserNotificationBatchCreateInput(
- userId=user_id,
- type=notification_type,
- Notifications={
- "create": [
- NotificationEventCreateInput(
- type=notification_type,
- data=json_data,
- )
- ]
+ # Prisma's upsert is find→INSERT/UPDATE under the hood, not a true
+ # SQL ON CONFLICT, so two concurrent calls on a missing row can both
+ # take the INSERT branch and one loses to @@unique([userId, type]).
+ # Retry once on UniqueViolationError: the row now exists, so the
+ # second pass takes the UPDATE branch. Avoids a write-skipping
+ # find_unique pre-check on the hot path. Skips the eager
+ # `Notifications` include that loaded thousands of rows for heavy
+ # AGENT_RUN users and tripped Postgres statement_timeout.
+ for attempt in range(2):
+ try:
+ resp = await UserNotificationBatch.prisma().upsert(
+ where={
+ "userId_type": {
+ "userId": user_id,
+ "type": notification_type,
+ }
+ },
+ data={
+ "create": UserNotificationBatchCreateInput(
+ userId=user_id,
+ type=notification_type,
+ Notifications={
+ "create": [
+ NotificationEventCreateInput(
+ type=notification_type,
+ data=json_data,
+ )
+ ]
+ },
+ ),
+ "update": {
+ "Notifications": {
+ "create": [
+ NotificationEventCreateInput(
+ type=notification_type,
+ data=json_data,
+ )
+ ]
+ }
+ },
},
- ),
- include={"Notifications": True},
- )
- return UserNotificationBatchDTO.from_db(resp)
- else:
- resp = await UserNotificationBatch.prisma().update(
- where={"id": existing_batch.id},
- data={
- "Notifications": {
- "create": [
- NotificationEventCreateInput(
- type=notification_type,
- data=json_data,
- )
- ]
- }
- },
- include={"Notifications": True},
- )
- if not resp:
- raise DatabaseError(
- f"Failed to add notification event to existing batch {existing_batch.id}"
)
- return UserNotificationBatchDTO.from_db(resp)
+ return UserNotificationBatchDTO.from_db(resp)
+ except UniqueViolationError:
+ if attempt == 0:
+ continue
+ raise
+ raise RuntimeError("unreachable") # for type-checkers
except Exception as e:
raise DatabaseError(
f"Failed to create or add to notification batch for user {user_id} and type {notification_type}: {e}"
diff --git a/autogpt_platform/backend/backend/data/notifications_test.py b/autogpt_platform/backend/backend/data/notifications_test.py
index 7469183a12..f4cf879b06 100644
--- a/autogpt_platform/backend/backend/data/notifications_test.py
+++ b/autogpt_platform/backend/backend/data/notifications_test.py
@@ -1,11 +1,28 @@
-"""Tests for notification data models."""
+"""Tests for notification data models and batch helpers."""
+import asyncio
+import logging
from datetime import datetime, timezone
+from uuid import uuid4
import pytest
+from prisma.actions import UserNotificationBatchActions
+from prisma.enums import NotificationType
+from prisma.errors import UniqueViolationError
+from prisma.models import NotificationEvent, User, UserNotificationBatch
from pydantic import ValidationError
-from backend.data.notifications import AgentApprovalData, AgentRejectionData
+from backend.data.notifications import (
+ AgentApprovalData,
+ AgentRejectionData,
+ AgentRunData,
+ NotificationEventModel,
+ create_or_add_to_user_notification_batch,
+ empty_user_notification_batch,
+)
+from backend.util.test import SpinTestServer
+
+logger = logging.getLogger(__name__)
class TestAgentApprovalData:
@@ -149,3 +166,233 @@ class TestAgentRejectionData:
assert restored_data.comments == original_data.comments
assert restored_data.reviewed_at == original_data.reviewed_at
assert restored_data.resubmit_url == original_data.resubmit_url
+
+
+def _make_agent_run_event(user_id: str, agent_name: str = "Test Agent"):
+ return NotificationEventModel[AgentRunData](
+ type=NotificationType.AGENT_RUN,
+ user_id=user_id,
+ data=AgentRunData(
+ agent_name=agent_name,
+ credits_used=10.0,
+ execution_time=5.0,
+ node_count=3,
+ graph_id="graph-" + agent_name,
+ outputs=[],
+ ),
+ )
+
+
+async def _create_test_user(user_id: str) -> None:
+ try:
+ await User.prisma().create(
+ data={
+ "id": user_id,
+ "email": f"test-{user_id}@example.com",
+ "name": f"Test User {user_id[:8]}",
+ }
+ )
+ except UniqueViolationError:
+ pass
+
+
+async def _cleanup_test_user(user_id: str) -> None:
+ try:
+ await empty_user_notification_batch(user_id, NotificationType.AGENT_RUN)
+ except Exception as exc:
+ logger.warning("notification cleanup for %s failed: %s", user_id, exc)
+ try:
+ await User.prisma().delete_many(where={"id": user_id})
+ except Exception as exc:
+ logger.warning("user cleanup for %s failed: %s", user_id, exc)
+
+
[email protected](loop_scope="session")
+async def test_upsert_creates_empty_batch_then_appends(server: SpinTestServer):
+ """Empty batch path creates a batch. Existing-batch path appends without
+ loading the existing notification rows."""
+ user_id = f"notif-create-{uuid4()}"
+ await _create_test_user(user_id)
+
+ try:
+ # First call → create path.
+ first = await create_or_add_to_user_notification_batch(
+ user_id=user_id,
+ notification_type=NotificationType.AGENT_RUN,
+ notification_data=_make_agent_run_event(user_id, agent_name="first"),
+ )
+ assert first.user_id == user_id
+ assert first.type == NotificationType.AGENT_RUN
+ # The returned DTO no longer eagerly loads notifications — callers
+ # that need them must fetch separately.
+ assert first.notifications == []
+
+ row_count = await NotificationEvent.prisma().count(
+ where={"UserNotificationBatch": {"is": {"userId": user_id}}}
+ )
+ assert row_count == 1
+
+ # Second call → update path.
+ second = await create_or_add_to_user_notification_batch(
+ user_id=user_id,
+ notification_type=NotificationType.AGENT_RUN,
+ notification_data=_make_agent_run_event(user_id, agent_name="second"),
+ )
+ assert second.user_id == user_id
+ # Still no eager include, even when the batch has rows.
+ assert second.notifications == []
+
+ row_count = await NotificationEvent.prisma().count(
+ where={"UserNotificationBatch": {"is": {"userId": user_id}}}
+ )
+ assert row_count == 2
+
+ # Exactly one batch for (user_id, AGENT_RUN).
+ batch_count = await UserNotificationBatch.prisma().count(
+ where={"userId": user_id, "type": NotificationType.AGENT_RUN}
+ )
+ assert batch_count == 1
+ finally:
+ await _cleanup_test_user(user_id)
+
+
[email protected](loop_scope="session")
+async def test_upsert_does_not_load_existing_notifications(server: SpinTestServer):
+ """Pre-seeding 25 notifications into the batch and then upserting one more
+ must not trip on the eager include that used to load every row."""
+ user_id = f"notif-bigbatch-{uuid4()}"
+ await _create_test_user(user_id)
+
+ try:
+ # Seed the batch by issuing 25 upserts. The first creates, the rest
+ # append.
+ for i in range(25):
+ await create_or_add_to_user_notification_batch(
+ user_id=user_id,
+ notification_type=NotificationType.AGENT_RUN,
+ notification_data=_make_agent_run_event(user_id, agent_name=f"a{i}"),
+ )
+
+ pre_count = await NotificationEvent.prisma().count(
+ where={"UserNotificationBatch": {"is": {"userId": user_id}}}
+ )
+ assert pre_count == 25
+
+ # The 26th upsert: must succeed and must NOT return the 25 prior rows.
+ resp = await create_or_add_to_user_notification_batch(
+ user_id=user_id,
+ notification_type=NotificationType.AGENT_RUN,
+ notification_data=_make_agent_run_event(user_id, agent_name="final"),
+ )
+ assert resp.notifications == [], (
+ "Upsert must not eagerly include Notifications; that is the "
+ "statement_timeout regression we are guarding against."
+ )
+
+ post_count = await NotificationEvent.prisma().count(
+ where={"UserNotificationBatch": {"is": {"userId": user_id}}}
+ )
+ assert post_count == 26
+ finally:
+ await _cleanup_test_user(user_id)
+
+
[email protected](loop_scope="session")
+async def test_upsert_concurrent_invocations_no_unique_violation(
+ server: SpinTestServer,
+):
+ """Two concurrent invocations on an empty batch must both succeed: one
+ creates, one appends. Neither loses its event, and no
+ @@unique([userId, type]) violation leaks out of the helper.
+
+ Prisma's upsert is find→INSERT/UPDATE, not a true SQL ON CONFLICT, so
+ two concurrent calls can both miss the existing row and both attempt
+ INSERT. The helper retries on UniqueViolationError so the loser
+ converges on the UPDATE path."""
+ user_id = f"notif-race-{uuid4()}"
+ await _create_test_user(user_id)
+
+ try:
+ results = await asyncio.gather(
+ create_or_add_to_user_notification_batch(
+ user_id=user_id,
+ notification_type=NotificationType.AGENT_RUN,
+ notification_data=_make_agent_run_event(user_id, agent_name="left"),
+ ),
+ create_or_add_to_user_notification_batch(
+ user_id=user_id,
+ notification_type=NotificationType.AGENT_RUN,
+ notification_data=_make_agent_run_event(user_id, agent_name="right"),
+ ),
+ return_exceptions=True,
+ )
+
+ assert all(not isinstance(r, Exception) for r in results), results
+
+ batch_count = await UserNotificationBatch.prisma().count(
+ where={"userId": user_id, "type": NotificationType.AGENT_RUN}
+ )
+ assert batch_count == 1
+
+ event_count = await NotificationEvent.prisma().count(
+ where={"UserNotificationBatch": {"is": {"userId": user_id}}}
+ )
+ assert event_count == 2, "both concurrent notifications must persist"
+ finally:
+ await _cleanup_test_user(user_id)
+
+
[email protected](loop_scope="session")
+async def test_upsert_retries_on_unique_violation(
+ server: SpinTestServer, monkeypatch: pytest.MonkeyPatch
+):
+ """Deterministic coverage for the retry path: when the first upsert
+ raises UniqueViolationError (the loser of a concurrent INSERT race),
+ the helper retries and the second attempt succeeds via UPDATE.
+
+ Async gather + a shared connection pool can serialize concurrent calls
+ in some test setups, so we patch the actions class to inject a unique
+ violation on the first call and prove the retry actually runs."""
+ user_id = f"notif-retry-{uuid4()}"
+ await _create_test_user(user_id)
+
+ try:
+ # Pre-create the batch so the retry's UPDATE path has a row to hit.
+ await create_or_add_to_user_notification_batch(
+ user_id=user_id,
+ notification_type=NotificationType.AGENT_RUN,
+ notification_data=_make_agent_run_event(user_id, agent_name="seed"),
+ )
+
+ original_upsert = UserNotificationBatchActions.upsert
+ call_count = {"n": 0}
+
+ async def flaky_upsert(self, *args, **kwargs):
+ call_count["n"] += 1
+ if call_count["n"] == 1:
+ raise UniqueViolationError(
+ {
+ "error": (
+ "Unique constraint failed on the fields: (`userId`,`type`)"
+ ),
+ "user_facing_error": {},
+ }
+ )
+ return await original_upsert(self, *args, **kwargs)
+
+ monkeypatch.setattr(UserNotificationBatchActions, "upsert", flaky_upsert)
+
+ resp = await create_or_add_to_user_notification_batch(
+ user_id=user_id,
+ notification_type=NotificationType.AGENT_RUN,
+ notification_data=_make_agent_run_event(user_id, agent_name="retry"),
+ )
+ assert resp.user_id == user_id
+ assert call_count["n"] == 2, "retry path must run exactly once"
+
+ event_count = await NotificationEvent.prisma().count(
+ where={"UserNotificationBatch": {"is": {"userId": user_id}}}
+ )
+ assert event_count == 2
+ finally:
+ await _cleanup_test_user(user_id)