Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): Ensure json column value serializable & remove excessive transaction and locking (#10441)

Zamil Majdyb71d0ec80581
diff --git a/autogpt_platform/backend/backend/data/analytics.py b/autogpt_platform/backend/backend/data/analytics.py
index 9ee0285221..fde2d3fd6e 100644
--- a/autogpt_platform/backend/backend/data/analytics.py
+++ b/autogpt_platform/backend/backend/data/analytics.py
@@ -2,6 +2,8 @@ import logging
import prisma.types
+from backend.util.json import SafeJson
+
logger = logging.getLogger(__name__)
@@ -15,7 +17,7 @@ async def log_raw_analytics(
data=prisma.types.AnalyticsDetailsCreateInput(
userId=user_id,
type=type,
- data=prisma.Json(data),
+ data=SafeJson(data),
dataIndex=data_index,
)
)
diff --git a/autogpt_platform/backend/backend/data/credit.py b/autogpt_platform/backend/backend/data/credit.py
index c8d29eb62f..5a3aac4e0c 100644
--- a/autogpt_platform/backend/backend/data/credit.py
+++ b/autogpt_platform/backend/backend/data/credit.py
@@ -37,6 +37,7 @@ from backend.notifications.notifications import queue_notification_async
from backend.server.model import Pagination
from backend.server.v2.admin.model import UserHistoryResponse
from backend.util.exceptions import InsufficientBalanceError
+from backend.util.json import SafeJson
from backend.util.retry import func_retry
from backend.util.settings import Settings
@@ -316,7 +317,7 @@ class UserCreditBase(ABC):
transaction_key: str | None = None,
ceiling_balance: int | None = None,
fail_insufficient_credits: bool = True,
- metadata: Json = Json({}),
+ metadata: Json = SafeJson({}),
) -> tuple[int, str]:
"""
Add a new transaction for the user.
@@ -399,7 +400,7 @@ class UserCredit(UserCreditBase):
user_id=user_id,
amount=-cost,
transaction_type=CreditTransactionType.USAGE,
- metadata=Json(metadata.model_dump()),
+ metadata=SafeJson(metadata.model_dump()),
)
# Auto top-up if balance is below threshold.
@@ -439,7 +440,7 @@ class UserCredit(UserCreditBase):
amount=credits,
transaction_type=CreditTransactionType.GRANT,
transaction_key=f"REWARD-{user_id}-{step.value}",
- metadata=Json(
+ metadata=SafeJson(
{"reason": f"Reward for completing {step.value} onboarding step."}
),
)
@@ -531,7 +532,7 @@ class UserCredit(UserCreditBase):
amount=-request.amount,
transaction_type=CreditTransactionType.REFUND,
transaction_key=request.id,
- metadata=Json(request),
+ metadata=SafeJson(request),
fail_insufficient_credits=False,
)
@@ -669,7 +670,7 @@ class UserCredit(UserCreditBase):
is_active=False,
transaction_key=key,
ceiling_balance=ceiling_balance,
- metadata=(Json(metadata)),
+ metadata=(SafeJson(metadata)),
)
customer_id = await get_stripe_customer_id(user_id)
@@ -693,7 +694,7 @@ class UserCredit(UserCreditBase):
},
)
if setup_intent.status == "succeeded":
- successful_transaction = Json({"setup_intent": setup_intent})
+ successful_transaction = SafeJson({"setup_intent": setup_intent})
new_transaction_key = setup_intent.id
break
else:
@@ -711,7 +712,9 @@ class UserCredit(UserCreditBase):
},
)
if payment_intent.status == "succeeded":
- successful_transaction = Json({"payment_intent": payment_intent})
+ successful_transaction = SafeJson(
+ {"payment_intent": payment_intent}
+ )
new_transaction_key = payment_intent.id
break
@@ -766,7 +769,7 @@ class UserCredit(UserCreditBase):
transaction_type=CreditTransactionType.TOP_UP,
transaction_key=checkout_session.id,
is_active=False,
- metadata=Json(checkout_session),
+ metadata=SafeJson(checkout_session),
)
return checkout_session.url or ""
@@ -822,7 +825,7 @@ class UserCredit(UserCreditBase):
transaction_key=credit_transaction.transactionKey,
new_transaction_key=new_transaction_key,
user_id=credit_transaction.userId,
- metadata=Json(checkout_session),
+ metadata=SafeJson(checkout_session),
)
async def get_credits(self, user_id: str) -> int:
@@ -935,7 +938,7 @@ class BetaUserCredit(UserCredit):
amount=max(self.num_user_credits_refill - balance, 0),
transaction_type=CreditTransactionType.GRANT,
transaction_key=f"MONTHLY-CREDIT-TOP-UP-{cur_time}",
- metadata=Json({"reason": "Monthly credit refill"}),
+ metadata=SafeJson({"reason": "Monthly credit refill"}),
)
return balance
except UniqueViolationError:
@@ -1012,7 +1015,7 @@ async def get_stripe_customer_id(user_id: str) -> str:
async def set_auto_top_up(user_id: str, config: AutoTopUpConfig):
await User.prisma().update(
where={"id": user_id},
- data={"topUpConfig": Json(config.model_dump())},
+ data={"topUpConfig": SafeJson(config.model_dump())},
)
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.py
index 6cb27b468f..5c69d465a3 100644
--- a/autogpt_platform/backend/backend/data/execution.py
+++ b/autogpt_platform/backend/backend/data/execution.py
@@ -16,7 +16,6 @@ from typing import (
overload,
)
-from prisma import Json
from prisma.enums import AgentExecutionStatus
from prisma.models import (
AgentGraphExecution,
@@ -39,6 +38,7 @@ from pydantic.fields import Field
from backend.server.v2.store.exceptions import DatabaseError
from backend.util import type as type_utils
+from backend.util.json import SafeJson
from backend.util.settings import Config
from backend.util.truncate import truncate
@@ -482,7 +482,7 @@ async def create_graph_execution(
queuedTime=datetime.now(tz=timezone.utc),
Input={
"create": [
- {"name": name, "data": Json(data)}
+ {"name": name, "data": SafeJson(data)}
for name, data in node_input.items()
]
},
@@ -540,7 +540,7 @@ async def upsert_execution_input(
order={"addedTime": "asc"},
include={"Input": True},
)
- json_input_data = Json(input_data)
+ json_input_data = SafeJson(input_data)
if existing_execution:
await AgentNodeExecutionInputOutput.prisma().create(
@@ -588,7 +588,7 @@ async def upsert_execution_output(
referencedByOutputExecId=node_exec_id,
)
if output_data is not None:
- data["data"] = Json(output_data)
+ data["data"] = SafeJson(output_data)
await AgentNodeExecutionInputOutput.prisma().create(data=data)
@@ -619,7 +619,7 @@ async def update_graph_execution_stats(
stats_dict = stats.model_dump()
if isinstance(stats_dict.get("error"), Exception):
stats_dict["error"] = str(stats_dict["error"])
- update_data["stats"] = Json(stats_dict)
+ update_data["stats"] = SafeJson(stats_dict)
if status:
update_data["executionStatus"] = status
@@ -656,7 +656,7 @@ async def update_node_execution_stats(
res = await AgentNodeExecution.prisma().update(
where={"id": node_exec_id},
data={
- "stats": Json(data),
+ "stats": SafeJson(data),
"endedTime": datetime.now(tz=timezone.utc),
},
include=EXECUTION_RESULT_INCLUDE,
@@ -714,9 +714,9 @@ def _get_update_status_data(
update_data["endedTime"] = now
if execution_data:
- update_data["executionData"] = Json(execution_data)
+ update_data["executionData"] = SafeJson(execution_data)
if stats:
- update_data["stats"] = Json(stats)
+ update_data["stats"] = SafeJson(stats)
return update_data
@@ -1019,11 +1019,11 @@ async def set_execution_kv_data(
userId=user_id,
agentNodeExecutionId=node_exec_id,
key=key,
- data=Json(data) if data is not None else None,
+ data=SafeJson(data) if data is not None else None,
),
"update": {
"agentNodeExecutionId": node_exec_id,
- "data": Json(data) if data is not None else None,
+ "data": SafeJson(data) if data is not None else None,
},
},
)
diff --git a/autogpt_platform/backend/backend/data/graph.py b/autogpt_platform/backend/backend/data/graph.py
index c301e3a5c1..df8597b218 100644
--- a/autogpt_platform/backend/backend/data/graph.py
+++ b/autogpt_platform/backend/backend/data/graph.py
@@ -3,7 +3,6 @@ import uuid
from collections import defaultdict
from typing import TYPE_CHECKING, Any, Literal, Optional, cast
-from prisma import Json
from prisma.enums import SubmissionStatus
from prisma.models import AgentGraph, AgentNode, AgentNodeLink, StoreListingVersion
from prisma.types import (
@@ -28,6 +27,7 @@ from backend.data.model import (
)
from backend.integrations.providers import ProviderName
from backend.util import type as type_utils
+from backend.util.json import SafeJson
from .block import Block, BlockInput, BlockSchema, BlockType, get_block, get_blocks
from .db import BaseDbModel, query_raw_with_schema, transaction
@@ -952,18 +952,18 @@ async def fork_graph(graph_id: str, graph_version: int, user_id: str) -> GraphMo
"""
Forks a graph by copying it and all its nodes and links to a new graph.
"""
- async with transaction() as tx:
- graph = await get_graph(graph_id, graph_version, user_id, True)
- if not graph:
- raise ValueError(f"Graph {graph_id} v{graph_version} not found")
+ graph = await get_graph(graph_id, graph_version, user_id, True)
+ if not graph:
+ raise ValueError(f"Graph {graph_id} v{graph_version} not found")
- # Set forked from ID and version as itself as it's about ot be copied
- graph.forked_from_id = graph.id
- graph.forked_from_version = graph.version
- graph.name = f"{graph.name} (copy)"
- graph.reassign_ids(user_id=user_id, reassign_graph_id=True)
- graph.validate_graph(for_run=False)
+ # Set forked from ID and version as itself as it's about ot be copied
+ graph.forked_from_id = graph.id
+ graph.forked_from_version = graph.version
+ graph.name = f"{graph.name} (copy)"
+ graph.reassign_ids(user_id=user_id, reassign_graph_id=True)
+ graph.validate_graph(for_run=False)
+ async with transaction() as tx:
await __create_graph(tx, graph, user_id)
return graph
@@ -995,8 +995,8 @@ async def __create_graph(tx, graph: Graph, user_id: str):
agentGraphId=graph.id,
agentGraphVersion=graph.version,
agentBlockId=node.block_id,
- constantInput=Json(node.input_default),
- metadata=Json(node.metadata),
+ constantInput=SafeJson(node.input_default),
+ metadata=SafeJson(node.metadata),
)
for graph in graphs
for node in graph.nodes
@@ -1121,7 +1121,7 @@ async def fix_llm_provider_credentials():
await store.update_creds(user_id, credentials)
await AgentNode.prisma().update(
where={"id": node_id},
- data={"constantInput": Json(node_preset_input)},
+ data={"constantInput": SafeJson(node_preset_input)},
)
diff --git a/autogpt_platform/backend/backend/data/integrations.py b/autogpt_platform/backend/backend/data/integrations.py
index 7bc4b3d93f..3efb96f1b3 100644
--- a/autogpt_platform/backend/backend/data/integrations.py
+++ b/autogpt_platform/backend/backend/data/integrations.py
@@ -1,7 +1,6 @@
import logging
from typing import AsyncGenerator, Literal, Optional, overload
-from prisma import Json
from prisma.models import IntegrationWebhook
from prisma.types import (
IntegrationWebhookCreateInput,
@@ -17,6 +16,7 @@ from backend.integrations.providers import ProviderName
from backend.integrations.webhooks.utils import webhook_ingress_url
from backend.server.v2.library.model import LibraryAgentPreset
from backend.util.exceptions import NotFoundError
+from backend.util.json import SafeJson
from .db import BaseDbModel
from .graph import NodeModel
@@ -90,7 +90,7 @@ async def create_webhook(webhook: Webhook) -> Webhook:
webhookType=webhook.webhook_type,
resource=webhook.resource,
events=webhook.events,
- config=Json(webhook.config),
+ config=SafeJson(webhook.config),
secret=webhook.secret,
providerWebhookId=webhook.provider_webhook_id,
)
@@ -205,7 +205,7 @@ async def update_webhook(
"""⚠️ No `user_id` check: DO NOT USE without check in user-facing endpoints."""
data: IntegrationWebhookUpdateInput = {}
if config is not None:
- data["config"] = Json(config)
+ data["config"] = SafeJson(config)
if events is not None:
data["events"] = events
if not data:
diff --git a/autogpt_platform/backend/backend/data/notifications.py b/autogpt_platform/backend/backend/data/notifications.py
index 80b194bedb..9e64ef0a6a 100644
--- a/autogpt_platform/backend/backend/data/notifications.py
+++ b/autogpt_platform/backend/backend/data/notifications.py
@@ -16,6 +16,7 @@ from prisma.types import (
from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_validator
from backend.server.v2.store.exceptions import DatabaseError
+from backend.util.json import SafeJson
from backend.util.logging import TruncatedLogger
from .db import transaction
@@ -395,7 +396,7 @@ async def create_or_add_to_user_notification_batch(
raise ValueError("Notification data must be provided")
# Serialize the data
- json_data: Json = Json(notification_data.data.model_dump())
+ json_data: Json = SafeJson(notification_data.data.model_dump())
# First try to find existing batch
existing_batch = await UserNotificationBatch.prisma().find_unique(
@@ -409,44 +410,40 @@ async def create_or_add_to_user_notification_batch(
)
if not existing_batch:
- async with transaction() as tx:
- notification_event = await tx.notificationevent.create(
- data=NotificationEventCreateInput(
- type=notification_type,
- data=json_data,
- )
- )
-
- # Create new batch
- resp = await tx.usernotificationbatch.create(
- data=UserNotificationBatchCreateInput(
- userId=user_id,
- type=notification_type,
- Notifications={"connect": [{"id": notification_event.id}]},
- ),
- include={"Notifications": True},
- )
- return UserNotificationBatchDTO.from_db(resp)
- else:
- async with transaction() as tx:
- notification_event = await tx.notificationevent.create(
- data=NotificationEventCreateInput(
- type=notification_type,
- data=json_data,
- UserNotificationBatch={"connect": {"id": existing_batch.id}},
- )
- )
- # Add to existing batch
- resp = await tx.usernotificationbatch.update(
- where={"id": existing_batch.id},
- data={
- "Notifications": {"connect": [{"id": notification_event.id}]}
+ resp = await UserNotificationBatch.prisma().create(
+ data=UserNotificationBatchCreateInput(
+ userId=user_id,
+ type=notification_type,
+ Notifications={
+ "create": [
+ NotificationEventCreateInput(
+ type=notification_type,
+ data=json_data,
+ )
+ ]
},
- include={"Notifications": True},
- )
+ ),
+ include={"Notifications": True},
+ )
+ return UserNotificationBatchDTO.from_db(resp)
+ else:
+ resp = await UserNotificationBatch.prisma().update(
+ where={"id": existing_batch.id},
+ data={
+ "Notifications": {
+ "create": [
+ NotificationEventCreateInput(
+ type=notification_type,
+ data=json_data,
+ )
+ ]
+ }
+ },
+ include={"Notifications": True},
+ )
if not resp:
raise DatabaseError(
- f"Failed to add notification event {notification_event.id} to existing batch {existing_batch.id}"
+ f"Failed to add notification event to existing batch {existing_batch.id}"
)
return UserNotificationBatchDTO.from_db(resp)
except Exception as e:
diff --git a/autogpt_platform/backend/backend/data/onboarding.py b/autogpt_platform/backend/backend/data/onboarding.py
index 0bdab3b905..817122f90a 100644
--- a/autogpt_platform/backend/backend/data/onboarding.py
+++ b/autogpt_platform/backend/backend/data/onboarding.py
@@ -3,17 +3,16 @@ from typing import Any, Optional
import prisma
import pydantic
-from prisma import Json
from prisma.enums import OnboardingStep
from prisma.models import UserOnboarding
from prisma.types import UserOnboardingCreateInput, UserOnboardingUpdateInput
-from backend.data import db
from backend.data.block import get_blocks
from backend.data.credit import get_user_credit_model
from backend.data.graph import GraphModel
from backend.data.model import CredentialsMetaInput
from backend.server.v2.store.model import StoreAgentDetails
+from backend.util.json import SafeJson
# Mapping from user reason id to categories to search for when choosing agent to show
REASON_MAPPING: dict[str, list[str]] = {
@@ -79,7 +78,7 @@ async def update_user_onboarding(user_id: str, data: UserOnboardingUpdate):
if data.selectedStoreListingVersionId is not None:
update["selectedStoreListingVersionId"] = data.selectedStoreListingVersionId
if data.agentInput is not None:
- update["agentInput"] = Json(data.agentInput)
+ update["agentInput"] = SafeJson(data.agentInput)
if data.onboardingAgentExecutionId is not None:
update["onboardingAgentExecutionId"] = data.onboardingAgentExecutionId
if data.agentRuns is not None:
@@ -95,43 +94,42 @@ async def update_user_onboarding(user_id: str, data: UserOnboardingUpdate):
async def reward_user(user_id: str, step: OnboardingStep):
- async with db.locked_transaction(f"usr_trx_{user_id}-reward"):
- reward = 0
- match step:
- # Reward user when they clicked New Run during onboarding
- # This is because they need credits before scheduling a run (next step)
- # This is seen as a reward for the GET_RESULTS step in the wallet
- case OnboardingStep.AGENT_NEW_RUN:
- reward = 300
- case OnboardingStep.RUN_AGENTS:
- reward = 300
- case OnboardingStep.MARKETPLACE_ADD_AGENT:
- reward = 100
- case OnboardingStep.MARKETPLACE_RUN_AGENT:
- reward = 100
- case OnboardingStep.BUILDER_SAVE_AGENT:
- reward = 100
- case OnboardingStep.BUILDER_RUN_AGENT:
- reward = 100
-
- if reward == 0:
- return
-
- onboarding = await get_user_onboarding(user_id)
-
- # Skip if already rewarded
- if step in onboarding.rewardedFor:
- return
-
- onboarding.rewardedFor.append(step)
- await user_credit.onboarding_reward(user_id, reward, step)
- await UserOnboarding.prisma().update(
- where={"userId": user_id},
- data={
- "completedSteps": list(set(onboarding.completedSteps + [step])),
- "rewardedFor": onboarding.rewardedFor,
- },
- )
+ reward = 0
+ match step:
+ # Reward user when they clicked New Run during onboarding
+ # This is because they need credits before scheduling a run (next step)
+ # This is seen as a reward for the GET_RESULTS step in the wallet
+ case OnboardingStep.AGENT_NEW_RUN:
+ reward = 300
+ case OnboardingStep.RUN_AGENTS:
+ reward = 300
+ case OnboardingStep.MARKETPLACE_ADD_AGENT:
+ reward = 100
+ case OnboardingStep.MARKETPLACE_RUN_AGENT:
+ reward = 100
+ case OnboardingStep.BUILDER_SAVE_AGENT:
+ reward = 100
+ case OnboardingStep.BUILDER_RUN_AGENT:
+ reward = 100
+
+ if reward == 0:
+ return
+
+ onboarding = await get_user_onboarding(user_id)
+
+ # Skip if already rewarded
+ if step in onboarding.rewardedFor:
+ return
+
+ onboarding.rewardedFor.append(step)
+ await user_credit.onboarding_reward(user_id, reward, step)
+ await UserOnboarding.prisma().update(
+ where={"userId": user_id},
+ data={
+ "completedSteps": list(set(onboarding.completedSteps + [step])),
+ "rewardedFor": onboarding.rewardedFor,
+ },
+ )
def clean_and_split(text: str) -> list[str]:
diff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.py
index 6e92203769..7102afb0ef 100644
--- a/autogpt_platform/backend/backend/data/user.py
+++ b/autogpt_platform/backend/backend/data/user.py
@@ -8,7 +8,6 @@ from urllib.parse import quote_plus
from autogpt_libs.auth.models import DEFAULT_USER_ID
from fastapi import HTTPException
-from prisma import Json
from prisma.enums import NotificationType
from prisma.models import User
from prisma.types import JsonFilter, UserCreateInput, UserUpdateInput
@@ -18,6 +17,7 @@ from backend.data.model import UserIntegrations, UserMetadata, UserMetadataRaw
from backend.data.notifications import NotificationPreference, NotificationPreferenceDTO
from backend.server.v2.store.exceptions import DatabaseError
from backend.util.encryption import JSONCryptor
+from backend.util.json import SafeJson
from backend.util.settings import Settings
logger = logging.getLogger(__name__)
@@ -105,7 +105,7 @@ async def get_user_metadata(user_id: str) -> UserMetadata:
async def update_user_metadata(user_id: str, metadata: UserMetadata):
await User.prisma().update(
where={"id": user_id},
- data={"metadata": Json(metadata.model_dump())},
+ data={"metadata": SafeJson(metadata.model_dump())},
)
@@ -139,7 +139,7 @@ async def migrate_and_encrypt_user_integrations():
JsonFilter,
{
"path": ["integration_credentials"],
- "not": Json(
+ "not": SafeJson(
{"a": "yolo"}
), # bogus value works to check if key exists
},
@@ -171,7 +171,7 @@ async def migrate_and_encrypt_user_integrations():
# Update metadata without integration data
await User.prisma().update(
where={"id": user.id},
- data={"metadata": Json(raw_metadata)},
+ data={"metadata": SafeJson(raw_metadata)},
)
diff --git a/autogpt_platform/backend/backend/server/v2/admin/credit_admin_routes.py b/autogpt_platform/backend/backend/server/v2/admin/credit_admin_routes.py
index 009c541432..36e05f1298 100644
--- a/autogpt_platform/backend/backend/server/v2/admin/credit_admin_routes.py
+++ b/autogpt_platform/backend/backend/server/v2/admin/credit_admin_routes.py
@@ -4,11 +4,11 @@ import typing
from autogpt_libs.auth import requires_admin_user
from autogpt_libs.auth.depends import get_user_id
from fastapi import APIRouter, Body, Depends
-from prisma import Json
from prisma.enums import CreditTransactionType
from backend.data.credit import admin_get_user_history, get_user_credit_model
from backend.server.v2.admin.model import AddUserCreditsResponse, UserHistoryResponse
+from backend.util.json import SafeJson
logger = logging.getLogger(__name__)
@@ -40,7 +40,7 @@ async def add_user_credits(
user_id,
amount,
transaction_type=CreditTransactionType.GRANT,
- metadata=Json({"admin_id": admin_user, "reason": comments}),
+ metadata=SafeJson({"admin_id": admin_user, "reason": comments}),
)
return {
"new_balance": new_balance,
diff --git a/autogpt_platform/backend/backend/server/v2/library/db.py b/autogpt_platform/backend/backend/server/v2/library/db.py
index b4368b4296..89028100e4 100644
--- a/autogpt_platform/backend/backend/server/v2/library/db.py
+++ b/autogpt_platform/backend/backend/server/v2/library/db.py
@@ -15,13 +15,14 @@ import backend.server.v2.store.exceptions as store_exceptions
import backend.server.v2.store.image_gen as store_image_gen
import backend.server.v2.store.media as store_media
from backend.data.block import BlockInput
-from backend.data.db import locked_transaction, transaction
+from backend.data.db import transaction
from backend.data.execution import get_graph_execution
from backend.data.includes import library_agent_include
from backend.data.model import CredentialsMetaInput
from backend.integrations.creds_manager import IntegrationCredentialsManager
from backend.integrations.webhooks.graph_lifecycle_hooks import on_graph_activate
from backend.util.exceptions import NotFoundError
+from backend.util.json import SafeJson
from backend.util.settings import Config
logger = logging.getLogger(__name__)
@@ -501,68 +502,64 @@ async def add_store_agent_to_library(
)
try:
- async with locked_transaction(f"add_agent_trx_{user_id}"):
- store_listing_version = (
- await prisma.models.StoreListingVersion.prisma().find_unique(
- where={"id": store_listing_version_id}, include={"AgentGraph": True}
- )
+ store_listing_version = (
+ await prisma.models.StoreListingVersion.prisma().find_unique(
+ where={"id": store_listing_version_id}, include={"AgentGraph": True}
+ )
+ )
+ if not store_listing_version or not store_listing_version.AgentGraph:
+ logger.warning(
+ f"Store listing version not found: {store_listing_version_id}"
+ )
+ raise store_exceptions.AgentNotFoundError(
+ f"Store listing version {store_listing_version_id} not found or invalid"
)
- if not store_listing_version or not store_listing_version.AgentGraph:
- logger.warning(
- f"Store listing version not found: {store_listing_version_id}"
- )
- raise store_exceptions.AgentNotFoundError(
- f"Store listing version {store_listing_version_id} not found or invalid"
- )
- graph = store_listing_version.AgentGraph
+ graph = store_listing_version.AgentGraph
- # Check if user already has this agent
- existing_library_agent = (
- await prisma.models.LibraryAgent.prisma().find_unique(
- where={
- "userId_agentGraphId_agentGraphVersion": {
- "userId": user_id,
- "agentGraphId": graph.id,
- "agentGraphVersion": graph.version,
- }
- },
- include={"AgentGraph": True},
+ # Check if user already has this agent
+ existing_library_agent = await prisma.models.LibraryAgent.prisma().find_unique(
+ where={
+ "userId_agentGraphId_agentGraphVersion": {
+ "userId": user_id,
+ "agentGraphId": graph.id,
+ "agentGraphVersion": graph.version,
+ }
+ },
+ include={"AgentGraph": True},
+ )
+ if existing_library_agent:
+ if existing_library_agent.isDeleted:
+ # Even if agent exists it needs to be marked as not deleted
+ await update_library_agent(
+ existing_library_agent.id, user_id, is_deleted=False
)
- )
- if existing_library_agent:
- if existing_library_agent.isDeleted:
- # Even if agent exists it needs to be marked as not deleted
- await update_library_agent(
- existing_library_agent.id, user_id, is_deleted=False
- )
- else:
- logger.debug(
- f"User #{user_id} already has graph #{graph.id} "
- f"v{graph.version} in their library"
- )
- return library_model.LibraryAgent.from_db(existing_library_agent)
-
- # Create LibraryAgent entry
- added_agent = await prisma.models.LibraryAgent.prisma().create(
- data={
- "User": {"connect": {"id": user_id}},
- "AgentGraph": {
- "connect": {
- "graphVersionId": {"id": graph.id, "version": graph.version}
- }
- },
- "isCreatedByUser": False,
- },
- include=library_agent_include(user_id),
- )
- logger.debug(
- f"Added graph #{graph.id} v{graph.version}"
- f"for store listing version #{store_listing_version.id} "
- f"to library for user #{user_id}"
- )
- return library_model.LibraryAgent.from_db(added_agent)
+ else:
+ logger.debug(
+ f"User #{user_id} already has graph #{graph.id} "
+ f"v{graph.version} in their library"
+ )
+ return library_model.LibraryAgent.from_db(existing_library_agent)
+ # Create LibraryAgent entry
+ added_agent = await prisma.models.LibraryAgent.prisma().create(
+ data={
+ "User": {"connect": {"id": user_id}},
+ "AgentGraph": {
+ "connect": {
+ "graphVersionId": {"id": graph.id, "version": graph.version}
+ }
+ },
+ "isCreatedByUser": False,
+ },
+ include=library_agent_include(user_id),
+ )
+ logger.debug(
+ f"Added graph #{graph.id} v{graph.version}"
+ f"for store listing version #{store_listing_version.id} "
+ f"to library for user #{user_id}"
+ )
+ return library_model.LibraryAgent.from_db(added_agent)
except store_exceptions.AgentNotFoundError:
# Reraise for external handling.
raise
@@ -704,7 +701,7 @@ async def create_preset(
InputPresets={
"create": [
prisma.types.AgentNodeExecutionInputOutputCreateWithoutRelationsInput( # noqa
- name=name, data=prisma.fields.Json(data)
+ name=name, data=SafeJson(data)
)
for name, data in {
**preset.inputs,
@@ -814,7 +811,7 @@ async def update_preset(
update_data["InputPresets"] = {
"create": [
prisma.types.AgentNodeExecutionInputOutputCreateWithoutRelationsInput( # noqa
- name=name, data=prisma.fields.Json(data)
+ name=name, data=SafeJson(data)
)
for name, data in {
**inputs,
@@ -907,26 +904,25 @@ async def fork_library_agent(
"""
logger.debug(f"Forking library agent {library_agent_id} for user {user_id}")
try:
- async with locked_transaction(f"usr_trx_{user_id}-fork_agent"):
- # Fetch the original agent
- original_agent = await get_library_agent(library_agent_id, user_id)
-
- # Check if user owns the library agent
- # TODO: once we have open/closed sourced agents this needs to be enabled ~kcze
- # + update library/agents/[id]/page.tsx agent actions
- # if not original_agent.can_access_graph:
- # raise store_exceptions.DatabaseError(
- # f"User {user_id} cannot access library agent graph {library_agent_id}"
- # )
-
- # Fork the underlying graph and nodes
- new_graph = await graph_db.fork_graph(
- original_agent.graph_id, original_agent.graph_version, user_id
- )
- new_graph = await on_graph_activate(new_graph, user_id=user_id)
+ # Fetch the original agent
+ original_agent = await get_library_agent(library_agent_id, user_id)
+
+ # Check if user owns the library agent
+ # TODO: once we have open/closed sourced agents this needs to be enabled ~kcze
+ # + update library/agents/[id]/page.tsx agent actions
+ # if not original_agent.can_access_graph:
+ # raise store_exceptions.DatabaseError(
+ # f"User {user_id} cannot access library agent graph {library_agent_id}"
+ # )
+
+ # Fork the underlying graph and nodes
+ new_graph = await graph_db.fork_graph(
+ original_agent.graph_id, original_agent.graph_version, user_id
+ )
+ new_graph = await on_graph_activate(new_graph, user_id=user_id)
- # Create a library agent for the new graph
- return (await create_library_agent(new_graph, user_id))[0]
+ # Create a library agent for the new graph
+ return (await create_library_agent(new_graph, user_id))[0]
except prisma.errors.PrismaError as e:
logger.error(f"Database error cloning library agent: {e}")
raise store_exceptions.DatabaseError("Failed to fork library agent") from e
diff --git a/autogpt_platform/backend/backend/server/v2/library/db_test.py b/autogpt_platform/backend/backend/server/v2/library/db_test.py
index 0b39b384ea..7cb34ed75a 100644
--- a/autogpt_platform/backend/backend/server/v2/library/db_test.py
+++ b/autogpt_platform/backend/backend/server/v2/library/db_test.py
@@ -87,7 +87,7 @@ async def test_add_agent_to_library(mocker):
await connect()
# Mock the transaction context
- mock_transaction = mocker.patch("backend.server.v2.library.db.locked_transaction")
+ mock_transaction = mocker.patch("backend.server.v2.library.db.transaction")
mock_transaction.return_value.__aenter__ = mocker.AsyncMock(return_value=None)
mock_transaction.return_value.__aexit__ = mocker.AsyncMock(return_value=None)
# Mock data
diff --git a/autogpt_platform/backend/backend/util/json.py b/autogpt_platform/backend/backend/util/json.py
index 505a7b27c8..e425483c31 100644
--- a/autogpt_platform/backend/backend/util/json.py
+++ b/autogpt_platform/backend/backend/util/json.py
@@ -3,6 +3,7 @@ from typing import Any, Type, TypeGuard, TypeVar, overload
import jsonschema
from fastapi.encoders import jsonable_encoder
+from prisma import Json
from pydantic import BaseModel
from .type import type_match
@@ -95,3 +96,19 @@ def convert_pydantic_to_json(output_data: Any) -> Any:
if is_list_of_basemodels(output_data):
return [item.model_dump() for item in output_data]
return output_data
+
+
+def SafeJson(data: Any) -> Json:
+ """Safely serialize data and return Prisma's Json type."""
+ if isinstance(data, BaseModel):
+ return Json(
+ data.model_dump(
+ mode="json",
+ warnings="error",
+ exclude_none=True,
+ fallback=lambda v: None,
+ )
+ )
+ # Round-trip through JSON to ensure proper serialization with fallback for non-serializable values
+ json_string = dumps(data, default=lambda v: None)
+ return Json(json.loads(json_string))
diff --git a/autogpt_platform/backend/backend/util/test_json.py b/autogpt_platform/backend/backend/util/test_json.py
new file mode 100644
index 0000000000..9e8602ea08
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/test_json.py
@@ -0,0 +1,217 @@
+import datetime
+from typing import Any, Optional
+
+from prisma import Json
+from pydantic import BaseModel
+
+from backend.util.json import SafeJson
+
+
+class SamplePydanticModel(BaseModel):
+ name: str
+ age: Optional[int] = None
+ timestamp: Optional[datetime.datetime] = None
+ metadata: Optional[dict] = None
+
+
+class SampleModelWithNonSerializable(BaseModel):
+ name: str
+ func: Any = None # Could contain non-serializable data
+ data: Optional[dict] = None
+
+
+class TestSafeJson:
+ """Test cases for SafeJson function."""
+
+ def test_safejson_returns_json_type(self):
+ """Test that SafeJson returns a proper Json instance."""
+ data = {"test": "value"}
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+
+ def test_simple_dict_serialization(self):
+ """Test basic dictionary serialization."""
+ data = {"name": "John", "age": 30, "active": True}
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+
+ def test_unicode_handling(self):
+ """Test that Unicode characters are handled properly."""
+ data = {
+ "name": "café",
+ "emoji": "🎉",
+ "chinese": "你好",
+ "arabic": "مرحبا",
+ }
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+
+ def test_nested_data_structures(self):
+ """Test complex nested data structures."""
+ data = {
+ "user": {
+ "name": "Alice",
+ "preferences": {
+ "theme": "dark",
+ "notifications": ["email", "push"],
+ },
+ },
+ "metadata": {
+ "tags": ["important", "urgent"],
+ "scores": [8.5, 9.2, 7.8],
+ },
+ }
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+
+ def test_pydantic_model_basic(self):
+ """Test basic Pydantic model serialization."""
+ model = SamplePydanticModel(name="John", age=30)
+ result = SafeJson(model)
+ assert isinstance(result, Json)
+
+ def test_pydantic_model_with_none_values(self):
+ """Test Pydantic model with None values (should be excluded)."""
+ model = SamplePydanticModel(name="John", age=None, timestamp=None)
+ result = SafeJson(model)
+ assert isinstance(result, Json)
+ # The actual Json content should exclude None values due to exclude_none=True
+
+ def test_pydantic_model_with_datetime(self):
+ """Test Pydantic model with datetime field."""
+ now = datetime.datetime.now()
+ model = SamplePydanticModel(name="John", age=25, timestamp=now)
+ result = SafeJson(model)
+ assert isinstance(result, Json)
+
+ def test_non_serializable_values_in_dict(self):
+ """Test that non-serializable values in dict are converted to None."""
+ data = {
+ "name": "test",
+ "function": lambda x: x, # Non-serializable
+ "datetime": datetime.datetime.now(), # Non-serializable
+ "valid_data": "this should work",
+ }
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+
+ def test_pydantic_model_with_non_serializable_fallback(self):
+ """Test Pydantic model with non-serializable field using fallback."""
+ model = SampleModelWithNonSerializable(
+ name="test",
+ func=lambda x: x, # Non-serializable
+ data={"valid": "data"},
+ )
+ result = SafeJson(model)
+ assert isinstance(result, Json)
+
+ def test_empty_data_structures(self):
+ """Test empty data structures."""
+ test_cases = [
+ {}, # Empty dict
+ [], # Empty list
+ "", # Empty string
+ None, # None value
+ ]
+
+ for data in test_cases:
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+
+ def test_complex_mixed_data(self):
+ """Test complex mixed data with various types."""
+ data = {
+ "string": "test",
+ "integer": 42,
+ "float": 3.14,
+ "boolean": True,
+ "none_value": None,
+ "list": [1, 2, "three", {"nested": "dict"}],
+ "nested_dict": {
+ "level2": {
+ "level3": ["deep", "nesting", 123],
+ }
+ },
+ }
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+
+ def test_list_of_pydantic_models(self):
+ """Test list containing Pydantic models."""
+ models = [
+ SamplePydanticModel(name="Alice", age=25),
+ SamplePydanticModel(name="Bob", age=30),
+ ]
+ data = {"users": models}
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+
+ def test_edge_case_circular_reference_protection(self):
+ """Test that circular references don't cause infinite loops."""
+ # Note: This test assumes the underlying json.dumps handles circular refs
+ # by raising an exception, which our fallback should handle
+ data = {}
+ data["self"] = data # Create circular reference
+
+ # This should either work with fallback or raise a reasonable error
+ try:
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+ except (ValueError, RecursionError):
+ # If it raises an error, that's also acceptable behavior
+ pass
+
+ def test_large_data_structure(self):
+ """Test with a reasonably large data structure."""
+ data = {
+ "items": [
+ {"id": i, "name": f"item_{i}", "active": i % 2 == 0} for i in range(100)
+ ],
+ "metadata": {
+ "total": 100,
+ "generated_at": "2024-01-01T00:00:00Z",
+ "tags": ["auto", "generated", "test"],
+ },
+ }
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+
+ def test_special_characters_and_encoding(self):
+ """Test various special characters and encoding scenarios."""
+ data = {
+ "quotes": 'He said "Hello world!"',
+ "backslashes": "C:\\Users\\test\\file.txt",
+ "newlines": "Line 1\nLine 2\nLine 3",
+ "tabs": "Column1\tColumn2\tColumn3",
+ "unicode_escape": "\u0048\u0065\u006c\u006c\u006f", # "Hello"
+ "mixed": "Test with émojis 🚀 and ñúméríçs",
+ }
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+
+ def test_numeric_edge_cases(self):
+ """Test various numeric edge cases."""
+ data = {
+ "zero": 0,
+ "negative": -42,
+ "large_int": 999999999999999999,
+ "small_float": 0.000001,
+ "large_float": 1e10,
+ "infinity": float("inf"), # This might become None due to fallback
+ "negative_infinity": float(
+ "-inf"
+ ), # This might become None due to fallback
+ }
+ result = SafeJson(data)
+ assert isinstance(result, Json)
+
+ def test_boolean_and_null_values(self):
+ """Test boolean and null value handling."""
+ data = {
+ "true_value": True,
+ "false_value": False,
+ "null_value": None,
+ "mixed_list": [True, False, None, "string", 42],
+ }
+ result = SafeJson(data)
+ assert isinstance(result, Json)