AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): optimize FastAPI endpoints performance and alert system (#11000)
Zamil Majdya97ff641c362
diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/auth/dependencies.py b/autogpt_platform/autogpt_libs/autogpt_libs/auth/dependencies.pyindex 3f7411b1bc..2fbc3da0e7 100644--- a/autogpt_platform/autogpt_libs/autogpt_libs/auth/dependencies.py+++ b/autogpt_platform/autogpt_libs/autogpt_libs/auth/dependencies.py@@ -10,7 +10,7 @@ from .jwt_utils import get_jwt_payload, verify_userfrom .models import User-def requires_user(jwt_payload: dict = fastapi.Security(get_jwt_payload)) -> User:+async def requires_user(jwt_payload: dict = fastapi.Security(get_jwt_payload)) -> User:"""FastAPI dependency that requires a valid authenticated user.@@ -20,7 +20,9 @@ def requires_user(jwt_payload: dict = fastapi.Security(get_jwt_payload)) -> Userreturn verify_user(jwt_payload, admin_only=False)-def requires_admin_user(jwt_payload: dict = fastapi.Security(get_jwt_payload)) -> User:+async def requires_admin_user(+ jwt_payload: dict = fastapi.Security(get_jwt_payload),+) -> User:"""FastAPI dependency that requires a valid admin user.@@ -30,7 +32,7 @@ def requires_admin_user(jwt_payload: dict = fastapi.Security(get_jwt_payload)) -return verify_user(jwt_payload, admin_only=True)-def get_user_id(jwt_payload: dict = fastapi.Security(get_jwt_payload)) -> str:+async def get_user_id(jwt_payload: dict = fastapi.Security(get_jwt_payload)) -> str:"""FastAPI dependency that returns the ID of the authenticated user.diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/auth/dependencies_test.py b/autogpt_platform/autogpt_libs/autogpt_libs/auth/dependencies_test.pyindex c09f286104..0b9cd6f866 100644--- a/autogpt_platform/autogpt_libs/autogpt_libs/auth/dependencies_test.py+++ b/autogpt_platform/autogpt_libs/autogpt_libs/auth/dependencies_test.py@@ -45,7 +45,7 @@ class TestAuthDependencies:"""Create a test client."""return TestClient(app)- def test_requires_user_with_valid_jwt_payload(self, mocker: MockerFixture):+ async def test_requires_user_with_valid_jwt_payload(self, mocker: MockerFixture):"""Test requires_user with valid JWT payload."""jwt_payload = {"sub": "user-123", "role": "user", "email": "[email protected]"}@@ -53,12 +53,12 @@ class TestAuthDependencies:mocker.patch("autogpt_libs.auth.dependencies.get_jwt_payload", return_value=jwt_payload)- user = requires_user(jwt_payload)+ user = await requires_user(jwt_payload)assert isinstance(user, User)assert user.user_id == "user-123"assert user.role == "user"- def test_requires_user_with_admin_jwt_payload(self, mocker: MockerFixture):+ async def test_requires_user_with_admin_jwt_payload(self, mocker: MockerFixture):"""Test requires_user accepts admin users."""jwt_payload = {"sub": "admin-456",@@ -69,28 +69,28 @@ class TestAuthDependencies:mocker.patch("autogpt_libs.auth.dependencies.get_jwt_payload", return_value=jwt_payload)- user = requires_user(jwt_payload)+ user = await requires_user(jwt_payload)assert user.user_id == "admin-456"assert user.role == "admin"- def test_requires_user_missing_sub(self):+ async def test_requires_user_missing_sub(self):"""Test requires_user with missing user ID."""jwt_payload = {"role": "user", "email": "[email protected]"}with pytest.raises(HTTPException) as exc_info:- requires_user(jwt_payload)+ await requires_user(jwt_payload)assert exc_info.value.status_code == 401assert "User ID not found" in exc_info.value.detail- def test_requires_user_empty_sub(self):+ async def test_requires_user_empty_sub(self):"""Test requires_user with empty user ID."""jwt_payload = {"sub": "", "role": "user"}with pytest.raises(HTTPException) as exc_info:- requires_user(jwt_payload)+ await requires_user(jwt_payload)assert exc_info.value.status_code == 401- def test_requires_admin_user_with_admin(self, mocker: MockerFixture):+ async def test_requires_admin_user_with_admin(self, mocker: MockerFixture):"""Test requires_admin_user with admin role."""jwt_payload = {"sub": "admin-789",@@ -101,51 +101,51 @@ class TestAuthDependencies:mocker.patch("autogpt_libs.auth.dependencies.get_jwt_payload", return_value=jwt_payload)- user = requires_admin_user(jwt_payload)+ user = await requires_admin_user(jwt_payload)assert user.user_id == "admin-789"assert user.role == "admin"- def test_requires_admin_user_with_regular_user(self):+ async def test_requires_admin_user_with_regular_user(self):"""Test requires_admin_user rejects regular users."""jwt_payload = {"sub": "user-123", "role": "user", "email": "[email protected]"}with pytest.raises(HTTPException) as exc_info:- requires_admin_user(jwt_payload)+ await requires_admin_user(jwt_payload)assert exc_info.value.status_code == 403assert "Admin access required" in exc_info.value.detail- def test_requires_admin_user_missing_role(self):+ async def test_requires_admin_user_missing_role(self):"""Test requires_admin_user with missing role."""jwt_payload = {"sub": "user-123", "email": "[email protected]"}with pytest.raises(KeyError):- requires_admin_user(jwt_payload)+ await requires_admin_user(jwt_payload)- def test_get_user_id_with_valid_payload(self, mocker: MockerFixture):+ async def test_get_user_id_with_valid_payload(self, mocker: MockerFixture):"""Test get_user_id extracts user ID correctly."""jwt_payload = {"sub": "user-id-xyz", "role": "user"}mocker.patch("autogpt_libs.auth.dependencies.get_jwt_payload", return_value=jwt_payload)- user_id = get_user_id(jwt_payload)+ user_id = await get_user_id(jwt_payload)assert user_id == "user-id-xyz"- def test_get_user_id_missing_sub(self):+ async def test_get_user_id_missing_sub(self):"""Test get_user_id with missing user ID."""jwt_payload = {"role": "user"}with pytest.raises(HTTPException) as exc_info:- get_user_id(jwt_payload)+ await get_user_id(jwt_payload)assert exc_info.value.status_code == 401assert "User ID not found" in exc_info.value.detail- def test_get_user_id_none_sub(self):+ async def test_get_user_id_none_sub(self):"""Test get_user_id with None user ID."""jwt_payload = {"sub": None, "role": "user"}with pytest.raises(HTTPException) as exc_info:- get_user_id(jwt_payload)+ await get_user_id(jwt_payload)assert exc_info.value.status_code == 401@@ -170,7 +170,7 @@ class TestAuthDependenciesIntegration:return _create_token- def test_endpoint_auth_enabled_no_token(self):+ async def test_endpoint_auth_enabled_no_token(self):"""Test endpoints require token when auth is enabled."""app = FastAPI()@@ -184,7 +184,7 @@ class TestAuthDependenciesIntegration:response = client.get("/test")assert response.status_code == 401- def test_endpoint_with_valid_token(self, create_token):+ async def test_endpoint_with_valid_token(self, create_token):"""Test endpoint with valid JWT token."""app = FastAPI()@@ -203,7 +203,7 @@ class TestAuthDependenciesIntegration:assert response.status_code == 200assert response.json()["user_id"] == "test-user"- def test_admin_endpoint_requires_admin_role(self, create_token):+ async def test_admin_endpoint_requires_admin_role(self, create_token):"""Test admin endpoint rejects non-admin users."""app = FastAPI()@@ -240,7 +240,7 @@ class TestAuthDependenciesIntegration:class TestAuthDependenciesEdgeCases:"""Edge case tests for authentication dependencies."""- def test_dependency_with_complex_payload(self):+ async def test_dependency_with_complex_payload(self):"""Test dependencies handle complex JWT payloads."""complex_payload = {"sub": "user-123",@@ -256,14 +256,14 @@ class TestAuthDependenciesEdgeCases:"exp": 9999999999,}- user = requires_user(complex_payload)+ user = await requires_user(complex_payload)assert user.user_id == "user-123"assert user.email == "[email protected]"- admin = requires_admin_user(complex_payload)+ admin = await requires_admin_user(complex_payload)assert admin.role == "admin"- def test_dependency_with_unicode_in_payload(self):+ async def test_dependency_with_unicode_in_payload(self):"""Test dependencies handle unicode in JWT payloads."""unicode_payload = {"sub": "user-π-123",@@ -272,11 +272,11 @@ class TestAuthDependenciesEdgeCases:"name": "ζ₯ζ¬θͺ",}- user = requires_user(unicode_payload)+ user = await requires_user(unicode_payload)assert "π" in user.user_idassert user.email == "ζ΅θ―@example.com"- def test_dependency_with_null_values(self):+ async def test_dependency_with_null_values(self):"""Test dependencies handle null values in payload."""null_payload = {"sub": "user-123",@@ -286,18 +286,18 @@ class TestAuthDependenciesEdgeCases:"metadata": None,}- user = requires_user(null_payload)+ user = await requires_user(null_payload)assert user.user_id == "user-123"assert user.email is None- def test_concurrent_requests_isolation(self):+ async def test_concurrent_requests_isolation(self):"""Test that concurrent requests don't interfere with each other."""payload1 = {"sub": "user-1", "role": "user"}payload2 = {"sub": "user-2", "role": "admin"}# Simulate concurrent processing- user1 = requires_user(payload1)- user2 = requires_admin_user(payload2)+ user1 = await requires_user(payload1)+ user2 = await requires_admin_user(payload2)assert user1.user_id == "user-1"assert user2.user_id == "user-2"@@ -314,7 +314,7 @@ class TestAuthDependenciesEdgeCases:({"sub": "user", "role": "user"}, "Admin access required", True),],)- def test_dependency_error_cases(+ async def test_dependency_error_cases(self, payload, expected_error: str, admin_only: bool):"""Test that errors propagate correctly through dependencies."""@@ -325,7 +325,7 @@ class TestAuthDependenciesEdgeCases:verify_user(payload, admin_only=admin_only)assert expected_error in exc_info.value.detail- def test_dependency_valid_user(self):+ async def test_dependency_valid_user(self):"""Test valid user case for dependency."""# Import verify_user to test it directly since dependencies use FastAPI Securityfrom autogpt_libs.auth.jwt_utils import verify_userdiff --git a/autogpt_platform/autogpt_libs/autogpt_libs/auth/jwt_utils.py b/autogpt_platform/autogpt_libs/autogpt_libs/auth/jwt_utils.pyindex abc96a3552..1341b21781 100644--- a/autogpt_platform/autogpt_libs/autogpt_libs/auth/jwt_utils.py+++ b/autogpt_platform/autogpt_libs/autogpt_libs/auth/jwt_utils.py@@ -16,7 +16,7 @@ bearer_jwt_auth = HTTPBearer()-def get_jwt_payload(+async def get_jwt_payload(credentials: HTTPAuthorizationCredentials | None = Security(bearer_jwt_auth),) -> dict[str, Any]:"""diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/auth/jwt_utils_test.py b/autogpt_platform/autogpt_libs/autogpt_libs/auth/jwt_utils_test.pyindex 96b56c4c31..c76dee66b5 100644--- a/autogpt_platform/autogpt_libs/autogpt_libs/auth/jwt_utils_test.py+++ b/autogpt_platform/autogpt_libs/autogpt_libs/auth/jwt_utils_test.py@@ -116,32 +116,32 @@ def test_parse_jwt_token_missing_audience():assert "Invalid token" in str(exc_info.value)-def test_get_jwt_payload_with_valid_token():+async def test_get_jwt_payload_with_valid_token():"""Test extracting JWT payload with valid bearer token."""token = create_token(TEST_USER_PAYLOAD)credentials = HTTPAuthorizationCredentials(scheme="Bearer", credentials=token)- result = jwt_utils.get_jwt_payload(credentials)+ result = await jwt_utils.get_jwt_payload(credentials)assert result["sub"] == "test-user-id"assert result["role"] == "user"-def test_get_jwt_payload_no_credentials():+async def test_get_jwt_payload_no_credentials():"""Test JWT payload when no credentials provided."""with pytest.raises(HTTPException) as exc_info:- jwt_utils.get_jwt_payload(None)+ await jwt_utils.get_jwt_payload(None)assert exc_info.value.status_code == 401assert "Authorization header is missing" in exc_info.value.detail-def test_get_jwt_payload_invalid_token():+async def test_get_jwt_payload_invalid_token():"""Test JWT payload extraction with invalid token."""credentials = HTTPAuthorizationCredentials(scheme="Bearer", credentials="invalid.token.here")with pytest.raises(HTTPException) as exc_info:- jwt_utils.get_jwt_payload(credentials)+ await jwt_utils.get_jwt_payload(credentials)assert exc_info.value.status_code == 401assert "Invalid token" in exc_info.value.detaildiff --git a/autogpt_platform/autogpt_libs/autogpt_libs/utils/cache.py b/autogpt_platform/autogpt_libs/autogpt_libs/utils/cache.pyindex 2d174afc61..57fd3b0382 100644--- a/autogpt_platform/autogpt_libs/autogpt_libs/utils/cache.py+++ b/autogpt_platform/autogpt_libs/autogpt_libs/utils/cache.py@@ -115,12 +115,23 @@ def cached("""def decorator(target_func):- # Cache storage and locks+ # Cache storage and per-event-loop lockscache_storage = {}+ _event_loop_locks = {} # Maps event loop to its asyncio.Lockif inspect.iscoroutinefunction(target_func):- # Async function with asyncio.Lock- cache_lock = asyncio.Lock()++ def _get_cache_lock():+ """Get or create an asyncio.Lock for the current event loop."""+ try:+ loop = asyncio.get_running_loop()+ except RuntimeError:+ # No event loop, use None as default key+ loop = None++ if loop not in _event_loop_locks:+ return _event_loop_locks.setdefault(loop, asyncio.Lock())+ return _event_loop_locks[loop]@wraps(target_func)async def async_wrapper(*args: P.args, **kwargs: P.kwargs):@@ -141,7 +152,7 @@ def cached(return result# Slow path: acquire lock for cache miss/expiry- async with cache_lock:+ async with _get_cache_lock():# Double-check: another coroutine might have populated cacheif key in cache_storage:if ttl_seconds is None:diff --git a/autogpt_platform/backend/backend/data/api_key.py b/autogpt_platform/backend/backend/data/api_key.pyindex ac00b98522..45194897de 100644--- a/autogpt_platform/backend/backend/data/api_key.py+++ b/autogpt_platform/backend/backend/data/api_key.py@@ -9,6 +9,7 @@ from prisma.models import APIKey as PrismaAPIKeyfrom prisma.types import APIKeyWhereUniqueInputfrom pydantic import BaseModel, Field+from backend.data.includes import MAX_USER_API_KEYS_FETCHfrom backend.util.exceptions import NotAuthorizedError, NotFoundErrorlogger = logging.getLogger(__name__)@@ -178,9 +179,13 @@ async def revoke_api_key(key_id: str, user_id: str) -> APIKeyInfo:return APIKeyInfo.from_db(updated_api_key)-async def list_user_api_keys(user_id: str) -> list[APIKeyInfo]:+async def list_user_api_keys(+ user_id: str, limit: int = MAX_USER_API_KEYS_FETCH+) -> list[APIKeyInfo]:api_keys = await PrismaAPIKey.prisma().find_many(- where={"userId": user_id}, order={"createdAt": "desc"}+ where={"userId": user_id},+ order={"createdAt": "desc"},+ take=limit,)return [APIKeyInfo.from_db(key) for key in api_keys]diff --git a/autogpt_platform/backend/backend/data/credit.py b/autogpt_platform/backend/backend/data/credit.pyindex e9fd682431..b118e5d853 100644--- a/autogpt_platform/backend/backend/data/credit.py+++ b/autogpt_platform/backend/backend/data/credit.py@@ -23,6 +23,7 @@ from pydantic import BaseModelfrom backend.data import dbfrom backend.data.block_cost_config import BLOCK_COSTS+from backend.data.includes import MAX_CREDIT_REFUND_REQUESTS_FETCHfrom backend.data.model import (AutoTopUpConfig,RefundRequest,@@ -905,7 +906,9 @@ class UserCredit(UserCreditBase):),)- async def get_refund_requests(self, user_id: str) -> list[RefundRequest]:+ async def get_refund_requests(+ self, user_id: str, limit: int = MAX_CREDIT_REFUND_REQUESTS_FETCH+ ) -> list[RefundRequest]:return [RefundRequest(id=r.id,@@ -921,6 +924,7 @@ class UserCredit(UserCreditBase):for r in await CreditRefundRequest.prisma().find_many(where={"userId": user_id},order={"createdAt": "desc"},+ take=limit,)]diff --git a/autogpt_platform/backend/backend/data/graph.py b/autogpt_platform/backend/backend/data/graph.pyindex 3ff9a6c418..8c045de2f7 100644--- a/autogpt_platform/backend/backend/data/graph.py+++ b/autogpt_platform/backend/backend/data/graph.py@@ -20,6 +20,7 @@ from backend.blocks.agent import AgentExecutorBlockfrom backend.blocks.io import AgentInputBlock, AgentOutputBlockfrom backend.blocks.llm import LlmModelfrom backend.data.db import prisma as db+from backend.data.includes import MAX_GRAPH_VERSIONS_FETCHfrom backend.data.model import (CredentialsField,CredentialsFieldInfo,@@ -1059,11 +1060,14 @@ async def set_graph_active_version(graph_id: str, version: int, user_id: str) ->)-async def get_graph_all_versions(graph_id: str, user_id: str) -> list[GraphModel]:+async def get_graph_all_versions(+ graph_id: str, user_id: str, limit: int = MAX_GRAPH_VERSIONS_FETCH+) -> list[GraphModel]:graph_versions = await AgentGraph.prisma().find_many(where={"id": graph_id, "userId": user_id},order={"version": "desc"},include=AGENT_GRAPH_INCLUDE,+ take=limit,)if not graph_versions:diff --git a/autogpt_platform/backend/backend/data/includes.py b/autogpt_platform/backend/backend/data/includes.pyindex ceae4bb62e..bc3d5c9dd1 100644--- a/autogpt_platform/backend/backend/data/includes.py+++ b/autogpt_platform/backend/backend/data/includes.py@@ -14,6 +14,7 @@ AGENT_GRAPH_INCLUDE: prisma.types.AgentGraphInclude = {"Nodes": {"include": AGENT_NODE_INCLUDE}}+EXECUTION_RESULT_ORDER: list[prisma.types.AgentNodeExecutionOrderByInput] = [{"queuedTime": "desc"},# Fallback: Incomplete execs has no queuedTime.@@ -28,6 +29,13 @@ EXECUTION_RESULT_INCLUDE: prisma.types.AgentNodeExecutionInclude = {}MAX_NODE_EXECUTIONS_FETCH = 1000+MAX_LIBRARY_AGENT_EXECUTIONS_FETCH = 10++# Default limits for potentially large result sets+MAX_CREDIT_REFUND_REQUESTS_FETCH = 100+MAX_INTEGRATION_WEBHOOKS_FETCH = 100+MAX_USER_API_KEYS_FETCH = 500+MAX_GRAPH_VERSIONS_FETCH = 50GRAPH_EXECUTION_INCLUDE_WITH_NODES: prisma.types.AgentGraphExecutionInclude = {"NodeExecutions": {@@ -71,13 +79,56 @@ INTEGRATION_WEBHOOK_INCLUDE: prisma.types.IntegrationWebhookInclude = {}-def library_agent_include(user_id: str) -> prisma.types.LibraryAgentInclude:- return {- "AgentGraph": {- "include": {- **AGENT_GRAPH_INCLUDE,- "Executions": {"where": {"userId": user_id}},- }- },- "Creator": True,+def library_agent_include(+ user_id: str,+ include_nodes: bool = True,+ include_executions: bool = True,+ execution_limit: int = MAX_LIBRARY_AGENT_EXECUTIONS_FETCH,+) -> prisma.types.LibraryAgentInclude:+ """+ Fully configurable includes for library agent queries with performance optimization.++ Args:+ user_id: User ID for filtering user-specific data+ include_nodes: Whether to include graph nodes (default: True, needed for get_sub_graphs)+ include_executions: Whether to include executions (default: True, safe with execution_limit)+ execution_limit: Limit on executions to fetch (default: MAX_LIBRARY_AGENT_EXECUTIONS_FETCH)++ Defaults maintain backward compatibility and safety - includes everything needed for all functionality.+ For performance optimization, explicitly set include_nodes=False and include_executions=False+ for listing views where frontend fetches data separately.++ Performance impact:+ - Default (full nodes + limited executions): Original performance, works everywhere+ - Listing optimization (no nodes/executions): ~2s for 15 agents vs potential timeouts+ - Unlimited executions: varies by user (thousands of executions = timeouts)+ """+ result: prisma.types.LibraryAgentInclude = {+ "Creator": True, # Always needed for creator info}++ # Build AgentGraph include based on requested options+ if include_nodes or include_executions:+ agent_graph_include = {}++ # Add nodes if requested (always full nodes)+ if include_nodes:+ agent_graph_include.update(AGENT_GRAPH_INCLUDE) # Full nodes++ # Add executions if requested+ if include_executions:+ agent_graph_include["Executions"] = {+ "where": {"userId": user_id},+ "order_by": {"createdAt": "desc"},+ "take": execution_limit,+ }++ result["AgentGraph"] = cast(+ prisma.types.AgentGraphArgsFromLibraryAgent,+ {"include": agent_graph_include},+ )+ else:+ # Default: Basic metadata only (fast - recommended for most use cases)+ result["AgentGraph"] = True # Basic graph metadata (name, description, id)++ return resultdiff --git a/autogpt_platform/backend/backend/data/integrations.py b/autogpt_platform/backend/backend/data/integrations.pyindex 3efb96f1b3..82f9d7a8bb 100644--- a/autogpt_platform/backend/backend/data/integrations.py+++ b/autogpt_platform/backend/backend/data/integrations.py@@ -11,7 +11,10 @@ from prisma.types import (from pydantic import Field, computed_fieldfrom backend.data.event_bus import AsyncRedisEventBus-from backend.data.includes import INTEGRATION_WEBHOOK_INCLUDE+from backend.data.includes import (+ INTEGRATION_WEBHOOK_INCLUDE,+ MAX_INTEGRATION_WEBHOOKS_FETCH,+)from backend.integrations.providers import ProviderNamefrom backend.integrations.webhooks.utils import webhook_ingress_urlfrom backend.server.v2.library.model import LibraryAgentPreset@@ -128,22 +131,36 @@ async def get_webhook(@overloadasync def get_all_webhooks_by_creds(- user_id: str, credentials_id: str, *, include_relations: Literal[True]+ user_id: str,+ credentials_id: str,+ *,+ include_relations: Literal[True],+ limit: int = MAX_INTEGRATION_WEBHOOKS_FETCH,) -> list[WebhookWithRelations]: ...@overloadasync def get_all_webhooks_by_creds(- user_id: str, credentials_id: str, *, include_relations: Literal[False] = False+ user_id: str,+ credentials_id: str,+ *,+ include_relations: Literal[False] = False,+ limit: int = MAX_INTEGRATION_WEBHOOKS_FETCH,) -> list[Webhook]: ...async def get_all_webhooks_by_creds(- user_id: str, credentials_id: str, *, include_relations: bool = False+ user_id: str,+ credentials_id: str,+ *,+ include_relations: bool = False,+ limit: int = MAX_INTEGRATION_WEBHOOKS_FETCH,) -> list[Webhook] | list[WebhookWithRelations]:if not credentials_id:raise ValueError("credentials_id must not be empty")webhooks = await IntegrationWebhook.prisma().find_many(where={"userId": user_id, "credentialsId": credentials_id},include=INTEGRATION_WEBHOOK_INCLUDE if include_relations else None,+ order={"createdAt": "desc"},+ take=limit,)return [(WebhookWithRelations if include_relations else Webhook).from_db(webhook)diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 95ce5699a9..50436ef575 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -436,7 +436,7 @@ class ExecutionProcessor:graph_id=node_exec.graph_id,node_eid=node_exec.node_exec_id,node_id=node_exec.node_id,- block_name="-",+ block_name=b.name if (b := get_block(node_exec.block_id)) else "-",)db_client = get_db_async_client()node = await db_client.get_node(node_exec.node_id)diff --git a/autogpt_platform/backend/backend/server/middleware/security.py b/autogpt_platform/backend/backend/server/middleware/security.pyindex ad551c55c2..1a3b3715aa 100644--- a/autogpt_platform/backend/backend/server/middleware/security.py+++ b/autogpt_platform/backend/backend/server/middleware/security.py@@ -1,12 +1,10 @@import refrom typing import Set-from fastapi import Request, Response-from starlette.middleware.base import BaseHTTPMiddleware-from starlette.types import ASGIApp+from starlette.types import ASGIApp, Message, Receive, Scope, Send-class SecurityHeadersMiddleware(BaseHTTPMiddleware):+class SecurityHeadersMiddleware:"""Middleware to add security headers to responses, with cache controldisabled by default for all endpoints except those explicitly allowed.@@ -25,6 +23,8 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):"/api/health","/api/v1/health","/api/status",+ "/api/blocks",+ "/api/v1/blocks",# Public store/marketplace pages (read-only)"/api/store/agents","/api/v1/store/agents",@@ -49,7 +49,7 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):}def __init__(self, app: ASGIApp):- super().__init__(app)+ self.app = app# Compile regex patterns for wildcard matchingself.cacheable_patterns = [re.compile(pattern.replace("*", "[^/]+"))@@ -72,26 +72,42 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):return False- async def dispatch(self, request: Request, call_next):- response: Response = await call_next(request)-- # Add general security headers- response.headers["X-Content-Type-Options"] = "nosniff"- response.headers["X-Frame-Options"] = "DENY"- response.headers["X-XSS-Protection"] = "1; mode=block"- response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"-- # Add noindex header for shared execution pages- if "/public/shared" in request.url.path:- response.headers["X-Robots-Tag"] = "noindex, nofollow"-- # Default: Disable caching for all endpoints- # Only allow caching for explicitly permitted paths- if not self.is_cacheable_path(request.url.path):- response.headers["Cache-Control"] = (- "no-store, no-cache, must-revalidate, private"- )- response.headers["Pragma"] = "no-cache"- response.headers["Expires"] = "0"-- return response+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:+ """Pure ASGI middleware implementation for better performance than BaseHTTPMiddleware."""+ if scope["type"] != "http":+ await self.app(scope, receive, send)+ return++ # Extract path from scope+ path = scope["path"]++ async def send_wrapper(message: Message) -> None:+ if message["type"] == "http.response.start":+ # Add security headers to the response+ headers = dict(message.get("headers", []))++ # Add general security headers (HTTP spec requires proper capitalization)+ headers[b"X-Content-Type-Options"] = b"nosniff"+ headers[b"X-Frame-Options"] = b"DENY"+ headers[b"X-XSS-Protection"] = b"1; mode=block"+ headers[b"Referrer-Policy"] = b"strict-origin-when-cross-origin"++ # Add noindex header for shared execution pages+ if "/public/shared" in path:+ headers[b"X-Robots-Tag"] = b"noindex, nofollow"++ # Default: Disable caching for all endpoints+ # Only allow caching for explicitly permitted paths+ if not self.is_cacheable_path(path):+ headers[b"Cache-Control"] = (+ b"no-store, no-cache, must-revalidate, private"+ )+ headers[b"Pragma"] = b"no-cache"+ headers[b"Expires"] = b"0"++ # Convert headers back to list format+ message["headers"] = list(headers.items())++ await send(message)++ await self.app(scope, receive, send_wrapper)diff --git a/autogpt_platform/backend/backend/server/rest_api.py b/autogpt_platform/backend/backend/server/rest_api.pyindex 402ad28e96..7c3d97b748 100644--- a/autogpt_platform/backend/backend/server/rest_api.py+++ b/autogpt_platform/backend/backend/server/rest_api.py@@ -1,5 +1,6 @@import contextlibimport logging+import platformfrom enum import Enumfrom typing import Any, Optional@@ -11,6 +12,7 @@ import uvicornfrom autogpt_libs.auth import add_auth_responses_to_openapifrom autogpt_libs.auth import verify_settings as verify_auth_settingsfrom fastapi.exceptions import RequestValidationError+from fastapi.middleware.gzip import GZipMiddlewarefrom fastapi.routing import APIRoutefrom prisma.errors import PrismaError@@ -70,6 +72,26 @@ async def lifespan_context(app: fastapi.FastAPI):await backend.data.db.connect()+ # Configure thread pool for FastAPI sync operation performance+ # CRITICAL: FastAPI automatically runs ALL sync functions in this thread pool:+ # - Any endpoint defined with 'def' (not async def)+ # - Any dependency function defined with 'def' (not async def)+ # - Manual run_in_threadpool() calls (like JWT decoding)+ # Default pool size is only 40 threads, causing bottlenecks under high concurrency+ config = backend.util.settings.Config()+ try:+ import anyio.to_thread++ anyio.to_thread.current_default_thread_limiter().total_tokens = (+ config.fastapi_thread_pool_size+ )+ logger.info(+ f"Thread pool size set to {config.fastapi_thread_pool_size} for sync endpoint/dependency performance"+ )+ except (ImportError, AttributeError) as e:+ logger.warning(f"Could not configure thread pool size: {e}")+ # Continue without thread pool configuration+# Ensure SDK auto-registration is patched before initializing blocksfrom backend.sdk.registry import AutoRegistry@@ -140,6 +162,9 @@ app = fastapi.FastAPI(app.add_middleware(SecurityHeadersMiddleware)+# Add GZip compression middleware for large responses (like /api/blocks)+app.add_middleware(GZipMiddleware, minimum_size=50_000) # 50KB threshold+# Add 401 responses to authenticated endpoints in OpenAPI specadd_auth_responses_to_openapi(app)@@ -273,12 +298,28 @@ class AgentServer(backend.util.service.AppProcess):allow_methods=["*"], # Allows all methodsallow_headers=["*"], # Allows all headers)- uvicorn.run(- server_app,- host=backend.util.settings.Config().agent_api_host,- port=backend.util.settings.Config().agent_api_port,- log_config=None,- )+ config = backend.util.settings.Config()++ # Configure uvicorn with performance optimizations from Kludex FastAPI tips+ uvicorn_config = {+ "app": server_app,+ "host": config.agent_api_host,+ "port": config.agent_api_port,+ "log_config": None,+ # Use httptools for HTTP parsing (if available)+ "http": "httptools",+ # Only use uvloop on Unix-like systems (not supported on Windows)+ "loop": "uvloop" if platform.system() != "Windows" else "auto",+ }++ # Only add debug in local environment (not supported in all uvicorn versions)+ if config.app_env == backend.util.settings.AppEnvironment.LOCAL:+ import os++ # Enable asyncio debug mode via environment variable+ os.environ["PYTHONASYNCIODEBUG"] = "1"++ uvicorn.run(**uvicorn_config)def cleanup(self):super().cleanup()diff --git a/autogpt_platform/backend/backend/server/routers/v1.py b/autogpt_platform/backend/backend/server/routers/v1.pyindex c8ff30351e..82463a78c9 100644--- a/autogpt_platform/backend/backend/server/routers/v1.py+++ b/autogpt_platform/backend/backend/server/routers/v1.py@@ -24,6 +24,8 @@ from fastapi import (Security,UploadFile,)+from fastapi.concurrency import run_in_threadpool+from pydantic import BaseModelfrom starlette.status import HTTP_204_NO_CONTENT, HTTP_404_NOT_FOUNDfrom typing_extensions import Optional, TypedDict@@ -85,6 +87,7 @@ from backend.server.model import (from backend.util.clients import get_scheduler_clientfrom backend.util.cloud_storage import get_cloud_storage_handlerfrom backend.util.exceptions import GraphValidationError, NotFoundError+from backend.util.json import dumpsfrom backend.util.settings import Settingsfrom backend.util.timezone_utils import (convert_utc_time_to_user_timezone,@@ -263,13 +266,10 @@ async def is_onboarding_enabled():########################################################-@cached()-def _get_cached_blocks() -> Sequence[dict[Any, Any]]:+def _compute_blocks_sync() -> str:"""- Get cached blocks with thundering herd protection.-- Uses sync_cache decorator to prevent multiple concurrent requests- from all executing the expensive block loading operation.+ Synchronous function to compute blocks data.+ This does the heavy lifting: instantiate 226+ blocks, compute costs, serialize."""from backend.data.credit import get_block_cost@@ -279,11 +279,27 @@ def _get_cached_blocks() -> Sequence[dict[Any, Any]]:for block_class in block_classes.values():block_instance = block_class()if not block_instance.disabled:- # Get costs for this specific block class without creating another instancecosts = get_block_cost(block_instance)- result.append({**block_instance.to_dict(), "costs": costs})+ # Convert BlockCost BaseModel objects to dictionaries for JSON serialization+ costs_dict = [+ cost.model_dump() if isinstance(cost, BaseModel) else cost+ for cost in costs+ ]+ result.append({**block_instance.to_dict(), "costs": costs_dict})- return result+ # Use our JSON utility which properly handles complex types through to_dict conversion+ return dumps(result)+++@cached()+async def _get_cached_blocks() -> str:+ """+ Async cached function with thundering herd protection.+ On cache miss: runs heavy work in thread pool+ On cache hit: returns cached string immediately (no thread pool needed)+ """+ # Only run in thread pool on cache miss - cache hits return immediately+ return await run_in_threadpool(_compute_blocks_sync)@v1_router.get(@@ -291,9 +307,28 @@ def _get_cached_blocks() -> Sequence[dict[Any, Any]]:summary="List available blocks",tags=["blocks"],dependencies=[Security(requires_user)],+ responses={+ 200: {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "items": {"additionalProperties": True, "type": "object"},+ "type": "array",+ "title": "Response Getv1List Available Blocks",+ }+ }+ },+ }+ },)-async def get_graph_blocks() -> Sequence[dict[Any, Any]]:- return _get_cached_blocks()+async def get_graph_blocks() -> Response:+ # Cache hit: returns immediately, Cache miss: runs in thread pool+ content = await _get_cached_blocks()+ return Response(+ content=content,+ media_type="application/json",+ )@v1_router.post(diff --git a/autogpt_platform/backend/backend/server/v2/library/db.py b/autogpt_platform/backend/backend/server/v2/library/db.pyindex 0a549326ff..61ecd6305c 100644--- a/autogpt_platform/backend/backend/server/v2/library/db.py+++ b/autogpt_platform/backend/backend/server/v2/library/db.py@@ -101,7 +101,9 @@ async def list_library_agents(try:library_agents = await prisma.models.LibraryAgent.prisma().find_many(where=where_clause,- include=library_agent_include(user_id),+ include=library_agent_include(+ user_id, include_nodes=False, include_executions=False+ ),order=order_by,skip=(page - 1) * page_size,take=page_size,@@ -185,7 +187,9 @@ async def list_favorite_library_agents(try:library_agents = await prisma.models.LibraryAgent.prisma().find_many(where=where_clause,- include=library_agent_include(user_id),+ include=library_agent_include(+ user_id, include_nodes=False, include_executions=False+ ),order=order_by,skip=(page - 1) * page_size,take=page_size,@@ -417,7 +421,9 @@ async def create_library_agent(}},),- include=library_agent_include(user_id),+ include=library_agent_include(+ user_id, include_nodes=False, include_executions=False+ ),)for graph_entry in graph_entries)@@ -642,7 +648,9 @@ async def add_store_agent_to_library(},"isCreatedByUser": False,},- include=library_agent_include(user_id),+ include=library_agent_include(+ user_id, include_nodes=False, include_executions=False+ ),)logger.debug(f"Added graph #{graph.id} v{graph.version}"diff --git a/autogpt_platform/backend/backend/server/v2/library/db_test.py b/autogpt_platform/backend/backend/server/v2/library/db_test.pyindex 7cb34ed75a..2d42d26cfa 100644--- a/autogpt_platform/backend/backend/server/v2/library/db_test.py+++ b/autogpt_platform/backend/backend/server/v2/library/db_test.py@@ -177,7 +177,9 @@ async def test_add_agent_to_library(mocker):},"isCreatedByUser": False,},- include=library_agent_include("test-user"),+ include=library_agent_include(+ "test-user", include_nodes=False, include_executions=False+ ),)diff --git a/autogpt_platform/backend/backend/util/json.py b/autogpt_platform/backend/backend/util/json.pyindex e425483c31..afa0ad5eac 100644--- a/autogpt_platform/backend/backend/util/json.py+++ b/autogpt_platform/backend/backend/util/json.py@@ -1,13 +1,19 @@import json+import refrom typing import Any, Type, TypeGuard, TypeVar, overloadimport jsonschema+import orjsonfrom fastapi.encoders import jsonable_encoderfrom prisma import Jsonfrom pydantic import BaseModelfrom .type import type_match+# Precompiled regex to remove PostgreSQL-incompatible control characters+# Removes \u0000-\u0008, \u000B-\u000C, \u000E-\u001F, \u007F (keeps tab \u0009, newline \u000A, carriage return \u000D)+POSTGRES_CONTROL_CHARS = re.compile(r"[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]")+def to_dict(data) -> dict:if isinstance(data, BaseModel):@@ -15,7 +21,9 @@ def to_dict(data) -> dict:return jsonable_encoder(data)-def dumps(data: Any, *args: Any, **kwargs: Any) -> str:+def dumps(+ data: Any, *args: Any, indent: int | None = None, option: int = 0, **kwargs: Any+) -> str:"""Serialize data to JSON string with automatic conversion of Pydantic models and complex types.@@ -28,9 +36,13 @@ def dumps(data: Any, *args: Any, **kwargs: Any) -> str:data : AnyThe data to serialize. Can be any type including Pydantic models, dicts, lists, etc.*args : Any- Additional positional arguments passed to json.dumps()+ Additional positional arguments+ indent : int | None+ If not None, pretty-print with indentation+ option : int+ orjson option flags (default: 0)**kwargs : Any- Additional keyword arguments passed to json.dumps() (e.g., indent, separators)+ Additional keyword arguments. Supported: default, ensure_ascii, separators, indentReturns-------@@ -45,7 +57,21 @@ def dumps(data: Any, *args: Any, **kwargs: Any) -> str:>>> dumps(pydantic_model_instance, indent=2)'{\n "field1": "value1",\n "field2": "value2"\n}'"""- return json.dumps(to_dict(data), *args, **kwargs)+ serializable_data = to_dict(data)++ # Handle indent parameter+ if indent is not None or kwargs.get("indent") is not None:+ option |= orjson.OPT_INDENT_2++ # orjson only accepts specific parameters, filter out stdlib json params+ # ensure_ascii: orjson always produces UTF-8 (better than ASCII)+ # separators: orjson uses compact separators by default+ supported_orjson_params = {"default"}+ orjson_kwargs = {k: v for k, v in kwargs.items() if k in supported_orjson_params}++ return orjson.dumps(serializable_data, option=option, **orjson_kwargs).decode(+ "utf-8"+ )T = TypeVar("T")@@ -62,9 +88,8 @@ def loads(data: str | bytes, *args, **kwargs) -> Any: ...def loads(data: str | bytes, *args, target_type: Type[T] | None = None, **kwargs) -> Any:- if isinstance(data, bytes):- data = data.decode("utf-8")- parsed = json.loads(data, *args, **kwargs)+ parsed = orjson.loads(data)+if target_type:return type_match(parsed, target_type)return parsed@@ -99,16 +124,19 @@ def convert_pydantic_to_json(output_data: Any) -> Any:def SafeJson(data: Any) -> Json:- """Safely serialize data and return Prisma's Json type."""+ """+ Safely serialize data and return Prisma's Json type.+ Sanitizes null bytes to prevent PostgreSQL 22P05 errors.+ """if isinstance(data, BaseModel):- return Json(- data.model_dump(- mode="json",- warnings="error",- exclude_none=True,- fallback=lambda v: None,- )+ json_string = data.model_dump_json(+ warnings="error",+ exclude_none=True,+ fallback=lambda v: None,)- # Round-trip through JSON to ensure proper serialization with fallback for non-serializable values- json_string = dumps(data, default=lambda v: None)- return Json(json.loads(json_string))+ else:+ json_string = dumps(data, default=lambda v: None)++ # Remove PostgreSQL-incompatible control characters in single regex operation+ sanitized_json = POSTGRES_CONTROL_CHARS.sub("", json_string)+ return Json(json.loads(sanitized_json))diff --git a/autogpt_platform/backend/backend/util/retry.py b/autogpt_platform/backend/backend/util/retry.pyindex c586281af9..937e411fa3 100644--- a/autogpt_platform/backend/backend/util/retry.py+++ b/autogpt_platform/backend/backend/util/retry.py@@ -17,6 +17,37 @@ from backend.util.process import get_service_namelogger = logging.getLogger(__name__)+# Alert threshold for excessive retries+EXCESSIVE_RETRY_THRESHOLD = 50+++def _send_critical_retry_alert(+ func_name: str, attempt_number: int, exception: Exception, context: str = ""+):+ """Send alert when a function is approaching the retry failure threshold."""+ try:+ # Import here to avoid circular imports+ from backend.util.clients import get_notification_manager_client++ notification_client = get_notification_manager_client()++ prefix = f"{context}: " if context else ""+ alert_msg = (+ f"π¨ CRITICAL: Operation Approaching Failure Threshold: {prefix}'{func_name}'\n\n"+ f"Current attempt: {attempt_number}/{EXCESSIVE_RETRY_THRESHOLD}\n"+ f"Error: {type(exception).__name__}: {exception}\n\n"+ f"This operation is about to fail permanently. Investigate immediately."+ )++ notification_client.discord_system_alert(alert_msg)+ logger.critical(+ f"CRITICAL ALERT SENT: Operation {func_name} at attempt {attempt_number}"+ )++ except Exception as alert_error:+ logger.error(f"Failed to send critical retry alert: {alert_error}")+ # Don't let alerting failures break the main flow+def _create_retry_callback(context: str = ""):"""Create a retry callback with optional context."""@@ -29,17 +60,22 @@ def _create_retry_callback(context: str = ""):prefix = f"{context}: " if context else ""if retry_state.outcome.failed and retry_state.next_action is None:- # Final failure+ # Final failure - just log the error (alert was already sent at excessive threshold)logger.error(f"{prefix}Giving up after {attempt_number} attempts for '{func_name}': "f"{type(exception).__name__}: {exception}")else:- # Retry attempt- logger.warning(- f"{prefix}Retry attempt {attempt_number} for '{func_name}': "- f"{type(exception).__name__}: {exception}"- )+ # Retry attempt - send critical alert only once at threshold+ if attempt_number == EXCESSIVE_RETRY_THRESHOLD:+ _send_critical_retry_alert(+ func_name, attempt_number, exception, context+ )+ else:+ logger.warning(+ f"{prefix}Retry attempt {attempt_number} for '{func_name}': "+ f"{type(exception).__name__}: {exception}"+ )return callbackdiff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.pyindex 290d259ac6..1cbdf2ecf4 100644--- a/autogpt_platform/backend/backend/util/service.py+++ b/autogpt_platform/backend/backend/util/service.py@@ -7,7 +7,7 @@ import osimport threadingimport timefrom abc import ABC, abstractmethod-from functools import cached_property, update_wrapper+from functools import update_wrapperfrom typing import (Any,Awaitable,@@ -375,6 +375,8 @@ def get_service_client(self.base_url = f"http://{host}:{port}".rstrip("/")self._connection_failure_count = 0self._last_client_reset = 0+ self._async_clients = {} # None key for default async client+ self._sync_clients = {} # For sync clients (no event loop concept)def _create_sync_client(self) -> httpx.Client:return httpx.Client(@@ -398,13 +400,33 @@ def get_service_client(),)- @cached_property+ @propertydef sync_client(self) -> httpx.Client:- return self._create_sync_client()+ """Get the sync client (thread-safe singleton)."""+ # Use service name as key for better identification+ service_name = service_client_type.get_service_type().__name__+ if client := self._sync_clients.get(service_name):+ return client+ return self._sync_clients.setdefault(+ service_name, self._create_sync_client()+ )- @cached_property+ @propertydef async_client(self) -> httpx.AsyncClient:- return self._create_async_client()+ """Get the appropriate async client for the current context.++ Returns per-event-loop client when in async context,+ falls back to default client otherwise.+ """+ try:+ loop = asyncio.get_running_loop()+ except RuntimeError:+ # No event loop, use None as default key+ loop = None++ if client := self._async_clients.get(loop):+ return client+ return self._async_clients.setdefault(loop, self._create_async_client())def _handle_connection_error(self, error: Exception) -> None:"""Handle connection errors and implement self-healing"""@@ -423,10 +445,8 @@ def get_service_client(# Clear cached clients to force recreation on next access# Only recreate when there's actually a problem- if hasattr(self, "sync_client"):- delattr(self, "sync_client")- if hasattr(self, "async_client"):- delattr(self, "async_client")+ self._sync_clients.clear()+ self._async_clients.clear()# Reset countersself._connection_failure_count = 0@@ -492,28 +512,37 @@ def get_service_client(raiseasync def aclose(self) -> None:- if hasattr(self, "sync_client"):- self.sync_client.close()- if hasattr(self, "async_client"):- await self.async_client.aclose()+ # Close all sync clients+ for client in self._sync_clients.values():+ client.close()+ self._sync_clients.clear()++ # Close all async clients (including default with None key)+ for client in self._async_clients.values():+ await client.aclose()+ self._async_clients.clear()def close(self) -> None:- if hasattr(self, "sync_client"):- self.sync_client.close()- # Note: Cannot close async client synchronously+ # Close all sync clients+ for client in self._sync_clients.values():+ client.close()+ self._sync_clients.clear()+ # Note: Cannot close async clients synchronously+ # They will be cleaned up by garbage collectiondef __del__(self):"""Cleanup HTTP clients on garbage collection to prevent resource leaks."""try:- if hasattr(self, "sync_client"):- self.sync_client.close()- if hasattr(self, "async_client"):- # Note: Can't await in __del__, so we just close sync- # The async client will be cleaned up by garbage collection+ # Close any remaining sync clients+ for client in self._sync_clients.values():+ client.close()++ # Warn if async clients weren't properly closed+ if self._async_clients:import warningswarnings.warn(- "DynamicClient async client not explicitly closed. "+ "DynamicClient async clients not explicitly closed. ""Call aclose() before destroying the client.",ResourceWarning,stacklevel=2,diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.pyindex 49c3790df4..cf2dc4588a 100644--- a/autogpt_platform/backend/backend/util/settings.py+++ b/autogpt_platform/backend/backend/util/settings.py@@ -59,6 +59,19 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):le=1000,description="Maximum number of workers to use for graph execution.",)++ # FastAPI Thread Pool Configuration+ # IMPORTANT: FastAPI automatically offloads ALL sync functions to a thread pool:+ # - Sync endpoint functions (def instead of async def)+ # - Sync dependency functions (def instead of async def)+ # - Manually called run_in_threadpool() operations+ # Default thread pool size is only 40, which becomes a bottleneck under high concurrency+ fastapi_thread_pool_size: int = Field(+ default=60,+ ge=40,+ le=500,+ description="Thread pool size for FastAPI sync operations. All sync endpoints and dependencies automatically use this pool. Higher values support more concurrent sync operations but use more memory.",+ )pyro_host: str = Field(default="localhost",description="The default hostname of the Pyro server.",diff --git a/autogpt_platform/backend/backend/util/test_json.py b/autogpt_platform/backend/backend/util/test_json.pyindex 9e8602ea08..8ea033a5d4 100644--- a/autogpt_platform/backend/backend/util/test_json.py+++ b/autogpt_platform/backend/backend/util/test_json.py@@ -215,3 +215,29 @@ class TestSafeJson:}result = SafeJson(data)assert isinstance(result, Json)++ def test_control_character_sanitization(self):+ """Test that PostgreSQL-incompatible control characters are sanitized by SafeJson."""+ # Test data with problematic control characters that would cause PostgreSQL errors+ problematic_data = {+ "null_byte": "data with \x00 null",+ "bell_char": "data with \x07 bell",+ "form_feed": "data with \x0C feed",+ "escape_char": "data with \x1B escape",+ "delete_char": "data with \x7F delete",+ }++ # SafeJson should successfully process data with control characters+ result = SafeJson(problematic_data)+ assert isinstance(result, Json)++ # Test that safe whitespace characters are preserved+ safe_data = {+ "with_tab": "text with \t tab",+ "with_newline": "text with \n newline",+ "with_carriage_return": "text with \r carriage return",+ "normal_text": "completely normal text",+ }++ safe_result = SafeJson(safe_data)+ assert isinstance(safe_result, Json)diff --git a/autogpt_platform/backend/backend/util/virus_scanner.py b/autogpt_platform/backend/backend/util/virus_scanner.pyindex 4494c94790..1ea31cac95 100644--- a/autogpt_platform/backend/backend/util/virus_scanner.py+++ b/autogpt_platform/backend/backend/util/virus_scanner.py@@ -2,9 +2,16 @@ import asyncioimport ioimport loggingimport time+import warningsfrom typing import Optional, Tuple-import aioclamd+# Suppress the specific pkg_resources deprecation warning from aioclamd+with warnings.catch_warnings():+ warnings.filterwarnings(+ "ignore", message="pkg_resources is deprecated", category=UserWarning+ )+ import aioclamd+from pydantic import BaseModelfrom pydantic_settings import BaseSettingsdiff --git a/autogpt_platform/backend/clean_test_db.py b/autogpt_platform/backend/clean_test_db.pydeleted file mode 100644index da155c6f75..0000000000--- a/autogpt_platform/backend/clean_test_db.py+++ /dev/null@@ -1,101 +0,0 @@-#!/usr/bin/env python3-"""-Clean the test database by removing all data while preserving the schema.--Usage:- poetry run python clean_test_db.py [--yes]--Options:- --yes Skip confirmation prompt-"""--import asyncio-import sys--from prisma import Prisma---async def main():- db = Prisma()- await db.connect()-- print("=" * 60)- print("Cleaning Test Database")- print("=" * 60)- print()-- # Get initial counts- user_count = await db.user.count()- agent_count = await db.agentgraph.count()-- print(f"Current data: {user_count} users, {agent_count} agent graphs")-- if user_count == 0 and agent_count == 0:- print("Database is already clean!")- await db.disconnect()- return-- # Check for --yes flag- skip_confirm = "--yes" in sys.argv-- if not skip_confirm:- response = input("\nDo you want to clean all data? (yes/no): ")- if response.lower() != "yes":- print("Aborted.")- await db.disconnect()- return-- print("\nCleaning database...")-- # Delete in reverse order of dependencies- tables = [- ("UserNotificationBatch", db.usernotificationbatch),- ("NotificationEvent", db.notificationevent),- ("CreditRefundRequest", db.creditrefundrequest),- ("StoreListingReview", db.storelistingreview),- ("StoreListingVersion", db.storelistingversion),- ("StoreListing", db.storelisting),- ("AgentNodeExecutionInputOutput", db.agentnodeexecutioninputoutput),- ("AgentNodeExecution", db.agentnodeexecution),- ("AgentGraphExecution", db.agentgraphexecution),- ("AgentNodeLink", db.agentnodelink),- ("LibraryAgent", db.libraryagent),- ("AgentPreset", db.agentpreset),- ("IntegrationWebhook", db.integrationwebhook),- ("AgentNode", db.agentnode),- ("AgentGraph", db.agentgraph),- ("AgentBlock", db.agentblock),- ("APIKey", db.apikey),- ("CreditTransaction", db.credittransaction),- ("AnalyticsMetrics", db.analyticsmetrics),- ("AnalyticsDetails", db.analyticsdetails),- ("Profile", db.profile),- ("UserOnboarding", db.useronboarding),- ("User", db.user),- ]-- for table_name, table in tables:- try:- count = await table.count()- if count > 0:- await table.delete_many()- print(f"β Deleted {count} records from {table_name}")- except Exception as e:- print(f"β Error cleaning {table_name}: {e}")-- # Refresh materialized views (they should be empty now)- try:- await db.execute_raw("SELECT refresh_store_materialized_views();")- print("\nβ Refreshed materialized views")- except Exception as e:- print(f"\nβ Could not refresh materialized views: {e}")-- await db.disconnect()-- print("\n" + "=" * 60)- print("Database cleaned successfully!")- print("=" * 60)---if __name__ == "__main__":- asyncio.run(main())diff --git a/autogpt_platform/backend/load-tests/README.md b/autogpt_platform/backend/load-tests/README.mdindex 6bc9a4978e..ba4d62e844 100644--- a/autogpt_platform/backend/load-tests/README.md+++ b/autogpt_platform/backend/load-tests/README.md@@ -8,11 +8,11 @@ Clean, streamlined load testing infrastructure for the AutoGPT Platform using k6# 1. Set up Supabase service key (required for token generation)export SUPABASE_SERVICE_KEY="your-supabase-service-key"-# 2. Generate pre-authenticated tokens (first time setup - creates 150+ tokens with 24-hour expiry)-node generate-tokens.js+# 2. Generate pre-authenticated tokens (first time setup - creates 160+ tokens with 24-hour expiry)+node generate-tokens.js --count=160-# 3. Set up k6 cloud credentials (for cloud testing)-export K6_CLOUD_TOKEN="your-k6-cloud-token"+# 3. Set up k6 cloud credentials (for cloud testing - see Credential Setup section below)+export K6_CLOUD_TOKEN="your-k6-cloud-token"export K6_CLOUD_PROJECT_ID="4254406"# 4. Verify setup and run quick test@@ -53,7 +53,7 @@ The AutoGPT Platform uses a single unified test runner (`run-tests.js`) for both### Test Modes- **Local Mode**: 5 VUs Γ 30s - Quick validation and debugging-- **Cloud Mode**: 80-150 VUs Γ 3-5m - Real performance testing+- **Cloud Mode**: 80-160 VUs Γ 3-6m - Real performance testing## π οΈ Usage@@ -99,12 +99,12 @@ npm run cloud### Pre-Authenticated Tokens-- **Generation**: Run `node generate-tokens.js` to create tokens-- **File**: `configs/pre-authenticated-tokens.js` (gitignored for security)-- **Capacity**: 150+ tokens supporting high-concurrency testing+- **Generation**: Run `node generate-tokens.js --count=160` to create tokens+- **File**: `configs/pre-authenticated-tokens.js` (gitignored for security)+- **Capacity**: 160+ tokens supporting high-concurrency testing- **Expiry**: 24 hours (86400 seconds) - extended for long-duration testing- **Benefit**: Eliminates Supabase auth rate limiting at scale-- **Regeneration**: Run `node generate-tokens.js` when tokens expire after 24 hours+- **Regeneration**: Run `node generate-tokens.js --count=160` when tokens expire after 24 hours### Environment Configuration@@ -124,8 +124,8 @@ npm run cloud### Load Testing Capabilities-- **High Concurrency**: Up to 150+ virtual users per test-- **Authentication Scaling**: Pre-auth tokens support 150+ concurrent users (10 tokens generated by default)+- **High Concurrency**: Up to 160+ virtual users per test+- **Authentication Scaling**: Pre-auth tokens support 160+ concurrent users- **Sequential Execution**: Multiple tests run one after another with proper delays- **Cloud Infrastructure**: Tests run on k6 cloud servers for consistent results- **ES Module Support**: Full ES module compatibility with modern JavaScript features@@ -134,10 +134,11 @@ npm run cloud### Validated Performance Limits-- **Core API**: 100 VUs successfully handling `/api/credits`, `/api/graphs`, `/api/blocks`, `/api/executions`-- **Graph Execution**: 80 VUs for complete workflow pipeline-- **Marketplace Browsing**: 150 VUs for public marketplace access-- **Authentication**: 150+ concurrent users with pre-authenticated tokens+- **Core API**: 100+ VUs successfully handling `/api/credits`, `/api/graphs`, `/api/blocks`, `/api/executions`+- **Graph Execution**: 80+ VUs for complete workflow pipeline+- **Marketplace Browsing**: 160 VUs for public marketplace access (verified)+- **Marketplace Library**: 160 VUs for authenticated library operations (verified)+- **Authentication**: 160+ concurrent users with pre-authenticated tokens### Target Metrics@@ -146,6 +147,14 @@ npm run cloud- **Success Rate**: Target > 95% under normal load- **Error Rate**: Target < 5% for all endpoints+### Recent Performance Results (160 VU Test - Verified)++- **Marketplace Library Operations**: 500-1000ms response times at 160 VUs+- **Authentication**: 100% success rate with pre-authenticated tokens+- **Library Journeys**: 5 operations per journey completing successfully+- **Test Duration**: 6+ minutes sustained load without degradation+- **k6 Cloud Execution**: Stable performance on Amazon US Columbus infrastructure+## π Troubleshooting### Common Issues@@ -157,8 +166,8 @@ npm run cloudβ Token has expired```-- **Solution**: Run `node generate-tokens.js` to create fresh 24-hour tokens-- **Note**: Default generates 10 tokens (increase with `--count=50` for higher concurrency)+- **Solution**: Run `node generate-tokens.js --count=160` to create fresh 24-hour tokens+- **Note**: Use `--count` parameter to generate appropriate number of tokens for your test scale**2. Cloud Credentials Missing**@@ -168,7 +177,17 @@ npm run cloud- **Solution**: Set `K6_CLOUD_TOKEN` and `K6_CLOUD_PROJECT_ID=4254406`-**3. Setup Verification Failed**+**3. k6 Cloud VU Scaling Issue**++```+β Test shows only 5 VUs instead of requested 100+ VUs+```++- **Problem**: Using `K6_ENVIRONMENT=DEV VUS=160 k6 cloud run test.js` (incorrect)+- **Solution**: Use `k6 cloud run --env K6_ENVIRONMENT=DEV --env VUS=160 test.js` (correct)+- **Note**: The unified test runner (`run-tests.js`) already uses the correct syntax++**4. Setup Verification Failed**```β Verification failed@@ -181,28 +200,38 @@ npm run cloud**1. Supabase Service Key (Required for all testing):**```bash-# Get service key from environment or Kubernetes+# Option 1: From your local environment (if available)export SUPABASE_SERVICE_KEY="your-supabase-service-key"++# Option 2: From Kubernetes secret (for platform developers)+kubectl get secret supabase-service-key -o jsonpath='{.data.service-key}' | base64 -d++# Option 3: From Supabase dashboard+# Go to Project Settings > API > service_role key (never commit this!)```**2. Generate Pre-Authenticated Tokens (Required):**```bash-# Creates 10 tokens with 24-hour expiry - prevents auth rate limiting-node generate-tokens.js+# Creates 160 tokens with 24-hour expiry - prevents auth rate limiting+node generate-tokens.js --count=160-# Generate more tokens for higher concurrency+# Generate fewer tokens for smaller tests (minimum 10)node generate-tokens.js --count=50# Regenerate when tokens expire (every 24 hours)-node generate-tokens.js+node generate-tokens.js --count=160```**3. k6 Cloud Credentials (Required for cloud testing):**```bash+# Get from k6 cloud dashboard: https://app.k6.io/account/api-tokenexport K6_CLOUD_TOKEN="your-k6-cloud-token"export K6_CLOUD_PROJECT_ID="4254406" # AutoGPT Platform project ID++# Verify credentials work+node run-tests.js list # Shows β k6 cloud credentials configured```## π File Structure@@ -252,8 +281,8 @@ For granular control over individual test scripts:```bash# k6 Cloud execution (recommended for performance testing)-K6_ENVIRONMENT=DEV VUS=100 DURATION=5m \-k6 cloud run --env K6_ENVIRONMENT=DEV --env VUS=100 --env DURATION=5m tests/api/core-api-test.js+# IMPORTANT: Use --env syntax for k6 cloud to ensure proper VU scaling+k6 cloud run --env K6_ENVIRONMENT=DEV --env VUS=160 --env DURATION=5m --env RAMP_UP=30s --env RAMP_DOWN=30s tests/marketplace/library-access-test.js# Local execution with cloud output (debugging)K6_ENVIRONMENT=DEV VUS=10 DURATION=1m \diff --git a/autogpt_platform/backend/poetry.lock b/autogpt_platform/backend/poetry.lockindex 059eb1d14f..341f4379fa 100644--- a/autogpt_platform/backend/poetry.lock+++ b/autogpt_platform/backend/poetry.lock@@ -3451,6 +3451,99 @@ files = [importlib-metadata = ">=6.0,<8.8.0"typing-extensions = ">=4.5.0"+[[package]]+name = "orjson"+version = "3.11.3"+description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy"+optional = false+python-versions = ">=3.9"+groups = ["main"]+files = [+ {file = "orjson-3.11.3-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:29cb1f1b008d936803e2da3d7cba726fc47232c45df531b29edf0b232dd737e7"},+ {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97dceed87ed9139884a55db8722428e27bd8452817fbf1869c58b49fecab1120"},+ {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:58533f9e8266cb0ac298e259ed7b4d42ed3fa0b78ce76860626164de49e0d467"},+ {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c212cfdd90512fe722fa9bd620de4d46cda691415be86b2e02243242ae81873"},+ {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff835b5d3e67d9207343effb03760c00335f8b5285bfceefd4dc967b0e48f6a"},+ {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5aa4682912a450c2db89cbd92d356fef47e115dffba07992555542f344d301b"},+ {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7d18dd34ea2e860553a579df02041845dee0af8985dff7f8661306f95504ddf"},+ {file = "orjson-3.11.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d8b11701bc43be92ea42bd454910437b355dfb63696c06fe953ffb40b5f763b4"},+ {file = "orjson-3.11.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:90368277087d4af32d38bd55f9da2ff466d25325bf6167c8f382d8ee40cb2bbc"},+ {file = "orjson-3.11.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fd7ff459fb393358d3a155d25b275c60b07a2c83dcd7ea962b1923f5a1134569"},+ {file = "orjson-3.11.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f8d902867b699bcd09c176a280b1acdab57f924489033e53d0afe79817da37e6"},+ {file = "orjson-3.11.3-cp310-cp310-win32.whl", hash = "sha256:bb93562146120bb51e6b154962d3dadc678ed0fce96513fa6bc06599bb6f6edc"},+ {file = "orjson-3.11.3-cp310-cp310-win_amd64.whl", hash = "sha256:976c6f1975032cc327161c65d4194c549f2589d88b105a5e3499429a54479770"},+ {file = "orjson-3.11.3-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9d2ae0cc6aeb669633e0124531f342a17d8e97ea999e42f12a5ad4adaa304c5f"},+ {file = "orjson-3.11.3-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:ba21dbb2493e9c653eaffdc38819b004b7b1b246fb77bfc93dc016fe664eac91"},+ {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00f1a271e56d511d1569937c0447d7dce5a99a33ea0dec76673706360a051904"},+ {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b67e71e47caa6680d1b6f075a396d04fa6ca8ca09aafb428731da9b3ea32a5a6"},+ {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d7d012ebddffcce8c85734a6d9e5f08180cd3857c5f5a3ac70185b43775d043d"},+ {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd759f75d6b8d1b62012b7f5ef9461d03c804f94d539a5515b454ba3a6588038"},+ {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6890ace0809627b0dff19cfad92d69d0fa3f089d3e359a2a532507bb6ba34efb"},+ {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9d4a5e041ae435b815e568537755773d05dac031fee6a57b4ba70897a44d9d2"},+ {file = "orjson-3.11.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2d68bf97a771836687107abfca089743885fb664b90138d8761cce61d5625d55"},+ {file = "orjson-3.11.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:bfc27516ec46f4520b18ef645864cee168d2a027dbf32c5537cb1f3e3c22dac1"},+ {file = "orjson-3.11.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f66b001332a017d7945e177e282a40b6997056394e3ed7ddb41fb1813b83e824"},+ {file = "orjson-3.11.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:212e67806525d2561efbfe9e799633b17eb668b8964abed6b5319b2f1cfbae1f"},+ {file = "orjson-3.11.3-cp311-cp311-win32.whl", hash = "sha256:6e8e0c3b85575a32f2ffa59de455f85ce002b8bdc0662d6b9c2ed6d80ab5d204"},+ {file = "orjson-3.11.3-cp311-cp311-win_amd64.whl", hash = "sha256:6be2f1b5d3dc99a5ce5ce162fc741c22ba9f3443d3dd586e6a1211b7bc87bc7b"},+ {file = "orjson-3.11.3-cp311-cp311-win_arm64.whl", hash = "sha256:fafb1a99d740523d964b15c8db4eabbfc86ff29f84898262bf6e3e4c9e97e43e"},+ {file = "orjson-3.11.3-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:8c752089db84333e36d754c4baf19c0e1437012242048439c7e80eb0e6426e3b"},+ {file = "orjson-3.11.3-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:9b8761b6cf04a856eb544acdd82fc594b978f12ac3602d6374a7edb9d86fd2c2"},+ {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b13974dc8ac6ba22feaa867fc19135a3e01a134b4f7c9c28162fed4d615008a"},+ {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f83abab5bacb76d9c821fd5c07728ff224ed0e52d7a71b7b3de822f3df04e15c"},+ {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6fbaf48a744b94091a56c62897b27c31ee2da93d826aa5b207131a1e13d4064"},+ {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc779b4f4bba2847d0d2940081a7b6f7b5877e05408ffbb74fa1faf4a136c424"},+ {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd4b909ce4c50faa2192da6bb684d9848d4510b736b0611b6ab4020ea6fd2d23"},+ {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:524b765ad888dc5518bbce12c77c2e83dee1ed6b0992c1790cc5fb49bb4b6667"},+ {file = "orjson-3.11.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:84fd82870b97ae3cdcea9d8746e592b6d40e1e4d4527835fc520c588d2ded04f"},+ {file = "orjson-3.11.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:fbecb9709111be913ae6879b07bafd4b0785b44c1eb5cac8ac76da048b3885a1"},+ {file = "orjson-3.11.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9dba358d55aee552bd868de348f4736ca5a4086d9a62e2bfbbeeb5629fe8b0cc"},+ {file = "orjson-3.11.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eabcf2e84f1d7105f84580e03012270c7e97ecb1fb1618bda395061b2a84a049"},+ {file = "orjson-3.11.3-cp312-cp312-win32.whl", hash = "sha256:3782d2c60b8116772aea8d9b7905221437fdf53e7277282e8d8b07c220f96cca"},+ {file = "orjson-3.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:79b44319268af2eaa3e315b92298de9a0067ade6e6003ddaef72f8e0bedb94f1"},+ {file = "orjson-3.11.3-cp312-cp312-win_arm64.whl", hash = "sha256:0e92a4e83341ef79d835ca21b8bd13e27c859e4e9e4d7b63defc6e58462a3710"},+ {file = "orjson-3.11.3-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:af40c6612fd2a4b00de648aa26d18186cd1322330bd3a3cc52f87c699e995810"},+ {file = "orjson-3.11.3-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:9f1587f26c235894c09e8b5b7636a38091a9e6e7fe4531937534749c04face43"},+ {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61dcdad16da5bb486d7227a37a2e789c429397793a6955227cedbd7252eb5a27"},+ {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:11c6d71478e2cbea0a709e8a06365fa63da81da6498a53e4c4f065881d21ae8f"},+ {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff94112e0098470b665cb0ed06efb187154b63649403b8d5e9aedeb482b4548c"},+ {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae8b756575aaa2a855a75192f356bbda11a89169830e1439cfb1a3e1a6dde7be"},+ {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c9416cc19a349c167ef76135b2fe40d03cea93680428efee8771f3e9fb66079d"},+ {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b822caf5b9752bc6f246eb08124c3d12bf2175b66ab74bac2ef3bbf9221ce1b2"},+ {file = "orjson-3.11.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:414f71e3bdd5573893bf5ecdf35c32b213ed20aa15536fe2f588f946c318824f"},+ {file = "orjson-3.11.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:828e3149ad8815dc14468f36ab2a4b819237c155ee1370341b91ea4c8672d2ee"},+ {file = "orjson-3.11.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ac9e05f25627ffc714c21f8dfe3a579445a5c392a9c8ae7ba1d0e9fb5333f56e"},+ {file = "orjson-3.11.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e44fbe4000bd321d9f3b648ae46e0196d21577cf66ae684a96ff90b1f7c93633"},+ {file = "orjson-3.11.3-cp313-cp313-win32.whl", hash = "sha256:2039b7847ba3eec1f5886e75e6763a16e18c68a63efc4b029ddf994821e2e66b"},+ {file = "orjson-3.11.3-cp313-cp313-win_amd64.whl", hash = "sha256:29be5ac4164aa8bdcba5fa0700a3c9c316b411d8ed9d39ef8a882541bd452fae"},+ {file = "orjson-3.11.3-cp313-cp313-win_arm64.whl", hash = "sha256:18bd1435cb1f2857ceb59cfb7de6f92593ef7b831ccd1b9bfb28ca530e539dce"},+ {file = "orjson-3.11.3-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cf4b81227ec86935568c7edd78352a92e97af8da7bd70bdfdaa0d2e0011a1ab4"},+ {file = "orjson-3.11.3-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:bc8bc85b81b6ac9fc4dae393a8c159b817f4c2c9dee5d12b773bddb3b95fc07e"},+ {file = "orjson-3.11.3-cp314-cp314-manylinux_2_34_aarch64.whl", hash = "sha256:88dcfc514cfd1b0de038443c7b3e6a9797ffb1b3674ef1fd14f701a13397f82d"},+ {file = "orjson-3.11.3-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:d61cd543d69715d5fc0a690c7c6f8dcc307bc23abef9738957981885f5f38229"},+ {file = "orjson-3.11.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2b7b153ed90ababadbef5c3eb39549f9476890d339cf47af563aea7e07db2451"},+ {file = "orjson-3.11.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:7909ae2460f5f494fecbcd10613beafe40381fd0316e35d6acb5f3a05bfda167"},+ {file = "orjson-3.11.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:2030c01cbf77bc67bee7eef1e7e31ecf28649353987775e3583062c752da0077"},+ {file = "orjson-3.11.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a0169ebd1cbd94b26c7a7ad282cf5c2744fce054133f959e02eb5265deae1872"},+ {file = "orjson-3.11.3-cp314-cp314-win32.whl", hash = "sha256:0c6d7328c200c349e3a4c6d8c83e0a5ad029bdc2d417f234152bf34842d0fc8d"},+ {file = "orjson-3.11.3-cp314-cp314-win_amd64.whl", hash = "sha256:317bbe2c069bbc757b1a2e4105b64aacd3bc78279b66a6b9e51e846e4809f804"},+ {file = "orjson-3.11.3-cp314-cp314-win_arm64.whl", hash = "sha256:e8f6a7a27d7b7bec81bd5924163e9af03d49bbb63013f107b48eb5d16db711bc"},+ {file = "orjson-3.11.3-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:56afaf1e9b02302ba636151cfc49929c1bb66b98794291afd0e5f20fecaf757c"},+ {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:913f629adef31d2d350d41c051ce7e33cf0fd06a5d1cb28d49b1899b23b903aa"},+ {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e0a23b41f8f98b4e61150a03f83e4f0d566880fe53519d445a962929a4d21045"},+ {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d721fee37380a44f9d9ce6c701b3960239f4fb3d5ceea7f31cbd43882edaa2f"},+ {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73b92a5b69f31b1a58c0c7e31080aeaec49c6e01b9522e71ff38d08f15aa56de"},+ {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2489b241c19582b3f1430cc5d732caefc1aaf378d97e7fb95b9e56bed11725f"},+ {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5189a5dab8b0312eadaf9d58d3049b6a52c454256493a557405e77a3d67ab7f"},+ {file = "orjson-3.11.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9d8787bdfbb65a85ea76d0e96a3b1bed7bf0fbcb16d40408dc1172ad784a49d2"},+ {file = "orjson-3.11.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:8e531abd745f51f8035e207e75e049553a86823d189a51809c078412cefb399a"},+ {file = "orjson-3.11.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8ab962931015f170b97a3dd7bd933399c1bae8ed8ad0fb2a7151a5654b6941c7"},+ {file = "orjson-3.11.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:124d5ba71fee9c9902c4a7baa9425e663f7f0aecf73d31d54fe3dd357d62c1a7"},+ {file = "orjson-3.11.3-cp39-cp39-win32.whl", hash = "sha256:22724d80ee5a815a44fc76274bb7ba2e7464f5564aacb6ecddaa9970a83e3225"},+ {file = "orjson-3.11.3-cp39-cp39-win_amd64.whl", hash = "sha256:215c595c792a87d4407cb72dd5e0f6ee8e694ceeb7f9102b533c5a9bf2a916bb"},+ {file = "orjson-3.11.3.tar.gz", hash = "sha256:1c0603b1d2ffcd43a411d64797a19556ef76958aef1c182f22dc30860152a98a"},+]+[[package]]name = "packaging"version = "24.2"@@ -7159,4 +7252,4 @@ cffi = ["cffi (>=1.11)"][metadata]lock-version = "2.1"python-versions = ">=3.10,<3.14"-content-hash = "2c7e9370f500039b99868376021627c5a120e0ee31c5c5e6de39db2c3d82f414"+content-hash = "b2363edeebb91f410039c8d4b563f683c1edb0cf4bda4f3e6c287040e93639bc"diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.tomlindex 13d6591e25..5af2114718 100644--- a/autogpt_platform/backend/pyproject.toml+++ b/autogpt_platform/backend/pyproject.toml@@ -38,6 +38,7 @@ mem0ai = "^0.1.115"moviepy = "^2.1.2"ollama = "^0.5.1"openai = "^1.97.1"+orjson = "^3.10.0"pika = "^1.3.2"pinecone = "^7.3.0"poetry = "2.1.1" # CHECK DEPENDABOT SUPPORT BEFORE UPGRADINGdiff --git a/autogpt_platform/backend/run_test_data.py b/autogpt_platform/backend/run_test_data.pydeleted file mode 100644index b376607292..0000000000--- a/autogpt_platform/backend/run_test_data.py+++ /dev/null@@ -1,110 +0,0 @@-#!/usr/bin/env python3-"""-Run test data creation and update scripts in sequence.--Usage:- poetry run python run_test_data.py-"""--import asyncio-import subprocess-import sys-from pathlib import Path---def run_command(cmd: list[str], cwd: Path | None = None) -> bool:- """Run a command and return True if successful."""- try:- result = subprocess.run(- cmd, check=True, capture_output=True, text=True, cwd=cwd- )- if result.stdout:- print(result.stdout)- return True- except subprocess.CalledProcessError as e:- print(f"Error running command: {' '.join(cmd)}")- print(f"Error: {e.stderr}")- return False---async def main():- """Main function to run test data scripts."""- print("=" * 60)- print("Running Test Data Scripts for AutoGPT Platform")- print("=" * 60)- print()-- # Get the backend directory- backend_dir = Path(__file__).parent- test_dir = backend_dir / "test"-- # Check if we're in the right directory- if not (backend_dir / "pyproject.toml").exists():- print("ERROR: This script must be run from the backend directory")- sys.exit(1)-- print("1. Checking database connection...")- print("-" * 40)-- # Import here to ensure proper environment setup- try:- from prisma import Prisma-- db = Prisma()- await db.connect()- print("β Database connection successful")- await db.disconnect()- except Exception as e:- print(f"β Database connection failed: {e}")- print("\nPlease ensure:")- print("1. The database services are running (docker compose up -d)")- print("2. The DATABASE_URL in .env is correct")- print("3. Migrations have been run (poetry run prisma migrate deploy)")- sys.exit(1)-- print()- print("2. Running test data creator...")- print("-" * 40)-- # Run test_data_creator.py- if run_command(["poetry", "run", "python", "test_data_creator.py"], cwd=test_dir):- print()- print("β Test data created successfully!")-- print()- print("3. Running test data updater...")- print("-" * 40)-- # Run test_data_updater.py- if run_command(- ["poetry", "run", "python", "test_data_updater.py"], cwd=test_dir- ):- print()- print("β Test data updated successfully!")- else:- print()- print("β Test data updater failed!")- sys.exit(1)- else:- print()- print("β Test data creator failed!")- sys.exit(1)-- print()- print("=" * 60)- print("Test data setup completed successfully!")- print("=" * 60)- print()- print("The materialized views have been populated with test data:")- print("- mv_agent_run_counts: Agent execution statistics")- print("- mv_review_stats: Store listing review statistics")- print()- print("You can now:")- print("1. Run tests: poetry run test")- print("2. Start the backend: poetry run serve")- print("3. View data in the database")- print()---if __name__ == "__main__":- asyncio.run(main())