AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): implement user rate limiting for concurrent graph executions (#11128)
Zamil Majdy05a72f418530
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex 873ae6e314..e03b5f7386 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -478,6 +478,41 @@ async def get_graph_executions(return [GraphExecutionMeta.from_db(execution) for execution in executions]+async def get_graph_executions_count(+ user_id: str,+ statuses: Optional[list[ExecutionStatus]] = None,+ created_time_gte: Optional[datetime] = None,+ created_time_lte: Optional[datetime] = None,+) -> int:+ """+ Get count of graph executions for a user with optional filters.++ Args:+ user_id: The user ID to filter by+ statuses: Optional list of execution statuses to filter by+ created_time_gte: Optional minimum creation time+ created_time_lte: Optional maximum creation time++ Returns:+ Count of matching graph executions+ """+ where_filter: AgentGraphExecutionWhereInput = {+ "isDeleted": False,+ "userId": user_id,+ }++ if created_time_gte or created_time_lte:+ where_filter["createdAt"] = {+ "gte": created_time_gte or datetime.min.replace(tzinfo=timezone.utc),+ "lte": created_time_lte or datetime.max.replace(tzinfo=timezone.utc),+ }+ if statuses:+ where_filter["OR"] = [{"executionStatus": status} for status in statuses]++ count = await AgentGraphExecution.prisma().count(where=where_filter)+ return count++class GraphExecutionsPaginated(BaseModel):"""Response schema for paginated graph executions."""diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.pyindex d2190197f9..ea09b888ba 100644--- a/autogpt_platform/backend/backend/executor/database.py+++ b/autogpt_platform/backend/backend/executor/database.py@@ -9,6 +9,7 @@ from backend.data.execution import (get_execution_kv_data,get_graph_execution_meta,get_graph_executions,+ get_graph_executions_count,get_latest_node_execution,get_node_execution,get_node_executions,@@ -71,7 +72,6 @@ async def _get_credits(user_id: str) -> int:class DatabaseManager(AppService):-def run_service(self) -> None:logger.info(f"[{self.service_name}] ⏳ Connecting to Database...")self.run_and_wait(db.connect())@@ -111,6 +111,7 @@ class DatabaseManager(AppService):# Executionsget_graph_executions = _(get_graph_executions)+ get_graph_executions_count = _(get_graph_executions_count)get_graph_execution_meta = _(get_graph_execution_meta)create_graph_execution = _(create_graph_execution)get_node_execution = _(get_node_execution)@@ -179,6 +180,7 @@ class DatabaseManagerClient(AppServiceClient):# Executionsget_graph_executions = _(d.get_graph_executions)+ get_graph_executions_count = _(d.get_graph_executions_count)get_graph_execution_meta = _(d.get_graph_execution_meta)get_node_executions = _(d.get_node_executions)update_node_execution_status = _(d.update_node_execution_status)diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 3724166364..dea8cffd30 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -7,6 +7,7 @@ import uuidfrom collections import defaultdictfrom concurrent.futures import Future, ThreadPoolExecutorfrom contextlib import asynccontextmanager+from datetime import datetime, timedelta, timezonefrom typing import TYPE_CHECKING, Any, Optional, TypeVar, castfrom pika.adapters.blocking_connection import BlockingChannel@@ -1183,9 +1184,9 @@ class ExecutionProcessor:f"❌ **Insufficient Funds Alert**\n"f"User: {user_email or user_id}\n"f"Agent: {metadata.name if metadata else 'Unknown Agent'}\n"- f"Current balance: ${e.balance/100:.2f}\n"- f"Attempted cost: ${abs(e.amount)/100:.2f}\n"- f"Shortfall: ${abs(shortfall)/100:.2f}\n"+ f"Current balance: ${e.balance / 100:.2f}\n"+ f"Attempted cost: ${abs(e.amount) / 100:.2f}\n"+ f"Shortfall: ${abs(shortfall) / 100:.2f}\n"f"[View User Details]({base_url}/admin/spending?search={user_email})")@@ -1232,9 +1233,9 @@ class ExecutionProcessor:alert_message = (f"⚠️ **Low Balance Alert**\n"f"User: {user_email or user_id}\n"- f"Balance dropped below ${LOW_BALANCE_THRESHOLD/100:.2f}\n"- f"Current balance: ${current_balance/100:.2f}\n"- f"Transaction cost: ${transaction_cost/100:.2f}\n"+ f"Balance dropped below ${LOW_BALANCE_THRESHOLD / 100:.2f}\n"+ f"Current balance: ${current_balance / 100:.2f}\n"+ f"Transaction cost: ${transaction_cost / 100:.2f}\n"f"[View User Details]({base_url}/admin/spending?search={user_email})")get_notification_manager_client().discord_system_alert(@@ -1465,10 +1466,37 @@ class ExecutionManager(AppProcess):returngraph_exec_id = graph_exec_entry.graph_exec_id+ user_id = graph_exec_entry.user_idlogger.info(- f"[{self.service_name}] Received RUN for graph_exec_id={graph_exec_id}"+ f"[{self.service_name}] Received RUN for graph_exec_id={graph_exec_id}, user_id={user_id}")+ # Check user rate limit before processing+ try:+ # Only check executions from the last 24 hours for performance+ current_running_count = get_db_client().get_graph_executions_count(+ user_id=user_id,+ statuses=[ExecutionStatus.RUNNING],+ created_time_gte=datetime.now(timezone.utc) - timedelta(hours=24),+ )++ if (+ current_running_count+ >= settings.config.max_concurrent_graph_executions_per_user+ ):+ logger.warning(+ f"[{self.service_name}] Rate limit exceeded for user {user_id}: "+ f"{current_running_count}/{settings.config.max_concurrent_graph_executions_per_user} running executions"+ )+ _ack_message(reject=True, requeue=True)+ return++ except Exception as e:+ logger.error(+ f"[{self.service_name}] Failed to check rate limit for user {user_id}: {e}, proceeding with execution"+ )+ # If rate limit check fails, proceed to avoid blocking executions+# Check for local duplicate execution firstif graph_exec_id in self.active_graph_runs:logger.warning(diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.pyindex cf2dc4588a..68dabceccd 100644--- a/autogpt_platform/backend/backend/util/settings.py+++ b/autogpt_platform/backend/backend/util/settings.py@@ -148,6 +148,12 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):default=60 * 60,description="Time in seconds for how far back to check for the late executions.",)+ max_concurrent_graph_executions_per_user: int = Field(+ default=50,+ ge=1,+ le=1000,+ description="Maximum number of concurrent graph executions allowed per user.",+ )block_error_rate_threshold: float = Field(default=0.5,