Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): implement user rate limiting for concurrent graph executions (#11128)

Zamil Majdy05a72f418530
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.py
index 873ae6e314..e03b5f7386 100644
--- a/autogpt_platform/backend/backend/data/execution.py
+++ b/autogpt_platform/backend/backend/data/execution.py
@@ -478,6 +478,41 @@ async def get_graph_executions(
return [GraphExecutionMeta.from_db(execution) for execution in executions]
+async def get_graph_executions_count(
+ user_id: str,
+ statuses: Optional[list[ExecutionStatus]] = None,
+ created_time_gte: Optional[datetime] = None,
+ created_time_lte: Optional[datetime] = None,
+) -> int:
+ """
+ Get count of graph executions for a user with optional filters.
+
+ Args:
+ user_id: The user ID to filter by
+ statuses: Optional list of execution statuses to filter by
+ created_time_gte: Optional minimum creation time
+ created_time_lte: Optional maximum creation time
+
+ Returns:
+ Count of matching graph executions
+ """
+ where_filter: AgentGraphExecutionWhereInput = {
+ "isDeleted": False,
+ "userId": user_id,
+ }
+
+ if created_time_gte or created_time_lte:
+ where_filter["createdAt"] = {
+ "gte": created_time_gte or datetime.min.replace(tzinfo=timezone.utc),
+ "lte": created_time_lte or datetime.max.replace(tzinfo=timezone.utc),
+ }
+ if statuses:
+ where_filter["OR"] = [{"executionStatus": status} for status in statuses]
+
+ count = await AgentGraphExecution.prisma().count(where=where_filter)
+ return count
+
+
class GraphExecutionsPaginated(BaseModel):
"""Response schema for paginated graph executions."""
diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.py
index d2190197f9..ea09b888ba 100644
--- a/autogpt_platform/backend/backend/executor/database.py
+++ b/autogpt_platform/backend/backend/executor/database.py
@@ -9,6 +9,7 @@ from backend.data.execution import (
get_execution_kv_data,
get_graph_execution_meta,
get_graph_executions,
+ get_graph_executions_count,
get_latest_node_execution,
get_node_execution,
get_node_executions,
@@ -71,7 +72,6 @@ async def _get_credits(user_id: str) -> int:
class DatabaseManager(AppService):
-
def run_service(self) -> None:
logger.info(f"[{self.service_name}] ⏳ Connecting to Database...")
self.run_and_wait(db.connect())
@@ -111,6 +111,7 @@ class DatabaseManager(AppService):
# Executions
get_graph_executions = _(get_graph_executions)
+ get_graph_executions_count = _(get_graph_executions_count)
get_graph_execution_meta = _(get_graph_execution_meta)
create_graph_execution = _(create_graph_execution)
get_node_execution = _(get_node_execution)
@@ -179,6 +180,7 @@ class DatabaseManagerClient(AppServiceClient):
# Executions
get_graph_executions = _(d.get_graph_executions)
+ get_graph_executions_count = _(d.get_graph_executions_count)
get_graph_execution_meta = _(d.get_graph_execution_meta)
get_node_executions = _(d.get_node_executions)
update_node_execution_status = _(d.update_node_execution_status)
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index 3724166364..dea8cffd30 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -7,6 +7,7 @@ import uuid
from collections import defaultdict
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import asynccontextmanager
+from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast
from pika.adapters.blocking_connection import BlockingChannel
@@ -1183,9 +1184,9 @@ class ExecutionProcessor:
f"❌ **Insufficient Funds Alert**\n"
f"User: {user_email or user_id}\n"
f"Agent: {metadata.name if metadata else 'Unknown Agent'}\n"
- f"Current balance: ${e.balance/100:.2f}\n"
- f"Attempted cost: ${abs(e.amount)/100:.2f}\n"
- f"Shortfall: ${abs(shortfall)/100:.2f}\n"
+ f"Current balance: ${e.balance / 100:.2f}\n"
+ f"Attempted cost: ${abs(e.amount) / 100:.2f}\n"
+ f"Shortfall: ${abs(shortfall) / 100:.2f}\n"
f"[View User Details]({base_url}/admin/spending?search={user_email})"
)
@@ -1232,9 +1233,9 @@ class ExecutionProcessor:
alert_message = (
f"⚠️ **Low Balance Alert**\n"
f"User: {user_email or user_id}\n"
- f"Balance dropped below ${LOW_BALANCE_THRESHOLD/100:.2f}\n"
- f"Current balance: ${current_balance/100:.2f}\n"
- f"Transaction cost: ${transaction_cost/100:.2f}\n"
+ f"Balance dropped below ${LOW_BALANCE_THRESHOLD / 100:.2f}\n"
+ f"Current balance: ${current_balance / 100:.2f}\n"
+ f"Transaction cost: ${transaction_cost / 100:.2f}\n"
f"[View User Details]({base_url}/admin/spending?search={user_email})"
)
get_notification_manager_client().discord_system_alert(
@@ -1465,10 +1466,37 @@ class ExecutionManager(AppProcess):
return
graph_exec_id = graph_exec_entry.graph_exec_id
+ user_id = graph_exec_entry.user_id
logger.info(
- f"[{self.service_name}] Received RUN for graph_exec_id={graph_exec_id}"
+ f"[{self.service_name}] Received RUN for graph_exec_id={graph_exec_id}, user_id={user_id}"
)
+ # Check user rate limit before processing
+ try:
+ # Only check executions from the last 24 hours for performance
+ current_running_count = get_db_client().get_graph_executions_count(
+ user_id=user_id,
+ statuses=[ExecutionStatus.RUNNING],
+ created_time_gte=datetime.now(timezone.utc) - timedelta(hours=24),
+ )
+
+ if (
+ current_running_count
+ >= settings.config.max_concurrent_graph_executions_per_user
+ ):
+ logger.warning(
+ f"[{self.service_name}] Rate limit exceeded for user {user_id}: "
+ f"{current_running_count}/{settings.config.max_concurrent_graph_executions_per_user} running executions"
+ )
+ _ack_message(reject=True, requeue=True)
+ return
+
+ except Exception as e:
+ logger.error(
+ f"[{self.service_name}] Failed to check rate limit for user {user_id}: {e}, proceeding with execution"
+ )
+ # If rate limit check fails, proceed to avoid blocking executions
+
# Check for local duplicate execution first
if graph_exec_id in self.active_graph_runs:
logger.warning(
diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.py
index cf2dc4588a..68dabceccd 100644
--- a/autogpt_platform/backend/backend/util/settings.py
+++ b/autogpt_platform/backend/backend/util/settings.py
@@ -148,6 +148,12 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):
default=60 * 60,
description="Time in seconds for how far back to check for the late executions.",
)
+ max_concurrent_graph_executions_per_user: int = Field(
+ default=50,
+ ge=1,
+ le=1000,
+ description="Maximum number of concurrent graph executions allowed per user.",
+ )
block_error_rate_threshold: float = Field(
default=0.5,