Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): implement comprehensive rate-limited Discord alerting system (#11106)

Zamil Majdy59c27fe2487a
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index c0d5c5f441..929a3f8071 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -84,7 +84,11 @@ from backend.util.file import clean_exec_files
from backend.util.logging import TruncatedLogger, configure_logging
from backend.util.metrics import DiscordChannel
from backend.util.process import AppProcess, set_service_name
-from backend.util.retry import continuous_retry, func_retry
+from backend.util.retry import (
+ continuous_retry,
+ func_retry,
+ send_rate_limited_discord_alert,
+)
from backend.util.settings import Settings
from .cluster_lock import ClusterLock
@@ -979,16 +983,31 @@ class ExecutionProcessor:
if isinstance(e, Exception)
else Exception(f"{e.__class__.__name__}: {e}")
)
+ if not execution_stats.error:
+ execution_stats.error = str(error)
known_errors = (InsufficientBalanceError, ModerationError)
if isinstance(error, known_errors):
- execution_stats.error = str(error)
return ExecutionStatus.FAILED
execution_status = ExecutionStatus.FAILED
log_metadata.exception(
f"Failed graph execution {graph_exec.graph_exec_id}: {error}"
)
+
+ # Send rate-limited Discord alert for unknown/unexpected errors
+ send_rate_limited_discord_alert(
+ "graph_execution",
+ error,
+ "unknown_error",
+ f"🚨 **Unknown Graph Execution Error**\n"
+ f"User: {graph_exec.user_id}\n"
+ f"Graph ID: {graph_exec.graph_id}\n"
+ f"Execution ID: {graph_exec.graph_exec_id}\n"
+ f"Error Type: {type(error).__name__}\n"
+ f"Error: {str(error)[:200]}{'...' if len(str(error)) > 200 else ''}\n",
+ )
+
raise
finally:
diff --git a/autogpt_platform/backend/backend/util/retry.py b/autogpt_platform/backend/backend/util/retry.py
index 3e8b26335e..fd4414319c 100644
--- a/autogpt_platform/backend/backend/util/retry.py
+++ b/autogpt_platform/backend/backend/util/retry.py
@@ -26,7 +26,7 @@ _rate_limiter_lock = threading.Lock()
ALERT_RATE_LIMIT_SECONDS = 300 # 5 minutes between same alerts
-def _should_send_alert(func_name: str, exception: Exception, context: str = "") -> bool:
+def should_send_alert(func_name: str, exception: Exception, context: str = "") -> bool:
"""Check if we should send an alert based on rate limiting."""
# Create a unique key for this function+error+context combination
error_signature = (
@@ -42,38 +42,51 @@ def _should_send_alert(func_name: str, exception: Exception, context: str = "")
return False
-def _send_critical_retry_alert(
- func_name: str, attempt_number: int, exception: Exception, context: str = ""
-):
- """Send alert when a function is approaching the retry failure threshold."""
+def send_rate_limited_discord_alert(
+ func_name: str, exception: Exception, context: str, alert_msg: str, channel=None
+) -> bool:
+ """
+ Send a Discord alert with rate limiting.
- # Rate limit alerts to prevent spam
- if not _should_send_alert(func_name, exception, context):
- return # Just silently skip, no extra logging
+ Returns True if alert was sent, False if rate limited.
+ """
+ if not should_send_alert(func_name, exception, context):
+ return False
try:
- # Import here to avoid circular imports
from backend.util.clients import get_notification_manager_client
+ from backend.util.metrics import DiscordChannel
notification_client = get_notification_manager_client()
-
- prefix = f"{context}: " if context else ""
- alert_msg = (
- f"🚨 CRITICAL: Operation Approaching Failure Threshold: {prefix}'{func_name}'\n\n"
- f"Current attempt: {attempt_number}/{EXCESSIVE_RETRY_THRESHOLD}\n"
- f"Error: {type(exception).__name__}: {exception}\n\n"
- f"This operation is about to fail permanently. Investigate immediately."
+ notification_client.discord_system_alert(
+ alert_msg, channel or DiscordChannel.PLATFORM
)
+ return True
+
+ except Exception as alert_error:
+ logger.error(f"Failed to send Discord alert: {alert_error}")
+ return False
+
- notification_client.discord_system_alert(alert_msg)
+def _send_critical_retry_alert(
+ func_name: str, attempt_number: int, exception: Exception, context: str = ""
+):
+ """Send alert when a function is approaching the retry failure threshold."""
+
+ prefix = f"{context}: " if context else ""
+ if send_rate_limited_discord_alert(
+ func_name,
+ exception,
+ context,
+ f"🚨 CRITICAL: Operation Approaching Failure Threshold: {prefix}'{func_name}'\n\n"
+ f"Current attempt: {attempt_number}/{EXCESSIVE_RETRY_THRESHOLD}\n"
+ f"Error: {type(exception).__name__}: {exception}\n\n"
+ f"This operation is about to fail permanently. Investigate immediately.",
+ ):
logger.critical(
f"CRITICAL ALERT SENT: Operation {func_name} at attempt {attempt_number}"
)
- except Exception as alert_error:
- logger.error(f"Failed to send critical retry alert: {alert_error}")
- # Don't let alerting failures break the main flow
-
def _create_retry_callback(context: str = ""):
"""Create a retry callback with optional context."""
@@ -157,7 +170,7 @@ def _log_prefix(resource_name: str, conn_id: str):
def conn_retry(
resource_name: str,
action_name: str,
- max_retry: int = 5,
+ max_retry: int = 100,
max_wait: float = 30,
):
conn_id = str(uuid4())
@@ -165,10 +178,29 @@ def conn_retry(
def on_retry(retry_state):
prefix = _log_prefix(resource_name, conn_id)
exception = retry_state.outcome.exception()
+ attempt_number = retry_state.attempt_number
+ func_name = getattr(retry_state.fn, "__name__", "unknown")
if retry_state.outcome.failed and retry_state.next_action is None:
logger.error(f"{prefix} {action_name} failed after retries: {exception}")
else:
+ if attempt_number == EXCESSIVE_RETRY_THRESHOLD:
+ if send_rate_limited_discord_alert(
+ func_name,
+ exception,
+ f"{resource_name}_infrastructure",
+ f"🚨 **Critical Infrastructure Connection Issue**\n"
+ f"Resource: {resource_name}\n"
+ f"Action: {action_name}\n"
+ f"Function: {func_name}\n"
+ f"Current attempt: {attempt_number}/{max_retry + 1}\n"
+ f"Error: {type(exception).__name__}: {str(exception)[:200]}{'...' if len(str(exception)) > 200 else ''}\n\n"
+ f"Infrastructure component is approaching failure threshold. Investigate immediately.",
+ ):
+ logger.critical(
+ f"INFRASTRUCTURE ALERT SENT: {resource_name} at {attempt_number} attempts"
+ )
+
logger.warning(
f"{prefix} {action_name} failed: {exception}. Retrying now..."
)
@@ -244,8 +276,8 @@ def continuous_retry(*, retry_delay: float = 1.0):
@wraps(func)
async def async_wrapper(*args, **kwargs):
+ counter = 0
while True:
- counter = 0
try:
return await func(*args, **kwargs)
except Exception as exc:
diff --git a/autogpt_platform/backend/backend/util/retry_test.py b/autogpt_platform/backend/backend/util/retry_test.py
index 31d23da060..092481caca 100644
--- a/autogpt_platform/backend/backend/util/retry_test.py
+++ b/autogpt_platform/backend/backend/util/retry_test.py
@@ -10,9 +10,9 @@ from backend.util.retry import (
_alert_rate_limiter,
_rate_limiter_lock,
_send_critical_retry_alert,
- _should_send_alert,
conn_retry,
create_retry_decorator,
+ should_send_alert,
)
@@ -71,17 +71,17 @@ class TestRetryRateLimiting:
def test_should_send_alert_allows_first_occurrence(self):
"""Test that the first occurrence of an error allows alert."""
exc = ValueError("test error")
- assert _should_send_alert("test_func", exc, "test_context") is True
+ assert should_send_alert("test_func", exc, "test_context") is True
def test_should_send_alert_rate_limits_duplicate(self):
"""Test that duplicate errors are rate limited."""
exc = ValueError("test error")
# First call should be allowed
- assert _should_send_alert("test_func", exc, "test_context") is True
+ assert should_send_alert("test_func", exc, "test_context") is True
# Second call should be rate limited
- assert _should_send_alert("test_func", exc, "test_context") is False
+ assert should_send_alert("test_func", exc, "test_context") is False
def test_should_send_alert_allows_different_errors(self):
"""Test that different errors are allowed even if same function."""
@@ -89,47 +89,47 @@ class TestRetryRateLimiting:
exc2 = ValueError("error 2")
# First error should be allowed
- assert _should_send_alert("test_func", exc1, "test_context") is True
+ assert should_send_alert("test_func", exc1, "test_context") is True
# Different error should also be allowed
- assert _should_send_alert("test_func", exc2, "test_context") is True
+ assert should_send_alert("test_func", exc2, "test_context") is True
def test_should_send_alert_allows_different_contexts(self):
"""Test that same error in different contexts is allowed."""
exc = ValueError("test error")
# First context should be allowed
- assert _should_send_alert("test_func", exc, "context1") is True
+ assert should_send_alert("test_func", exc, "context1") is True
# Different context should also be allowed
- assert _should_send_alert("test_func", exc, "context2") is True
+ assert should_send_alert("test_func", exc, "context2") is True
def test_should_send_alert_allows_different_functions(self):
"""Test that same error in different functions is allowed."""
exc = ValueError("test error")
# First function should be allowed
- assert _should_send_alert("func1", exc, "test_context") is True
+ assert should_send_alert("func1", exc, "test_context") is True
# Different function should also be allowed
- assert _should_send_alert("func2", exc, "test_context") is True
+ assert should_send_alert("func2", exc, "test_context") is True
def test_should_send_alert_respects_time_window(self):
"""Test that alerts are allowed again after the rate limit window."""
exc = ValueError("test error")
# First call should be allowed
- assert _should_send_alert("test_func", exc, "test_context") is True
+ assert should_send_alert("test_func", exc, "test_context") is True
# Immediately after should be rate limited
- assert _should_send_alert("test_func", exc, "test_context") is False
+ assert should_send_alert("test_func", exc, "test_context") is False
# Mock time to simulate passage of rate limit window
current_time = time.time()
with patch("backend.util.retry.time.time") as mock_time:
# Simulate time passing beyond rate limit window
mock_time.return_value = current_time + ALERT_RATE_LIMIT_SECONDS + 1
- assert _should_send_alert("test_func", exc, "test_context") is True
+ assert should_send_alert("test_func", exc, "test_context") is True
def test_should_send_alert_thread_safety(self):
"""Test that rate limiting is thread-safe."""
@@ -137,7 +137,7 @@ class TestRetryRateLimiting:
results = []
def check_alert():
- result = _should_send_alert("test_func", exc, "test_context")
+ result = should_send_alert("test_func", exc, "test_context")
results.append(result)
# Create multiple threads trying to send the same alert
@@ -192,7 +192,7 @@ class TestRetryRateLimiting:
_send_critical_retry_alert("test_func", 50, exc, "test_context")
# Rate limiter should still work for subsequent calls
- assert _should_send_alert("test_func", exc, "test_context") is False
+ assert should_send_alert("test_func", exc, "test_context") is False
def test_error_signature_generation(self):
"""Test that error signatures are generated correctly for rate limiting."""
@@ -201,8 +201,8 @@ class TestRetryRateLimiting:
exc = ValueError(long_message)
# Should not raise exception and should work normally
- assert _should_send_alert("test_func", exc, "test_context") is True
- assert _should_send_alert("test_func", exc, "test_context") is False
+ assert should_send_alert("test_func", exc, "test_context") is True
+ assert should_send_alert("test_func", exc, "test_context") is False
def test_real_world_scenario_spend_credits_spam(self):
"""Test the real-world scenario that was causing spam."""