AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): implement rate limiting for critical retry alerts to prevent spam (#11127)
Zamil Majdyc7575dc5791e
diff --git a/autogpt_platform/backend/backend/util/retry.py b/autogpt_platform/backend/backend/util/retry.pyindex 937e411fa3..3e8b26335e 100644--- a/autogpt_platform/backend/backend/util/retry.py+++ b/autogpt_platform/backend/backend/util/retry.py@@ -20,11 +20,37 @@ logger = logging.getLogger(__name__)# Alert threshold for excessive retriesEXCESSIVE_RETRY_THRESHOLD = 50+# Rate limiting for alerts - track last alert time per function+error combination+_alert_rate_limiter = {}+_rate_limiter_lock = threading.Lock()+ALERT_RATE_LIMIT_SECONDS = 300 # 5 minutes between same alerts+++def _should_send_alert(func_name: str, exception: Exception, context: str = "") -> bool:+ """Check if we should send an alert based on rate limiting."""+ # Create a unique key for this function+error+context combination+ error_signature = (+ f"{context}:{func_name}:{type(exception).__name__}:{str(exception)[:100]}"+ )+ current_time = time.time()++ with _rate_limiter_lock:+ last_alert_time = _alert_rate_limiter.get(error_signature, 0)+ if current_time - last_alert_time >= ALERT_RATE_LIMIT_SECONDS:+ _alert_rate_limiter[error_signature] = current_time+ return True+ return False+def _send_critical_retry_alert(func_name: str, attempt_number: int, exception: Exception, context: str = ""):"""Send alert when a function is approaching the retry failure threshold."""++ # Rate limit alerts to prevent spam+ if not _should_send_alert(func_name, exception, context):+ return # Just silently skip, no extra logging+try:# Import here to avoid circular importsfrom backend.util.clients import get_notification_manager_client@@ -66,7 +92,7 @@ def _create_retry_callback(context: str = ""):f"{type(exception).__name__}: {exception}")else:- # Retry attempt - send critical alert only once at threshold+ # Retry attempt - send critical alert only once at threshold (rate limited)if attempt_number == EXCESSIVE_RETRY_THRESHOLD:_send_critical_retry_alert(func_name, attempt_number, exception, contextdiff --git a/autogpt_platform/backend/backend/util/retry_test.py b/autogpt_platform/backend/backend/util/retry_test.pyindex 29b6a192aa..31d23da060 100644--- a/autogpt_platform/backend/backend/util/retry_test.py+++ b/autogpt_platform/backend/backend/util/retry_test.py@@ -1,8 +1,19 @@import asyncio+import threading+import time+from unittest.mock import Mock, patchimport pytest-from backend.util.retry import conn_retry+from backend.util.retry import (+ ALERT_RATE_LIMIT_SECONDS,+ _alert_rate_limiter,+ _rate_limiter_lock,+ _send_critical_retry_alert,+ _should_send_alert,+ conn_retry,+ create_retry_decorator,+)def test_conn_retry_sync_function():@@ -47,3 +58,194 @@ async def test_conn_retry_async_function():with pytest.raises(ValueError) as e:await test_function()assert str(e.value) == "Test error"+++class TestRetryRateLimiting:+ """Test the rate limiting functionality for critical retry alerts."""++ def setup_method(self):+ """Reset rate limiter state before each test."""+ with _rate_limiter_lock:+ _alert_rate_limiter.clear()++ def test_should_send_alert_allows_first_occurrence(self):+ """Test that the first occurrence of an error allows alert."""+ exc = ValueError("test error")+ assert _should_send_alert("test_func", exc, "test_context") is True++ def test_should_send_alert_rate_limits_duplicate(self):+ """Test that duplicate errors are rate limited."""+ exc = ValueError("test error")++ # First call should be allowed+ assert _should_send_alert("test_func", exc, "test_context") is True++ # Second call should be rate limited+ assert _should_send_alert("test_func", exc, "test_context") is False++ def test_should_send_alert_allows_different_errors(self):+ """Test that different errors are allowed even if same function."""+ exc1 = ValueError("error 1")+ exc2 = ValueError("error 2")++ # First error should be allowed+ assert _should_send_alert("test_func", exc1, "test_context") is True++ # Different error should also be allowed+ assert _should_send_alert("test_func", exc2, "test_context") is True++ def test_should_send_alert_allows_different_contexts(self):+ """Test that same error in different contexts is allowed."""+ exc = ValueError("test error")++ # First context should be allowed+ assert _should_send_alert("test_func", exc, "context1") is True++ # Different context should also be allowed+ assert _should_send_alert("test_func", exc, "context2") is True++ def test_should_send_alert_allows_different_functions(self):+ """Test that same error in different functions is allowed."""+ exc = ValueError("test error")++ # First function should be allowed+ assert _should_send_alert("func1", exc, "test_context") is True++ # Different function should also be allowed+ assert _should_send_alert("func2", exc, "test_context") is True++ def test_should_send_alert_respects_time_window(self):+ """Test that alerts are allowed again after the rate limit window."""+ exc = ValueError("test error")++ # First call should be allowed+ assert _should_send_alert("test_func", exc, "test_context") is True++ # Immediately after should be rate limited+ assert _should_send_alert("test_func", exc, "test_context") is False++ # Mock time to simulate passage of rate limit window+ current_time = time.time()+ with patch("backend.util.retry.time.time") as mock_time:+ # Simulate time passing beyond rate limit window+ mock_time.return_value = current_time + ALERT_RATE_LIMIT_SECONDS + 1+ assert _should_send_alert("test_func", exc, "test_context") is True++ def test_should_send_alert_thread_safety(self):+ """Test that rate limiting is thread-safe."""+ exc = ValueError("test error")+ results = []++ def check_alert():+ result = _should_send_alert("test_func", exc, "test_context")+ results.append(result)++ # Create multiple threads trying to send the same alert+ threads = [threading.Thread(target=check_alert) for _ in range(10)]++ # Start all threads+ for thread in threads:+ thread.start()++ # Wait for all threads to complete+ for thread in threads:+ thread.join()++ # Only one thread should have been allowed to send the alert+ assert sum(results) == 1+ assert len([r for r in results if r is True]) == 1+ assert len([r for r in results if r is False]) == 9++ @patch("backend.util.clients.get_notification_manager_client")+ def test_send_critical_retry_alert_rate_limiting(self, mock_get_client):+ """Test that _send_critical_retry_alert respects rate limiting."""+ mock_client = Mock()+ mock_get_client.return_value = mock_client++ exc = ValueError("spend_credits API error")++ # First alert should be sent+ _send_critical_retry_alert("spend_credits", 50, exc, "Service communication")+ assert mock_client.discord_system_alert.call_count == 1++ # Second identical alert should be rate limited (not sent)+ _send_critical_retry_alert("spend_credits", 50, exc, "Service communication")+ assert mock_client.discord_system_alert.call_count == 1 # Still 1, not 2++ # Different error should be allowed+ exc2 = ValueError("different API error")+ _send_critical_retry_alert("spend_credits", 50, exc2, "Service communication")+ assert mock_client.discord_system_alert.call_count == 2++ @patch("backend.util.clients.get_notification_manager_client")+ def test_send_critical_retry_alert_handles_notification_failure(+ self, mock_get_client+ ):+ """Test that notification failures don't break the rate limiter."""+ mock_client = Mock()+ mock_client.discord_system_alert.side_effect = Exception("Notification failed")+ mock_get_client.return_value = mock_client++ exc = ValueError("test error")++ # Should not raise exception even if notification fails+ _send_critical_retry_alert("test_func", 50, exc, "test_context")++ # Rate limiter should still work for subsequent calls+ assert _should_send_alert("test_func", exc, "test_context") is False++ def test_error_signature_generation(self):+ """Test that error signatures are generated correctly for rate limiting."""+ # Test with long exception message (should be truncated to 100 chars)+ long_message = "x" * 200+ exc = ValueError(long_message)++ # Should not raise exception and should work normally+ assert _should_send_alert("test_func", exc, "test_context") is True+ assert _should_send_alert("test_func", exc, "test_context") is False++ def test_real_world_scenario_spend_credits_spam(self):+ """Test the real-world scenario that was causing spam."""+ # Simulate the exact error that was causing issues+ exc = Exception(+ "HTTP 500: Server error '500 Internal Server Error' for url 'http://autogpt-database-manager.prod-agpt.svc.cluster.local:8005/spend_credits'"+ )++ # First 50 attempts reach threshold - should send alert+ with patch(+ "backend.util.clients.get_notification_manager_client"+ ) as mock_get_client:+ mock_client = Mock()+ mock_get_client.return_value = mock_client++ _send_critical_retry_alert(+ "_call_method_sync", 50, exc, "Service communication"+ )+ assert mock_client.discord_system_alert.call_count == 1++ # Next 950 failures should not send alerts (rate limited)+ for _ in range(950):+ _send_critical_retry_alert(+ "_call_method_sync", 50, exc, "Service communication"+ )++ # Still only 1 alert sent total+ assert mock_client.discord_system_alert.call_count == 1++ @patch("backend.util.clients.get_notification_manager_client")+ def test_retry_decorator_with_excessive_failures(self, mock_get_client):+ """Test retry decorator behavior when it hits the alert threshold."""+ mock_client = Mock()+ mock_get_client.return_value = mock_client++ @create_retry_decorator(+ max_attempts=60, max_wait=0.1+ ) # More than EXCESSIVE_RETRY_THRESHOLD, but fast+ def always_failing_function():+ raise ValueError("persistent failure")++ with pytest.raises(ValueError):+ always_failing_function()++ # Should have sent exactly one alert at the threshold+ assert mock_client.discord_system_alert.call_count == 1