Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): Add request retry on block execution and RPC (#10183)

Zamil Majdy1e0a3d3c1ba3
diff --git a/autogpt_platform/backend/backend/data/notifications.py b/autogpt_platform/backend/backend/data/notifications.py
index 14cf77d3c4..f575081be7 100644
--- a/autogpt_platform/backend/backend/data/notifications.py
+++ b/autogpt_platform/backend/backend/data/notifications.py
@@ -16,10 +16,11 @@ from prisma.types import (
from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_validator
from backend.server.v2.store.exceptions import DatabaseError
+from backend.util.logging import TruncatedLogger
from .db import transaction
-logger = logging.getLogger(__name__)
+logger = TruncatedLogger(logging.getLogger(__name__), prefix="[NotificationService]")
NotificationDataType_co = TypeVar(
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index aabdb94411..2d5e169742 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -85,7 +85,8 @@ from backend.util.retry import continuous_retry, func_retry
from backend.util.service import get_service_client
from backend.util.settings import Settings
-logger = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
+logger = TruncatedLogger(_logger, prefix="[GraphExecutor]")
settings = Settings()
active_runs_gauge = Gauge(
@@ -122,7 +123,7 @@ class LogMetadata(TruncatedLogger):
}
prefix = f"[ExecutionManager|uid:{user_id}|gid:{graph_id}|nid:{node_id}]|geid:{graph_eid}|neid:{node_eid}|{block_name}]"
super().__init__(
- logger,
+ _logger,
max_length=max_length,
prefix=prefix,
metadata=metadata,
@@ -1214,7 +1215,9 @@ def get_db_client() -> "DatabaseManagerClient":
from backend.executor import DatabaseManagerClient
# Disable health check for the service client to avoid breaking process initializer.
- return get_service_client(DatabaseManagerClient, health_check=False)
+ return get_service_client(
+ DatabaseManagerClient, health_check=False, request_retry=True
+ )
@thread_cached
@@ -1222,7 +1225,9 @@ def get_db_async_client() -> "DatabaseManagerAsyncClient":
from backend.executor import DatabaseManagerAsyncClient
# Disable health check for the service client to avoid breaking process initializer.
- return get_service_client(DatabaseManagerAsyncClient, health_check=False)
+ return get_service_client(
+ DatabaseManagerAsyncClient, health_check=False, request_retry=True
+ )
async def send_async_execution_update(
diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.py
index 025528909d..82f1659169 100644
--- a/autogpt_platform/backend/backend/executor/utils.py
+++ b/autogpt_platform/backend/backend/executor/utils.py
@@ -39,6 +39,7 @@ from backend.data.rabbitmq import (
SyncRabbitMQ,
)
from backend.util.exceptions import NotFoundError
+from backend.util.logging import TruncatedLogger
from backend.util.mock import MockObject
from backend.util.service import get_service_client
from backend.util.settings import Config
@@ -49,7 +50,7 @@ if TYPE_CHECKING:
from backend.integrations.credentials_store import IntegrationCredentialsStore
config = Config()
-logger = logging.getLogger(__name__)
+logger = TruncatedLogger(logging.getLogger(__name__), prefix="[GraphExecutorUtil]")
# ============ Resource Helpers ============ #
diff --git a/autogpt_platform/backend/backend/util/logging.py b/autogpt_platform/backend/backend/util/logging.py
index 40dfb52a9f..9b6a64deac 100644
--- a/autogpt_platform/backend/backend/util/logging.py
+++ b/autogpt_platform/backend/backend/util/logging.py
@@ -61,6 +61,7 @@ class TruncatedLogger:
def _wrap(self, msg: str, **extra):
extra_msg = str(extra or "")
- if len(extra_msg) > 1000:
- extra_msg = extra_msg[:1000] + "..."
- return f"{self.prefix} {msg} {extra_msg}"
+ text = f"{self.prefix} {msg} {extra_msg}"
+ if len(text) > self.max_length:
+ text = text[: self.max_length] + "..."
+ return text
diff --git a/autogpt_platform/backend/backend/util/request.py b/autogpt_platform/backend/backend/util/request.py
index a3327c137a..9a4ee81392 100644
--- a/autogpt_platform/backend/backend/util/request.py
+++ b/autogpt_platform/backend/backend/util/request.py
@@ -11,9 +11,13 @@ from urllib.parse import quote, urljoin, urlparse
import aiohttp
import idna
from aiohttp import FormData, abc
+from tenacity import retry, retry_if_result, wait_exponential_jitter
from backend.util.json import json
+# Retry status codes for which we will automatically retry the request
+THROTTLE_RETRY_STATUS_CODES: set[int] = {429, 500, 502, 503, 504, 408}
+
# List of IP networks to block
BLOCKED_IP_NETWORKS = [
# --8<-- [start:BLOCKED_IP_NETWORKS]
@@ -289,6 +293,7 @@ class Requests:
raise_for_status: bool = True,
extra_url_validator: Callable[[URL], URL] | None = None,
extra_headers: dict[str, str] | None = None,
+ retry_max_wait: float = 300.0,
):
self.trusted_origins = []
for url in trusted_origins or []:
@@ -300,6 +305,7 @@ class Requests:
self.raise_for_status = raise_for_status
self.extra_url_validator = extra_url_validator
self.extra_headers = extra_headers
+ self.retry_max_wait = retry_max_wait
async def request(
self,
@@ -313,6 +319,39 @@ class Requests:
allow_redirects: bool = True,
max_redirects: int = 10,
**kwargs,
+ ) -> Response:
+ @retry(
+ wait=wait_exponential_jitter(max=self.retry_max_wait),
+ retry=retry_if_result(lambda r: r.status in THROTTLE_RETRY_STATUS_CODES),
+ reraise=True,
+ )
+ async def _make_request() -> Response:
+ return await self._request(
+ method=method,
+ url=url,
+ headers=headers,
+ files=files,
+ data=data,
+ json=json,
+ allow_redirects=allow_redirects,
+ max_redirects=max_redirects,
+ **kwargs,
+ )
+
+ return await _make_request()
+
+ async def _request(
+ self,
+ method: str,
+ url: str,
+ *,
+ headers: Optional[dict] = None,
+ files: list[tuple[str, tuple[str, BytesIO, str]]] | None = None,
+ data: Any | None = None,
+ json: Any | None = None,
+ allow_redirects: bool = True,
+ max_redirects: int = 10,
+ **kwargs,
) -> Response:
if files is not None:
if json is not None:
diff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.py
index 91eef36def..abcfd75dd3 100644
--- a/autogpt_platform/backend/backend/util/service.py
+++ b/autogpt_platform/backend/backend/util/service.py
@@ -24,6 +24,12 @@ import httpx
import uvicorn
from fastapi import FastAPI, Request, responses
from pydantic import BaseModel, TypeAdapter, create_model
+from tenacity import (
+ retry,
+ retry_if_exception_type,
+ stop_after_attempt,
+ wait_exponential_jitter,
+)
from backend.util.exceptions import InsufficientBalanceError
from backend.util.json import to_dict
@@ -248,9 +254,39 @@ def get_service_client(
service_client_type: Type[ASC],
call_timeout: int | None = api_call_timeout,
health_check: bool = True,
+ request_retry: bool | int = False,
) -> ASC:
+
+ def _maybe_retry(fn: Callable[..., R]) -> Callable[..., R]:
+ """Decorate *fn* with tenacity retry when enabled."""
+ nonlocal request_retry
+
+ if isinstance(request_retry, int):
+ retry_attempts = request_retry
+ request_retry = True
+ else:
+ retry_attempts = api_comm_retry
+
+ if not request_retry:
+ return fn
+
+ return retry(
+ reraise=True,
+ stop=stop_after_attempt(retry_attempts),
+ wait=wait_exponential_jitter(max=4.0),
+ retry=retry_if_exception_type(
+ (
+ httpx.ConnectError,
+ httpx.ReadTimeout,
+ httpx.WriteTimeout,
+ httpx.ConnectTimeout,
+ httpx.RemoteProtocolError,
+ )
+ ),
+ )(fn)
+
class DynamicClient:
- def __init__(self):
+ def __init__(self) -> None:
service_type = service_client_type.get_service_type()
host = service_type.get_host()
port = service_type.get_port()
@@ -271,7 +307,7 @@ def get_service_client(
)
def _handle_call_method_response(
- self, response: httpx.Response, method_name: str
+ self, *, response: httpx.Response, method_name: str
) -> Any:
try:
response.raise_for_status()
@@ -284,13 +320,15 @@ def get_service_client(
*(error.args or [str(e)])
)
- def _call_method_sync(self, method_name: str, **kwargs) -> Any:
+ @_maybe_retry
+ def _call_method_sync(self, method_name: str, **kwargs: Any) -> Any:
return self._handle_call_method_response(
method_name=method_name,
response=self.sync_client.post(method_name, json=to_dict(kwargs)),
)
- async def _call_method_async(self, method_name: str, **kwargs) -> Any:
+ @_maybe_retry
+ async def _call_method_async(self, method_name: str, **kwargs: Any) -> Any:
return self._handle_call_method_response(
method_name=method_name,
response=await self.async_client.post(
@@ -298,17 +336,19 @@ def get_service_client(
),
)
- async def aclose(self):
+ async def aclose(self) -> None:
self.sync_client.close()
await self.async_client.aclose()
- def close(self):
+ def close(self) -> None:
self.sync_client.close()
- def _get_params(self, signature: inspect.Signature, *args, **kwargs) -> dict:
+ def _get_params(
+ self, signature: inspect.Signature, *args: Any, **kwargs: Any
+ ) -> dict[str, Any]:
if args:
arg_names = list(signature.parameters.keys())
- if arg_names[0] in ("self", "cls"):
+ if arg_names and arg_names[0] in ("self", "cls"):
arg_names = arg_names[1:]
kwargs.update(dict(zip(arg_names, args)))
return kwargs
@@ -324,35 +364,34 @@ def get_service_client(
raise AttributeError(
f"Method {name} not found in {service_client_type}"
)
- else:
- name = original_func.__name__
+ rpc_name = original_func.__name__
sig = inspect.signature(original_func)
ret_ann = sig.return_annotation
- if ret_ann != inspect.Signature.empty:
- expected_return = TypeAdapter(ret_ann)
- else:
- expected_return = None
+ expected_return = (
+ None if ret_ann is inspect.Signature.empty else TypeAdapter(ret_ann)
+ )
if inspect.iscoroutinefunction(original_func):
- async def async_method(*args, **kwargs) -> Any:
+ async def async_method(*args: P.args, **kwargs: P.kwargs):
params = self._get_params(sig, *args, **kwargs)
- result = await self._call_method_async(name, **params)
+ result = await self._call_method_async(rpc_name, **params)
return self._get_return(expected_return, result)
return async_method
+
else:
- def sync_method(*args, **kwargs) -> Any:
+ def sync_method(*args: P.args, **kwargs: P.kwargs):
params = self._get_params(sig, *args, **kwargs)
- result = self._call_method_sync(name, **params)
+ result = self._call_method_sync(rpc_name, **params)
return self._get_return(expected_return, result)
return sync_method
client = cast(ASC, DynamicClient())
- if health_check:
+ if health_check and hasattr(client, "health_check"):
client.health_check()
return client