AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): Add request retry on block execution and RPC (#10183)
Zamil Majdy1e0a3d3c1ba3
diff --git a/autogpt_platform/backend/backend/data/notifications.py b/autogpt_platform/backend/backend/data/notifications.pyindex 14cf77d3c4..f575081be7 100644--- a/autogpt_platform/backend/backend/data/notifications.py+++ b/autogpt_platform/backend/backend/data/notifications.py@@ -16,10 +16,11 @@ from prisma.types import (from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_validatorfrom backend.server.v2.store.exceptions import DatabaseError+from backend.util.logging import TruncatedLoggerfrom .db import transaction-logger = logging.getLogger(__name__)+logger = TruncatedLogger(logging.getLogger(__name__), prefix="[NotificationService]")NotificationDataType_co = TypeVar(diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex aabdb94411..2d5e169742 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -85,7 +85,8 @@ from backend.util.retry import continuous_retry, func_retryfrom backend.util.service import get_service_clientfrom backend.util.settings import Settings-logger = logging.getLogger(__name__)+_logger = logging.getLogger(__name__)+logger = TruncatedLogger(_logger, prefix="[GraphExecutor]")settings = Settings()active_runs_gauge = Gauge(@@ -122,7 +123,7 @@ class LogMetadata(TruncatedLogger):}prefix = f"[ExecutionManager|uid:{user_id}|gid:{graph_id}|nid:{node_id}]|geid:{graph_eid}|neid:{node_eid}|{block_name}]"super().__init__(- logger,+ _logger,max_length=max_length,prefix=prefix,metadata=metadata,@@ -1214,7 +1215,9 @@ def get_db_client() -> "DatabaseManagerClient":from backend.executor import DatabaseManagerClient# Disable health check for the service client to avoid breaking process initializer.- return get_service_client(DatabaseManagerClient, health_check=False)+ return get_service_client(+ DatabaseManagerClient, health_check=False, request_retry=True+ )@thread_cached@@ -1222,7 +1225,9 @@ def get_db_async_client() -> "DatabaseManagerAsyncClient":from backend.executor import DatabaseManagerAsyncClient# Disable health check for the service client to avoid breaking process initializer.- return get_service_client(DatabaseManagerAsyncClient, health_check=False)+ return get_service_client(+ DatabaseManagerAsyncClient, health_check=False, request_retry=True+ )async def send_async_execution_update(diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex 025528909d..82f1659169 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -39,6 +39,7 @@ from backend.data.rabbitmq import (SyncRabbitMQ,)from backend.util.exceptions import NotFoundError+from backend.util.logging import TruncatedLoggerfrom backend.util.mock import MockObjectfrom backend.util.service import get_service_clientfrom backend.util.settings import Config@@ -49,7 +50,7 @@ if TYPE_CHECKING:from backend.integrations.credentials_store import IntegrationCredentialsStoreconfig = Config()-logger = logging.getLogger(__name__)+logger = TruncatedLogger(logging.getLogger(__name__), prefix="[GraphExecutorUtil]")# ============ Resource Helpers ============ #diff --git a/autogpt_platform/backend/backend/util/logging.py b/autogpt_platform/backend/backend/util/logging.pyindex 40dfb52a9f..9b6a64deac 100644--- a/autogpt_platform/backend/backend/util/logging.py+++ b/autogpt_platform/backend/backend/util/logging.py@@ -61,6 +61,7 @@ class TruncatedLogger:def _wrap(self, msg: str, **extra):extra_msg = str(extra or "")- if len(extra_msg) > 1000:- extra_msg = extra_msg[:1000] + "..."- return f"{self.prefix} {msg} {extra_msg}"+ text = f"{self.prefix} {msg} {extra_msg}"+ if len(text) > self.max_length:+ text = text[: self.max_length] + "..."+ return textdiff --git a/autogpt_platform/backend/backend/util/request.py b/autogpt_platform/backend/backend/util/request.pyindex a3327c137a..9a4ee81392 100644--- a/autogpt_platform/backend/backend/util/request.py+++ b/autogpt_platform/backend/backend/util/request.py@@ -11,9 +11,13 @@ from urllib.parse import quote, urljoin, urlparseimport aiohttpimport idnafrom aiohttp import FormData, abc+from tenacity import retry, retry_if_result, wait_exponential_jitterfrom backend.util.json import json+# Retry status codes for which we will automatically retry the request+THROTTLE_RETRY_STATUS_CODES: set[int] = {429, 500, 502, 503, 504, 408}+# List of IP networks to blockBLOCKED_IP_NETWORKS = [# --8<-- [start:BLOCKED_IP_NETWORKS]@@ -289,6 +293,7 @@ class Requests:raise_for_status: bool = True,extra_url_validator: Callable[[URL], URL] | None = None,extra_headers: dict[str, str] | None = None,+ retry_max_wait: float = 300.0,):self.trusted_origins = []for url in trusted_origins or []:@@ -300,6 +305,7 @@ class Requests:self.raise_for_status = raise_for_statusself.extra_url_validator = extra_url_validatorself.extra_headers = extra_headers+ self.retry_max_wait = retry_max_waitasync def request(self,@@ -313,6 +319,39 @@ class Requests:allow_redirects: bool = True,max_redirects: int = 10,**kwargs,+ ) -> Response:+ @retry(+ wait=wait_exponential_jitter(max=self.retry_max_wait),+ retry=retry_if_result(lambda r: r.status in THROTTLE_RETRY_STATUS_CODES),+ reraise=True,+ )+ async def _make_request() -> Response:+ return await self._request(+ method=method,+ url=url,+ headers=headers,+ files=files,+ data=data,+ json=json,+ allow_redirects=allow_redirects,+ max_redirects=max_redirects,+ **kwargs,+ )++ return await _make_request()++ async def _request(+ self,+ method: str,+ url: str,+ *,+ headers: Optional[dict] = None,+ files: list[tuple[str, tuple[str, BytesIO, str]]] | None = None,+ data: Any | None = None,+ json: Any | None = None,+ allow_redirects: bool = True,+ max_redirects: int = 10,+ **kwargs,) -> Response:if files is not None:if json is not None:diff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.pyindex 91eef36def..abcfd75dd3 100644--- a/autogpt_platform/backend/backend/util/service.py+++ b/autogpt_platform/backend/backend/util/service.py@@ -24,6 +24,12 @@ import httpximport uvicornfrom fastapi import FastAPI, Request, responsesfrom pydantic import BaseModel, TypeAdapter, create_model+from tenacity import (+ retry,+ retry_if_exception_type,+ stop_after_attempt,+ wait_exponential_jitter,+)from backend.util.exceptions import InsufficientBalanceErrorfrom backend.util.json import to_dict@@ -248,9 +254,39 @@ def get_service_client(service_client_type: Type[ASC],call_timeout: int | None = api_call_timeout,health_check: bool = True,+ request_retry: bool | int = False,) -> ASC:++ def _maybe_retry(fn: Callable[..., R]) -> Callable[..., R]:+ """Decorate *fn* with tenacity retry when enabled."""+ nonlocal request_retry++ if isinstance(request_retry, int):+ retry_attempts = request_retry+ request_retry = True+ else:+ retry_attempts = api_comm_retry++ if not request_retry:+ return fn++ return retry(+ reraise=True,+ stop=stop_after_attempt(retry_attempts),+ wait=wait_exponential_jitter(max=4.0),+ retry=retry_if_exception_type(+ (+ httpx.ConnectError,+ httpx.ReadTimeout,+ httpx.WriteTimeout,+ httpx.ConnectTimeout,+ httpx.RemoteProtocolError,+ )+ ),+ )(fn)+class DynamicClient:- def __init__(self):+ def __init__(self) -> None:service_type = service_client_type.get_service_type()host = service_type.get_host()port = service_type.get_port()@@ -271,7 +307,7 @@ def get_service_client()def _handle_call_method_response(- self, response: httpx.Response, method_name: str+ self, *, response: httpx.Response, method_name: str) -> Any:try:response.raise_for_status()@@ -284,13 +320,15 @@ def get_service_client(*(error.args or [str(e)]))- def _call_method_sync(self, method_name: str, **kwargs) -> Any:+ @_maybe_retry+ def _call_method_sync(self, method_name: str, **kwargs: Any) -> Any:return self._handle_call_method_response(method_name=method_name,response=self.sync_client.post(method_name, json=to_dict(kwargs)),)- async def _call_method_async(self, method_name: str, **kwargs) -> Any:+ @_maybe_retry+ async def _call_method_async(self, method_name: str, **kwargs: Any) -> Any:return self._handle_call_method_response(method_name=method_name,response=await self.async_client.post(@@ -298,17 +336,19 @@ def get_service_client(),)- async def aclose(self):+ async def aclose(self) -> None:self.sync_client.close()await self.async_client.aclose()- def close(self):+ def close(self) -> None:self.sync_client.close()- def _get_params(self, signature: inspect.Signature, *args, **kwargs) -> dict:+ def _get_params(+ self, signature: inspect.Signature, *args: Any, **kwargs: Any+ ) -> dict[str, Any]:if args:arg_names = list(signature.parameters.keys())- if arg_names[0] in ("self", "cls"):+ if arg_names and arg_names[0] in ("self", "cls"):arg_names = arg_names[1:]kwargs.update(dict(zip(arg_names, args)))return kwargs@@ -324,35 +364,34 @@ def get_service_client(raise AttributeError(f"Method {name} not found in {service_client_type}")- else:- name = original_func.__name__+ rpc_name = original_func.__name__sig = inspect.signature(original_func)ret_ann = sig.return_annotation- if ret_ann != inspect.Signature.empty:- expected_return = TypeAdapter(ret_ann)- else:- expected_return = None+ expected_return = (+ None if ret_ann is inspect.Signature.empty else TypeAdapter(ret_ann)+ )if inspect.iscoroutinefunction(original_func):- async def async_method(*args, **kwargs) -> Any:+ async def async_method(*args: P.args, **kwargs: P.kwargs):params = self._get_params(sig, *args, **kwargs)- result = await self._call_method_async(name, **params)+ result = await self._call_method_async(rpc_name, **params)return self._get_return(expected_return, result)return async_method+else:- def sync_method(*args, **kwargs) -> Any:+ def sync_method(*args: P.args, **kwargs: P.kwargs):params = self._get_params(sig, *args, **kwargs)- result = self._call_method_sync(name, **params)+ result = self._call_method_sync(rpc_name, **params)return self._get_return(expected_return, result)return sync_methodclient = cast(ASC, DynamicClient())- if health_check:+ if health_check and hasattr(client, "health_check"):client.health_check()return client