AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): Reduce excessive application log, Make DB configurable, Set DB connection limit & timeout (#9605)
Zamil Majdy020bf02e483d
diff --git a/autogpt_platform/backend/.env.example b/autogpt_platform/backend/.env.exampleindex 9fe12b3050..60e09a03cf 100644--- a/autogpt_platform/backend/.env.example+++ b/autogpt_platform/backend/.env.example@@ -2,9 +2,18 @@ DB_USER=postgresDB_PASS=your-super-secret-and-long-postgres-passwordDB_NAME=postgresDB_PORT=5432-DATABASE_URL="postgresql://${DB_USER}:${DB_PASS}@localhost:${DB_PORT}/${DB_NAME}?connect_timeout=60&schema=platform"+DB_HOST=localhost+DB_CONNECTION_LIMIT=12+DB_CONNECT_TIMEOUT=60+DB_POOL_TIMEOUT=300+DB_SCHEMA=platform+DATABASE_URL="postgresql://${DB_USER}:${DB_PASS}@${DB_HOST}:${DB_PORT}/${DB_NAME}?schema=${DB_SCHEMA}&connect_timeout=${DB_CONNECT_TIMEOUT}"PRISMA_SCHEMA="postgres/schema.prisma"+# EXECUTOR+NUM_GRAPH_WORKERS=10+NUM_NODE_WORKERS=3+BACKEND_CORS_ALLOW_ORIGINS=["http://localhost:3000"]# generate using `from cryptography.fernet import Fernet;Fernet.generate_key().decode()`diff --git a/autogpt_platform/backend/backend/data/db.py b/autogpt_platform/backend/backend/data/db.pyindex 66d4f167ad..058a74f8f4 100644--- a/autogpt_platform/backend/backend/data/db.py+++ b/autogpt_platform/backend/backend/data/db.py@@ -2,6 +2,7 @@ import loggingimport osimport zlibfrom contextlib import asynccontextmanager+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparsefrom uuid import uuid4from dotenv import load_dotenv@@ -15,7 +16,38 @@ load_dotenv()PRISMA_SCHEMA = os.getenv("PRISMA_SCHEMA", "schema.prisma")os.environ["PRISMA_SCHEMA_PATH"] = PRISMA_SCHEMA-prisma = Prisma(auto_register=True)++def add_param(url: str, key: str, value: str) -> str:+ p = urlparse(url)+ qs = dict(parse_qsl(p.query))+ qs[key] = value+ return urlunparse(p._replace(query=urlencode(qs)))+++DATABASE_URL = os.getenv("DATABASE_URL")+if not DATABASE_URL:+ raise ValueError("DATABASE_URL is not set.")++CONN_LIMIT = os.getenv("DB_CONNECTION_LIMIT")+if CONN_LIMIT:+ DATABASE_URL = add_param(DATABASE_URL, "connection_limit", CONN_LIMIT)++CONN_TIMEOUT = os.getenv("DB_CONNECT_TIMEOUT")+if CONN_TIMEOUT:+ DATABASE_URL = add_param(DATABASE_URL, "connect_timeout", CONN_TIMEOUT)++POOL_TIMEOUT = os.getenv("DB_POOL_TIMEOUT")+if POOL_TIMEOUT:+ DATABASE_URL = add_param(DATABASE_URL, "pool_timeout", POOL_TIMEOUT)++HTTP_TIMEOUT = int(POOL_TIMEOUT) if POOL_TIMEOUT else None++prisma = Prisma(+ auto_register=True,+ http={"timeout": HTTP_TIMEOUT},+ datasource={"url": DATABASE_URL},+)+logger = logging.getLogger(__name__)diff --git a/autogpt_platform/backend/backend/data/queue.py b/autogpt_platform/backend/backend/data/queue.pyindex 8fbdb03cd6..f9b06412e4 100644--- a/autogpt_platform/backend/backend/data/queue.py+++ b/autogpt_platform/backend/backend/data/queue.py@@ -35,7 +35,7 @@ class BaseRedisEventBus(Generic[M], ABC):def _serialize_message(self, item: M, channel_key: str) -> tuple[str, str]:message = json.dumps(item.model_dump(), cls=DateTimeEncoder)channel_name = f"{self.event_bus_name}/{channel_key}"- logger.info(f"[{channel_name}] Publishing an event to Redis {message}")+ logger.debug(f"[{channel_name}] Publishing an event to Redis {message}")return message, channel_namedef _deserialize_message(self, msg: Any, channel_key: str) -> M | None:@@ -44,7 +44,7 @@ class BaseRedisEventBus(Generic[M], ABC):return Nonetry:data = json.loads(msg["data"])- logger.info(f"Consuming an event from Redis {data}")+ logger.debug(f"Consuming an event from Redis {data}")return self.Model(**data)except Exception as e:logger.error(f"Failed to parse event result from Redis {msg} {e}")diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 79225f9b34..039c09b2ee 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -109,7 +109,10 @@ class LogMetadata:logger.exception(msg, extra={"json_fields": {**self.metadata, **extra}})def _wrap(self, msg: str, **extra):- return f"{self.prefix} {msg} {extra}"+ extra_msg = str(extra or "")+ if len(extra_msg) > 1000:+ extra_msg = extra_msg[:1000] + "..."+ return f"{self.prefix} {msg} {extra_msg}"T = TypeVar("T")diff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.pyindex d1601dbf5f..3a45581c05 100644--- a/autogpt_platform/backend/backend/util/service.py+++ b/autogpt_platform/backend/backend/util/service.py@@ -56,6 +56,7 @@ config = Config()api_host = config.pyro_hostapi_comm_retry = config.pyro_client_comm_retryapi_comm_timeout = config.pyro_client_comm_timeout+api_call_timeout = config.rpc_client_call_timeoutpyro_config.MAX_RETRIES = api_comm_retry # type: ignorepyro_config.COMMTIMEOUT = api_comm_timeout # type: ignore@@ -264,7 +265,11 @@ class FastApiAppService(BaseAppService, ABC):def _handle_internal_http_error(status_code: int = 500, log_error: bool = True):def handler(request: Request, exc: Exception):if log_error:- logger.exception(f"{request.method} {request.url.path} failed: {exc}")+ if status_code == 500:+ log = logger.exception+ else:+ log = logger.error+ log(f"{request.method} {request.url.path} failed: {exc}")return responses.JSONResponse(status_code=status_code,content=RemoteCallError(@@ -429,7 +434,10 @@ def fastapi_close_service_client(client: Any) -> None:@conn_retry("FastAPI client", "Creating service client", max_retry=api_comm_retry)-def fastapi_get_service_client(service_type: Type[AS]) -> AS:+def fastapi_get_service_client(+ service_type: Type[AS],+ call_timeout: int | None = api_call_timeout,+) -> AS:class DynamicClient:def __init__(self):host = service_type.get_host()@@ -437,7 +445,7 @@ def fastapi_get_service_client(service_type: Type[AS]) -> AS:self.base_url = f"http://{host}:{port}".rstrip("/")self.client = httpx.Client(base_url=self.base_url,- timeout=api_comm_timeout,+ timeout=call_timeout,)def _call_method(self, method_name: str, **kwargs) -> Any:diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.pyindex d1f5d72843..cc926a57ca 100644--- a/autogpt_platform/backend/backend/util/settings.py+++ b/autogpt_platform/backend/backend/util/settings.py@@ -81,6 +81,10 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):default=3,description="The default number of retries for Pyro client connections.",)+ rpc_client_call_timeout: int = Field(+ default=300,+ description="The default timeout in seconds, for RPC client calls.",+ )enable_auth: bool = Field(default=True,description="If authentication is enabled or not",diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prismaindex 6c8a3ef048..bbb54302b0 100644--- a/autogpt_platform/backend/schema.prisma+++ b/autogpt_platform/backend/schema.prisma@@ -1,4 +1,3 @@-// THIS FILE IS AUTO-GENERATED, RUN `poetry run schema` TO UPDATEdatasource db {provider = "postgresql"url = env("DATABASE_URL")