AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
refactor(backend): Refactor log client and resource cleanup (#10558)
Zamil Majdy3fe88b61064f
diff --git a/autogpt_platform/backend/backend/blocks/agent.py b/autogpt_platform/backend/backend/blocks/agent.pyindex c777084597..2f72527bdf 100644--- a/autogpt_platform/backend/backend/blocks/agent.py+++ b/autogpt_platform/backend/backend/blocks/agent.py@@ -14,7 +14,8 @@ from backend.data.block import ()from backend.data.execution import ExecutionStatusfrom backend.data.model import NodeExecutionStats, SchemaField-from backend.util import json, retry+from backend.util.json import validate_with_jsonschema+from backend.util.retry import func_retry_logger = logging.getLogger(__name__)@@ -48,7 +49,7 @@ class AgentExecutorBlock(Block):@classmethoddef get_mismatch_error(cls, data: BlockInput) -> str | None:- return json.validate_with_jsonschema(cls.get_input_schema(data), data)+ return validate_with_jsonschema(cls.get_input_schema(data), data)class Output(BlockSchema):pass@@ -180,7 +181,7 @@ class AgentExecutorBlock(Block):)yield output_name, output_data- @retry.func_retry+ @func_retryasync def _stop(self,graph_exec_id: str,diff --git a/autogpt_platform/backend/backend/blocks/ayrshare/_util.py b/autogpt_platform/backend/backend/blocks/ayrshare/_util.pyindex 46e16b23aa..a647e933df 100644--- a/autogpt_platform/backend/backend/blocks/ayrshare/_util.py+++ b/autogpt_platform/backend/backend/blocks/ayrshare/_util.py@@ -1,26 +1,18 @@from datetime import datetimefrom typing import Optional-from autogpt_libs.utils.cache import thread_cachedfrom pydantic import BaseModel, Fieldfrom backend.data.block import BlockSchemafrom backend.data.model import SchemaField, UserIntegrationsfrom backend.integrations.ayrshare import AyrshareClient+from backend.util.clients import get_database_manager_async_clientfrom backend.util.exceptions import MissingConfigError-@thread_cached-def get_database_manager_client():- from backend.executor import DatabaseManagerAsyncClient- from backend.util.service import get_service_client-- return get_service_client(DatabaseManagerAsyncClient, health_check=False)--async def get_profile_key(user_id: str):user_integrations: UserIntegrations = (- await get_database_manager_client().get_user_integrations(user_id)+ await get_database_manager_async_client().get_user_integrations(user_id))return user_integrations.managed_credentials.ayrshare_profile_keydiff --git a/autogpt_platform/backend/backend/blocks/persistence.py b/autogpt_platform/backend/backend/blocks/persistence.pyindex 42634fdc9a..8b165569b5 100644--- a/autogpt_platform/backend/backend/blocks/persistence.py+++ b/autogpt_platform/backend/backend/blocks/persistence.py@@ -1,22 +1,13 @@import loggingfrom typing import Any, Literal-from autogpt_libs.utils.cache import thread_cached-from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchemafrom backend.data.model import SchemaField+from backend.util.clients import get_database_manager_async_clientlogger = logging.getLogger(__name__)-@thread_cached-def get_database_manager_client():- from backend.executor import DatabaseManagerAsyncClient- from backend.util.service import get_service_client-- return get_service_client(DatabaseManagerAsyncClient, health_check=False)--StorageScope = Literal["within_agent", "across_agents"]@@ -88,7 +79,7 @@ class PersistInformationBlock(Block):async def _store_data(self, user_id: str, node_exec_id: str, key: str, data: Any) -> Any | None:- return await get_database_manager_client().set_execution_kv_data(+ return await get_database_manager_async_client().set_execution_kv_data(user_id=user_id,node_exec_id=node_exec_id,key=key,@@ -149,7 +140,7 @@ class RetrieveInformationBlock(Block):yield "value", input_data.default_valueasync def _retrieve_data(self, user_id: str, key: str) -> Any | None:- return await get_database_manager_client().get_execution_kv_data(+ return await get_database_manager_async_client().get_execution_kv_data(user_id=user_id,key=key,)diff --git a/autogpt_platform/backend/backend/blocks/smart_decision_maker.py b/autogpt_platform/backend/backend/blocks/smart_decision_maker.pyindex c53d5c0147..0876bfba86 100644--- a/autogpt_platform/backend/backend/blocks/smart_decision_maker.py+++ b/autogpt_platform/backend/backend/blocks/smart_decision_maker.py@@ -3,8 +3,6 @@ import refrom collections import Counterfrom typing import TYPE_CHECKING, Any-from autogpt_libs.utils.cache import thread_cached-import backend.blocks.llm as llmfrom backend.blocks.agent import AgentExecutorBlockfrom backend.data.block import (@@ -17,6 +15,7 @@ from backend.data.block import ()from backend.data.model import NodeExecutionStats, SchemaFieldfrom backend.util import json+from backend.util.clients import get_database_manager_async_clientif TYPE_CHECKING:from backend.data.graph import Link, Node@@ -24,14 +23,6 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)-@thread_cached-def get_database_manager_client():- from backend.executor import DatabaseManagerAsyncClient- from backend.util.service import get_service_client-- return get_service_client(DatabaseManagerAsyncClient, health_check=False)--def _get_tool_requests(entry: dict[str, Any]) -> list[str]:"""Return a list of tool_call_ids if the entry is a tool request.@@ -333,7 +324,7 @@ class SmartDecisionMakerBlock(Block):if not graph_id or not graph_version:raise ValueError("Graph ID or Graph Version not found in sink node.")- db_client = get_database_manager_client()+ db_client = get_database_manager_async_client()sink_graph_meta = await db_client.get_graph_metadata(graph_id, graph_version)if not sink_graph_meta:raise ValueError(@@ -393,7 +384,7 @@ class SmartDecisionMakerBlock(Block):ValueError: If no tool links are found for the specified node_id, or if a sink nodeor its metadata cannot be found."""- db_client = get_database_manager_client()+ db_client = get_database_manager_async_client()tools = [(link, node)for link, node in await db_client.get_connected_output_nodes(node_id)diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex 97892212eb..3a2fc38ac9 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -39,6 +39,7 @@ from pydantic.fields import Fieldfrom backend.server.v2.store.exceptions import DatabaseErrorfrom backend.util import type as type_utilsfrom backend.util.json import SafeJson+from backend.util.retry import func_retryfrom backend.util.settings import Configfrom backend.util.truncate import truncate@@ -883,15 +884,15 @@ class RedisExecutionEventBus(RedisEventBus[ExecutionEvent]):def publish(self, res: GraphExecution | NodeExecutionResult):if isinstance(res, GraphExecution):- self.publish_graph_exec_update(res)+ self._publish_graph_exec_update(res)else:- self.publish_node_exec_update(res)+ self._publish_node_exec_update(res)- def publish_node_exec_update(self, res: NodeExecutionResult):+ def _publish_node_exec_update(self, res: NodeExecutionResult):event = NodeExecutionEvent.model_validate(res.model_dump())self._publish(event, f"{res.user_id}/{res.graph_id}/{res.graph_exec_id}")- def publish_graph_exec_update(self, res: GraphExecution):+ def _publish_graph_exec_update(self, res: GraphExecution):event = GraphExecutionEvent.model_validate(res.model_dump())self._publish(event, f"{res.user_id}/{res.graph_id}/{res.id}")@@ -923,17 +924,18 @@ class AsyncRedisExecutionEventBus(AsyncRedisEventBus[ExecutionEvent]):def event_bus_name(self) -> str:return config.execution_event_bus_name+ @func_retryasync def publish(self, res: GraphExecutionMeta | NodeExecutionResult):if isinstance(res, GraphExecutionMeta):- await self.publish_graph_exec_update(res)+ await self._publish_graph_exec_update(res)else:- await self.publish_node_exec_update(res)+ await self._publish_node_exec_update(res)- async def publish_node_exec_update(self, res: NodeExecutionResult):+ async def _publish_node_exec_update(self, res: NodeExecutionResult):event = NodeExecutionEvent.model_validate(res.model_dump())await self._publish(event, f"{res.user_id}/{res.graph_id}/{res.graph_exec_id}")- async def publish_graph_exec_update(self, res: GraphExecutionMeta):+ async def _publish_graph_exec_update(self, res: GraphExecutionMeta):# GraphExecutionEvent requires inputs and outputs fields that GraphExecutionMeta doesn't have# Add default empty values for compatibilityevent_data = res.model_dump()diff --git a/autogpt_platform/backend/backend/data/rabbitmq.py b/autogpt_platform/backend/backend/data/rabbitmq.pyindex 13bdbe44de..e50b86b0d4 100644--- a/autogpt_platform/backend/backend/data/rabbitmq.py+++ b/autogpt_platform/backend/backend/data/rabbitmq.py@@ -4,20 +4,12 @@ from enum import Enumfrom typing import Awaitable, Optionalimport aio_pika-import aio_pika.exceptions as aio_eximport pikaimport pika.adapters.blocking_connection-from pika.exceptions import AMQPErrorfrom pika.spec import BasicPropertiesfrom pydantic import BaseModel-from tenacity import (- retry,- retry_if_exception_type,- stop_after_attempt,- wait_random_exponential,-)--from backend.util.retry import conn_retry++from backend.util.retry import conn_retry, func_retryfrom backend.util.settings import Settingslogger = logging.getLogger(__name__)@@ -198,12 +190,7 @@ class SyncRabbitMQ(RabbitMQBase):routing_key=queue.routing_key or queue.name,)- @retry(- retry=retry_if_exception_type((AMQPError, ConnectionError)),- wait=wait_random_exponential(multiplier=1, max=5),- stop=stop_after_attempt(5),- reraise=True,- )+ @func_retrydef publish_message(self,routing_key: str,@@ -302,12 +289,7 @@ class AsyncRabbitMQ(RabbitMQBase):exchange, routing_key=queue.routing_key or queue.name)- @retry(- retry=retry_if_exception_type((aio_ex.AMQPError, ConnectionError)),- wait=wait_random_exponential(multiplier=1, max=5),- stop=stop_after_attempt(5),- reraise=True,- )+ @func_retryasync def publish_message(self,routing_key: str,diff --git a/autogpt_platform/backend/backend/executor/activity_status_generator.py b/autogpt_platform/backend/backend/executor/activity_status_generator.pyindex 1ac46aa420..3ad13d29d3 100644--- a/autogpt_platform/backend/backend/executor/activity_status_generator.py+++ b/autogpt_platform/backend/backend/executor/activity_status_generator.py@@ -13,6 +13,7 @@ from backend.blocks.llm import LlmModel, llm_callfrom backend.data.block import get_blockfrom backend.data.execution import ExecutionStatus, NodeExecutionResultfrom backend.data.model import APIKeyCredentials, GraphExecutionStats+from backend.util.retry import func_retryfrom backend.util.settings import Settingsfrom backend.util.truncate import truncate@@ -415,6 +416,7 @@ def _build_execution_summary(}+@func_retryasync def _call_llm_direct(credentials: APIKeyCredentials, prompt: list[dict[str, str]]) -> str:diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex c61b9cb0a6..39192e2802 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -26,14 +26,13 @@ from backend.data.rabbitmq import SyncRabbitMQfrom backend.executor.activity_status_generator import (generate_activity_status_for_execution,)-from backend.executor.utils import LogMetadata, create_execution_queue_config+from backend.executor.utils import LogMetadatafrom backend.notifications.notifications import queue_notificationfrom backend.util.exceptions import InsufficientBalanceErrorif TYPE_CHECKING:from backend.executor import DatabaseManagerClient, DatabaseManagerAsyncClient-from autogpt_libs.utils.cache import thread_cachedfrom prometheus_client import Gauge, start_http_serverfrom backend.blocks.agent import AgentExecutorBlock@@ -63,14 +62,19 @@ from backend.executor.utils import (ExecutionOutputEntry,NodeExecutionProgress,block_usage_cost,+ create_execution_queue_config,execution_usage_cost,- get_async_execution_event_bus,- get_execution_event_bus,parse_execution_output,validate_exec,)from backend.integrations.creds_manager import IntegrationCredentialsManagerfrom backend.util import json+from backend.util.clients import (+ get_async_execution_event_bus,+ get_database_manager_async_client,+ get_database_manager_client,+ get_execution_event_bus,+)from backend.util.decorator import (async_error_logged,async_time_measured,@@ -81,7 +85,6 @@ from backend.util.file import clean_exec_filesfrom backend.util.logging import TruncatedLogger, configure_loggingfrom backend.util.process import AppProcess, set_service_namefrom backend.util.retry import continuous_retry, func_retry-from backend.util.service import get_service_clientfrom backend.util.settings import Settings_logger = logging.getLogger(__name__)@@ -1088,11 +1091,33 @@ class ExecutionManager(AppProcess):super().__init__()self.pool_size = settings.config.num_graph_workersself.active_graph_runs: dict[str, tuple[Future, threading.Event]] = {}- self._stop_consuming = None+self._executor = None+ self._stop_consuming = None++ self._cancel_thread = Noneself._cancel_client = None+ self._run_thread = Noneself._run_client = None+ @property+ def cancel_thread(self) -> threading.Thread:+ if self._cancel_thread is None:+ self._cancel_thread = threading.Thread(+ target=lambda: self._consume_execution_cancel(),+ daemon=True,+ )+ return self._cancel_thread++ @property+ def run_thread(self) -> threading.Thread:+ if self._run_thread is None:+ self._run_thread = threading.Thread(+ target=lambda: self._consume_execution_run(),+ daemon=True,+ )+ return self._run_thread+@propertydef stop_consuming(self) -> threading.Event:if self._stop_consuming is None:@@ -1108,44 +1133,55 @@ class ExecutionManager(AppProcess):)return self._executor+ @property+ def cancel_client(self) -> SyncRabbitMQ:+ if self._cancel_client is None:+ self._cancel_client = SyncRabbitMQ(create_execution_queue_config())+ return self._cancel_client++ @property+ def run_client(self) -> SyncRabbitMQ:+ if self._run_client is None:+ self._run_client = SyncRabbitMQ(create_execution_queue_config())+ return self._run_client+def run(self):logger.info(f"[{self.service_name}] ⏳ Spawn max-{self.pool_size} workers...")pool_size_gauge.set(self.pool_size)self._update_prompt_metrics()+ start_http_server(settings.config.execution_manager_port)- threading.Thread(- target=lambda: self._consume_execution_cancel(),- daemon=True,- ).start()-- threading.Thread(- target=lambda: self._consume_execution_run(),- daemon=True,- ).start()-- threading.Thread(- target=start_http_server,- args=(settings.config.execution_manager_port,),- daemon=True,- ).start()+ self.cancel_thread.start()+ self.run_thread.start()- while not self.stop_consuming.is_set():+ while True:time.sleep(1e5)@continuous_retry()def _consume_execution_cancel(self):- self._cancel_client = SyncRabbitMQ(create_execution_queue_config())- self._cancel_client.connect()- cancel_channel = self._cancel_client.get_channel()- logger.info(f"[{self.service_name}] ⏳ Starting cancel message consumer...")+ if self.stop_consuming.is_set() and not self.active_graph_runs:+ logger.info(+ f"[{self.service_name}] Stop reconnecting cancel consumer since the service is cleaned up."+ )+ return++ self.cancel_client.connect()+ cancel_channel = self.cancel_client.get_channel()cancel_channel.basic_consume(queue=GRAPH_EXECUTION_CANCEL_QUEUE_NAME,on_message_callback=self._handle_cancel_message,auto_ack=True,)+ logger.info(f"[{self.service_name}] ⏳ Starting cancel message consumer...")cancel_channel.start_consuming()- raise RuntimeError(f"❌ cancel message consumer is stopped: {cancel_channel}")+ if not self.stop_consuming.is_set() or self.active_graph_runs:+ raise RuntimeError(+ f"[{self.service_name}] ❌ cancel message consumer is stopped: {cancel_channel}"+ )+ logger.info(+ f"[{self.service_name}] ✅ Cancel message consumer stopped gracefully"+ )@continuous_retry()def _consume_execution_run(self):@@ -1159,9 +1195,8 @@ class ExecutionManager(AppProcess):)return- self._run_client = SyncRabbitMQ(create_execution_queue_config())- self._run_client.connect()- run_channel = self._run_client.get_channel()+ self.run_client.connect()+ run_channel = self.run_client.get_channel()run_channel.basic_qos(prefetch_count=self.pool_size)# Configure consumer for long-running graph executions@@ -1173,21 +1208,12 @@ class ExecutionManager(AppProcess):consumer_tag="graph_execution_consumer",)run_channel.confirm_delivery()-logger.info(f"[{self.service_name}] ⏳ Starting to consume run messages...")-- # Continue consuming messages until stop flag is set- # This keeps the connection alive but rejects new messages in _handle_run_message- while not self.stop_consuming.is_set():- try:- run_channel.connection.process_data_events(time_limit=1)- except Exception as e:- if self.stop_consuming.is_set():- # Expected during shutdown- break- logger.error(f"[{self.service_name}] Error processing events: {e}")- raise-+ run_channel.start_consuming()+ if not self.stop_consuming.is_set():+ raise RuntimeError(+ f"[{self.service_name}] ❌ run message consumer is stopped: {run_channel}"+ )logger.info(f"[{self.service_name}] ✅ Run message consumer stopped gracefully")@error_logged(swallow=True)@@ -1233,18 +1259,30 @@ class ExecutionManager(AppProcess):):delivery_tag = method.delivery_tag+ @func_retry+ def _ack_message(reject: bool = False):+ """Acknowledge or reject the message based on execution status."""+ if reject:+ channel.connection.add_callback_threadsafe(+ lambda: channel.basic_nack(delivery_tag, requeue=True)+ )+ else:+ channel.connection.add_callback_threadsafe(+ lambda: channel.basic_ack(delivery_tag)+ )+# Check if we're shutting down - reject new messages but keep connection aliveif self.stop_consuming.is_set():logger.info(f"[{self.service_name}] Rejecting new execution during shutdown")- channel.basic_nack(delivery_tag, requeue=True)+ _ack_message(reject=True)return# Check if we can accept more runsself._cleanup_completed_runs()if len(self.active_graph_runs) >= self.pool_size:- channel.basic_nack(delivery_tag, requeue=True)+ _ack_message(reject=True)returntry:@@ -1253,7 +1291,7 @@ class ExecutionManager(AppProcess):logger.error(f"[{self.service_name}] Could not parse run message: {e}, body={body}")- channel.basic_nack(delivery_tag, requeue=False)+ _ack_message(reject=True)returngraph_exec_id = graph_exec_entry.graph_exec_id@@ -1265,7 +1303,7 @@ class ExecutionManager(AppProcess):logger.error(f"[{self.service_name}] Graph {graph_exec_id} already running; rejecting duplicate run.")- channel.basic_nack(delivery_tag, requeue=False)+ _ack_message(reject=True)returncancel_event = multiprocessing.Manager().Event()@@ -1283,23 +1321,9 @@ class ExecutionManager(AppProcess):logger.error(f"[{self.service_name}] Execution for {graph_exec_id} failed: {type(exec_error)} {exec_error}")- try:- channel.connection.add_callback_threadsafe(- lambda: channel.basic_nack(delivery_tag, requeue=True)- )- except Exception as ack_error:- logger.error(- f"[{self.service_name}] Failed to NACK message for {graph_exec_id}: {ack_error}"- )+ _ack_message(reject=True)else:- try:- channel.connection.add_callback_threadsafe(- lambda: channel.basic_ack(delivery_tag)- )- except Exception as ack_error:- logger.error(- f"[{self.service_name}] Failed to ACK message for {graph_exec_id}: {ack_error}"- )+ _ack_message(reject=False)except BaseException as e:logger.exception(f"[{self.service_name}] Error in run completion callback: {e}"@@ -1326,7 +1350,7 @@ class ExecutionManager(AppProcess):def _update_prompt_metrics(self):active_count = len(self.active_graph_runs)active_runs_gauge.set(active_count)- if self._stop_consuming and self._stop_consuming.is_set():+ if self.stop_consuming.is_set():utilization_gauge.set(1.0)else:utilization_gauge.set(active_count / self.pool_size)@@ -1337,8 +1361,15 @@ class ExecutionManager(AppProcess):logger.info(f"{prefix} 🧹 Starting graceful shutdown...")# Signal the consumer thread to stop (thread-safe)- self.stop_consuming.set()- logger.info(f"{prefix} ✅ Signaled execution message consumer to stop")+ try:+ self.stop_consuming.set()+ run_channel = self.run_client.get_channel()+ run_channel.connection.add_callback_threadsafe(+ lambda: run_channel.stop_consuming()+ )+ logger.info(f"{prefix} ✅ Exec consumer has been signaled to stop")+ except Exception as e:+ logger.error(f"{prefix} ⚠️ Error signaling consumer to stop: {type(e)} {e}")# Wait for active executions to completeif self.active_graph_runs:@@ -1371,34 +1402,34 @@ class ExecutionManager(AppProcess):else:logger.info(f"{prefix} ✅ All executions completed gracefully")- # NOW shutdown executor pool after all executions and cleanup are complete- if self._executor:- logger.info(f"{prefix} ⏳ Shutting down GraphExec pool...")- try:- # All active executions are done, safe to shutdown workers- self._executor.shutdown(cancel_futures=True, wait=False)- logger.info(f"{prefix} ✅ Executor shutdown completed")- except Exception as e:- logger.error(f"{prefix} ⚠️ Error during executor shutdown: {e}")+ # Shutdown the executor+ try:+ self.executor.shutdown(cancel_futures=True, wait=False)+ logger.info(f"{prefix} ✅ Executor shutdown completed")+ except Exception as e:+ logger.error(f"{prefix} ⚠️ Error during executor shutdown: {type(e)} {e}")- # Clean up RabbitMQ connections- if self._cancel_client:- logger.info(f"{prefix} ⏳ Disconnecting cancel RabbitMQ client...")- try:- self._cancel_client.disconnect()- logger.info(f"{prefix} ✅ Cancel RabbitMQ client disconnected")- except Exception as e:- logger.error(- f"{prefix} ⚠️ Error disconnecting cancel RabbitMQ client: {e}"- )+ # Disconnect the run execution consumer+ try:+ run_channel = self.run_client.get_channel()+ run_channel.connection.add_callback_threadsafe(+ lambda: self.run_client.disconnect()+ )+ self.run_thread.join()+ logger.info(f"{prefix} ✅ Run client disconnected")+ except Exception as e:+ logger.error(f"{prefix} ⚠️ Error disconnecting run client: {type(e)} {e}")- if self._run_client:- logger.info(f"{prefix} ⏳ Disconnecting run RabbitMQ client...")- try:- self._run_client.disconnect()- logger.info(f"{prefix} ✅ Run RabbitMQ client disconnected")- except Exception as e:- logger.error(f"{prefix} ⚠️ Error disconnecting run RabbitMQ client: {e}")+ # Disconnect the cancel execution consumer+ try:+ cancel_channel = self.cancel_client.get_channel()+ cancel_channel.connection.add_callback_threadsafe(+ lambda: self.cancel_client.disconnect()+ )+ self.cancel_thread.join()+ logger.info(f"{prefix} ✅ Cancel client disconnected")+ except Exception as e:+ logger.error(f"{prefix} ⚠️ Error disconnecting cancel client: {type(e)} {e}")logger.info(f"{prefix} ✅ Finished GraphExec cleanup")@@ -1406,26 +1437,15 @@ class ExecutionManager(AppProcess):# ------- UTILITIES ------- #-@thread_cacheddef get_db_client() -> "DatabaseManagerClient":- from backend.executor import DatabaseManagerClient-- # Disable health check for the service client to avoid breaking process initializer.- return get_service_client(- DatabaseManagerClient, health_check=False, request_retry=True- )+ return get_database_manager_client()-@thread_cacheddef get_db_async_client() -> "DatabaseManagerAsyncClient":- from backend.executor import DatabaseManagerAsyncClient-- # Disable health check for the service client to avoid breaking process initializer.- return get_service_client(- DatabaseManagerAsyncClient, health_check=False, request_retry=True- )+ return get_database_manager_async_client()+@func_retryasync def send_async_execution_update(entry: GraphExecution | NodeExecutionResult | None,) -> None:@@ -1434,6 +1454,7 @@ async def send_async_execution_update(await get_async_execution_event_bus().publish(entry)+@func_retrydef send_execution_update(entry: GraphExecution | NodeExecutionResult | None):if entry is None:returndiff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex 2d6a85a52a..cde3e2144c 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -93,6 +93,7 @@ async def _execute_graph(**kwargs):f"Graph execution started with ID {graph_exec.id} for graph {args.graph_id}")except Exception as e:+ # TODO: We need to communicate this error to the user somehow.logger.error(f"Error executing graph {args.graph_id}: {e}")diff --git a/autogpt_platform/backend/backend/executor/scheduler_test.py b/autogpt_platform/backend/backend/executor/scheduler_test.pyindex f6b2b028c2..c4fa35d46c 100644--- a/autogpt_platform/backend/backend/executor/scheduler_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_test.py@@ -1,10 +1,9 @@import pytestfrom backend.data import db-from backend.executor.scheduler import SchedulerClientfrom backend.server.model import CreateGraphfrom backend.usecases.sample import create_test_graph, create_test_user-from backend.util.service import get_service_client+from backend.util.clients import get_scheduler_clientfrom backend.util.test import SpinTestServer@@ -17,7 +16,7 @@ async def test_agent_schedule(server: SpinTestServer):user_id=test_user.id,)- scheduler = get_service_client(SchedulerClient)+ scheduler = get_scheduler_client()schedules = await scheduler.get_execution_schedules(test_graph.id, test_user.id)assert len(schedules) == 0diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex 2fbec30b9e..e92eabfd23 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -4,9 +4,8 @@ import threadingimport timefrom collections import defaultdictfrom concurrent.futures import Future-from typing import TYPE_CHECKING, Any, Optional+from typing import Any, Optional-from autogpt_libs.utils.cache import thread_cachedfrom pydantic import BaseModel, JsonValue, ValidationErrorfrom backend.data import execution as execution_db@@ -16,33 +15,25 @@ from backend.data.block_cost_config import BLOCK_COSTSfrom backend.data.cost import BlockCostTypefrom backend.data.db import prismafrom backend.data.execution import (- AsyncRedisExecutionEventBus,ExecutionStatus,GraphExecutionStats,GraphExecutionWithNodes,- RedisExecutionEventBus,)from backend.data.graph import GraphModel, Nodefrom backend.data.model import CredentialsMetaInput-from backend.data.rabbitmq import (- AsyncRabbitMQ,- Exchange,- ExchangeType,- Queue,- RabbitMQConfig,- SyncRabbitMQ,+from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig+from backend.util.clients import (+ get_async_execution_event_bus,+ get_async_execution_queue,+ get_database_manager_async_client,+ get_integration_credentials_store,)from backend.util.exceptions import GraphValidationError, NotFoundErrorfrom backend.util.logging import TruncatedLoggerfrom backend.util.mock import MockObject-from backend.util.service import get_service_clientfrom backend.util.settings import Configfrom backend.util.type import convert-if TYPE_CHECKING:- from backend.executor import DatabaseManagerAsyncClient, DatabaseManagerClient- from backend.integrations.credentials_store import IntegrationCredentialsStore-config = Config()logger = TruncatedLogger(logging.getLogger(__name__), prefix="[GraphExecutorUtil]")@@ -79,51 +70,6 @@ class LogMetadata(TruncatedLogger):)-@thread_cached-def get_execution_event_bus() -> RedisExecutionEventBus:- return RedisExecutionEventBus()---@thread_cached-def get_async_execution_event_bus() -> AsyncRedisExecutionEventBus:- return AsyncRedisExecutionEventBus()---@thread_cached-def get_execution_queue() -> SyncRabbitMQ:- client = SyncRabbitMQ(create_execution_queue_config())- client.connect()- return client---@thread_cached-async def get_async_execution_queue() -> AsyncRabbitMQ:- client = AsyncRabbitMQ(create_execution_queue_config())- await client.connect()- return client---@thread_cached-def get_integration_credentials_store() -> "IntegrationCredentialsStore":- from backend.integrations.credentials_store import IntegrationCredentialsStore-- return IntegrationCredentialsStore()---@thread_cached-def get_db_client() -> "DatabaseManagerClient":- from backend.executor import DatabaseManagerClient-- return get_service_client(DatabaseManagerClient)---@thread_cached-def get_db_async_client() -> "DatabaseManagerAsyncClient":- from backend.executor import DatabaseManagerAsyncClient-- return get_service_client(DatabaseManagerAsyncClient)--# ============ Execution Cost Helpers ============ #@@ -450,7 +396,7 @@ def validate_exec(# Last validation: Validate the input values against the schema.if error := schema.get_mismatch_error(data):error_message = f"{error_prefix} {error}"- logger.error(error_message)+ logger.warning(error_message)return None, error_messagereturn data, node_block.name@@ -685,11 +631,6 @@ def _merge_nodes_input_masks(# ============ Execution Queue Helpers ============ #--class CancelExecutionEvent(BaseModel):- graph_exec_id: str--GRAPH_EXECUTION_EXCHANGE = Exchange(name="graph_execution",type=ExchangeType.DIRECT,@@ -750,6 +691,10 @@ def create_execution_queue_config() -> RabbitMQConfig:)+class CancelExecutionEvent(BaseModel):+ graph_exec_id: str++async def stop_graph_execution(user_id: str,graph_exec_id: str,@@ -763,7 +708,7 @@ async def stop_graph_execution(3. Update execution statuses in DB and set `error` outputs to `"TERMINATED"`."""queue_client = await get_async_execution_queue()- db = execution_db if prisma.is_connected() else get_db_async_client()+ db = execution_db if prisma.is_connected() else get_database_manager_async_client()await queue_client.publish_message(routing_key="",message=CancelExecutionEvent(graph_exec_id=graph_exec_id).model_dump_json(),@@ -849,8 +794,8 @@ async def add_graph_execution(gdb = graph_dbedb = execution_dbelse:- gdb = get_db_async_client()- edb = get_db_async_client()+ gdb = get_database_manager_async_client()+ edb = get_database_manager_async_client()graph: GraphModel | None = await gdb.get_graph(graph_id=graph_id,@@ -903,7 +848,7 @@ async def add_graph_execution(except BaseException as e:err = str(e) or type(e).__name__if not graph_exec:- logger.error(f"Graph execution #{graph_id} failed: {err}")+ logger.error(f"Unable to execute graph #{graph_id} failed: {err}")raiselogger.error(diff --git a/autogpt_platform/backend/backend/integrations/credentials_store.py b/autogpt_platform/backend/backend/integrations/credentials_store.pyindex 4d3a9b2ae6..eedb38bf4b 100644--- a/autogpt_platform/backend/backend/integrations/credentials_store.py+++ b/autogpt_platform/backend/backend/integrations/credentials_store.py@@ -5,7 +5,6 @@ from contextlib import asynccontextmanagerfrom datetime import datetime, timedelta, timezonefrom typing import Optional-from autogpt_libs.utils.cache import thread_cachedfrom autogpt_libs.utils.synchronize import AsyncRedisKeyedMutexfrom pydantic import SecretStr@@ -229,17 +228,15 @@ class IntegrationCredentialsStore:return self._locks@property- @thread_cacheddef db_manager(self):if prisma.is_connected():from backend.data import userreturn userelse:- from backend.executor.database import DatabaseManagerAsyncClient- from backend.util.service import get_service_client+ from backend.util.clients import get_database_manager_async_client- return get_service_client(DatabaseManagerAsyncClient)+ return get_database_manager_async_client()# =============== USER-MANAGED CREDENTIALS =============== #async def add_creds(self, user_id: str, credentials: Credentials) -> None:diff --git a/autogpt_platform/backend/backend/monitoring/block_error_monitor.py b/autogpt_platform/backend/backend/monitoring/block_error_monitor.pyindex 8de813d06b..ffd2ffc888 100644--- a/autogpt_platform/backend/backend/monitoring/block_error_monitor.py+++ b/autogpt_platform/backend/backend/monitoring/block_error_monitor.py@@ -8,10 +8,11 @@ from pydantic import BaseModelfrom backend.data.block import get_blockfrom backend.data.execution import ExecutionStatus, NodeExecutionResult-from backend.executor import utils as execution_utils-from backend.notifications.notifications import NotificationManagerClient+from backend.util.clients import (+ get_database_manager_client,+ get_notification_manager_client,+)from backend.util.metrics import sentry_capture_error-from backend.util.service import get_service_clientfrom backend.util.settings import Configlogger = logging.getLogger(__name__)@@ -40,7 +41,7 @@ class BlockErrorMonitor:def __init__(self, include_top_blocks: int | None = None):self.config = config- self.notification_client = get_service_client(NotificationManagerClient)+ self.notification_client = get_notification_manager_client()self.include_top_blocks = (include_top_blocksif include_top_blocks is not None@@ -107,7 +108,7 @@ class BlockErrorMonitor:) -> dict[str, BlockStatsWithSamples]:"""Get block execution stats using efficient SQL aggregation."""- result = execution_utils.get_db_client().get_block_error_stats(+ result = get_database_manager_client().get_block_error_stats(start_time, end_time)@@ -197,7 +198,7 @@ class BlockErrorMonitor:) -> list[str]:"""Get error samples for a specific block - just a few recent ones."""# Only fetch a small number of recent failed executions for this specific block- executions = execution_utils.get_db_client().get_node_executions(+ executions = get_database_manager_client().get_node_executions(block_ids=[block_id],statuses=[ExecutionStatus.FAILED],created_time_gte=start_time,diff --git a/autogpt_platform/backend/backend/monitoring/late_execution_monitor.py b/autogpt_platform/backend/backend/monitoring/late_execution_monitor.pyindex 7cace69908..1e0c99cac3 100644--- a/autogpt_platform/backend/backend/monitoring/late_execution_monitor.py+++ b/autogpt_platform/backend/backend/monitoring/late_execution_monitor.py@@ -4,10 +4,11 @@ import loggingfrom datetime import datetime, timedelta, timezonefrom backend.data.execution import ExecutionStatus-from backend.executor import utils as execution_utils-from backend.notifications.notifications import NotificationManagerClient+from backend.util.clients import (+ get_database_manager_client,+ get_notification_manager_client,+)from backend.util.metrics import sentry_capture_error-from backend.util.service import get_service_clientfrom backend.util.settings import Configlogger = logging.getLogger(__name__)@@ -25,13 +26,13 @@ class LateExecutionMonitor:def __init__(self):self.config = config- self.notification_client = get_service_client(NotificationManagerClient)+ self.notification_client = get_notification_manager_client()def check_late_executions(self) -> str:"""Check for late executions and send alerts if found."""# Check for QUEUED executions- queued_late_executions = execution_utils.get_db_client().get_graph_executions(+ queued_late_executions = get_database_manager_client().get_graph_executions(statuses=[ExecutionStatus.QUEUED],created_time_gte=datetime.now(timezone.utc)- timedelta(@@ -43,7 +44,7 @@ class LateExecutionMonitor:)# Check for RUNNING executions stuck for more than 24 hours- running_late_executions = execution_utils.get_db_client().get_graph_executions(+ running_late_executions = get_database_manager_client().get_graph_executions(statuses=[ExecutionStatus.RUNNING],created_time_gte=datetime.now(timezone.utc)- timedelta(hours=24)diff --git a/autogpt_platform/backend/backend/monitoring/notification_monitor.py b/autogpt_platform/backend/backend/monitoring/notification_monitor.pyindex 3326dd91a2..db69b4d9fb 100644--- a/autogpt_platform/backend/backend/monitoring/notification_monitor.py+++ b/autogpt_platform/backend/backend/monitoring/notification_monitor.py@@ -2,12 +2,10 @@import logging-from autogpt_libs.utils.cache import thread_cachedfrom prisma.enums import NotificationTypefrom pydantic import BaseModel-from backend.notifications.notifications import NotificationManagerClient-from backend.util.service import get_service_client+from backend.util.clients import get_notification_manager_clientlogger = logging.getLogger(__name__)@@ -17,11 +15,6 @@ class NotificationJobArgs(BaseModel):cron: str-@thread_cached-def get_notification_manager_client():- return get_service_client(NotificationManagerClient)--def process_existing_batches(**kwargs):"""Process existing notification batches."""args = NotificationJobArgs(**kwargs)diff --git a/autogpt_platform/backend/backend/notifications/notifications.py b/autogpt_platform/backend/backend/notifications/notifications.pyindex d830bccfdf..67327d2985 100644--- a/autogpt_platform/backend/backend/notifications/notifications.py+++ b/autogpt_platform/backend/backend/notifications/notifications.py@@ -5,7 +5,6 @@ from datetime import datetime, timedelta, timezonefrom typing import Callableimport aio_pika-from autogpt_libs.utils.cache import thread_cachedfrom prisma.enums import NotificationTypefrom backend.data import rabbitmq@@ -26,26 +25,14 @@ from backend.data.notifications import (get_notif_data_type,get_summary_params_type,)-from backend.data.rabbitmq import (- AsyncRabbitMQ,- Exchange,- ExchangeType,- Queue,- RabbitMQConfig,- SyncRabbitMQ,-)+from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfigfrom backend.data.user import generate_unsubscribe_linkfrom backend.notifications.email import EmailSender+from backend.util.clients import get_database_manager_clientfrom backend.util.logging import TruncatedLoggerfrom backend.util.metrics import discord_send_alertfrom backend.util.retry import continuous_retry-from backend.util.service import (- AppService,- AppServiceClient,- endpoint_to_sync,- expose,- get_service_client,-)+from backend.util.service import AppService, AppServiceClient, endpoint_to_sync, exposefrom backend.util.settings import Settingslogger = TruncatedLogger(logging.getLogger(__name__), "[NotificationManager]")@@ -116,27 +103,6 @@ def create_notification_config() -> RabbitMQConfig:)-@thread_cached-def get_db():- from backend.executor.database import DatabaseManagerClient-- return get_service_client(DatabaseManagerClient)---@thread_cached-def get_notification_queue() -> SyncRabbitMQ:- client = SyncRabbitMQ(create_notification_config())- client.connect()- return client---@thread_cached-async def get_async_notification_queue() -> AsyncRabbitMQ:- client = AsyncRabbitMQ(create_notification_config())- await client.connect()- return client--def get_routing_key(event_type: NotificationType) -> str:strategy = NotificationTypeOverride(event_type).strategy"""Get the appropriate routing key for an event"""@@ -161,6 +127,8 @@ def queue_notification(event: NotificationEventModel) -> NotificationResult:exchange = "notifications"routing_key = get_routing_key(event.type)+ from backend.util.clients import get_notification_queue+queue = get_notification_queue()queue.publish_message(routing_key=routing_key,@@ -186,6 +154,8 @@ async def queue_notification_async(event: NotificationEventModel) -> Notificatioexchange = "notifications"routing_key = get_routing_key(event.type)+ from backend.util.clients import get_async_notification_queue+queue = await get_async_notification_queue()await queue.publish_message(routing_key=routing_key,@@ -241,7 +211,7 @@ class NotificationManager(AppService):processed_count = 0current_time = datetime.now(tz=timezone.utc)start_time = current_time - timedelta(days=7)- users = get_db().get_active_user_ids_in_timerange(+ users = get_database_manager_client().get_active_user_ids_in_timerange(end_time=current_time.isoformat(),start_time=start_time.isoformat(),)@@ -275,14 +245,14 @@ class NotificationManager(AppService):for notification_type in notification_types:# Get all batches for this notification type- batches = get_db().get_all_batches_by_type(notification_type)+ batches = get_database_manager_client().get_all_batches_by_type(+ notification_type+ )for batch in batches:# Check if batch has aged out- oldest_message = (- get_db().get_user_notification_oldest_message_in_batch(- batch.user_id, notification_type- )+ oldest_message = get_database_manager_client().get_user_notification_oldest_message_in_batch(+ batch.user_id, notification_type)if not oldest_message:@@ -296,7 +266,11 @@ class NotificationManager(AppService):# If batch has aged out, process itif oldest_message.created_at + max_delay < current_time:- recipient_email = get_db().get_user_email_by_id(batch.user_id)+ recipient_email = (+ get_database_manager_client().get_user_email_by_id(+ batch.user_id+ )+ )if not recipient_email:logger.error(@@ -313,13 +287,15 @@ class NotificationManager(AppService):f"User {batch.user_id} does not want to receive {notification_type} notifications")# Clear the batch- get_db().empty_user_notification_batch(+ get_database_manager_client().empty_user_notification_batch(batch.user_id, notification_type)continue- batch_data = get_db().get_user_notification_batch(- batch.user_id, notification_type+ batch_data = (+ get_database_manager_client().get_user_notification_batch(+ batch.user_id, notification_type+ ))if not batch_data or not batch_data.notifications:@@ -327,7 +303,7 @@ class NotificationManager(AppService):f"Batch data not found for user {batch.user_id}")# Clear the batch- get_db().empty_user_notification_batch(+ get_database_manager_client().empty_user_notification_batch(batch.user_id, notification_type)continue@@ -363,7 +339,7 @@ class NotificationManager(AppService):)# Clear the batch- get_db().empty_user_notification_batch(+ get_database_manager_client().empty_user_notification_batch(batch.user_id, notification_type)@@ -412,9 +388,11 @@ class NotificationManager(AppService):self, user_id: str, event_type: NotificationType) -> bool:"""Check if a user wants to receive a notification based on their preferences and email verification status"""- validated_email = get_db().get_user_email_verification(user_id)+ validated_email = get_database_manager_client().get_user_email_verification(+ user_id+ )preference = (- get_db()+ get_database_manager_client().get_user_notification_preference(user_id).preferences.get(event_type, True))@@ -505,10 +483,14 @@ class NotificationManager(AppService):self, user_id: str, event_type: NotificationType, event: NotificationEventModel) -> bool:- get_db().create_or_add_to_user_notification_batch(user_id, event_type, event)+ get_database_manager_client().create_or_add_to_user_notification_batch(+ user_id, event_type, event+ )- oldest_message = get_db().get_user_notification_oldest_message_in_batch(- user_id, event_type+ oldest_message = (+ get_database_manager_client().get_user_notification_oldest_message_in_batch(+ user_id, event_type+ ))if not oldest_message:logger.error(@@ -559,7 +541,9 @@ class NotificationManager(AppService):return Falselogger.debug(f"Processing immediate notification: {event}")- recipient_email = get_db().get_user_email_by_id(event.user_id)+ recipient_email = get_database_manager_client().get_user_email_by_id(+ event.user_id+ )if not recipient_email:logger.error(f"User email not found for user {event.user_id}")return False@@ -594,7 +578,9 @@ class NotificationManager(AppService):return Falselogger.info(f"Processing batch notification: {event}")- recipient_email = get_db().get_user_email_by_id(event.user_id)+ recipient_email = get_database_manager_client().get_user_email_by_id(+ event.user_id+ )if not recipient_email:logger.error(f"User email not found for user {event.user_id}")return False@@ -613,7 +599,9 @@ class NotificationManager(AppService):if not should_send:logger.info("Batch not old enough to send")return False- batch = get_db().get_user_notification_batch(event.user_id, event.type)+ batch = get_database_manager_client().get_user_notification_batch(+ event.user_id, event.type+ )if not batch or not batch.notifications:logger.error(f"Batch not found for user {event.user_id}")return False@@ -714,7 +702,9 @@ class NotificationManager(AppService):logger.info(f"Successfully sent all {successfully_sent_count} notifications, clearing batch")- get_db().empty_user_notification_batch(event.user_id, event.type)+ get_database_manager_client().empty_user_notification_batch(+ event.user_id, event.type+ )else:logger.warning(f"Only sent {successfully_sent_count} of {len(batch_messages)} notifications. "@@ -736,7 +726,9 @@ class NotificationManager(AppService):logger.info(f"Processing summary notification: {model}")- recipient_email = get_db().get_user_email_by_id(event.user_id)+ recipient_email = get_database_manager_client().get_user_email_by_id(+ event.user_id+ )if not recipient_email:logger.error(f"User email not found for user {event.user_id}")return Falsediff --git a/autogpt_platform/backend/backend/server/routers/v1.py b/autogpt_platform/backend/backend/server/routers/v1.pyindex e6904d7b05..eef333aa2a 100644--- a/autogpt_platform/backend/backend/server/routers/v1.py+++ b/autogpt_platform/backend/backend/server/routers/v1.py@@ -9,7 +9,6 @@ import pydanticimport stripefrom autogpt_libs.auth.middleware import auth_middlewarefrom autogpt_libs.feature_flag.client import feature_flag-from autogpt_libs.utils.cache import thread_cachedfrom fastapi import (APIRouter,Body,@@ -51,7 +50,6 @@ from backend.data.credit import (get_user_credit_model,set_auto_top_up,)-from backend.data.execution import AsyncRedisExecutionEventBusfrom backend.data.model import CredentialsMetaInputfrom backend.data.notifications import NotificationPreference, NotificationPreferenceDTOfrom backend.data.onboarding import (@@ -84,18 +82,13 @@ from backend.server.model import (UploadFileResponse,)from backend.server.utils import get_user_id+from backend.util.clients import get_scheduler_clientfrom backend.util.cloud_storage import get_cloud_storage_handlerfrom backend.util.exceptions import GraphValidationError, NotFoundError-from backend.util.service import get_service_clientfrom backend.util.settings import Settingsfrom backend.util.virus_scanner import scan_content_safe-@thread_cached-def execution_scheduler_client() -> scheduler.SchedulerClient:- return get_service_client(scheduler.SchedulerClient, health_check=False)--def _create_file_size_error(size_bytes: int, max_size_mb: int) -> HTTPException:"""Create standardized file size error response."""return HTTPException(@@ -104,11 +97,6 @@ def _create_file_size_error(size_bytes: int, max_size_mb: int) -> HTTPException:)-@thread_cached-def execution_event_bus() -> AsyncRedisExecutionEventBus:- return AsyncRedisExecutionEventBus()--settings = Settings()logger = logging.getLogger(__name__)@@ -924,7 +912,7 @@ async def create_graph_execution_schedule(detail=f"Graph #{graph_id} v{schedule_params.graph_version} not found.",)- return await execution_scheduler_client().add_execution_schedule(+ return await get_scheduler_client().add_execution_schedule(user_id=user_id,graph_id=graph_id,graph_version=graph.version,@@ -945,7 +933,7 @@ async def list_graph_execution_schedules(user_id: Annotated[str, Depends(get_user_id)],graph_id: str = Path(),) -> list[scheduler.GraphExecutionJobInfo]:- return await execution_scheduler_client().get_execution_schedules(+ return await get_scheduler_client().get_execution_schedules(user_id=user_id,graph_id=graph_id,)@@ -960,7 +948,7 @@ async def list_graph_execution_schedules(async def list_all_graphs_execution_schedules(user_id: Annotated[str, Depends(get_user_id)],) -> list[scheduler.GraphExecutionJobInfo]:- return await execution_scheduler_client().get_execution_schedules(user_id=user_id)+ return await get_scheduler_client().get_execution_schedules(user_id=user_id)@v1_router.delete(@@ -974,7 +962,7 @@ async def delete_graph_execution_schedule(schedule_id: str = Path(..., description="ID of the schedule to delete"),) -> dict[str, Any]:try:- await execution_scheduler_client().delete_schedule(schedule_id, user_id=user_id)+ await get_scheduler_client().delete_schedule(schedule_id, user_id=user_id)except NotFoundError:raise HTTPException(status_code=HTTP_404_NOT_FOUND,diff --git a/autogpt_platform/backend/backend/util/clients.py b/autogpt_platform/backend/backend/util/clients.pynew file mode 100644index 0000000000..b30615c283--- /dev/null+++ b/autogpt_platform/backend/backend/util/clients.py@@ -0,0 +1,134 @@+"""+Centralized service client helpers with thread caching.+"""++from typing import TYPE_CHECKING++from autogpt_libs.utils.cache import thread_cached++if TYPE_CHECKING:+ from backend.data.execution import (+ AsyncRedisExecutionEventBus,+ RedisExecutionEventBus,+ )+ from backend.data.rabbitmq import AsyncRabbitMQ, SyncRabbitMQ+ from backend.executor import DatabaseManagerAsyncClient, DatabaseManagerClient+ from backend.executor.scheduler import SchedulerClient+ from backend.integrations.credentials_store import IntegrationCredentialsStore+ from backend.notifications.notifications import NotificationManagerClient+++@thread_cached+def get_database_manager_client() -> "DatabaseManagerClient":+ """Get a thread-cached DatabaseManagerClient with request retry enabled."""+ from backend.executor import DatabaseManagerClient+ from backend.util.service import get_service_client++ return get_service_client(DatabaseManagerClient, request_retry=True)+++@thread_cached+def get_database_manager_async_client() -> "DatabaseManagerAsyncClient":+ """Get a thread-cached DatabaseManagerAsyncClient with request retry enabled."""+ from backend.executor import DatabaseManagerAsyncClient+ from backend.util.service import get_service_client++ return get_service_client(DatabaseManagerAsyncClient, request_retry=True)+++@thread_cached+def get_scheduler_client() -> "SchedulerClient":+ """Get a thread-cached SchedulerClient."""+ from backend.executor.scheduler import SchedulerClient+ from backend.util.service import get_service_client++ return get_service_client(SchedulerClient)+++@thread_cached+def get_notification_manager_client() -> "NotificationManagerClient":+ """Get a thread-cached NotificationManagerClient."""+ from backend.notifications.notifications import NotificationManagerClient+ from backend.util.service import get_service_client++ return get_service_client(NotificationManagerClient)+++# ============ Execution Event Bus Helpers ============ #+++@thread_cached+def get_execution_event_bus() -> "RedisExecutionEventBus":+ """Get a thread-cached RedisExecutionEventBus."""+ from backend.data.execution import RedisExecutionEventBus++ return RedisExecutionEventBus()+++@thread_cached+def get_async_execution_event_bus() -> "AsyncRedisExecutionEventBus":+ """Get a thread-cached AsyncRedisExecutionEventBus."""+ from backend.data.execution import AsyncRedisExecutionEventBus++ return AsyncRedisExecutionEventBus()+++# ============ Execution Queue Helpers ============ #+++@thread_cached+def get_execution_queue() -> "SyncRabbitMQ":+ """Get a thread-cached SyncRabbitMQ execution queue client."""+ from backend.data.rabbitmq import SyncRabbitMQ+ from backend.executor.utils import create_execution_queue_config++ client = SyncRabbitMQ(create_execution_queue_config())+ client.connect()+ return client+++@thread_cached+async def get_async_execution_queue() -> "AsyncRabbitMQ":+ """Get a thread-cached AsyncRabbitMQ execution queue client."""+ from backend.data.rabbitmq import AsyncRabbitMQ+ from backend.executor.utils import create_execution_queue_config++ client = AsyncRabbitMQ(create_execution_queue_config())+ await client.connect()+ return client+++# ============ Integration Credentials Store ============ #+++@thread_cached+def get_integration_credentials_store() -> "IntegrationCredentialsStore":+ """Get a thread-cached IntegrationCredentialsStore."""+ from backend.integrations.credentials_store import IntegrationCredentialsStore++ return IntegrationCredentialsStore()+++# ============ Notification Queue Helpers ============ #+++@thread_cached+def get_notification_queue() -> "SyncRabbitMQ":+ """Get a thread-cached SyncRabbitMQ notification queue client."""+ from backend.data.rabbitmq import SyncRabbitMQ+ from backend.notifications.notifications import create_notification_config++ client = SyncRabbitMQ(create_notification_config())+ client.connect()+ return client+++@thread_cached+async def get_async_notification_queue() -> "AsyncRabbitMQ":+ """Get a thread-cached AsyncRabbitMQ notification queue client."""+ from backend.data.rabbitmq import AsyncRabbitMQ+ from backend.notifications.notifications import create_notification_config++ client = AsyncRabbitMQ(create_notification_config())+ await client.connect()+ return clientdiff --git a/autogpt_platform/backend/backend/util/process.py b/autogpt_platform/backend/backend/util/process.pyindex 1460ee9f99..3f8204838c 100644--- a/autogpt_platform/backend/backend/util/process.py+++ b/autogpt_platform/backend/backend/util/process.py@@ -75,7 +75,7 @@ class AppProcess(ABC):self.run()except BaseException as e:logger.warning(- f"[{self.service_name}] Termination request: {type(e).__name__}; executing cleanup."+ f"[{self.service_name}] Termination request: {type(e).__name__}; {e} executing cleanup.")finally:self.cleanup()diff --git a/autogpt_platform/backend/backend/util/retry.py b/autogpt_platform/backend/backend/util/retry.pyindex 79f3d715cf..c586281af9 100644--- a/autogpt_platform/backend/backend/util/retry.py+++ b/autogpt_platform/backend/backend/util/retry.py@@ -161,14 +161,21 @@ def continuous_retry(*, retry_delay: float = 1.0):@wraps(func)def sync_wrapper(*args, **kwargs):+ counter = 0while True:try:return func(*args, **kwargs)except Exception as exc:- logger.exception(- "%s failed with %s — retrying in %.2f s",+ counter += 1+ if counter % 10 == 0:+ log = logger.exception+ else:+ log = logger.warning+ log(+ "%s failed for the %s times, error: [%s] — retrying in %.2fs",func.__name__,- exc,+ counter,+ str(exc) or type(exc).__name__,retry_delay,)time.sleep(retry_delay)@@ -176,13 +183,20 @@ def continuous_retry(*, retry_delay: float = 1.0):@wraps(func)async def async_wrapper(*args, **kwargs):while True:+ counter = 0try:return await func(*args, **kwargs)except Exception as exc:- logger.exception(- "%s failed with %s — retrying in %.2f s",+ counter += 1+ if counter % 10 == 0:+ log = logger.exception+ else:+ log = logger.warning+ log(+ "%s failed for the %s times, error: [%s] — retrying in %.2fs",func.__name__,- exc,+ counter,+ str(exc) or type(exc).__name__,retry_delay,)await asyncio.sleep(retry_delay)diff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.pyindex c0f4a1fb88..2715378f9f 100644--- a/autogpt_platform/backend/backend/util/service.py+++ b/autogpt_platform/backend/backend/util/service.py@@ -277,7 +277,6 @@ ASC = TypeVar("ASC", bound=AppServiceClient)def get_service_client(service_client_type: Type[ASC],call_timeout: int | None = api_call_timeout,- health_check: bool = True,request_retry: bool = False,) -> ASC:@@ -461,8 +460,6 @@ def get_service_client(return sync_methodclient = cast(ASC, DynamicClient())- if health_check and hasattr(client, "health_check"):- client.health_check()return clientdiff --git a/autogpt_platform/backend/backend/util/service_test.py b/autogpt_platform/backend/backend/util/service_test.pyindex 00fd206179..3a594fed25 100644--- a/autogpt_platform/backend/backend/util/service_test.py+++ b/autogpt_platform/backend/backend/util/service_test.py@@ -16,6 +16,12 @@ from backend.util.service import (TEST_SERVICE_PORT = 8765+def wait_for_service_ready(service_client_type, timeout_seconds=30):+ """Helper method to wait for a service to be ready using health check with retry."""+ client = get_service_client(service_client_type, request_retry=True)+ client.health_check() # This will retry until service is ready++class ServiceTest(AppService):def __init__(self):super().__init__()@@ -28,6 +34,15 @@ class ServiceTest(AppService):def get_port(cls) -> int:return TEST_SERVICE_PORT+ def __enter__(self):+ # Start the service+ result = super().__enter__()++ # Wait for the service to be ready+ wait_for_service_ready(ServiceTestClient)++ return result+@exposedef add(self, a: int, b: int) -> int:return a + b@@ -48,13 +63,13 @@ class ServiceTest(AppService):"""Method that fails 2 times then succeeds - for testing retry logic"""self.fail_count += 1if self.fail_count <= 2:- raise RuntimeError("Database connection failed")+ raise RuntimeError(f"Intended error for testing {self.fail_count}/2")return a + b@exposedef always_failing_add(self, a: int, b: int) -> int:"""Method that always fails - for testing no retry when disabled"""- raise RuntimeError("Database connection failed")+ raise RuntimeError("Intended error for testing")class ServiceTestClient(AppServiceClient):@@ -349,5 +364,5 @@ def test_service_no_retry_when_disabled(server):client = get_service_client(ServiceTestClient, request_retry=False)# This should fail immediately without retry- with pytest.raises(RuntimeError, match="Database connection failed"):+ with pytest.raises(RuntimeError, match="Intended error for testing"):client.always_failing_add(5, 3)