AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): add graph validation before scheduling recurring jobs (#10568)
Zamil Majdy378d256b5878
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex a67e27973c..187951b271 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -61,6 +61,9 @@ apscheduler_logger.addFilter(PrefixFilter("[Scheduler] [APScheduler]"))config = Config()+# Timeout constants+SCHEDULER_OPERATION_TIMEOUT_SECONDS = 300 # 5 minutes for scheduler operations+def job_listener(event):"""Logs job execution outcomes for better monitoring."""@@ -71,6 +74,7 @@ def job_listener(event):_event_loop: asyncio.AbstractEventLoop | None = None+_event_loop_thread: threading.Thread | None = None@func_retry@@ -81,12 +85,17 @@ def get_event_loop():return _event_loop+def run_async(coro, timeout: float = SCHEDULER_OPERATION_TIMEOUT_SECONDS):+ """Run a coroutine in the shared event loop and wait for completion."""+ loop = get_event_loop()+ future = asyncio.run_coroutine_threadsafe(coro, loop)+ return future.result(timeout=timeout)++def execute_graph(**kwargs):"""Execute graph in the shared event loop and wait for completion."""- loop = get_event_loop()- future = asyncio.run_coroutine_threadsafe(_execute_graph(**kwargs), loop)# Wait for completion to ensure job doesn't exit prematurely- future.result(timeout=300) # 5 minute timeout for graph execution+ run_async(_execute_graph(**kwargs))async def _execute_graph(**kwargs):@@ -110,10 +119,8 @@ async def _execute_graph(**kwargs):def cleanup_expired_files():"""Clean up expired files from cloud storage."""- loop = get_event_loop()- future = asyncio.run_coroutine_threadsafe(cleanup_expired_files_async(), loop)# Wait for completion- future.result(timeout=300) # 5 minute timeout for cleanup+ run_async(cleanup_expired_files_async())# Monitoring functions are now imported from monitoring module@@ -195,10 +202,11 @@ class Scheduler(AppService):_event_loop = asyncio.new_event_loop()# Use daemon thread since it should die with the main service- event_loop_thread = threading.Thread(+ global _event_loop_thread+ _event_loop_thread = threading.Thread(target=_event_loop.run_forever, daemon=True, name="SchedulerEventLoop")- event_loop_thread.start()+ _event_loop_thread.start()db_schema, db_url = _extract_schema_from_url(os.getenv("DIRECT_URL"))self.scheduler = BlockingScheduler(@@ -287,7 +295,19 @@ class Scheduler(AppService):super().cleanup()if self.scheduler:logger.info("⏳ Shutting down scheduler...")- self.scheduler.shutdown(wait=False)+ self.scheduler.shutdown(wait=True)++ global _event_loop+ if _event_loop:+ logger.info("⏳ Closing event loop...")+ _event_loop.call_soon_threadsafe(_event_loop.stop)++ global _event_loop_thread+ if _event_loop_thread:+ logger.info("⏳ Waiting for event loop thread to finish...")+ _event_loop_thread.join(timeout=SCHEDULER_OPERATION_TIMEOUT_SECONDS)++ logger.info("Scheduler cleanup complete.")@exposedef add_graph_execution_schedule(@@ -300,6 +320,18 @@ class Scheduler(AppService):input_credentials: dict[str, CredentialsMetaInput],name: Optional[str] = None,) -> GraphExecutionJobInfo:+ # Validate the graph before scheduling to prevent runtime failures+ # We don't need the return value, just want the validation to run+ run_async(+ execution_utils.validate_and_construct_node_execution_input(+ graph_id=graph_id,+ user_id=user_id,+ graph_inputs=input_data,+ graph_version=graph_version,+ graph_credentials_inputs=input_credentials,+ )+ )+job_args = GraphExecutionJobArgs(user_id=user_id,graph_id=graph_id,diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex e92eabfd23..34eaab4519 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -548,7 +548,7 @@ async def validate_graph_with_credentials(return node_input_errors-async def construct_node_execution_input(+async def _construct_node_execution_input(graph: GraphModel,user_id: str,graph_inputs: BlockInput,@@ -615,6 +615,67 @@ async def construct_node_execution_input(return nodes_input+async def validate_and_construct_node_execution_input(+ graph_id: str,+ user_id: str,+ graph_inputs: BlockInput,+ graph_version: Optional[int] = None,+ graph_credentials_inputs: Optional[dict[str, CredentialsMetaInput]] = None,+ nodes_input_masks: Optional[dict[str, dict[str, JsonValue]]] = None,+) -> tuple[GraphModel, list[tuple[str, BlockInput]]]:+ """+ Public wrapper that handles graph fetching, credential mapping, and validation+construction.+ This centralizes the logic used by both scheduler validation and actual execution.++ Args:+ graph_id: The ID of the graph to validate/construct.+ user_id: The ID of the user.+ graph_inputs: The input data for the graph execution.+ graph_version: The version of the graph to use.+ graph_credentials_inputs: Credentials inputs to use.+ nodes_input_masks: Node inputs to use.++ Returns:+ tuple[GraphModel, list[tuple[str, BlockInput]]]: Graph model and list of tuples for node execution input.++ Raises:+ NotFoundError: If the graph is not found.+ GraphValidationError: If the graph has validation issues.+ ValueError: If there are other validation errors.+ """+ if prisma.is_connected():+ gdb = graph_db+ else:+ gdb = get_database_manager_async_client()++ graph: GraphModel | None = await gdb.get_graph(+ graph_id=graph_id,+ user_id=user_id,+ version=graph_version,+ include_subgraphs=True,+ )+ if not graph:+ raise NotFoundError(f"Graph #{graph_id} not found.")++ nodes_input_masks = _merge_nodes_input_masks(+ (+ make_node_credentials_input_map(graph, graph_credentials_inputs)+ if graph_credentials_inputs+ else {}+ ),+ nodes_input_masks or {},+ )++ starting_nodes_input = await _construct_node_execution_input(+ graph=graph,+ user_id=user_id,+ graph_inputs=graph_inputs,+ nodes_input_masks=nodes_input_masks,+ )++ return graph, starting_nodes_input++def _merge_nodes_input_masks(overrides_map_1: dict[str, dict[str, JsonValue]],overrides_map_2: dict[str, dict[str, JsonValue]],@@ -791,33 +852,16 @@ async def add_graph_execution(ValueError: If the graph is not found or if there are validation errors."""if prisma.is_connected():- gdb = graph_dbedb = execution_dbelse:- gdb = get_database_manager_async_client()edb = get_database_manager_async_client()- graph: GraphModel | None = await gdb.get_graph(+ graph, starting_nodes_input = await validate_and_construct_node_execution_input(graph_id=graph_id,user_id=user_id,- version=graph_version,- include_subgraphs=True,- )- if not graph:- raise NotFoundError(f"Graph #{graph_id} not found.")-- nodes_input_masks = _merge_nodes_input_masks(- (- make_node_credentials_input_map(graph, graph_credentials_inputs)- if graph_credentials_inputs- else {}- ),- nodes_input_masks or {},- )- starting_nodes_input = await construct_node_execution_input(- graph=graph,- user_id=user_id,graph_inputs=inputs or {},+ graph_version=graph_version,+ graph_credentials_inputs=graph_credentials_inputs,nodes_input_masks=nodes_input_masks,)graph_exec = None