Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): Add thread safety to NodeExecutionProgress output handling (#10415)

Zamil Majdyf4a179e5d695
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index 709b375604..df246abb05 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -783,11 +783,13 @@ class Executor:
# node evaluation future -----------------
if inflight_eval := running_node_evaluation.get(node_id):
+ if not inflight_eval.done():
+ continue
try:
- inflight_eval.result()
+ inflight_eval.result(timeout=0)
running_node_evaluation.pop(node_id)
- except TimeoutError:
- continue
+ except Exception as e:
+ log_metadata.error(f"Node eval #{node_id} failed: {e}")
# node execution future ---------------------------
if inflight_exec.is_done():
diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.py
index e27c43d63f..8568c41394 100644
--- a/autogpt_platform/backend/backend/executor/utils.py
+++ b/autogpt_platform/backend/backend/executor/utils.py
@@ -1,5 +1,6 @@
import asyncio
import logging
+import threading
import time
from collections import defaultdict
from concurrent.futures import Future
@@ -885,12 +886,14 @@ class NodeExecutionProgress:
self.output: dict[str, list[ExecutionOutputEntry]] = defaultdict(list)
self.tasks: dict[str, Future] = {}
self.on_done_task = on_done_task
+ self._lock = threading.Lock()
def add_task(self, node_exec_id: str, task: Future):
self.tasks[node_exec_id] = task
def add_output(self, output: ExecutionOutputEntry):
- self.output[output.node_exec_id].append(output)
+ with self._lock:
+ self.output[output.node_exec_id].append(output)
def pop_output(self) -> ExecutionOutputEntry | None:
exec_id = self._next_exec()
@@ -900,8 +903,9 @@ class NodeExecutionProgress:
if self._pop_done_task(exec_id):
return self.pop_output()
- if next_output := self.output[exec_id]:
- return next_output.pop(0)
+ with self._lock:
+ if next_output := self.output[exec_id]:
+ return next_output.pop(0)
return None
@@ -966,8 +970,9 @@ class NodeExecutionProgress:
if not task.done():
return False
- if self.output[exec_id]:
- return False
+ with self._lock:
+ if self.output[exec_id]:
+ return False
if task := self.tasks.pop(exec_id):
try: