Skip to content

Agent Zero AI framework

Fix: proper task cancellation in scheduler, leakage in defer.py

Rafael Uzarowski784fe5589a76
diff --git a/python/api/chat_remove.py b/python/api/chat_remove.py
index c7b75f02a..671e43d9e 100644
--- a/python/api/chat_remove.py
+++ b/python/api/chat_remove.py
@@ -8,6 +8,9 @@ class RemoveChat(ApiHandler):
async def process(self, input: Input, request: Request) -> Output:
ctxid = input.get("context", "")
+ scheduler = TaskScheduler.get()
+ scheduler.cancel_tasks_by_context(ctxid, terminate_thread=True)
+
context = AgentContext.use(ctxid)
if context:
# stop processing any tasks
@@ -16,7 +19,6 @@ class RemoveChat(ApiHandler):
AgentContext.remove(ctxid)
persist_chat.remove_chat(ctxid)
- scheduler = TaskScheduler.get()
await scheduler.reload()
tasks = scheduler.get_tasks_by_context_id(ctxid)
diff --git a/python/api/chat_reset.py b/python/api/chat_reset.py
index a5ce1284e..668b08e26 100644
--- a/python/api/chat_reset.py
+++ b/python/api/chat_reset.py
@@ -2,12 +2,16 @@ from python.helpers.api import ApiHandler, Input, Output, Request, Response
from python.helpers import persist_chat
+from python.helpers.task_scheduler import TaskScheduler
class Reset(ApiHandler):
async def process(self, input: Input, request: Request) -> Output:
ctxid = input.get("context", "")
+ # attempt to stop any scheduler tasks bound to this context
+ TaskScheduler.get().cancel_tasks_by_context(ctxid, terminate_thread=True)
+
# context instance - get or create
context = self.use_context(ctxid)
context.reset()
diff --git a/python/api/scheduler_task_delete.py b/python/api/scheduler_task_delete.py
index f9b52ac03..5e41a0bd6 100644
--- a/python/api/scheduler_task_delete.py
+++ b/python/api/scheduler_task_delete.py
@@ -34,6 +34,7 @@ class SchedulerTaskDelete(ApiHandler):
# If the task is running, update its state to IDLE first
if task.state == TaskState.RUNNING:
+ scheduler.cancel_running_task(task_id, terminate_thread=True)
if context:
context.reset()
# Update the state to IDLE so any ongoing processes know to terminate
diff --git a/python/helpers/defer.py b/python/helpers/defer.py
index dc96efe5a..8c2c7e86e 100644
--- a/python/helpers/defer.py
+++ b/python/helpers/defer.py
@@ -6,8 +6,9 @@ from typing import Any, Callable, Optional, Coroutine, TypeVar, Awaitable
T = TypeVar("T")
+
class EventLoopThread:
- _instances = {}
+ _instances: dict[str, "EventLoopThread"] = {}
_lock = threading.Lock()
def __init__(self, thread_name: str = "Background") -> None:
@@ -38,8 +39,29 @@ class EventLoopThread:
self.loop.run_forever()
def terminate(self):
- if self.loop and self.loop.is_running():
- self.loop.stop()
+ loop = getattr(self, "loop", None)
+ thread = getattr(self, "thread", None)
+
+ if not loop:
+ return
+
+ if loop.is_running():
+ if thread and thread is threading.current_thread():
+ loop.stop()
+ else:
+ loop.call_soon_threadsafe(loop.stop)
+ if thread:
+ thread.join()
+ elif thread and thread.is_alive() and thread is not threading.current_thread():
+ thread.join()
+
+ if not loop.is_closed():
+ loop.close()
+
+ with self.__class__._lock:
+ if self.thread_name in self.__class__._instances:
+ del self.__class__._instances[self.thread_name]
+
self.loop = None
self.thread = None
@@ -79,6 +101,12 @@ class DeferredTask:
def _start_task(self):
self._future = self.event_loop_thread.run_coroutine(self._run())
+ if self._future:
+ self._future.add_done_callback(self._on_task_done)
+
+ def _on_task_done(self, _future: Future):
+ # Ensure child background tasks are always cleaned up once the parent finishes
+ self.kill_children()
async def _run(self):
return await self.func(*self.args, **self.kwargs)
@@ -120,30 +148,16 @@ class DeferredTask:
if self._future and not self._future.done():
self._future.cancel()
- if (
- terminate_thread
- and self.event_loop_thread.loop
- and self.event_loop_thread.loop.is_running()
- ):
-
- def cleanup():
- tasks = [
- t
- for t in asyncio.all_tasks(self.event_loop_thread.loop)
- if t is not asyncio.current_task(self.event_loop_thread.loop)
- ]
- for task in tasks:
- task.cancel()
- try:
- # Give tasks a chance to cleanup
- if self.event_loop_thread.loop:
- self.event_loop_thread.loop.run_until_complete(
- asyncio.gather(task, return_exceptions=True)
- )
- except Exception:
- pass # Ignore cleanup errors
-
- self.event_loop_thread.loop.call_soon_threadsafe(cleanup)
+ if terminate_thread and self.event_loop_thread.loop:
+ if self.event_loop_thread.loop.is_running():
+ try:
+ cleanup_future = asyncio.run_coroutine_threadsafe(
+ self._drain_event_loop_tasks(), self.event_loop_thread.loop
+ )
+ cleanup_future.result()
+ except Exception:
+ pass
+
self.event_loop_thread.terminate()
def kill_children(self) -> None:
@@ -196,3 +210,19 @@ class DeferredTask:
asyncio.run_coroutine_threadsafe(wrapped(), self.event_loop_thread.loop)
return asyncio.wrap_future(future)
+
+ @staticmethod
+ async def _drain_event_loop_tasks():
+ """Cancel and await all pending tasks on the current event loop."""
+ loop = asyncio.get_running_loop()
+ current_task = asyncio.current_task(loop=loop)
+ pending = [
+ task
+ for task in asyncio.all_tasks(loop=loop)
+ if task is not current_task
+ ]
+ if not pending:
+ return
+ for task in pending:
+ task.cancel()
+ await asyncio.gather(*pending, return_exceptions=True)
diff --git a/python/helpers/task_scheduler.py b/python/helpers/task_scheduler.py
index a707777af..249945bb2 100644
--- a/python/helpers/task_scheduler.py
+++ b/python/helpers/task_scheduler.py
@@ -619,6 +619,8 @@ class TaskScheduler:
_tasks: SchedulerTaskList
_printer: PrintStyle
_instance = None
+ _running_deferred_tasks: Dict[str, DeferredTask]
+ _running_tasks_lock: threading.RLock
@classmethod
def get(cls) -> "TaskScheduler":
@@ -631,8 +633,38 @@ class TaskScheduler:
if not hasattr(self, '_initialized'):
self._tasks = SchedulerTaskList.get()
self._printer = PrintStyle(italic=True, font_color="green", padding=False)
+ self._running_deferred_tasks = {}
+ self._running_tasks_lock = threading.RLock()
self._initialized = True
+ def _register_running_task(self, task_uuid: str, deferred_task: DeferredTask) -> None:
+ with self._running_tasks_lock:
+ self._running_deferred_tasks[task_uuid] = deferred_task
+
+ def _unregister_running_task(self, task_uuid: str) -> None:
+ with self._running_tasks_lock:
+ self._running_deferred_tasks.pop(task_uuid, None)
+
+ def cancel_running_task(self, task_uuid: str, terminate_thread: bool = False) -> bool:
+ with self._running_tasks_lock:
+ deferred_task = self._running_deferred_tasks.get(task_uuid)
+ if not deferred_task:
+ return False
+ self._printer.print(f"Scheduler cancelling task {task_uuid}")
+ deferred_task.kill(terminate_thread=terminate_thread)
+ return True
+
+ def cancel_tasks_by_context(self, context_id: str, terminate_thread: bool = False) -> bool:
+ cancelled_any = False
+ with self._running_tasks_lock:
+ running_tasks = list(self._running_deferred_tasks.keys())
+ for task_uuid in running_tasks:
+ task = self.get_task_by_uuid(task_uuid)
+ if task and task.context_id == context_id:
+ if self.cancel_running_task(task_uuid, terminate_thread=terminate_thread):
+ cancelled_any = True
+ return cancelled_any
+
async def reload(self):
await self._tasks.reload()
@@ -774,19 +806,23 @@ class TaskScheduler:
task_snapshot: Union[ScheduledTask, AdHocTask, PlannedTask] | None = self.get_task_by_uuid(task_uuid)
if task_snapshot is None:
self._printer.print(f"Scheduler Task with UUID '{task_uuid}' not found")
+ self._unregister_running_task(task_uuid)
return
if task_snapshot.state == TaskState.RUNNING:
self._printer.print(f"Scheduler Task '{task_snapshot.name}' already running, skipping")
+ self._unregister_running_task(task_uuid)
return
# Atomically fetch and check the task's current state
current_task = await self.update_task_checked(task_uuid, lambda task: task.state != TaskState.RUNNING, state=TaskState.RUNNING)
if not current_task:
self._printer.print(f"Scheduler Task with UUID '{task_uuid}' not found or updated by another process")
+ self._unregister_running_task(task_uuid)
return
if current_task.state != TaskState.RUNNING:
# This means the update failed due to state conflict
self._printer.print(f"Scheduler Task '{current_task.name}' state is '{current_task.state}', skipping")
+ self._unregister_running_task(task_uuid)
return
await current_task.on_run()
@@ -868,6 +904,13 @@ class TaskScheduler:
self._printer.print(f"Fixing task state consistency: '{current_task.name}' state is not IDLE after success")
await self.update_task(task_uuid, state=TaskState.IDLE)
+ except asyncio.CancelledError:
+ self._printer.print(f"Scheduler Task '{current_task.name}' cancelled by user")
+ try:
+ await asyncio.shield(self.update_task(task_uuid, state=TaskState.IDLE))
+ except Exception:
+ pass
+ raise
except Exception as e:
# Error
self._printer.print(f"Scheduler Task '{current_task.name}' failed: {e}")
@@ -884,17 +927,31 @@ class TaskScheduler:
agent.handle_critical_exception(e)
finally:
# Call on_finish for task-specific cleanup
- await current_task.on_finish()
+ try:
+ await asyncio.shield(current_task.on_finish())
+ except asyncio.CancelledError:
+ pass
+ except Exception:
+ pass
# Make one final save to ensure all states are persisted
- await self._tasks.save()
+ try:
+ await asyncio.shield(self._tasks.save())
+ except asyncio.CancelledError:
+ pass
+ except Exception:
+ pass
+
+ self._unregister_running_task(task_uuid)
deferred_task = DeferredTask(thread_name=self.__class__.__name__)
+ self._register_running_task(task.uuid, deferred_task)
deferred_task.start_task(_run_task_wrapper, task.uuid, task_context)
- # Ensure background execution doesn't exit immediately on async await, especially in script contexts
- # This helps prevent premature exits when running from non-event-loop contexts
- asyncio.create_task(asyncio.sleep(0.1))
+ # Ensure background execution doesn't exit immediately on async await, especially in script contexts.
+ # Yielding briefly keeps callers like CLI scripts alive long enough for the DeferredTask thread to spin up
+ # without leaving stray pending tasks that trigger \"Task was destroyed\" warnings when the loop shuts down.
+ await asyncio.sleep(0.1)
def serialize_all_tasks(self) -> list[Dict[str, Any]]:
"""