Agent Zero AI framework
Fix: proper task cancellation in scheduler, leakage in defer.py
Rafael Uzarowski784fe5589a76
diff --git a/python/api/chat_remove.py b/python/api/chat_remove.pyindex c7b75f02a..671e43d9e 100644--- a/python/api/chat_remove.py+++ b/python/api/chat_remove.py@@ -8,6 +8,9 @@ class RemoveChat(ApiHandler):async def process(self, input: Input, request: Request) -> Output:ctxid = input.get("context", "")+ scheduler = TaskScheduler.get()+ scheduler.cancel_tasks_by_context(ctxid, terminate_thread=True)+context = AgentContext.use(ctxid)if context:# stop processing any tasks@@ -16,7 +19,6 @@ class RemoveChat(ApiHandler):AgentContext.remove(ctxid)persist_chat.remove_chat(ctxid)- scheduler = TaskScheduler.get()await scheduler.reload()tasks = scheduler.get_tasks_by_context_id(ctxid)diff --git a/python/api/chat_reset.py b/python/api/chat_reset.pyindex a5ce1284e..668b08e26 100644--- a/python/api/chat_reset.py+++ b/python/api/chat_reset.py@@ -2,12 +2,16 @@ from python.helpers.api import ApiHandler, Input, Output, Request, Responsefrom python.helpers import persist_chat+from python.helpers.task_scheduler import TaskSchedulerclass Reset(ApiHandler):async def process(self, input: Input, request: Request) -> Output:ctxid = input.get("context", "")+ # attempt to stop any scheduler tasks bound to this context+ TaskScheduler.get().cancel_tasks_by_context(ctxid, terminate_thread=True)+# context instance - get or createcontext = self.use_context(ctxid)context.reset()diff --git a/python/api/scheduler_task_delete.py b/python/api/scheduler_task_delete.pyindex f9b52ac03..5e41a0bd6 100644--- a/python/api/scheduler_task_delete.py+++ b/python/api/scheduler_task_delete.py@@ -34,6 +34,7 @@ class SchedulerTaskDelete(ApiHandler):# If the task is running, update its state to IDLE firstif task.state == TaskState.RUNNING:+ scheduler.cancel_running_task(task_id, terminate_thread=True)if context:context.reset()# Update the state to IDLE so any ongoing processes know to terminatediff --git a/python/helpers/defer.py b/python/helpers/defer.pyindex dc96efe5a..8c2c7e86e 100644--- a/python/helpers/defer.py+++ b/python/helpers/defer.py@@ -6,8 +6,9 @@ from typing import Any, Callable, Optional, Coroutine, TypeVar, AwaitableT = TypeVar("T")+class EventLoopThread:- _instances = {}+ _instances: dict[str, "EventLoopThread"] = {}_lock = threading.Lock()def __init__(self, thread_name: str = "Background") -> None:@@ -38,8 +39,29 @@ class EventLoopThread:self.loop.run_forever()def terminate(self):- if self.loop and self.loop.is_running():- self.loop.stop()+ loop = getattr(self, "loop", None)+ thread = getattr(self, "thread", None)++ if not loop:+ return++ if loop.is_running():+ if thread and thread is threading.current_thread():+ loop.stop()+ else:+ loop.call_soon_threadsafe(loop.stop)+ if thread:+ thread.join()+ elif thread and thread.is_alive() and thread is not threading.current_thread():+ thread.join()++ if not loop.is_closed():+ loop.close()++ with self.__class__._lock:+ if self.thread_name in self.__class__._instances:+ del self.__class__._instances[self.thread_name]+self.loop = Noneself.thread = None@@ -79,6 +101,12 @@ class DeferredTask:def _start_task(self):self._future = self.event_loop_thread.run_coroutine(self._run())+ if self._future:+ self._future.add_done_callback(self._on_task_done)++ def _on_task_done(self, _future: Future):+ # Ensure child background tasks are always cleaned up once the parent finishes+ self.kill_children()async def _run(self):return await self.func(*self.args, **self.kwargs)@@ -120,30 +148,16 @@ class DeferredTask:if self._future and not self._future.done():self._future.cancel()- if (- terminate_thread- and self.event_loop_thread.loop- and self.event_loop_thread.loop.is_running()- ):-- def cleanup():- tasks = [- t- for t in asyncio.all_tasks(self.event_loop_thread.loop)- if t is not asyncio.current_task(self.event_loop_thread.loop)- ]- for task in tasks:- task.cancel()- try:- # Give tasks a chance to cleanup- if self.event_loop_thread.loop:- self.event_loop_thread.loop.run_until_complete(- asyncio.gather(task, return_exceptions=True)- )- except Exception:- pass # Ignore cleanup errors-- self.event_loop_thread.loop.call_soon_threadsafe(cleanup)+ if terminate_thread and self.event_loop_thread.loop:+ if self.event_loop_thread.loop.is_running():+ try:+ cleanup_future = asyncio.run_coroutine_threadsafe(+ self._drain_event_loop_tasks(), self.event_loop_thread.loop+ )+ cleanup_future.result()+ except Exception:+ pass+self.event_loop_thread.terminate()def kill_children(self) -> None:@@ -196,3 +210,19 @@ class DeferredTask:asyncio.run_coroutine_threadsafe(wrapped(), self.event_loop_thread.loop)return asyncio.wrap_future(future)++ @staticmethod+ async def _drain_event_loop_tasks():+ """Cancel and await all pending tasks on the current event loop."""+ loop = asyncio.get_running_loop()+ current_task = asyncio.current_task(loop=loop)+ pending = [+ task+ for task in asyncio.all_tasks(loop=loop)+ if task is not current_task+ ]+ if not pending:+ return+ for task in pending:+ task.cancel()+ await asyncio.gather(*pending, return_exceptions=True)diff --git a/python/helpers/task_scheduler.py b/python/helpers/task_scheduler.pyindex a707777af..249945bb2 100644--- a/python/helpers/task_scheduler.py+++ b/python/helpers/task_scheduler.py@@ -619,6 +619,8 @@ class TaskScheduler:_tasks: SchedulerTaskList_printer: PrintStyle_instance = None+ _running_deferred_tasks: Dict[str, DeferredTask]+ _running_tasks_lock: threading.RLock@classmethoddef get(cls) -> "TaskScheduler":@@ -631,8 +633,38 @@ class TaskScheduler:if not hasattr(self, '_initialized'):self._tasks = SchedulerTaskList.get()self._printer = PrintStyle(italic=True, font_color="green", padding=False)+ self._running_deferred_tasks = {}+ self._running_tasks_lock = threading.RLock()self._initialized = True+ def _register_running_task(self, task_uuid: str, deferred_task: DeferredTask) -> None:+ with self._running_tasks_lock:+ self._running_deferred_tasks[task_uuid] = deferred_task++ def _unregister_running_task(self, task_uuid: str) -> None:+ with self._running_tasks_lock:+ self._running_deferred_tasks.pop(task_uuid, None)++ def cancel_running_task(self, task_uuid: str, terminate_thread: bool = False) -> bool:+ with self._running_tasks_lock:+ deferred_task = self._running_deferred_tasks.get(task_uuid)+ if not deferred_task:+ return False+ self._printer.print(f"Scheduler cancelling task {task_uuid}")+ deferred_task.kill(terminate_thread=terminate_thread)+ return True++ def cancel_tasks_by_context(self, context_id: str, terminate_thread: bool = False) -> bool:+ cancelled_any = False+ with self._running_tasks_lock:+ running_tasks = list(self._running_deferred_tasks.keys())+ for task_uuid in running_tasks:+ task = self.get_task_by_uuid(task_uuid)+ if task and task.context_id == context_id:+ if self.cancel_running_task(task_uuid, terminate_thread=terminate_thread):+ cancelled_any = True+ return cancelled_any+async def reload(self):await self._tasks.reload()@@ -774,19 +806,23 @@ class TaskScheduler:task_snapshot: Union[ScheduledTask, AdHocTask, PlannedTask] | None = self.get_task_by_uuid(task_uuid)if task_snapshot is None:self._printer.print(f"Scheduler Task with UUID '{task_uuid}' not found")+ self._unregister_running_task(task_uuid)returnif task_snapshot.state == TaskState.RUNNING:self._printer.print(f"Scheduler Task '{task_snapshot.name}' already running, skipping")+ self._unregister_running_task(task_uuid)return# Atomically fetch and check the task's current statecurrent_task = await self.update_task_checked(task_uuid, lambda task: task.state != TaskState.RUNNING, state=TaskState.RUNNING)if not current_task:self._printer.print(f"Scheduler Task with UUID '{task_uuid}' not found or updated by another process")+ self._unregister_running_task(task_uuid)returnif current_task.state != TaskState.RUNNING:# This means the update failed due to state conflictself._printer.print(f"Scheduler Task '{current_task.name}' state is '{current_task.state}', skipping")+ self._unregister_running_task(task_uuid)returnawait current_task.on_run()@@ -868,6 +904,13 @@ class TaskScheduler:self._printer.print(f"Fixing task state consistency: '{current_task.name}' state is not IDLE after success")await self.update_task(task_uuid, state=TaskState.IDLE)+ except asyncio.CancelledError:+ self._printer.print(f"Scheduler Task '{current_task.name}' cancelled by user")+ try:+ await asyncio.shield(self.update_task(task_uuid, state=TaskState.IDLE))+ except Exception:+ pass+ raiseexcept Exception as e:# Errorself._printer.print(f"Scheduler Task '{current_task.name}' failed: {e}")@@ -884,17 +927,31 @@ class TaskScheduler:agent.handle_critical_exception(e)finally:# Call on_finish for task-specific cleanup- await current_task.on_finish()+ try:+ await asyncio.shield(current_task.on_finish())+ except asyncio.CancelledError:+ pass+ except Exception:+ pass# Make one final save to ensure all states are persisted- await self._tasks.save()+ try:+ await asyncio.shield(self._tasks.save())+ except asyncio.CancelledError:+ pass+ except Exception:+ pass++ self._unregister_running_task(task_uuid)deferred_task = DeferredTask(thread_name=self.__class__.__name__)+ self._register_running_task(task.uuid, deferred_task)deferred_task.start_task(_run_task_wrapper, task.uuid, task_context)- # Ensure background execution doesn't exit immediately on async await, especially in script contexts- # This helps prevent premature exits when running from non-event-loop contexts- asyncio.create_task(asyncio.sleep(0.1))+ # Ensure background execution doesn't exit immediately on async await, especially in script contexts.+ # Yielding briefly keeps callers like CLI scripts alive long enough for the DeferredTask thread to spin up+ # without leaving stray pending tasks that trigger \"Task was destroyed\" warnings when the loop shuts down.+ await asyncio.sleep(0.1)def serialize_all_tasks(self) -> list[Dict[str, Any]]:"""