"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(agent): reap owned background processes on cancellation and shutdown
Yufeng He04efdafdcd00
diff --git a/agent/api_server.py b/agent/api_server.pyindex 8fddc250f..63446c981 100644--- a/agent/api_server.py+++ b/agent/api_server.py@@ -156,6 +156,14 @@ async def _lifespan(_: FastAPI) -> AsyncIterator[None]:yieldfinally:await _stop_scheduled_research_on_shutdown()+ # Reap background work this host owns before its runtime disappears.+ # Read the existing service rather than _get_session_service(): a+ # shutdown must not create the very object it is trying to release.+ import src.api.state as _api_state++ service = getattr(_api_state, "_session_service", None)+ if service is not None:+ service.shutdown(timeout=5.0)app = FastAPI(diff --git a/agent/src/agent/loop.py b/agent/src/agent/loop.pyindex e0eb5dac6..1c2b9dc65 100644--- a/agent/src/agent/loop.py+++ b/agent/src/agent/loop.py@@ -297,6 +297,10 @@ class AgentLoop(ToolCallLifecycle):next cooperative checkpoint instead of only at the next iteration."""self._cancel_event.set()+ # Stop must reach commands the current or an earlier turn spawned, not+ # only the ReAct cycle. Normal successful turns do not call cancel().+ if self._background_manager is not None:+ self._background_manager.cancel_all()def _write_run_manifest(self, trace_dir: "Path", messages: List[Dict[str, Any]]) -> None:"""Record what methodology produced this run, beside its trace.diff --git a/agent/src/session/service.py b/agent/src/session/service.pyindex e87be3f30..32731aa27 100644--- a/agent/src/session/service.py+++ b/agent/src/session/service.py@@ -117,6 +117,8 @@ class SessionService:self._inflight: set[str] = set()self._inflight_lock = threading.Lock()self._search_index = get_shared_index()+ self._background_handles: Dict[str, Any] = {}+ self._shutting_down = Falseself._recover_interrupted_attempts()def _recover_interrupted_attempts(self) -> None:@@ -213,6 +215,44 @@ class SessionService:)self._inflight.add(session_id)+ def _publish_background_completion(self, owner_id: str, payload: dict) -> None:+ """Route a background task's terminal state to its owning session.++ The AgentLoop that started the task may already have exited by the time+ the command finishes, so this cannot depend on a later iteration draining+ the queue. The payload keeps the attempt that started the task, which is+ the attempt the completion belongs to, not whichever one is current.+ """+ if not owner_id.startswith("session:"):+ return+ session_id = owner_id[len("session:"):]+ self.event_bus.emit(session_id, "background.completed", payload)++ def shutdown(self, *, timeout: float = 5.0) -> list[str]:+ """Stop accepting work, cancel owned background work, and wait for reaping.++ Does not tear down the event bus or thread pools this service may share+ with other hosts. Returns task ids still running after the wait, so an+ incomplete cleanup is reported instead of being called a success.+ """+ with self._inflight_lock:+ self._shutting_down = True+ with self._inflight_lock:+ tasks = [task for task in self._active_tasks.values() if not task.done()]+ for task in tasks:+ task.cancel()+ deadline = time.monotonic() + max(0.0, timeout)+ while any(not task.done() for task in tasks) and time.monotonic() < deadline:+ time.sleep(0.01)+ pending = []+ for background in list(self._background_handles.values()):+ remaining = max(0.0, deadline - time.monotonic())+ # cancel_all, not shutdown: these handles sit over the process-wide+ # table, and closing it would stop unrelated hosts in this process.+ # New work is refused by the _shutting_down gate in send_message.+ pending.extend(background.cancel_all(timeout=remaining)["pending"])+ return pending+def _release_session(self, session_id: str) -> None:"""Release a session claim. Safe to call when no claim is held.@@ -284,6 +324,9 @@ class SessionService:self._search_index.remove_session(session_id)deleted = self.store.delete_session(session_id)self.event_bus.clear(session_id)+ background = self._background_handles.pop(session_id, None)+ if background is not None:+ background.cancel_all()return deletedasync def send_message(@@ -314,6 +357,8 @@ class SessionService:session = self.store.get_session(session_id)if not session:raise ValueError(f"Session {session_id} not found")+ if self._shutting_down:+ raise SessionBusyError("shutting down; no new runs are accepted")# Claim the session before persisting anything. Reserving after the# user message is appended (or relying on _active_loops, which is only@@ -364,6 +409,9 @@ class SessionService:Returns:Whether cancellation succeeded. True means an active loop existed and received a cancel signal."""+ background_handle = self._background_handles.get(session_id)+ if background_handle is not None:+ background_handle.cancel_all()loop = self._active_loops.get(session_id)if loop is not None:loop.cancel()@@ -593,7 +641,11 @@ class SessionService:background_scope = BackgroundScope(owner_id=f"session:{session_id}", attempt_id=attempt_id)- background = get_background_manager().for_scope(background_scope)+ background = get_background_manager().for_scope(+ background_scope,+ on_completion=self._publish_background_completion,+ )+ self._background_handles[session_id] = backgroundregistry = await loop.run_in_executor(_AGENT_EXECUTOR,diff --git a/agent/src/tools/background_tools.py b/agent/src/tools/background_tools.pyindex ad06c0852..37c50732e 100644--- a/agent/src/tools/background_tools.py+++ b/agent/src/tools/background_tools.py@@ -3,6 +3,7 @@from __future__ import annotationsimport json+import loggingimport osimport signalimport subprocess@@ -11,7 +12,7 @@ import timeimport uuidfrom dataclasses import dataclassfrom pathlib import Path-from typing import Any, Dict, List, Optional+from typing import Any, Callable, Dict, List, Optionalfrom src.agent.tools import BaseToolfrom src.tools._shell_safety import broad_python_kill_error@@ -90,11 +91,10 @@ def _taskkill_windows_process_tree(process: subprocess.Popen[str]) -> None:def _force_stop_process_tree(process: subprocess.Popen[str]) -> None:"""Force-stop one tracked process tree without draining its pipes."""- if process.poll() is not None:- returnif os.name == "nt":_taskkill_windows_process_tree(process)else:+ # Descendants can keep the group's pipes open after its shell exits._signal_posix_process_group(process, signal.SIGKILL)@@ -136,12 +136,20 @@ class BackgroundManager:session sees none of them."""- def __init__(self, scope: Optional[BackgroundScope] = None) -> None:+ def __init__(+ self,+ scope: Optional[BackgroundScope] = None,+ *,+ on_completion: Optional[Callable[[str, dict], None]] = None,+ ) -> None:self.tasks: Dict[str, dict] = {}# (owner_id, payload). Kept as pairs so the payload shape consumers and# existing tests see stays exactly what it was; routing is internal.self._notifications: List[tuple[str, dict]] = []self._lock = threading.Lock()+ self._condition = threading.Condition(self._lock)+ self._accepting = [True]+ self._on_completion = on_completionself._scope = scope or BackgroundScope(owner_id=f"isolated:{uuid.uuid4().hex}")@property@@ -149,12 +157,22 @@ class BackgroundManager:"""The owner this handle reads and writes under."""return self._scope- def for_scope(self, scope: BackgroundScope) -> "BackgroundManager":+ def for_scope(+ self,+ scope: BackgroundScope,+ *,+ on_completion: Optional[Callable[[str, dict], None]] = None,+ ) -> "BackgroundManager":"""Return a handle over the same task table bound to another owner."""other = BackgroundManager(scope=scope)other.tasks = self.tasksother._notifications = self._notificationsother._lock = self._lock+ other._condition = self._condition+ other._accepting = self._accepting+ other._on_completion = (+ self._on_completion if on_completion is None else on_completion+ )return otherdef _owns(self, task: dict) -> bool:@@ -179,6 +197,8 @@ class BackgroundManager:task_id = uuid.uuid4().hex[:8]started_at = time.monotonic()with self._lock:+ if not self._accepting[0]:+ return json.dumps({"status": "error", "error": "Background host is shutting down"})self.tasks[task_id] = {"status": "running","result": None,@@ -199,35 +219,40 @@ class BackgroundManager:output = ""status = "error"try:- process = _start_process(command)with self._lock:task = self.tasks.get(task_id)- if task is None:+ skip_spawn = task is None or bool(task.get("cancel_requested"))+ if not skip_spawn:+ process = _start_process(command)+ with self._lock:+ task = self.tasks[task_id]+ task["process"] = process+ cancel_requested = bool(task.get("cancel_requested"))+ if cancel_requested:_force_stop_process_tree(process)- return- task["process"] = process- cancel_requested = bool(task.get("cancel_requested"))- if cancel_requested:- _force_stop_process_tree(process)- try:- stdout, stderr = process.communicate(timeout=_COMMAND_TIMEOUT_SECONDS)- output = _combined_output(stdout, stderr)- exit_code = process.returncode- status = "completed" if exit_code == 0 else "error"- except subprocess.TimeoutExpired:- stdout, stderr = _terminate_process_tree(process)- partial_output = _combined_output(stdout, stderr)- timeout_message = f"Timeout ({_COMMAND_TIMEOUT_SECONDS:g}s)"- output = (- f"{partial_output}\n{timeout_message}"- if partial_output- else timeout_message- )- output = output[:_MAX_OUTPUT_CHARS]- status = "timeout"+ try:+ stdout, stderr = process.communicate(timeout=_COMMAND_TIMEOUT_SECONDS)+ output = _combined_output(stdout, stderr)+ exit_code = process.returncode+ status = "completed" if exit_code == 0 else "error"+ except subprocess.TimeoutExpired:+ stdout, stderr = _terminate_process_tree(process)+ partial_output = _combined_output(stdout, stderr)+ timeout_message = f"Timeout ({_COMMAND_TIMEOUT_SECONDS:g}s)"+ output = (+ f"{partial_output}\n{timeout_message}"+ if partial_output+ else timeout_message+ )[:_MAX_OUTPUT_CHARS]+ exit_code = process.returncode+ status = "timeout"except Exception as e:output, status = str(e), "error"+ if process is not None:+ # Do not publish a terminal cancellation before its root is reaped.+ _terminate_process_tree(process)+ exit_code = process.returncodewith self._lock:task = self.tasks.get(task_id)@@ -252,14 +277,51 @@ class BackgroundManager:max(0.0, time.monotonic() - started_at),3,)- self._notifications.append((- task["owner_id"],- {- "task_id": task_id, "status": status,- "command": command[:80], "result": (output or "")[:500],- "exit_code": exit_code,- },- ))+ payload = {+ "task_id": task_id, "status": status,+ "command": command[:80], "result": (output or "")[:500],+ "exit_code": exit_code,+ }+ self._notifications.append((task["owner_id"], payload))+ self._condition.notify_all()+ if self._on_completion is not None:+ try:+ self._on_completion(task["owner_id"], {**payload, "attempt_id": task.get("attempt_id")})+ except Exception:+ logging.getLogger(__name__).exception("Could not publish background task completion")++ def cancel_all(self, *, timeout: float = 0.0) -> dict:+ """Request owner-scoped cancellation and optionally wait for actual reaping."""+ with self._lock:+ task_ids = [tid for tid, task in self.tasks.items()+ if self._owns(task) and task["status"] in {"running", "cancelling"}]+ for task_id in task_ids:+ self.cancel(task_id)+ with self._condition:+ def pending():+ return [tid for tid in task_ids+ if self.tasks[tid]["status"] in {"running", "cancelling"}]++ self._condition.wait_for(lambda: not pending(), timeout=max(0.0, timeout))+ return {"requested": len(task_ids), "pending": pending()}++ def shutdown(self, *, timeout: float = 5.0) -> list[str]:+ """Close this host's shared table, not other managers in the process."""+ with self._lock:+ self._accepting[0] = False+ owners = {task["owner_id"] for task in self.tasks.values()+ if task["status"] in {"running", "cancelling"}}+ deadline = time.monotonic() + max(0.0, timeout)+ for owner in owners:+ remaining = max(0.0, deadline - time.monotonic())+ self.for_scope(BackgroundScope(owner)).cancel_all(timeout=remaining)+ with self._condition:+ def pending():+ return [tid for tid, task in self.tasks.items()+ if task["status"] in {"running", "cancelling"}]++ self._condition.wait_for(lambda: not pending(), timeout=max(0.0, deadline - time.monotonic()))+ return pending()def cancel(self, task_id: str) -> str:"""Cancel one task by its tracked process group.@@ -296,13 +358,6 @@ class BackgroundManager:})process = task.get("process")- if process is not None and process.poll() is not None:- return json.dumps({- "status": "error",- "error": f"Task {task_id} is already finishing",- "task_status": task_status,- })-task["cancel_requested"] = Truetask["status"] = "cancelling"diff --git a/agent/tests/test_background_host_lifecycle.py b/agent/tests/test_background_host_lifecycle.pynew file mode 100644index 000000000..4d514045f--- /dev/null+++ b/agent/tests/test_background_host_lifecycle.py@@ -0,0 +1,233 @@+"""Real, harmless process fixtures for owner cancellation and host shutdown."""++from __future__ import annotations++import json+import os+import shlex+import subprocess+import sys+import threading+import time+from pathlib import Path+from types import SimpleNamespace++import pytest++from src.tools import background_tools as bg+++def command(source: str) -> str:+ args = [sys.executable, "-c", source]+ return subprocess.list2cmdline(args) if os.name == "nt" else shlex.join(args)+++def wait_until(predicate, timeout=8.0):+ deadline = time.monotonic() + timeout+ while time.monotonic() < deadline:+ if predicate():+ return+ time.sleep(0.01)+ raise AssertionError("process fixture did not reach the expected state")+++def manager(tmp_path, monkeypatch):+ monkeypatch.setattr(bg, "WORKDIR", tmp_path)+ instance = bg.BackgroundManager(bg.BackgroundScope("host-a"))+ yield instance+ if hasattr(instance, "shutdown"):+ assert instance.shutdown(timeout=8.0) == []+ else:+ for task_id in list(instance.tasks):+ instance.cancel(task_id)+++def launch(manager, ready: Path) -> str:+ task_id = json.loads(+ manager.run(command(f"from pathlib import Path; import time; Path({str(ready)!r}).touch(); time.sleep(60)"))+ )["task_id"]+ wait_until(ready.exists)+ return task_id+++def test_cancel_all_reaps_only_its_owner(manager, tmp_path):+ sibling = manager.for_scope(bg.BackgroundScope("host-b"))+ mine = launch(manager, tmp_path / "a")+ theirs = launch(sibling, tmp_path / "b")+ outcome = manager.cancel_all(timeout=8.0)+ assert outcome == {"requested": 1, "pending": []}+ assert json.loads(manager.check(mine))["status"] == "cancelled"+ assert json.loads(sibling.check(theirs))["status"] == "running"+ assert manager.cancel_all(timeout=0) == {"requested": 0, "pending": []}+++def test_shutdown_closes_all_handles_and_reaps(manager, tmp_path):+ sibling = manager.for_scope(bg.BackgroundScope("host-b"))+ launch(manager, tmp_path / "a")+ launch(sibling, tmp_path / "b")+ assert manager.shutdown(timeout=8.0) == []+ assert json.loads(sibling.run(command("print('must not run')")))["status"] == "error"+ assert all(t["status"] == "cancelled" and t["process"] is None for t in manager.tasks.values())+++def test_cancellation_during_spawn_is_reaped(manager, monkeypatch):+ entered, release = threading.Event(), threading.Event()+ original = bg._start_process++ def delayed_start(cmd):+ entered.set()+ assert release.wait(5)+ return original(cmd)++ monkeypatch.setattr(bg, "_start_process", delayed_start)+ task_id = json.loads(manager.run(command("import time; time.sleep(60)")))["task_id"]+ try:+ assert entered.wait(5)+ assert manager.cancel_all(timeout=0)["pending"] == [task_id]+ assert json.loads(manager.check(task_id))["status"] == "cancelling"+ finally:+ release.set()+ assert manager.cancel_all(timeout=8)["pending"] == []+ assert json.loads(manager.check(task_id))["status"] == "cancelled"+++def test_cancel_before_spawn_never_starts_command(manager, monkeypatch):+ class DeferredThread:+ def __init__(self, **kwargs):+ pass++ def start(self):+ pass++ monkeypatch.setattr(bg.threading, "Thread", DeferredThread)+ cmd = command("print('must not run')")+ task_id = json.loads(manager.run(cmd))["task_id"]+ manager.cancel(task_id)+ monkeypatch.setattr(bg, "_start_process", lambda _: pytest.fail("cancelled before spawn"))+ manager._execute(task_id, cmd)+ assert json.loads(manager.check(task_id))["status"] == "cancelled"++[email protected](os.name == "nt", reason="POSIX process-group contract; Windows has native CI")+def test_cancel_reaps_descendant_even_after_shell_exits(manager, tmp_path):+ ready = tmp_path / "child"+ child = f"import time; from pathlib import Path; Path({str(ready)!r}).touch(); time.sleep(60)"+ source = f"import subprocess, sys; subprocess.Popen([sys.executable, '-c', {child!r}])"+ task_id = json.loads(manager.run(command(source)))["task_id"]+ wait_until(ready.exists)+ wait_until(lambda: manager.tasks[task_id]["process"].poll() is not None)+ assert manager.cancel_all(timeout=8)["pending"] == []+ assert json.loads(manager.check(task_id))["status"] == "cancelled"+++def test_session_cancel_reaps_background_tasks_before_its_turn_ends():+ from types import SimpleNamespace+ from src.session.service import SessionService++ calls = []+ service = SessionService.__new__(SessionService)+ manager = bg.BackgroundManager(bg.BackgroundScope("session:s1"))+ service._background_handles = {"s1": manager}+ service._active_tasks = {"s1": SimpleNamespace(done=lambda: False, cancel=lambda: calls.append("cancelled"))}+ service._active_loops = {}+ service._user_cancel_requests = set()+ service._inflight_lock = __import__("threading").Lock()+ service._inflight = set()+ service._shutting_down = False+ task_id = json.loads(manager.run(command("import time; time.sleep(60)")))["task_id"]+ assert service.cancel_current("s1") is True+ wait_until(lambda: json.loads(manager.check(task_id))["status"] == "cancelled")+ assert calls == ["cancelled"]+++def test_session_shutdown_stops_new_work_and_reaps_tasks():+ import threading+ from src.session.service import SessionService++ service = SessionService.__new__(SessionService)+ service._background_handles = {"s1": bg.BackgroundManager(bg.BackgroundScope("session:s1"))}+ service._active_tasks = {}+ service._inflight_lock = threading.Lock()+ service._inflight = set()+ service._shutting_down = False+ manager = service._background_handles["s1"]+ task_id = json.loads(manager.run(command("import time; time.sleep(60)")))["task_id"]+ assert service.shutdown(timeout=8.0) == []+ assert json.loads(manager.check(task_id))["status"] == "cancelled"+ # Refusal lives at the service gate: these handles share the process-wide+ # table, so closing that table would stop unrelated hosts in this process.+ service.store = SimpleNamespace(+ get_session=lambda session_id: SimpleNamespace(+ session_id=session_id,+ config={},+ last_attempt_id=None,+ )+ )+ service._search_index = SimpleNamespace(index_message=lambda *a, **k: None)+ service.event_bus = SimpleNamespace(emit=lambda *a, **k: None)+ with pytest.raises(RuntimeError, match="shutting down"):+ __import__("asyncio").run(service.send_message("s1", "hello"))+++def test_session_attempt_registers_completion_with_its_owner():+ from src.session.service import SessionService++ emitted = []+ service = SessionService.__new__(SessionService)+ service.event_bus = SimpleNamespace(+ emit=lambda session_id, event_type, data: emitted.append((session_id, event_type, data)),+ )+ service._publish_background_completion(+ "session:s1",+ {+ "task_id": "task-1",+ "status": "completed",+ "attempt_id": "a1",+ "command": "echo done",+ "exit_code": 0,+ "result": "done",+ },+ )+ # A swarm worker's owner is not a session; it must not leak onto a bus.+ service._publish_background_completion("swarm:run:agent:task", {"task_id": "t"})+ assert emitted == [+ (+ "s1",+ "background.completed",+ {+ "task_id": "task-1",+ "status": "completed",+ "attempt_id": "a1",+ "command": "echo done",+ "exit_code": 0,+ "result": "done",+ },+ )+ ]+++def test_agent_stop_reaps_tasks_it_started_or_carried():+ from src.agent.loop import AgentLoop++ registry = bg.BackgroundManager(bg.BackgroundScope("session:s1"))+ task_id = json.loads(registry.run(command("import time; time.sleep(60)")))["task_id"]+ loop = AgentLoop(+ registry=SimpleNamespace(get=lambda name: None, background=registry),+ llm=SimpleNamespace(),+ max_iterations=1,+ background_manager=registry,+ )+ loop.cancel()+ wait_until(lambda: json.loads(registry.check(task_id))["status"] == "cancelled")+++def test_completion_callback_keeps_originating_attempt(manager):+ seen = []+ manager._on_completion = lambda owner, payload: seen.append((owner, payload))+ handle = manager.for_scope(bg.BackgroundScope("host-a", "attempt-1"))+ task_id = json.loads(handle.run(command("print('done')")))["task_id"]+ wait_until(lambda: bool(seen))+ assert seen[0][0] == "host-a"+ assert seen[0][1]["task_id"] == task_id+ assert seen[0][1]["attempt_id"] == "attempt-1"diff --git a/agent/tests/test_swarm_cancel_cascade.py b/agent/tests/test_swarm_cancel_cascade.pyindex d095a3666..458283dd9 100644--- a/agent/tests/test_swarm_cancel_cascade.py+++ b/agent/tests/test_swarm_cancel_cascade.py@@ -200,6 +200,8 @@ def _bare_service():service._active_loops = {}service._active_tasks = {}service._user_cancel_requests = set()+ service._background_handles = {}+ service._shutting_down = Falsereturn service