Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(agent): reap owned background processes on cancellation and shutdown

Yufeng He04efdafdcd00
diff --git a/agent/api_server.py b/agent/api_server.py
index 8fddc250f..63446c981 100644
--- a/agent/api_server.py
+++ b/agent/api_server.py
@@ -156,6 +156,14 @@ async def _lifespan(_: FastAPI) -> AsyncIterator[None]:
yield
finally:
await _stop_scheduled_research_on_shutdown()
+ # Reap background work this host owns before its runtime disappears.
+ # Read the existing service rather than _get_session_service(): a
+ # shutdown must not create the very object it is trying to release.
+ import src.api.state as _api_state
+
+ service = getattr(_api_state, "_session_service", None)
+ if service is not None:
+ service.shutdown(timeout=5.0)
app = FastAPI(
diff --git a/agent/src/agent/loop.py b/agent/src/agent/loop.py
index e0eb5dac6..1c2b9dc65 100644
--- a/agent/src/agent/loop.py
+++ b/agent/src/agent/loop.py
@@ -297,6 +297,10 @@ class AgentLoop(ToolCallLifecycle):
next cooperative checkpoint instead of only at the next iteration.
"""
self._cancel_event.set()
+ # Stop must reach commands the current or an earlier turn spawned, not
+ # only the ReAct cycle. Normal successful turns do not call cancel().
+ if self._background_manager is not None:
+ self._background_manager.cancel_all()
def _write_run_manifest(self, trace_dir: "Path", messages: List[Dict[str, Any]]) -> None:
"""Record what methodology produced this run, beside its trace.
diff --git a/agent/src/session/service.py b/agent/src/session/service.py
index e87be3f30..32731aa27 100644
--- a/agent/src/session/service.py
+++ b/agent/src/session/service.py
@@ -117,6 +117,8 @@ class SessionService:
self._inflight: set[str] = set()
self._inflight_lock = threading.Lock()
self._search_index = get_shared_index()
+ self._background_handles: Dict[str, Any] = {}
+ self._shutting_down = False
self._recover_interrupted_attempts()
def _recover_interrupted_attempts(self) -> None:
@@ -213,6 +215,44 @@ class SessionService:
)
self._inflight.add(session_id)
+ def _publish_background_completion(self, owner_id: str, payload: dict) -> None:
+ """Route a background task's terminal state to its owning session.
+
+ The AgentLoop that started the task may already have exited by the time
+ the command finishes, so this cannot depend on a later iteration draining
+ the queue. The payload keeps the attempt that started the task, which is
+ the attempt the completion belongs to, not whichever one is current.
+ """
+ if not owner_id.startswith("session:"):
+ return
+ session_id = owner_id[len("session:"):]
+ self.event_bus.emit(session_id, "background.completed", payload)
+
+ def shutdown(self, *, timeout: float = 5.0) -> list[str]:
+ """Stop accepting work, cancel owned background work, and wait for reaping.
+
+ Does not tear down the event bus or thread pools this service may share
+ with other hosts. Returns task ids still running after the wait, so an
+ incomplete cleanup is reported instead of being called a success.
+ """
+ with self._inflight_lock:
+ self._shutting_down = True
+ with self._inflight_lock:
+ tasks = [task for task in self._active_tasks.values() if not task.done()]
+ for task in tasks:
+ task.cancel()
+ deadline = time.monotonic() + max(0.0, timeout)
+ while any(not task.done() for task in tasks) and time.monotonic() < deadline:
+ time.sleep(0.01)
+ pending = []
+ for background in list(self._background_handles.values()):
+ remaining = max(0.0, deadline - time.monotonic())
+ # cancel_all, not shutdown: these handles sit over the process-wide
+ # table, and closing it would stop unrelated hosts in this process.
+ # New work is refused by the _shutting_down gate in send_message.
+ pending.extend(background.cancel_all(timeout=remaining)["pending"])
+ return pending
+
def _release_session(self, session_id: str) -> None:
"""Release a session claim. Safe to call when no claim is held.
@@ -284,6 +324,9 @@ class SessionService:
self._search_index.remove_session(session_id)
deleted = self.store.delete_session(session_id)
self.event_bus.clear(session_id)
+ background = self._background_handles.pop(session_id, None)
+ if background is not None:
+ background.cancel_all()
return deleted
async def send_message(
@@ -314,6 +357,8 @@ class SessionService:
session = self.store.get_session(session_id)
if not session:
raise ValueError(f"Session {session_id} not found")
+ if self._shutting_down:
+ raise SessionBusyError("shutting down; no new runs are accepted")
# Claim the session before persisting anything. Reserving after the
# user message is appended (or relying on _active_loops, which is only
@@ -364,6 +409,9 @@ class SessionService:
Returns:
Whether cancellation succeeded. True means an active loop existed and received a cancel signal.
"""
+ background_handle = self._background_handles.get(session_id)
+ if background_handle is not None:
+ background_handle.cancel_all()
loop = self._active_loops.get(session_id)
if loop is not None:
loop.cancel()
@@ -593,7 +641,11 @@ class SessionService:
background_scope = BackgroundScope(
owner_id=f"session:{session_id}", attempt_id=attempt_id
)
- background = get_background_manager().for_scope(background_scope)
+ background = get_background_manager().for_scope(
+ background_scope,
+ on_completion=self._publish_background_completion,
+ )
+ self._background_handles[session_id] = background
registry = await loop.run_in_executor(
_AGENT_EXECUTOR,
diff --git a/agent/src/tools/background_tools.py b/agent/src/tools/background_tools.py
index ad06c0852..37c50732e 100644
--- a/agent/src/tools/background_tools.py
+++ b/agent/src/tools/background_tools.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
+import logging
import os
import signal
import subprocess
@@ -11,7 +12,7 @@ import time
import uuid
from dataclasses import dataclass
from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional
from src.agent.tools import BaseTool
from src.tools._shell_safety import broad_python_kill_error
@@ -90,11 +91,10 @@ def _taskkill_windows_process_tree(process: subprocess.Popen[str]) -> None:
def _force_stop_process_tree(process: subprocess.Popen[str]) -> None:
"""Force-stop one tracked process tree without draining its pipes."""
- if process.poll() is not None:
- return
if os.name == "nt":
_taskkill_windows_process_tree(process)
else:
+ # Descendants can keep the group's pipes open after its shell exits.
_signal_posix_process_group(process, signal.SIGKILL)
@@ -136,12 +136,20 @@ class BackgroundManager:
session sees none of them.
"""
- def __init__(self, scope: Optional[BackgroundScope] = None) -> None:
+ def __init__(
+ self,
+ scope: Optional[BackgroundScope] = None,
+ *,
+ on_completion: Optional[Callable[[str, dict], None]] = None,
+ ) -> None:
self.tasks: Dict[str, dict] = {}
# (owner_id, payload). Kept as pairs so the payload shape consumers and
# existing tests see stays exactly what it was; routing is internal.
self._notifications: List[tuple[str, dict]] = []
self._lock = threading.Lock()
+ self._condition = threading.Condition(self._lock)
+ self._accepting = [True]
+ self._on_completion = on_completion
self._scope = scope or BackgroundScope(owner_id=f"isolated:{uuid.uuid4().hex}")
@property
@@ -149,12 +157,22 @@ class BackgroundManager:
"""The owner this handle reads and writes under."""
return self._scope
- def for_scope(self, scope: BackgroundScope) -> "BackgroundManager":
+ def for_scope(
+ self,
+ scope: BackgroundScope,
+ *,
+ on_completion: Optional[Callable[[str, dict], None]] = None,
+ ) -> "BackgroundManager":
"""Return a handle over the same task table bound to another owner."""
other = BackgroundManager(scope=scope)
other.tasks = self.tasks
other._notifications = self._notifications
other._lock = self._lock
+ other._condition = self._condition
+ other._accepting = self._accepting
+ other._on_completion = (
+ self._on_completion if on_completion is None else on_completion
+ )
return other
def _owns(self, task: dict) -> bool:
@@ -179,6 +197,8 @@ class BackgroundManager:
task_id = uuid.uuid4().hex[:8]
started_at = time.monotonic()
with self._lock:
+ if not self._accepting[0]:
+ return json.dumps({"status": "error", "error": "Background host is shutting down"})
self.tasks[task_id] = {
"status": "running",
"result": None,
@@ -199,35 +219,40 @@ class BackgroundManager:
output = ""
status = "error"
try:
- process = _start_process(command)
with self._lock:
task = self.tasks.get(task_id)
- if task is None:
+ skip_spawn = task is None or bool(task.get("cancel_requested"))
+ if not skip_spawn:
+ process = _start_process(command)
+ with self._lock:
+ task = self.tasks[task_id]
+ task["process"] = process
+ cancel_requested = bool(task.get("cancel_requested"))
+ if cancel_requested:
_force_stop_process_tree(process)
- return
- task["process"] = process
- cancel_requested = bool(task.get("cancel_requested"))
- if cancel_requested:
- _force_stop_process_tree(process)
- try:
- stdout, stderr = process.communicate(timeout=_COMMAND_TIMEOUT_SECONDS)
- output = _combined_output(stdout, stderr)
- exit_code = process.returncode
- status = "completed" if exit_code == 0 else "error"
- except subprocess.TimeoutExpired:
- stdout, stderr = _terminate_process_tree(process)
- partial_output = _combined_output(stdout, stderr)
- timeout_message = f"Timeout ({_COMMAND_TIMEOUT_SECONDS:g}s)"
- output = (
- f"{partial_output}\n{timeout_message}"
- if partial_output
- else timeout_message
- )
- output = output[:_MAX_OUTPUT_CHARS]
- status = "timeout"
+ try:
+ stdout, stderr = process.communicate(timeout=_COMMAND_TIMEOUT_SECONDS)
+ output = _combined_output(stdout, stderr)
+ exit_code = process.returncode
+ status = "completed" if exit_code == 0 else "error"
+ except subprocess.TimeoutExpired:
+ stdout, stderr = _terminate_process_tree(process)
+ partial_output = _combined_output(stdout, stderr)
+ timeout_message = f"Timeout ({_COMMAND_TIMEOUT_SECONDS:g}s)"
+ output = (
+ f"{partial_output}\n{timeout_message}"
+ if partial_output
+ else timeout_message
+ )[:_MAX_OUTPUT_CHARS]
+ exit_code = process.returncode
+ status = "timeout"
except Exception as e:
output, status = str(e), "error"
+ if process is not None:
+ # Do not publish a terminal cancellation before its root is reaped.
+ _terminate_process_tree(process)
+ exit_code = process.returncode
with self._lock:
task = self.tasks.get(task_id)
@@ -252,14 +277,51 @@ class BackgroundManager:
max(0.0, time.monotonic() - started_at),
3,
)
- self._notifications.append((
- task["owner_id"],
- {
- "task_id": task_id, "status": status,
- "command": command[:80], "result": (output or "")[:500],
- "exit_code": exit_code,
- },
- ))
+ payload = {
+ "task_id": task_id, "status": status,
+ "command": command[:80], "result": (output or "")[:500],
+ "exit_code": exit_code,
+ }
+ self._notifications.append((task["owner_id"], payload))
+ self._condition.notify_all()
+ if self._on_completion is not None:
+ try:
+ self._on_completion(task["owner_id"], {**payload, "attempt_id": task.get("attempt_id")})
+ except Exception:
+ logging.getLogger(__name__).exception("Could not publish background task completion")
+
+ def cancel_all(self, *, timeout: float = 0.0) -> dict:
+ """Request owner-scoped cancellation and optionally wait for actual reaping."""
+ with self._lock:
+ task_ids = [tid for tid, task in self.tasks.items()
+ if self._owns(task) and task["status"] in {"running", "cancelling"}]
+ for task_id in task_ids:
+ self.cancel(task_id)
+ with self._condition:
+ def pending():
+ return [tid for tid in task_ids
+ if self.tasks[tid]["status"] in {"running", "cancelling"}]
+
+ self._condition.wait_for(lambda: not pending(), timeout=max(0.0, timeout))
+ return {"requested": len(task_ids), "pending": pending()}
+
+ def shutdown(self, *, timeout: float = 5.0) -> list[str]:
+ """Close this host's shared table, not other managers in the process."""
+ with self._lock:
+ self._accepting[0] = False
+ owners = {task["owner_id"] for task in self.tasks.values()
+ if task["status"] in {"running", "cancelling"}}
+ deadline = time.monotonic() + max(0.0, timeout)
+ for owner in owners:
+ remaining = max(0.0, deadline - time.monotonic())
+ self.for_scope(BackgroundScope(owner)).cancel_all(timeout=remaining)
+ with self._condition:
+ def pending():
+ return [tid for tid, task in self.tasks.items()
+ if task["status"] in {"running", "cancelling"}]
+
+ self._condition.wait_for(lambda: not pending(), timeout=max(0.0, deadline - time.monotonic()))
+ return pending()
def cancel(self, task_id: str) -> str:
"""Cancel one task by its tracked process group.
@@ -296,13 +358,6 @@ class BackgroundManager:
})
process = task.get("process")
- if process is not None and process.poll() is not None:
- return json.dumps({
- "status": "error",
- "error": f"Task {task_id} is already finishing",
- "task_status": task_status,
- })
-
task["cancel_requested"] = True
task["status"] = "cancelling"
diff --git a/agent/tests/test_background_host_lifecycle.py b/agent/tests/test_background_host_lifecycle.py
new file mode 100644
index 000000000..4d514045f
--- /dev/null
+++ b/agent/tests/test_background_host_lifecycle.py
@@ -0,0 +1,233 @@
+"""Real, harmless process fixtures for owner cancellation and host shutdown."""
+
+from __future__ import annotations
+
+import json
+import os
+import shlex
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from src.tools import background_tools as bg
+
+
+def command(source: str) -> str:
+ args = [sys.executable, "-c", source]
+ return subprocess.list2cmdline(args) if os.name == "nt" else shlex.join(args)
+
+
+def wait_until(predicate, timeout=8.0):
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if predicate():
+ return
+ time.sleep(0.01)
+ raise AssertionError("process fixture did not reach the expected state")
+
+
+def manager(tmp_path, monkeypatch):
+ monkeypatch.setattr(bg, "WORKDIR", tmp_path)
+ instance = bg.BackgroundManager(bg.BackgroundScope("host-a"))
+ yield instance
+ if hasattr(instance, "shutdown"):
+ assert instance.shutdown(timeout=8.0) == []
+ else:
+ for task_id in list(instance.tasks):
+ instance.cancel(task_id)
+
+
+def launch(manager, ready: Path) -> str:
+ task_id = json.loads(
+ manager.run(command(f"from pathlib import Path; import time; Path({str(ready)!r}).touch(); time.sleep(60)"))
+ )["task_id"]
+ wait_until(ready.exists)
+ return task_id
+
+
+def test_cancel_all_reaps_only_its_owner(manager, tmp_path):
+ sibling = manager.for_scope(bg.BackgroundScope("host-b"))
+ mine = launch(manager, tmp_path / "a")
+ theirs = launch(sibling, tmp_path / "b")
+ outcome = manager.cancel_all(timeout=8.0)
+ assert outcome == {"requested": 1, "pending": []}
+ assert json.loads(manager.check(mine))["status"] == "cancelled"
+ assert json.loads(sibling.check(theirs))["status"] == "running"
+ assert manager.cancel_all(timeout=0) == {"requested": 0, "pending": []}
+
+
+def test_shutdown_closes_all_handles_and_reaps(manager, tmp_path):
+ sibling = manager.for_scope(bg.BackgroundScope("host-b"))
+ launch(manager, tmp_path / "a")
+ launch(sibling, tmp_path / "b")
+ assert manager.shutdown(timeout=8.0) == []
+ assert json.loads(sibling.run(command("print('must not run')")))["status"] == "error"
+ assert all(t["status"] == "cancelled" and t["process"] is None for t in manager.tasks.values())
+
+
+def test_cancellation_during_spawn_is_reaped(manager, monkeypatch):
+ entered, release = threading.Event(), threading.Event()
+ original = bg._start_process
+
+ def delayed_start(cmd):
+ entered.set()
+ assert release.wait(5)
+ return original(cmd)
+
+ monkeypatch.setattr(bg, "_start_process", delayed_start)
+ task_id = json.loads(manager.run(command("import time; time.sleep(60)")))["task_id"]
+ try:
+ assert entered.wait(5)
+ assert manager.cancel_all(timeout=0)["pending"] == [task_id]
+ assert json.loads(manager.check(task_id))["status"] == "cancelling"
+ finally:
+ release.set()
+ assert manager.cancel_all(timeout=8)["pending"] == []
+ assert json.loads(manager.check(task_id))["status"] == "cancelled"
+
+
+def test_cancel_before_spawn_never_starts_command(manager, monkeypatch):
+ class DeferredThread:
+ def __init__(self, **kwargs):
+ pass
+
+ def start(self):
+ pass
+
+ monkeypatch.setattr(bg.threading, "Thread", DeferredThread)
+ cmd = command("print('must not run')")
+ task_id = json.loads(manager.run(cmd))["task_id"]
+ manager.cancel(task_id)
+ monkeypatch.setattr(bg, "_start_process", lambda _: pytest.fail("cancelled before spawn"))
+ manager._execute(task_id, cmd)
+ assert json.loads(manager.check(task_id))["status"] == "cancelled"
+
+
[email protected](os.name == "nt", reason="POSIX process-group contract; Windows has native CI")
+def test_cancel_reaps_descendant_even_after_shell_exits(manager, tmp_path):
+ ready = tmp_path / "child"
+ child = f"import time; from pathlib import Path; Path({str(ready)!r}).touch(); time.sleep(60)"
+ source = f"import subprocess, sys; subprocess.Popen([sys.executable, '-c', {child!r}])"
+ task_id = json.loads(manager.run(command(source)))["task_id"]
+ wait_until(ready.exists)
+ wait_until(lambda: manager.tasks[task_id]["process"].poll() is not None)
+ assert manager.cancel_all(timeout=8)["pending"] == []
+ assert json.loads(manager.check(task_id))["status"] == "cancelled"
+
+
+def test_session_cancel_reaps_background_tasks_before_its_turn_ends():
+ from types import SimpleNamespace
+ from src.session.service import SessionService
+
+ calls = []
+ service = SessionService.__new__(SessionService)
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:s1"))
+ service._background_handles = {"s1": manager}
+ service._active_tasks = {"s1": SimpleNamespace(done=lambda: False, cancel=lambda: calls.append("cancelled"))}
+ service._active_loops = {}
+ service._user_cancel_requests = set()
+ service._inflight_lock = __import__("threading").Lock()
+ service._inflight = set()
+ service._shutting_down = False
+ task_id = json.loads(manager.run(command("import time; time.sleep(60)")))["task_id"]
+ assert service.cancel_current("s1") is True
+ wait_until(lambda: json.loads(manager.check(task_id))["status"] == "cancelled")
+ assert calls == ["cancelled"]
+
+
+def test_session_shutdown_stops_new_work_and_reaps_tasks():
+ import threading
+ from src.session.service import SessionService
+
+ service = SessionService.__new__(SessionService)
+ service._background_handles = {"s1": bg.BackgroundManager(bg.BackgroundScope("session:s1"))}
+ service._active_tasks = {}
+ service._inflight_lock = threading.Lock()
+ service._inflight = set()
+ service._shutting_down = False
+ manager = service._background_handles["s1"]
+ task_id = json.loads(manager.run(command("import time; time.sleep(60)")))["task_id"]
+ assert service.shutdown(timeout=8.0) == []
+ assert json.loads(manager.check(task_id))["status"] == "cancelled"
+ # Refusal lives at the service gate: these handles share the process-wide
+ # table, so closing that table would stop unrelated hosts in this process.
+ service.store = SimpleNamespace(
+ get_session=lambda session_id: SimpleNamespace(
+ session_id=session_id,
+ config={},
+ last_attempt_id=None,
+ )
+ )
+ service._search_index = SimpleNamespace(index_message=lambda *a, **k: None)
+ service.event_bus = SimpleNamespace(emit=lambda *a, **k: None)
+ with pytest.raises(RuntimeError, match="shutting down"):
+ __import__("asyncio").run(service.send_message("s1", "hello"))
+
+
+def test_session_attempt_registers_completion_with_its_owner():
+ from src.session.service import SessionService
+
+ emitted = []
+ service = SessionService.__new__(SessionService)
+ service.event_bus = SimpleNamespace(
+ emit=lambda session_id, event_type, data: emitted.append((session_id, event_type, data)),
+ )
+ service._publish_background_completion(
+ "session:s1",
+ {
+ "task_id": "task-1",
+ "status": "completed",
+ "attempt_id": "a1",
+ "command": "echo done",
+ "exit_code": 0,
+ "result": "done",
+ },
+ )
+ # A swarm worker's owner is not a session; it must not leak onto a bus.
+ service._publish_background_completion("swarm:run:agent:task", {"task_id": "t"})
+ assert emitted == [
+ (
+ "s1",
+ "background.completed",
+ {
+ "task_id": "task-1",
+ "status": "completed",
+ "attempt_id": "a1",
+ "command": "echo done",
+ "exit_code": 0,
+ "result": "done",
+ },
+ )
+ ]
+
+
+def test_agent_stop_reaps_tasks_it_started_or_carried():
+ from src.agent.loop import AgentLoop
+
+ registry = bg.BackgroundManager(bg.BackgroundScope("session:s1"))
+ task_id = json.loads(registry.run(command("import time; time.sleep(60)")))["task_id"]
+ loop = AgentLoop(
+ registry=SimpleNamespace(get=lambda name: None, background=registry),
+ llm=SimpleNamespace(),
+ max_iterations=1,
+ background_manager=registry,
+ )
+ loop.cancel()
+ wait_until(lambda: json.loads(registry.check(task_id))["status"] == "cancelled")
+
+
+def test_completion_callback_keeps_originating_attempt(manager):
+ seen = []
+ manager._on_completion = lambda owner, payload: seen.append((owner, payload))
+ handle = manager.for_scope(bg.BackgroundScope("host-a", "attempt-1"))
+ task_id = json.loads(handle.run(command("print('done')")))["task_id"]
+ wait_until(lambda: bool(seen))
+ assert seen[0][0] == "host-a"
+ assert seen[0][1]["task_id"] == task_id
+ assert seen[0][1]["attempt_id"] == "attempt-1"
diff --git a/agent/tests/test_swarm_cancel_cascade.py b/agent/tests/test_swarm_cancel_cascade.py
index d095a3666..458283dd9 100644
--- a/agent/tests/test_swarm_cancel_cascade.py
+++ b/agent/tests/test_swarm_cancel_cascade.py
@@ -200,6 +200,8 @@ def _bare_service():
service._active_loops = {}
service._active_tasks = {}
service._user_cancel_requests = set()
+ service._background_handles = {}
+ service._shutting_down = False
return service