AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): reap the browser process tree when a turn ends (#14542)
Bentlyef37b05c8d80
diff --git a/autogpt_platform/backend/Dockerfile b/autogpt_platform/backend/Dockerfileindex 810333d2a2..030ecce4ad 100644--- a/autogpt_platform/backend/Dockerfile+++ b/autogpt_platform/backend/Dockerfile@@ -141,10 +141,13 @@ COPY --from=builder /root/.cache/prisma-python/binaries /root/.cache/prisma-pyth# Note: system chromium tracks the Debian package schedule rather than a pinned# Chrome for Testing release. If agent-browser requires a specific Chrome version,# verify compatibility against the chromium package version in the base image.+# agent-browser is pinned: the daemon's process lifecycle (idle shutdown,+# Chromium reaping on close) has changed across releases, and an unpinned+# install means two builds of the same commit can behave differently.RUN apt-get update \&& apt-get install -y --no-install-recommends chromium fonts-liberation \&& rm -rf /var/lib/apt/lists/* \- && npm install -g agent-browser \+ && npm install -g [email protected] \&& rm -rf /tmp/* /root/.npmENV AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromiumdiff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.pyindex 9c62c96b6a..849e23b361 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py@@ -27,6 +27,7 @@ from backend.copilot.response_model import StreamError, StreamStatusfrom backend.copilot.sdk import service as sdk_servicefrom backend.copilot.sdk.dummy import stream_chat_completion_dummyfrom backend.copilot.stream_heartbeat import wrap_stream_with_heartbeat+from backend.copilot.tools.agent_browser import close_browser_daemonfrom backend.copilot.trial_cost_context import trial_cost_contextfrom backend.data.model import OAuth2Credentialsfrom backend.executor.cluster_lock import ClusterLock@@ -783,3 +784,8 @@ class CoPilotProcessor:log.error(f"Failed to mark session completed: {mark_err}")finally:await cost_context_stack.aclose()+ # A browser turn leaves a Chromium tree on this pod that+ # nothing else ever stops; state is already persisted, so+ # the next turn restores it wherever it lands.+ if await close_browser_daemon(entry.session_id):+ log.info("Browser daemon teardown attempted for this turn")diff --git a/autogpt_platform/backend/backend/copilot/tools/agent_browser.py b/autogpt_platform/backend/backend/copilot/tools/agent_browser.pyindex 8951843484..7e5cb2f696 100644--- a/autogpt_platform/backend/backend/copilot/tools/agent_browser.py+++ b/autogpt_platform/backend/backend/copilot/tools/agent_browser.py@@ -82,6 +82,7 @@ async def _run(session_name,*args,]+ _touched_sessions.add(session_name)proc = Nonetry:proc = await asyncio.create_subprocess_exec(@@ -92,14 +93,13 @@ async def _run(stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)return proc.returncode or 0, stdout.decode(), stderr.decode()except asyncio.TimeoutError:- # Kill the orphaned subprocess so it does not linger in the process table.- if proc is not None and proc.returncode is None:- proc.kill()- try:- await proc.communicate()- except Exception:- pass # Best-effort reap; ignore errors during cleanup.+ await _reap(proc)return 1, "", f"Command timed out after {timeout}s."+ except asyncio.CancelledError:+ # Teardown cancels saves that overrun the drain; the CLI process+ # they were waiting on must not outlive them.+ await _reap(proc)+ raiseexcept FileNotFoundError:return (1,@@ -108,6 +108,18 @@ async def _run()+async def _reap(proc: asyncio.subprocess.Process | None) -> None:+ """Kill a CLI process we stopped waiting on so it does not linger in the+ process table. Best-effort: errors during cleanup are ignored."""+ if proc is None or proc.returncode is not None:+ return+ proc.kill()+ try:+ await proc.communicate()+ except Exception:+ pass++async def _snapshot(session_name: str) -> str:"""Return the current page's interactive accessibility tree, truncated."""rc, stdout, stderr = await _run(session_name, "snapshot", "-i", "-c")@@ -129,6 +141,12 @@ async def _snapshot(session_name: str) -> str:# Avoids the subprocess probe on every tool call within the same pod._alive_sessions: set[str] = set()+# Every session this pod has sent ANY agent-browser command for. The CLI+# starts a daemon (and a Chromium tree) on first contact, including a mere+# `get url` probe, so this is the set that can have left processes behind.+# Cleared per session by close_browser_daemon at the end of a turn.+_touched_sessions: set[str] = set()+# Per-session locks to prevent concurrent _ensure_session calls from# triggering duplicate _restore_browser_state for the same session.# Protected by _session_locks_mutex to ensure setdefault/pop are not@@ -147,9 +165,23 @@ _RESTORE_CONCURRENCY = 10# thousands of cookies; restoring them all would be slow and is rarely useful._MAX_RESTORE_COOKIES = 100-# Background tasks for fire-and-forget state persistence.-# Prevents GC from collecting tasks before they complete.-_background_tasks: set[asyncio.Task] = set()+# Background state saves, per session, so teardown can wait for the ones+# still running. Holding the task also stops GC collecting it mid-flight.+_pending_saves: dict[str, set[asyncio.Task[None]]] = {}++# Sessions whose daemon is being closed. A save scheduled now would send+# `get url` to a daemon that is about to go, or, if it lands after the close,+# start a new one that nothing will ever stop.+_closing_sessions: set[str] = set()++# Turn-end teardown budget. Drain plus close must stay under the executor's+# _CANCEL_GRACE_SECONDS (5s), with room for the rest of the turn's cleanup:+# past the grace the worker re-cancels the turn task, the teardown never+# reaps, and the cluster lock is released while it is still running. A save+# slower than the drain is dropped; the next turn restores from the last one+# that landed.+_SAVE_DRAIN_TIMEOUT = 2+_TURN_END_CLOSE_TIMEOUT = 2def _fire_and_forget_save(@@ -160,9 +192,33 @@ def _fire_and_forget_save(State save is already best-effort (errors are swallowed), so running itin the background avoids adding latency to tool responses."""+ if session_name in _closing_sessions:+ logger.debug(+ "[browser] Skipping state save for session %s: closing", session_name+ )+ returntask = asyncio.create_task(_save_browser_state(session_name, user_id, session))- _background_tasks.add(task)- task.add_done_callback(_background_tasks.discard)+ saves = _pending_saves.setdefault(session_name, set())+ saves.add(task)+ task.add_done_callback(saves.discard)+++async def _drain_saves(session_name: str) -> None:+ """Wait for the session's in-flight saves so the last tool call's state+ is on disk before the daemon that holds it is closed."""+ saves = _pending_saves.pop(session_name, set())+ if not saves:+ return+ _, still_running = await asyncio.wait(saves, timeout=_SAVE_DRAIN_TIMEOUT)+ for task in still_running:+ task.cancel()+ if still_running:+ logger.warning(+ "[browser] %d state save(s) for session %s did not finish in %ss",+ len(still_running),+ session_name,+ _SAVE_DRAIN_TIMEOUT,+ )async def _has_local_session(session_name: str) -> bool:@@ -349,6 +405,59 @@ async def _ensure_session(_alive_sessions.add(session_name)+async def close_browser_daemon(session_name: str) -> bool:+ """Stop this pod's agent-browser daemon for *session_name* at turn end.++ A daemon is one Chromium process tree, roughly 15 processes and most of+ a gigabyte, and nothing else ever stops it: ``close_browser_session``+ runs only on session deletion, from the API server, never on the+ executor pod that owns the processes. Left alone, every browser turn a+ pod serves adds a tree that outlives the turn, the session, and the+ user, until the pod is OOM killed.++ The persisted state file is deliberately kept. Cookies and storage are+ saved to the workspace after every tool call and ``_ensure_session``+ restores them on demand, so the next browser turn resumes where this one+ left off, on whichever pod it lands.++ Returns whether a close was attempted. Best-effort: never raises.+ """+ if session_name not in _touched_sessions:+ return False+ try:+ # Inside the try so a cancellation on the mutex still reaches the+ # finally; a session left in _closing_sessions never saves again.+ _closing_sessions.add(session_name)+ _alive_sessions.discard(session_name)+ async with _session_locks_mutex:+ _session_locks.pop(session_name, None)+ # The last tool call's save may still be running. It has to finish+ # first: it reads the daemon this is about to close, and if it ran+ # afterwards its `get url` would start a fresh one.+ await _drain_saves(session_name)+ rc, _, stderr = await _run(+ session_name, "close", timeout=_TURN_END_CLOSE_TIMEOUT+ )+ if rc != 0:+ logger.warning(+ "[browser] close at turn end failed for session %s: %s",+ session_name,+ stderr[:200],+ )+ except Exception:+ logger.warning(+ "[browser] Exception closing daemon for session %s",+ session_name,+ exc_info=True,+ )+ finally:+ # `_run` re-adds the session while sending `close`; nothing is+ # running for it now.+ _touched_sessions.discard(session_name)+ _closing_sessions.discard(session_name)+ return True++async def close_browser_session(session_name: str, user_id: str | None = None) -> None:"""Shut down the local agent-browser daemon and clean up stored state.@@ -389,6 +498,10 @@ async def close_browser_session(session_name: str, user_id: str | None = None) -session_name,exc_info=True,)+ finally:+ # After the close, not before: `_run` records the session again+ # while sending it.+ _touched_sessions.discard(session_name)# ---------------------------------------------------------------------------diff --git a/autogpt_platform/backend/backend/copilot/tools/agent_browser_test.py b/autogpt_platform/backend/backend/copilot/tools/agent_browser_test.pyindex cd453af900..7bbda92a91 100644--- a/autogpt_platform/backend/backend/copilot/tools/agent_browser_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/agent_browser_test.py@@ -9,6 +9,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest+from backend.copilot.executor.processor import _CANCEL_GRACE_SECONDSfrom backend.copilot.model import ChatSessionfrom .agent_browser import (@@ -16,9 +17,11 @@ from .agent_browser import (BrowserNavigateTool,BrowserScreenshotTool,_ensure_session,+ _fire_and_forget_save,_has_local_session,_restore_browser_state,_save_browser_state,+ close_browser_daemon,close_browser_session,)from .models import (@@ -43,6 +46,9 @@ def _reset_and_mock_state(monkeypatch):_mod._alive_sessions.clear()_mod._session_locks.clear()+ _mod._touched_sessions.clear()+ _mod._pending_saves.clear()+ _mod._closing_sessions.clear()monkeypatch.setattr("backend.copilot.tools.agent_browser._ensure_session", AsyncMock())@@ -52,6 +58,9 @@ def _reset_and_mock_state(monkeypatch):yield_mod._alive_sessions.clear()_mod._session_locks.clear()+ _mod._touched_sessions.clear()+ _mod._pending_saves.clear()+ _mod._closing_sessions.clear()# ---------------------------------------------------------------------------@@ -1280,6 +1289,212 @@ class TestCloseBrowserSession:await close_browser_session("bad-sess")+# ---------------------------------------------------------------------------+# close_browser_daemon (turn-end teardown on the executor pod)+# ---------------------------------------------------------------------------+++class TestCloseBrowserDaemon:+ @pytest.mark.asyncio+ async def test_untouched_session_is_left_alone(self):+ """No daemon to close: skip the subprocess on every non-browser turn."""+ from . import agent_browser as _mod++ _mod._touched_sessions.discard("never-sess")+ with patch(+ "backend.copilot.tools.agent_browser._run", new_callable=AsyncMock+ ) as mock_run:+ assert await close_browser_daemon("never-sess") is False++ mock_run.assert_not_called()++ @pytest.mark.asyncio+ async def test_a_cancelled_command_kills_its_cli_process(self):+ """Teardown cancels saves that overrun the drain; the CLI process a+ save was waiting on must go with it, as it does on a timeout."""+ from . import agent_browser as _mod++ started = asyncio.Event()+ killed = asyncio.Event()++ async def communicate():+ started.set()+ await killed.wait()+ return b"", b""++ proc = MagicMock(returncode=None, communicate=communicate)+ proc.kill = MagicMock(side_effect=killed.set)+ with patch(+ "backend.copilot.tools.agent_browser.asyncio.create_subprocess_exec",+ new_callable=AsyncMock,+ return_value=proc,+ ):+ task = asyncio.create_task(_mod._run("cancel-sess", "get", "url"))+ await started.wait()+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task++ proc.kill.assert_called_once()++ @pytest.mark.asyncio+ async def test_any_command_marks_the_session_touched(self):+ """A bare probe starts the daemon too, so `_run` itself must record it."""+ from . import agent_browser as _mod++ _mod._touched_sessions.discard("probe-sess")+ with patch(+ "backend.copilot.tools.agent_browser.asyncio.create_subprocess_exec",+ side_effect=FileNotFoundError,+ ):+ await _mod._run("probe-sess", "get", "url", timeout=5)++ assert "probe-sess" in _mod._touched_sessions++ @pytest.mark.asyncio+ async def test_touched_session_is_closed_and_forgotten(self):+ from . import agent_browser as _mod++ _mod._touched_sessions.add("turn-sess")+ _mod._alive_sessions.add("turn-sess")+ _mod._session_locks["turn-sess"] = asyncio.Lock()+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ) as mock_run:+ assert await close_browser_daemon("turn-sess") is True++ mock_run.assert_called_once_with(+ "turn-sess", "close", timeout=_mod._TURN_END_CLOSE_TIMEOUT+ )+ assert "turn-sess" not in _mod._touched_sessions+ assert "turn-sess" not in _mod._alive_sessions+ assert "turn-sess" not in _mod._session_locks++ @pytest.mark.asyncio+ async def test_persisted_state_is_kept(self):+ """Unlike session deletion, turn end must leave the workspace state+ file: the next browser turn restores cookies from it."""+ from . import agent_browser as _mod++ _mod._touched_sessions.add("keep-sess")+ with patch(_GET_MANAGER, new_callable=AsyncMock) as mock_get_mgr:+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ):+ await close_browser_daemon("keep-sess")++ mock_get_mgr.assert_not_called()++ @pytest.mark.asyncio+ async def test_failure_is_swallowed_and_session_still_forgotten(self):+ from . import agent_browser as _mod++ _mod._touched_sessions.add("bad-sess")+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ side_effect=RuntimeError("gone"),+ ):+ assert await close_browser_daemon("bad-sess") is True++ assert "bad-sess" not in _mod._touched_sessions++ @pytest.mark.asyncio+ async def test_close_waits_for_a_pending_state_save(self):+ """The save reads the daemon; run after the close, its `get url`+ would start a new one that nothing stops."""+ from . import agent_browser as _mod++ _mod._touched_sessions.add("late-sess")+ release = asyncio.Event()+ order: list[str] = []++ async def slow_save(*_):+ await release.wait()+ order.append("saved")++ async def run(_session, *args, **_kwargs):+ order.append(args[0])+ return _run_result(rc=0)++ with (+ patch(+ "backend.copilot.tools.agent_browser._save_browser_state",+ slow_save,+ ),+ patch("backend.copilot.tools.agent_browser._run", run),+ ):+ _fire_and_forget_save("late-sess", "user1", make_session("late-sess"))+ closing = asyncio.create_task(close_browser_daemon("late-sess"))+ await asyncio.sleep(0.01)+ assert order == [], "close must not run while a save is pending"+ release.set()+ assert await closing is True++ assert order == ["saved", "close"]+ assert "late-sess" not in _mod._pending_saves++ @pytest.mark.asyncio+ async def test_no_save_is_scheduled_once_the_session_is_closing(self):+ from . import agent_browser as _mod++ _mod._closing_sessions.add("closing-sess")+ with patch(+ "backend.copilot.tools.agent_browser._save_browser_state",+ new_callable=AsyncMock,+ ) as save:+ _fire_and_forget_save("closing-sess", "user1", make_session())+ await asyncio.sleep(0)++ save.assert_not_awaited()+ assert not _mod._pending_saves.get("closing-sess")++ def test_teardown_fits_inside_the_cancel_grace(self):+ """Past the grace the worker re-cancels the turn task: the teardown+ is interrupted before it reaps, and the cluster lock is released while+ it is still running. The other turn cleanup shares the same window."""+ from . import agent_browser as _mod++ budget = _mod._SAVE_DRAIN_TIMEOUT + _mod._TURN_END_CLOSE_TIMEOUT+ assert budget < _CANCEL_GRACE_SECONDS++ @pytest.mark.asyncio+ async def test_a_cancel_during_setup_still_clears_the_closing_flag(self):+ """Left set, the session would never persist state again on this pod."""+ from . import agent_browser as _mod++ _mod._touched_sessions.add("cancel-sess")+ mutex = MagicMock()+ mutex.__aenter__ = AsyncMock(side_effect=asyncio.CancelledError)+ mutex.__aexit__ = AsyncMock(return_value=False)++ with patch.object(_mod, "_session_locks_mutex", mutex):+ with pytest.raises(asyncio.CancelledError):+ await close_browser_daemon("cancel-sess")++ assert "cancel-sess" not in _mod._closing_sessions+ assert "cancel-sess" not in _mod._touched_sessions++ @pytest.mark.asyncio+ async def test_session_deletion_forgets_the_session(self):+ """The deletion path's own `close` records the session through `_run`;+ without a discard afterwards every deleted session stays forever."""+ from . import agent_browser as _mod++ async def run(session_name, *_args, **_kwargs):+ _mod._touched_sessions.add(session_name)+ return _run_result(rc=0)++ with patch("backend.copilot.tools.agent_browser._run", run):+ await close_browser_session("deleted-sess")++ assert "deleted-sess" not in _mod._touched_sessions++# ---------------------------------------------------------------------------# Integration: save/restore called from _execute methods# ---------------------------------------------------------------------------