The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(browser): activate hidden tabs only when the harness actually drives a cloud browser
Teknium9194256d0120
diff --git a/tests/tools/test_browser_use_shot.py b/tests/tools/test_browser_use_shot.pyindex a220104e741..6d5883d88d2 100644--- a/tests/tools/test_browser_use_shot.py+++ b/tests/tools/test_browser_use_shot.py@@ -9,6 +9,7 @@ import jsonimport osimport sysimport textwrap+from types import SimpleNamespaceimport pytestfrom PIL import Image@@ -61,6 +62,7 @@ _FAKE_TRANSPORT = textwrap.dedent("""\import browser_harness.admin as admin, browser_harness.telemetry as telemetryadmin.ensure_daemon = admin.print_update_banner = lambda *a, **k: Noneadmin.daemon_alive = lambda *a, **k: True+ admin.daemon_browser_kind = lambda *a, **k: os.environ.get("FAKE_DAEMON_KIND") or Nonetelemetry.is_enabled = lambda: Falsetelemetry.capture_cli_event = lambda *a, **k: None""")@@ -85,8 +87,14 @@ def run_exec(monkeypatch, tmp_path):monkeypatch.setattr(bu_cli, "_find_cli", lambda: [sys.executable, "-m", "browser_harness.run"])log = tmp_path / "cdp.jsonl"- def run(activate_works: bool, cloud: bool = False, code: str = "print(capture_screenshot({shot!r}))"):+ def run(activate_works: bool, cloud: bool = False, code: str = "print(capture_screenshot({shot!r}))",+ autospawn_daemon_kind: str = ""):monkeypatch.setattr(bs_cloud, "get_cloud_provider", (lambda: object()) if cloud else (lambda: None))+ if autospawn_daemon_kind: # direct-API Browser Use config: the harness picks its own browser+ monkeypatch.setattr(bs_cloud, "get_cloud_provider", lambda: SimpleNamespace(name="browser-use"))+ monkeypatch.setattr(bu_cli, "_use_gateway", lambda cfg: False)+ monkeypatch.setattr(bu_cli, "_browser_use_api_key_set", lambda: True)+ monkeypatch.setenv("FAKE_DAEMON_KIND", autospawn_daemon_kind)monkeypatch.setattr(bs, "get_session_info",lambda key: {"cdp_url": f"ws://127.0.0.1:47000/devtools/browser/{key}","features": {} if cloud else {"local": True}})@@ -151,3 +159,15 @@ def test_cloud_new_tab_survives_a_provider_rejecting_activation(run_exec):assert result.get("success") is True, result.get("stderr") or resultassert "TAB T2" in result.get("output", "") + result.get("stdout", ""), resultassert [c["targetId"] for c in calls if c["method"] == "Target.activateTarget"] == ["T2"]++[email protected]("daemon_kind", ["local", "cloud"])+def test_autospawn_route_activates_only_when_the_harness_actually_reached_cloud(run_exec, daemon_kind):+ # BU_AUTOSPAWN attaches to a local Chrome that is already listening instead of provisioning a cloud+ # browser: the configured cloud intent says nothing about which browser the daemon is driving.+ code = "print(capture_screenshot({shot!r})); print('TAB', new_tab())"+ result, _, calls = run_exec(True, code=code, autospawn_daemon_kind=daemon_kind)++ assert result.get("success") is True, result.get("stderr") or result+ activations = [c["targetId"] for c in calls if c["method"] == "Target.activateTarget"]+ assert activations == (["T1", "T2"] if daemon_kind == "cloud" else []), callsdiff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.pyindex 7948e45fe44..5f09da97ca7 100644--- a/tools/browser_use_cli.py+++ b/tools/browser_use_cli.py@@ -46,8 +46,7 @@ _BOT_DESKTOP_BROWSER_SENTINEL = "_HERMES_BU_BOT_DESKTOP_BROWSER"_REAL_PROFILE_FALLBACK_SENTINEL = "_HERMES_BU_REAL_PROFILE_FALLBACK"# local=True whose real profile could not load: run on the local sidecar browser. Popped before launch._LOCAL_FALLBACK_SENTINEL = "_HERMES_BU_LOCAL_FALLBACK"-# The resolved browser is a cloud provider's (no human watching it): new tabs are brought to the front.-# Popped before launch.+# Where new/hidden tabs may be brought to the front: a browser_use_shot.ACTIVATE_* mode. Popped before launch._CLOUD_BROWSER_SENTINEL = "_HERMES_BU_CLOUD_BROWSER"# Prepended to the model's code for named sessions on SHARED browsers (a /browser connect CDP override): the@@ -434,7 +433,10 @@ def _resolve_backend_cdp(env: dict, task_id: Optional[str], session_name: str =provider_key = str(getattr(provider, "name", "") or "").strip().lower()if provider_key == _BROWSER_USE_PROVIDER and not _use_gateway(_read_browser_cfg()) and _browser_use_api_key_set():env[_PRIVATE_BROWSER_SENTINEL] = "1" # named BU cloud browsers are exclusive to their daemon- env[_CLOUD_BROWSER_SENTINEL] = "1"+ # Only intent: the harness attaches to a local Chrome already listening instead of autospawning, so+ # the live daemon's own report decides.+ from tools.browser_use_shot import ACTIVATE_IF_CLOUD+ env[_CLOUD_BROWSER_SENTINEL] = ACTIVATE_IF_CLOUDreturn Noneprovider_name = type(provider).__name__@@ -461,7 +463,8 @@ def _resolve_backend_cdp(env: dict, task_id: Optional[str], session_name: str =_mark_local_browser(env, session, key)return errif err is None:- env[_CLOUD_BROWSER_SENTINEL] = "1"+ from tools.browser_use_shot import ACTIVATE_ALWAYS+ env[_CLOUD_BROWSER_SENTINEL] = ACTIVATE_ALWAYS # the provider's own session: the harness connects only there# A provider browser keyed bu-named-<name> is exclusive to this session — the# own-tab preamble would just leak a blank tab into it.if err is None and session_name:@@ -727,8 +730,8 @@ def browser_exec(code: str, session: str = "", timeout_s: int = _DEFAULT_TIMEOUTcode = _OWN_TAB_PREAMBLE + code# Every backend: capture_screenshot() falls back to jpeg instead of hanging on a png a hidden tab never# renders; only on a cloud browser does it (and new_tab) bring the tab to the front (tools/browser_use_shot.py).- from tools.browser_use_shot import harness_command- cmd = harness_command(cmd, activate_new_tabs=bool(env.pop(_CLOUD_BROWSER_SENTINEL, None)))+ from tools.browser_use_shot import ACTIVATE_NEVER, harness_command+ cmd = harness_command(cmd, activate=env.pop(_CLOUD_BROWSER_SENTINEL, None) or ACTIVATE_NEVER)workspace = _workspace_dir(task_id)if workspace:diff --git a/tools/browser_use_shot.py b/tools/browser_use_shot.pyindex 976d75f2a25..d06693e155f 100644--- a/tools/browser_use_shot.py+++ b/tools/browser_use_shot.py@@ -58,15 +58,15 @@ def _write_shot(data: bytes, path: str, fmt: str) -> None:def make_capture_screenshot(cdp: Callable[..., Any], current_tab: Callable[[], dict],- activate_hidden: bool = False) -> Callable[..., str]:- """``activate_hidden`` only on a browser no human watches (cloud): on the user's own or connected- Chrome, activating the agent's tab would switch the tab they are looking at. Every backend keeps the- bounded png attempt and the jpeg fallback."""+ activate_hidden: Callable[[], bool] = lambda: False) -> Callable[..., str]:+ """``activate_hidden()`` is true only on a browser no human watches (cloud): on the user's own or+ connected Chrome, activating the agent's tab would switch the tab they are looking at. Every backend+ keeps the bounded png attempt and the jpeg fallback."""def capture_screenshot(path=None, full=False, max_dim=None):"""Save a PNG of the current viewport. Set max_dim=1800 on a 2x display to keep the file underthe 2000px-per-side limit some image-aware LLMs enforce."""path = os.fspath(path or _default_shot_path()) # a pathlib.Path works too- if activate_hidden:+ if activate_hidden():_bring_to_front_if_hidden(cdp, current_tab)fmt = "png"try:@@ -92,11 +92,12 @@ def make_capture_screenshot(cdp: Callable[..., Any], current_tab: Callable[[], dreturn capture_screenshot-def make_new_tab(new_tab: Callable[..., Any], cdp: Callable[..., Any]) -> Callable[..., Any]:+def make_new_tab(new_tab: Callable[..., Any], cdp: Callable[..., Any],+ activate: Callable[[], bool] = lambda: True) -> Callable[..., Any]:def new_tab_in_front(url="about:blank"):- """Open (or reuse a blank) tab, navigate to url, and bring it to the front."""+ """Open (or reuse a blank) tab, navigate to url, and bring it to the front on a cloud browser."""tid = new_tab(url)- if tid:+ if tid and activate():try:cdp("Target.activateTarget", targetId=tid)# The tab exists either way; a provider that rejects activation leaves it in the background.@@ -112,17 +113,38 @@ def _is_harness_helper(fn: Any) -> bool:return callable(fn) and getattr(fn, "__module__", "") == _HARNESS_MODULE-def install(helpers: Any, activate_new_tabs: bool = False) -> None:+# Which browser the tabs may be activated on, decided by browser_exec from the route it resolved.+ACTIVATE_NEVER = "never" # local, connected or real-profile Chrome: a human may be looking at it+ACTIVATE_ALWAYS = "always" # a cloud provider's session whose CDP endpoint Hermes itself resolved+ACTIVATE_IF_CLOUD = "if-cloud" # BU_AUTOSPAWN: the harness picks the browser (a listening local Chrome wins)+++def _harness_daemon_is_cloud() -> bool:+ """True only when the live harness daemon says it drives a Browser Use cloud browser (it was started+ with that browser's id). Asked per call, after the harness has chosen or replaced its daemon; an+ unreachable or older daemon counts as not cloud."""+ from browser_harness import admin+ return admin.daemon_browser_kind() == "cloud"+++def _activation_check(mode: str) -> Callable[[], bool]:+ if mode == ACTIVATE_IF_CLOUD:+ return _harness_daemon_is_cloud+ return (lambda: True) if mode == ACTIVATE_ALWAYS else (lambda: False)+++def install(helpers: Any, activate: str = ACTIVATE_NEVER) -> None:"""Rebind the helpers on the ``browser_harness.helpers`` module before ``browser_harness.run`` star-imports- them, so the agent's code, the harness's step tracer and recorder all see these. ``activate_new_tabs`` is- for browsers no human is watching (cloud providers): there a background tab only costs throttling and- blank screenshots, so new tabs and hidden tabs being captured are brought to the front; on a user's own- Chrome the harness deliberately leaves the visible tab alone."""+ them, so the agent's code, the harness's step tracer and recorder all see these. Activation is for+ browsers no human is watching (cloud): there a background tab only costs throttling and blank+ screenshots, so new tabs and hidden tabs being captured are brought to the front; on a user's own+ Chrome the harness deliberately leaves the visible tab alone. ``activate`` is an ``ACTIVATE_*`` mode."""+ check = _activation_check(activate)if _is_harness_helper(helpers.capture_screenshot):helpers.capture_screenshot = make_capture_screenshot(helpers.cdp, helpers.current_tab,- activate_hidden=activate_new_tabs)- if activate_new_tabs and _is_harness_helper(helpers.new_tab):- helpers.new_tab = make_new_tab(helpers.new_tab, helpers.cdp)+ activate_hidden=check)+ if activate != ACTIVATE_NEVER and _is_harness_helper(helpers.new_tab):+ helpers.new_tab = make_new_tab(helpers.new_tab, helpers.cdp, activate=check)_HARNESS_MODULE_ARGS = ["-m", "browser_harness.run"]@@ -131,15 +153,15 @@ import importlib.util, runpyspec = importlib.util.spec_from_file_location('hermes_browser_use_shot', {path!r})shot = importlib.util.module_from_spec(spec); spec.loader.exec_module(shot)import browser_harness.helpers-shot.install(browser_harness.helpers, activate_new_tabs={activate!r})+shot.install(browser_harness.helpers, activate={activate!r})runpy.run_module('browser_harness.run', run_name='__main__', alter_sys=True)"""-def harness_command(cmd: list[str], activate_new_tabs: bool) -> list[str]:+def harness_command(cmd: list[str], activate: str) -> list[str]:"""``python -m browser_harness.run`` as a launcher that installs these helpers first. The model's codestill arrives untouched on stdin (no preamble shifting its traceback line numbers). This file is loadedby path: the harness interpreter has no Hermes checkout on sys.path."""if cmd[1:] != _HARNESS_MODULE_ARGS:return cmd- return [cmd[0], "-c", _LAUNCHER.format(path=os.path.abspath(__file__), activate=bool(activate_new_tabs))]+ return [cmd[0], "-c", _LAUNCHER.format(path=os.path.abspath(__file__), activate=activate)]