Skip to content

The agent that grows with you

fix(browser): label a locally-run browser_exec call 'local' in shared metrics

Tekniumc44c4e9f9d95
diff --git a/tests/hermes_cli/test_shared_metrics_loop.py b/tests/hermes_cli/test_shared_metrics_loop.py
index 9f067292309..aea49b28311 100644
--- a/tests/hermes_cli/test_shared_metrics_loop.py
+++ b/tests/hermes_cli/test_shared_metrics_loop.py
@@ -308,18 +308,63 @@ def test_browser_calls_name_the_backend_that_served_them(home):
]
-def test_browser_backend_is_resolved_from_the_profile_at_call_time(home, monkeypatch):
- from tools.browser_session_cloud import browser_backend_name
-
- def handler():
- return loop.record_browser_call(lambda: '{"success": true}', browser_backend_name)
-
- handler()
+def browser_exec_route(monkeypatch):
+ """The real ``browser_exec`` registry handler under a configured Browserbase provider, cut at the session
+ layer (``get_session_info``) and the CLI process boundary; nothing launches or connects."""
+ import socket
+ import subprocess
+ from types import SimpleNamespace
+
+ import tools.browser_session as bs
+ import tools.browser_session_cloud as bs_cloud
+ import tools.browser_supervisor as supervisor
+ import tools.browser_use_cli as bu
+
+ route = SimpleNamespace(fallback=False)
+
+ def session_info(key):
+ if bs.is_local_sidecar_key(key) or route.fallback:
+ return {"cdp_url": f"ws://127.0.0.1:47000/devtools/browser/{key}", "features": {"local": True},
+ **({"fallback_from_cloud": True, "fallback_reason": "429"} if route.fallback else {})}
+ return {"cdp_url": "wss://cloud.example/x"}
+
+ def getaddrinfo(host, *a, **kw):
+ return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("192.168.0.1" if host == "192.168.0.1" else "93.184.216.34", 0))]
+
+ bs_cloud.reset_caches()
+ monkeypatch.setattr(bs_cloud, "get_cloud_provider", lambda: SimpleNamespace(name="browserbase"))
+ monkeypatch.setattr(bs_cloud, "auto_local_for_private_urls", lambda: True)
+ monkeypatch.setattr(bs, "get_session_info", session_info)
+ monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo)
+ monkeypatch.setattr(supervisor, "SUPERVISOR_REGISTRY", SimpleNamespace(get_or_start=lambda **kw: None))
+ monkeypatch.setattr(bu, "_find_cli", lambda: ["browser-use"])
+ monkeypatch.setattr(bu, "_run_cli_killing_process_group",
+ lambda cmd, code, env, timeout: subprocess.CompletedProcess(cmd, 0, "ok", ""))
+ monkeypatch.setattr(bu, "_exec_route_is_sidecar", {})
+ monkeypatch.setattr(bu, "_driven_daemons", {})
+ yield route
+ bs_cloud.reset_caches()
+
+
+def test_browser_exec_counts_against_the_browser_that_actually_served_it(home, monkeypatch, browser_exec_route):
+ """A call the configured provider served names it; a hybrid ``::local`` sidecar call (private URL) or a
+ call the session layer fell back to local Chromium for ran locally and must not; a `/browser connect`
+ published mid-session is read at call time."""
+ from tools.browser_use_cli import _handle_browser_exec
+
+ def call(code, task_id):
+ assert json.loads(_handle_browser_exec({"code": code}, task_id=task_id))["success"] is True
+
+ call('new_tab("https://example.com/")', "cloud")
+ call('new_tab("http://192.168.0.1:9120/login")', "sidecar")
+ browser_exec_route.fallback = True
+ call('new_tab("https://example.com/")', "fallback")
monkeypatch.setenv("BROWSER_CDP_URL", "ws://127.0.0.1:9/devtools/browser/x") # `/browser connect` mid-session
- handler()
+ call('new_tab("https://example.com/")', "connected")
- rows = sorted((d["backend"], d["outcome"]) for d, _ in _rows(home, "hermes.execution_backend.count"))
- assert rows == [("cdp", "success"), ("local", "success")]
+ rows = {d["backend"]: v for d, v in _rows(home, "hermes.execution_backend.count") if d["outcome"] == "success"}
+ assert rows == {"browserbase": 1, "local": 2, "cdp": 1}
def _raise():
diff --git a/tools/browser_session_cloud.py b/tools/browser_session_cloud.py
index ec883661da9..46dfc846f7a 100644
--- a/tools/browser_session_cloud.py
+++ b/tools/browser_session_cloud.py
@@ -262,13 +262,15 @@ def is_local_backend() -> bool:
return terminal_env("TERMINAL_ENV", "local").strip().lower() in ("local", "")
-def browser_backend_name() -> str:
+def browser_backend_name(ran_local: bool = False) -> str:
"""Shared-metrics label for the active profile's backend in session-creation precedence
- (CDP override > cloud provider > local engine). Config reads only, no network I/O."""
+ (CDP override > cloud provider > local engine). ``ran_local``: the call ran on a LOCAL browser (the
+ hybrid ``::local`` sidecar, a cloud failure's local fallback, the real profile), so a configured
+ provider did not serve it. Config reads only, no network I/O."""
from tools.browser_session_cdp import get_cdp_override_raw
if get_cdp_override_raw():
return "cdp"
- provider = get_cloud_provider()
+ provider = None if ran_local else get_cloud_provider()
if provider is not None:
return str(getattr(provider, "name", "") or "other")
return "lightpanda" if get_browser_engine() == "lightpanda" else "local"
diff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.py
index c33bfa07fe0..7ee12ea618f 100644
--- a/tools/browser_use_cli.py
+++ b/tools/browser_use_cli.py
@@ -7,6 +7,7 @@ The harness speaks CDP to whatever ``tools.browser_session`` resolved for the ca
"""
import contextlib
+import contextvars
import functools
import importlib
import importlib.util
@@ -711,6 +712,7 @@ def browser_exec(code: str, session: str = "", timeout_s: int = _DEFAULT_TIMEOUT
env["BU_NAME"] = _sidecar_daemon_name(session)
_exec_route_is_sidecar[route_key] = local_sidecar
bot_desktop_browser = bool(env.pop(_BOT_DESKTOP_BROWSER_SENTINEL, None))
+ _exec_ran_local.set(bot_desktop_browser) # only local browsers carry the sentinel
# SHARED browser (/browser connect CDP override): pin each named session to its own tab (see
# _OWN_TAB_PREAMBLE). Private per-name browsers skip this — nothing to collide with.
@@ -889,15 +891,24 @@ BROWSER_EXEC_SCHEMA = {
from tools.registry import registry
+# Whether this context's browser_exec call ran on a LOCAL browser: a configured cloud provider did not
+# serve a hybrid ::local sidecar, cloud-fallback or real-profile call, so its metrics row must not name it.
+_exec_ran_local: contextvars.ContextVar[bool] = contextvars.ContextVar("browser_exec_ran_local", default=False)
+
+
def _handle_browser_exec(args: dict, **kw):
"""Registry handler; each call counts against the backend that served it (shared metrics, opt-in)."""
from hermes_cli.observability.shared_metrics_loop import record_browser_call
from tools.browser_session_cloud import browser_backend_name
- return record_browser_call(lambda: browser_exec(
- code=args.get("code", ""), session=args.get("session", "") or "",
- timeout_s=args.get("timeout_s", _DEFAULT_TIMEOUT_S), task_id=kw.get("task_id"),
- local=bool(args.get("local", False)),
- ), browser_backend_name)
+ token = _exec_ran_local.set(False)
+ try:
+ return record_browser_call(lambda: browser_exec(
+ code=args.get("code", ""), session=args.get("session", "") or "",
+ timeout_s=args.get("timeout_s", _DEFAULT_TIMEOUT_S), task_id=kw.get("task_id"),
+ local=bool(args.get("local", False)),
+ ), lambda: browser_backend_name(ran_local=_exec_ran_local.get()))
+ finally:
+ _exec_ran_local.reset(token)
registry.register(