Skip to content

The agent that grows with you

fix(browser): browser_exec says how screenshots reach the model instead of calling it blind

Tekniumbb216b95bd2e
diff --git a/tests/tools/test_browser_use_cli.py b/tests/tools/test_browser_use_cli.py
index 7b6395b45d2..9bd2fb574d2 100644
--- a/tests/tools/test_browser_use_cli.py
+++ b/tests/tools/test_browser_use_cli.py
@@ -730,25 +730,25 @@ class TestHeaderVariants:
monkeypatch.setattr(
"tools.vision_tools._should_use_native_vision_fast_path", lambda: False
)
+ monkeypatch.setattr("tools.vision_tools.check_vision_requirements", lambda: False)
monkeypatch.setattr(
"tools.browser_lightpanda.lightpanda_engine_status", lambda: (False, "")
)
assert bu_cli._description_header() == bu_cli._HEADER_BASE + bu_cli._HEADER_TEXT_ONLY
- def test_text_only_header_claims_nothing_about_model_vision(self, monkeypatch):
- """The text-only gate measures whether screenshots ride back in TOOL RESULTS, not whether
- the model has vision; a vision-capable model on a provider without tool-result media must
- not be told it "cannot view images" (#131568: it then skipped a user-attached image)."""
- monkeypatch.setattr(
- "tools.vision_tools._should_use_native_vision_fast_path", lambda: False
- )
- monkeypatch.setattr(
- "tools.browser_tool_lightpanda_fallback.lightpanda_engine_status", lambda: (False, "")
- )
- header = bu_cli._description_header().lower()
- assert "your model cannot" not in header
- assert "cannot view images" not in header
- assert "tool's results" in header and "text-first" in header
+ def test_non_native_header_never_calls_the_model_blind(self, monkeypatch):
+ """The gate measures whether screenshots ride back in THIS tool's results, not whether the
+ model sees (#131568: "your model cannot view images" made a model skip a user image). With a
+ vision route (an explicit auxiliary.vision pin) the header names it; without one it says so."""
+ monkeypatch.setattr("tools.vision_tools._should_use_native_vision_fast_path", lambda: False)
+ monkeypatch.setattr("tools.browser_lightpanda.lightpanda_engine_status", lambda: (False, ""))
+ for aux_route in (True, False):
+ monkeypatch.setattr("tools.vision_tools.check_vision_requirements", lambda: aux_route)
+ header = bu_cli._description_header().lower()
+ assert "cannot view images" not in header and "your model cannot" not in header
+ assert "described below" not in header # no screenshot section follows the header
+ assert "not attached to this tool's results" in header and "text-first" in header
+ assert ("vision_analyze" in header) is aux_route
class TestSkillTextDescription:
diff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.py
index 159de1b3d90..1c18831f1c9 100644
--- a/tools/browser_use_cli.py
+++ b/tools/browser_use_cli.py
@@ -808,25 +808,32 @@ _HEADER_VISION = (
"with your own vision — never send browser screenshots to a separate vision tool."
)
-# Chosen when screenshots cannot ride back inside THIS tool's results (provider/model
-# rejects tool-result media, or routing is not ``native``). That is a transport fact
-# about tool results, not a verdict on the model's vision: a vision-capable model still
-# sees images the user attaches to the conversation, so the header must never say
-# "your model cannot view images" — models quote that against themselves (#131568).
-_HEADER_TEXT_ONLY = (
- " Screenshots will not reach you through this tool's results in this setup, so work text-first: "
+_TEXT_FIRST = (
"page_info() for state, js() for reading/extracting DOM text, fill_input(selector, text) for inputs, and "
- "js(\"document.querySelector('…').click()\") for clicks — skip the screenshot-driven workflow described below. "
- "This says nothing about images the user attaches to the conversation."
+ "js(\"document.querySelector('…').click()\") for clicks."
+)
+
+# The non-native headers state a transport fact (screenshots do not ride back inside THIS
+# tool's results: routing is not ``native`` — e.g. an explicit ``auxiliary.vision`` pin — or
+# the route rejects tool-result media), never a verdict on the model's own vision: models
+# quote "your model cannot view images" against themselves and skip user images (#131568).
+_HEADER_VISION_TOOL = (
+ " Screenshots are not attached to this tool's results; to inspect one, pass its capture_screenshot() "
+ "path to vision_analyze. Otherwise work text-first: " + _TEXT_FIRST
+)
+
+_HEADER_TEXT_ONLY = (
+ " Screenshots are not attached to this tool's results and no vision route is configured, so work "
+ "text-first: " + _TEXT_FIRST
)
-# Appended when the local engine is Lightpanda: no graphical renderer, and one CDP
-# connection holds one page — a second Target.createTarget fails with
-# TargetAlreadyLoaded (drop the new_tab() sentence once lightpanda-io/browser#1962 lands).
+# Lightpanda: no graphical renderer, and one CDP connection holds one page — a second
+# Target.createTarget fails with TargetAlreadyLoaded (drop the new_tab() sentence once
+# lightpanda-io/browser#1962 lands).
_HEADER_LIGHTPANDA = (
" The local engine is Lightpanda (no graphical renderer, one page per session): capture_screenshot() "
- "is unavailable, so work text-first; navigate with new_tab(url) exactly once, then goto_url(url) for "
- "every later navigation — a second new_tab() fails with TargetAlreadyLoaded."
+ "is unavailable, so work text-first: " + _TEXT_FIRST + " Navigate with new_tab(url) exactly once, then "
+ "goto_url(url) for every later navigation — a second new_tab() fails with TargetAlreadyLoaded."
)
# Pinned quick-reference for the CLI's pre-imported helpers, replacing the live
@@ -847,12 +854,15 @@ _HELPERS_DIGEST = (
def _description_header() -> str:
- """Header tailored to whether the active model can see images natively"""
+ """Header tailored to how a screenshot reaches the model: attached natively, via
+ vision_analyze (the same gate that registers that tool), or not at all."""
if _lazy_call("tools.browser_lightpanda", "lightpanda_engine_status", (False, ""),
"lightpanda engine status unavailable")[0]: # no screenshots, whatever the model sees
- return _HEADER_BASE + _HEADER_TEXT_ONLY + _HEADER_LIGHTPANDA
- vision = _lazy_call("tools.vision_tools", "_should_use_native_vision_fast_path", False, "")
- return _HEADER_BASE + (_HEADER_VISION if vision else _HEADER_TEXT_ONLY)
+ return _HEADER_BASE + _HEADER_LIGHTPANDA
+ if _lazy_call("tools.vision_tools", "_should_use_native_vision_fast_path", False, ""):
+ return _HEADER_BASE + _HEADER_VISION
+ vision_tool = _lazy_call("tools.vision_tools", "check_vision_requirements", False, "vision route lookup failed")
+ return _HEADER_BASE + (_HEADER_VISION_TOOL if vision_tool else _HEADER_TEXT_ONLY)
def _dynamic_schema_overrides() -> dict: