The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""End-to-end regression for #24015 — capture routing via auxiliary.vision.
When ``computer_use(action='capture', mode='som'|'vision')`` returns a
screenshot, ``_capture_response`` previously always returned a
``_multimodal`` envelope. For non-vision main models, or when the user
explicitly configured ``auxiliary.vision`` in ``config.yaml``, that
envelope tripped HTTP 404 / 400 at the provider boundary even though a
perfectly good vision backend was sitting in config waiting to be used.
This file exercises the integrated ``_capture_response`` flow with
deterministic stubs for:
* ``_native_tool_result_images`` (the shared policy decision)
* ``_run_async`` (sync->async bridge)
* ``vision_analyze_tool`` (the aux LLM call)
* ``hermes_constants.get_hermes_dir`` (cache path)
…so the full code path is covered without a live cua-driver, a real
auxiliary client, or network access.
"""
from __future__ import annotations
import base64
import json
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Fixtures / helpers
# ---------------------------------------------------------------------------
# 8×8 PNG (transparent) — minimal provider-acceptable bytes that decode cleanly.
_PNG_B64 = (
"iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAADUlEQVR4nG"
"NgGAUgAAABCAABgukLHQAAAABJRU5ErkJggg=="
)
# 1×1 JPEG — used to verify mime detection works for either stream type.
_JPEG_B64 = (
"/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEB"
"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/"
)
@pytest.fixture
def tmp_cache_dir(tmp_path):
"""Override get_hermes_dir so cache writes land under tmp_path."""
cache_dir = tmp_path / "cache_vision"
cache_dir.mkdir()
def _fake_get(*_args, **_kw):
return cache_dir
with patch("hermes_constants.get_hermes_dir", _fake_get):
yield cache_dir
def _make_capture(
*,
png_b64: str = _PNG_B64,
mode: str = "som",
elements=None,
app: str = "Safari",
window_title: str = "GitHub – Issue #24015",
width: int = 1280,
height: int = 800,
):
from tools.computer_use.backend import CaptureResult, UIElement
elements = list(elements or [
UIElement(index=0, role="AXButton", label="Sign in",
bounds=(10, 20, 80, 30)),
UIElement(index=1, role="AXTextField", label="username",
bounds=(10, 60, 200, 24)),
])
raw = base64.b64decode(png_b64, validate=False)
return CaptureResult(
mode=mode,
width=width,
height=height,
png_b64=png_b64,
elements=elements,
app=app,
window_title=window_title,
png_bytes_len=len(raw),
)
def _stub_aux_analysis(text: str):
"""Return a fake vision_analyze_tool coroutine result (JSON envelope)."""
return json.dumps({"success": True, "analysis": text})
# ---------------------------------------------------------------------------
# _capture_response: routing OFF (current/native behaviour)
# ---------------------------------------------------------------------------
class TestCaptureResponseDefaultPath:
"""When routing helper says 'native', the existing multimodal envelope wins."""
def test_som_capture_returns_multimodal_envelope_when_native(self):
from tools.computer_use import tool as cu_tool
cap = _make_capture(png_b64=_PNG_B64, mode="som")
with patch.object(cu_tool, "_should_route_through_aux_vision",
return_value=False):
resp = cu_tool._capture_response(cap)
assert isinstance(resp, dict)
assert resp.get("_multimodal") is True
# Image part must use image/png MIME for a PNG payload.
image_part = next(
p for p in resp["content"] if p.get("type") == "image_url"
)
url = image_part["image_url"]["url"]
assert url.startswith("data:image/png;base64,")
assert "vision_analysis" not in resp
# ---------------------------------------------------------------------------
# _capture_response: routing ON (the #24015 fix)
# ---------------------------------------------------------------------------
class TestCaptureResponseRoutedToAuxVision:
"""When routing helper says 'aux', the PNG is pre-analysed and a text
response is returned with no image_url parts at all."""
def test_som_capture_returns_text_with_vision_analysis(
self, tmp_cache_dir,
):
from tools.computer_use import tool as cu_tool
cap = _make_capture(mode="som")
captured_calls = {}
def _fake_run_async(coro):
captured_calls["called"] = True
return _stub_aux_analysis(
"A Safari window showing a GitHub issue page with a 'Sign "
"in' button and a 'username' text field."
)
# vision_analyze_tool is async; force a sync MagicMock so we can
# assert positional args without dealing with awaitables.
fake_vat = MagicMock(return_value="<coro>")
with patch.object(cu_tool, "_should_route_through_aux_vision",
return_value=True), \
patch("model_tools._run_async", side_effect=_fake_run_async), \
patch("tools.vision_tools.vision_analyze_tool",
new_callable=lambda: fake_vat):
resp = cu_tool._capture_response(cap)
# Must be a JSON string, NOT a multimodal envelope. This is exactly
# the contract that prevents #24015's HTTP 404 from firing on the
# next agent turn.
assert isinstance(resp, str)
assert "data:image" not in resp and "image_url" not in resp
body = json.loads(resp)
assert body["mode"] == "som"
assert body["app"] == "Safari"
assert "Sign in" in body["vision_analysis"]
assert body["vision_analysis_routed_via"] == "auxiliary.vision"
# The original AX-only metadata (window title, element index, app)
# is preserved alongside the new vision analysis so the agent loses
# no context vs the multimodal path.
assert body["window_title"] == "GitHub – Issue #24015"
assert len(body["elements"]) == 2
assert captured_calls.get("called") is True
# vision_analyze_tool was invoked with a path under the patched cache
# and a non-empty prompt.
args, _kwargs = fake_vat.call_args
path_arg, prompt_arg = args[0], args[1]
assert str(tmp_cache_dir) in path_arg
# AX summary is included so the aux model can ground its description
# against the same set-of-mark index the agent will see.
assert "Sign in" in prompt_arg
def test_invalid_aux_response_degrades_to_text_payload(self, tmp_cache_dir):
from tools.computer_use import tool as cu_tool
cap = _make_capture(mode="som")
def _fake_run_async(_coro):
return 1234 # not a string at all
fake_vat = MagicMock(return_value="<coro>")
with patch.object(cu_tool, "_should_route_through_aux_vision",
return_value=True), \
patch("model_tools._run_async", side_effect=_fake_run_async), \
patch("tools.vision_tools.vision_analyze_tool",
new_callable=lambda: fake_vat):
resp = cu_tool._capture_response(cap)
assert isinstance(resp, str)
body = json.loads(resp)
assert body.get("vision_unavailable") is True
# ---------------------------------------------------------------------------
# _should_route_through_aux_vision: end-to-end with real config plumbing
# ---------------------------------------------------------------------------
class TestRoutingDecisionWiring:
"""Verify _should_route_through_aux_vision wires the right config + helper."""
def test_explicit_aux_vision_in_config_routes_to_aux(self):
from tools.computer_use import tool as cu_tool
cfg = {
"model": {"default": "tencent/hy3-preview", "provider": "openrouter"},
"auxiliary": {
"vision": {
"provider": "openrouter",
"model": "google/gemini-2.5-flash",
}
},
}
with patch("agent.auxiliary_client._read_main_provider",
return_value="openrouter"), \
patch("agent.auxiliary_client._read_main_model",
return_value="tencent/hy3-preview"), \
patch("hermes_cli.config.load_config_readonly", return_value=cfg):
assert cu_tool._should_route_through_aux_vision() is True
def test_helper_decision_exception_is_swallowed(self):
from tools.computer_use import tool as cu_tool
with patch("agent.auxiliary_client._read_main_provider",
return_value="openrouter"), \
patch("agent.auxiliary_client._read_main_model",
return_value="x"), \
patch("hermes_cli.config.load_config_readonly", return_value={}), \
patch("tools.vision_tools._native_tool_result_images",
side_effect=ValueError("policy bug")):
assert cu_tool._should_route_through_aux_vision() is False
def _route(provider, model, cfg, *, catalog, provider_takes_media, veto=False):
"""The capture route through the real shared gate; only the capability lookups underneath are patched."""
from tools.computer_use import tool as cu_tool
with patch("agent.auxiliary_client._read_main_provider", return_value=provider), \
patch("agent.auxiliary_client._read_main_model", return_value=model), \
patch("hermes_cli.config.load_config_readonly", return_value=cfg), \
patch("agent.image_routing._lookup_supports_vision", return_value=catalog), \
patch("tools.vision_tools._supports_media_in_tool_results", return_value=provider_takes_media), \
patch("tools.vision_tools._profile_rejects_tool_media", return_value=veto):
return cu_tool._should_route_through_aux_vision()
class TestRouteDecision:
"""True = pre-analyse via auxiliary.vision, False = the multimodal envelope (#115248)."""
def test_explicit_aux_backend_wins_over_a_vision_main_model(self):
"""#24015: a configured auxiliary.vision backend is the de-facto image route in auto mode."""
cfg = {"auxiliary": {"vision": {"provider": "openrouter", "model": "google/gemini-2.5-flash"}}}
assert _route("anthropic", "claude-opus-4-5", cfg, catalog=True, provider_takes_media=True) is True
def test_vision_main_model_without_aux_backend_stays_native(self):
assert _route("anthropic", "claude-opus-4-5", {}, catalog=True, provider_takes_media=True) is False
def test_catalog_text_only_model_on_a_media_provider_goes_to_aux(self):
"""deepseek-v3.2 on OpenRouter: the provider carries tool-result media, the model cannot see. Routing it
native made the agent backstop swap the screenshot for a 'switch to a vision model' error."""
assert _route("openrouter", "deepseek/deepseek-v3.2", {}, catalog=False, provider_takes_media=True) is True
def test_unknown_provider_and_model_fail_closed_to_aux(self):
assert _route("exotic-provider", "exotic-model", {}, catalog=None, provider_takes_media=False) is True
def test_image_input_mode_text_keeps_pixels_out(self):
cfg = {"agent": {"image_input_mode": "text"}}
assert _route("anthropic", "claude-opus-4-5", cfg, catalog=True, provider_takes_media=True) is True
def test_image_input_mode_native_keeps_a_catalog_unknown_model_native(self):
"""A proxy alias the catalog does not know: native mode is the user's explicit word for it."""
cfg = {"agent": {"image_input_mode": "native"}}
assert _route("anthropic", "my-proxy-claude", cfg, catalog=None, provider_takes_media=True) is False
def test_profile_veto_routes_a_vision_model_to_aux(self):
assert _route("xiaomi", "mimo-v2.5", {}, catalog=True, provider_takes_media=False, veto=True) is True