Enterprise agent platform for building, deploying, and operating AI teammates.
"""Tests for FetchWebContent tool."""
import ssl
from unittest.mock import AsyncMock, Mock, patch
import httpx
import pytest
from xagent.core.tools.adapters.vibe.fetch_web_content import (
FetchWebContentArgs,
FetchWebContentResult,
FetchWebContentTool,
)
from xagent.core.tools.core import web_content
from xagent.core.tools.core.web_content import (
WebContentFetcher,
build_isolated_httpx_client_kwargs,
get_trusted_proxy_url,
)
from xagent.core.utils.security import PrivateNetworkHostError
@pytest.fixture
def fetch_tool():
return FetchWebContentTool()
@pytest.fixture(autouse=True)
def allow_public_test_hosts():
with patch(
"xagent.core.utils.security.validate_public_http_url",
new=AsyncMock(return_value=["93.184.216.34"]),
) as validate:
yield validate
class _MockStreamResponse:
def __init__(
self,
*,
body: bytes = b"",
chunks: list[bytes] | None = None,
headers: dict[str, str] | None = None,
status_code: int = 200,
url: str = "https://example.com/page",
reason_phrase: str = "OK",
encoding: str | None = "utf-8",
raise_status: bool = False,
) -> None:
self._chunks = chunks if chunks is not None else [body]
self.headers = headers or {}
self.status_code = status_code
self.url = url
self.reason_phrase = reason_phrase
self.encoding = encoding
self._raise_status = raise_status
def raise_for_status(self) -> None:
if self._raise_status:
raise httpx.HTTPStatusError(
f"{self.status_code} {self.reason_phrase}",
request=Mock(),
response=self,
)
async def aiter_bytes(self):
for chunk in self._chunks:
yield chunk
class _MockStreamContext:
def __init__(self, response: _MockStreamResponse) -> None:
self._response = response
async def __aenter__(self) -> _MockStreamResponse:
return self._response
async def __aexit__(self, exc_type, exc, tb) -> bool:
return False
class TestFetchWebContentTool:
def test_tool_properties(self, fetch_tool):
assert fetch_tool.name == "fetch_web_content"
assert "web" in fetch_tool.tags
assert fetch_tool.args_type() == FetchWebContentArgs
assert fetch_tool.return_type() == FetchWebContentResult
def test_sync_not_implemented(self, fetch_tool):
with pytest.raises(NotImplementedError):
fetch_tool.run_json_sync({"url": "https://example.com"})
@pytest.mark.asyncio
async def test_fetch_webpage_content(self, fetch_tool):
html = """
<html>
<head><title>Example Title</title></head>
<body>
<script>console.log("remove me")</script>
<h1>Readable Heading</h1>
<p>Useful body text.</p>
<a href="/about">About</a>
</body>
</html>
"""
response = _MockStreamResponse(
body=html.encode("utf-8"),
headers={"content-type": "text/html; charset=utf-8"},
url="https://example.com/page",
)
with patch(
"httpx.AsyncClient.stream", return_value=_MockStreamContext(response)
):
result = await fetch_tool.run_json_async(
{"url": "https://example.com/page"}
)
assert result["success"] is True
assert result["url"] == "https://example.com/page"
assert result["title"] == "Example Title"
assert "Readable Heading" in result["content"]
assert "Useful body text." in result["content"]
assert "console.log" not in result["content"]
assert "https://example.com/about" in result["content"]
assert result["status_code"] == 200
assert result["content_type"] == "text/html; charset=utf-8"
assert result["error"] is None
@pytest.mark.asyncio
async def test_fetch_discovers_exact_html_assets_when_requested(self, fetch_tool):
html = """
<html>
<head>
<title>Brand</title>
<link rel="icon" href="/favicon.png">
<script defer src="/static/js/main.abc123.js"></script>
</head>
<body>
<img src="/assets/brand-logo.svg" alt="Brand logo">
<img src="/assets/product.png" alt="Product">
</body>
</html>
"""
response = _MockStreamResponse(
body=html.encode("utf-8"),
headers={"content-type": "text/html"},
url="https://example.com/campaign",
)
with (
patch(
"httpx.AsyncClient.stream", return_value=_MockStreamContext(response)
),
patch("httpx.AsyncClient.get") as mock_get,
):
result = await fetch_tool.run_json_async(
{
"url": "https://example.com/campaign",
"include_assets": True,
"asset_query": "logo",
}
)
assert result["success"] is True
assert result["assets"] == [
{
"url": "https://example.com/assets/brand-logo.svg",
"kind": "image",
"name": "",
"alt": "Brand logo",
"source": "html",
}
]
mock_get.assert_not_awaited()
@pytest.mark.asyncio
async def test_fetch_discovers_asset_name_joins_multi_valued_class(
self, fetch_tool
):
html = """
<html>
<body>
<img src="/assets/brand-logo.svg" class="logo primary" alt="Brand logo">
</body>
</html>
"""
response = _MockStreamResponse(
body=html.encode("utf-8"),
headers={"content-type": "text/html"},
url="https://example.com/campaign",
)
with (
patch(
"httpx.AsyncClient.stream", return_value=_MockStreamContext(response)
),
patch("httpx.AsyncClient.get"),
):
result = await fetch_tool.run_json_async(
{
"url": "https://example.com/campaign",
"include_assets": True,
}
)
assert result["success"] is True
assert result["assets"] == [
{
"url": "https://example.com/assets/brand-logo.svg",
"kind": "image",
"name": "logo primary",
"alt": "Brand logo",
"source": "html",
}
]
@pytest.mark.asyncio
async def test_fetch_follows_redirects(
self, fetch_tool, allow_public_test_hosts: AsyncMock
):
html = "<html><body><p>Redirect target</p></body></html>"
redirect = _MockStreamResponse(
headers={"location": "https://example.com/final"},
status_code=302,
url="https://example.com/start",
)
response = _MockStreamResponse(
body=html.encode("utf-8"),
headers={"content-type": "text/html"},
url="https://example.com/final",
)
with patch(
"httpx.AsyncClient.stream",
side_effect=[
_MockStreamContext(redirect),
_MockStreamContext(response),
],
) as mock_stream:
result = await fetch_tool.run_json_async(
{"url": "https://example.com/start"}
)
assert result["success"] is True
assert result["url"] == "https://example.com/final"
assert "Redirect target" in result["content"]
assert allow_public_test_hosts.await_args_list == [
(("https://example.com/start",),),
(("https://example.com/final",),),
]
assert all(
call.kwargs["follow_redirects"] is False
for call in mock_stream.call_args_list
)
@pytest.mark.asyncio
async def test_fetch_rejects_private_network_before_request(
self, fetch_tool, allow_public_test_hosts: AsyncMock
):
allow_public_test_hosts.side_effect = PrivateNetworkHostError(
"Host must not resolve to a private network."
)
with patch("httpx.AsyncClient.stream") as mock_stream:
result = await fetch_tool.run_json_async(
{"url": "http://169.254.169.254/latest/meta-data/"}
)
assert result["success"] is False
assert "private network" in result["error"]
mock_stream.assert_not_called()
@pytest.mark.asyncio
async def test_fetch_plain_text_content(self, fetch_tool):
response = _MockStreamResponse(
body=b"plain text body",
headers={"content-type": "text/plain; charset=utf-8"},
url="https://example.com/plain.txt",
)
with patch(
"httpx.AsyncClient.stream", return_value=_MockStreamContext(response)
):
result = await fetch_tool.run_json_async(
{"url": "https://example.com/plain.txt"}
)
assert result["success"] is True
assert result["title"] == ""
assert result["content"] == "plain text body"
assert result["content_type"] == "text/plain; charset=utf-8"
@pytest.mark.asyncio
async def test_fetch_rejects_unsupported_binary_content(self, fetch_tool):
response = _MockStreamResponse(
body=b"%PDF-1.7",
headers={"content-type": "application/pdf"},
url="https://example.com/file.pdf",
)
with patch(
"httpx.AsyncClient.stream", return_value=_MockStreamContext(response)
):
result = await fetch_tool.run_json_async(
{"url": "https://example.com/file.pdf"}
)
assert result["success"] is False
assert result["content"] == ""
assert result["content_type"] == "application/pdf"
assert "Unsupported non-text content type" in result["error"]
@pytest.mark.asyncio
async def test_fetch_rejects_large_content_length(self, fetch_tool):
response = _MockStreamResponse(
body=b"",
headers={
"content-type": "text/html",
"content-length": str(10 * 1024 * 1024 + 1),
},
)
with patch(
"httpx.AsyncClient.stream", return_value=_MockStreamContext(response)
):
result = await fetch_tool.run_json_async(
{"url": "https://example.com/large"}
)
assert result["success"] is False
assert result["content"] == ""
assert "exceeds maximum" in result["error"]
@pytest.mark.asyncio
async def test_fetch_rejects_stream_larger_than_limit(self):
response = _MockStreamResponse(
chunks=[b"1234", b"5678", b"9"],
headers={"content-type": "text/html"},
)
with patch(
"httpx.AsyncClient.stream", return_value=_MockStreamContext(response)
):
result = await WebContentFetcher(max_content_bytes=8).fetch(
"https://example.com/large"
)
assert result.success is False
assert result.content == ""
assert "exceeds maximum" in result.error
@pytest.mark.asyncio
async def test_fetch_webpage_http_error(self, fetch_tool):
response = _MockStreamResponse(
status_code=404,
reason_phrase="Not Found",
raise_status=True,
)
with patch(
"httpx.AsyncClient.stream", return_value=_MockStreamContext(response)
):
result = await fetch_tool.run_json_async(
{"url": "https://example.com/missing"}
)
assert result["success"] is False
assert result["content"] == ""
assert result["status_code"] == 404
assert "HTTP 404 error" in result["error"]
assert result["blocked_by_site"] is False
assert "refused automated access" not in result["error"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("status_code", "reason_phrase", "headers"),
[
(401, "Unauthorized", {}),
(403, "Forbidden", {}),
(429, "Too Many Requests", {}),
(451, "Unavailable For Legal Reasons", {}),
(503, "Service Unavailable", {"cf-mitigated": "challenge"}),
],
)
async def test_fetch_flags_site_blocked_responses(
self, fetch_tool, status_code, reason_phrase, headers
):
response = _MockStreamResponse(
status_code=status_code,
reason_phrase=reason_phrase,
headers=headers,
raise_status=True,
)
with patch(
"httpx.AsyncClient.stream", return_value=_MockStreamContext(response)
):
result = await fetch_tool.run_json_async(
{"url": "https://example.com/news"}
)
assert result["success"] is False
assert result["status_code"] == status_code
assert result["blocked_by_site"] is True
assert f"HTTP {status_code} error" in result["error"]
assert "refused automated access" in result["error"]
assert "Do not retry this URL" in result["error"]
@pytest.mark.asyncio
async def test_fetch_does_not_flag_plain_server_error(self, fetch_tool):
response = _MockStreamResponse(
status_code=503,
reason_phrase="Service Unavailable",
raise_status=True,
)
with patch(
"httpx.AsyncClient.stream", return_value=_MockStreamContext(response)
):
result = await fetch_tool.run_json_async(
{"url": "https://example.com/news"}
)
assert result["blocked_by_site"] is False
assert "refused automated access" not in result["error"]
def test_args_validation(self):
args = FetchWebContentArgs(url="https://example.com")
assert args.url == "https://example.com"
assert args.include_assets is False
assert args.asset_query is None
class TestGetTrustedProxyUrl:
"""An ambient HTTP(S)_PROXY must not silently reopen the DNS-rebinding
TOCTOU window that ``via_proxy`` pinning gives up on: proxied requests
only get IP pinning's protection if the proxy is explicitly marked
trusted to enforce its own private-range egress policy."""
def test_untrusted_proxy_raises(self, monkeypatch):
monkeypatch.delenv("XAGENT_TRUSTED_EGRESS_PROXY", raising=False)
monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example.com:8080")
monkeypatch.delenv("HTTP_PROXY", raising=False)
with pytest.raises(PrivateNetworkHostError):
get_trusted_proxy_url()
def test_trusted_proxy_returns_url(self, monkeypatch):
monkeypatch.setenv("XAGENT_TRUSTED_EGRESS_PROXY", "1")
monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example.com:8080")
monkeypatch.delenv("HTTP_PROXY", raising=False)
assert get_trusted_proxy_url() == "http://proxy.example.com:8080"
def test_no_proxy_configured_returns_none_regardless_of_trust_flag(
self, monkeypatch
):
monkeypatch.delenv("HTTPS_PROXY", raising=False)
monkeypatch.delenv("https_proxy", raising=False)
monkeypatch.delenv("HTTP_PROXY", raising=False)
monkeypatch.delenv("http_proxy", raising=False)
monkeypatch.delenv("XAGENT_TRUSTED_EGRESS_PROXY", raising=False)
assert get_trusted_proxy_url() is None
monkeypatch.setenv("XAGENT_TRUSTED_EGRESS_PROXY", "1")
assert get_trusted_proxy_url() is None
@pytest.mark.asyncio
async def test_fetch_web_content_surfaces_untrusted_proxy_as_tool_error(
self, fetch_tool, monkeypatch
):
"""End-to-end: an ambient but untrusted proxy must fail the tool
call with a clear error instead of silently fetching through it
with DNS pinning disabled."""
monkeypatch.delenv("XAGENT_TRUSTED_EGRESS_PROXY", raising=False)
monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example.com:8080")
monkeypatch.delenv("HTTP_PROXY", raising=False)
with patch("httpx.AsyncClient.stream") as mock_stream:
result = await fetch_tool.run_json_async({"url": "https://example.com"})
assert result["success"] is False
assert "XAGENT_TRUSTED_EGRESS_PROXY" in result["error"]
mock_stream.assert_not_called()
class TestBuildIsolatedHttpxClientKwargs:
"""httpx.AsyncClient defaults to trust_env=True, which (via
httpx._utils.get_environment_proxies() -> urllib.request.getproxies())
falls back to the OS's own proxy configuration
(getproxies_macosx_sysconf()/getproxies_registry() on macOS/Windows)
once no HTTP(S)_PROXY/ALL_PROXY env var is set -- the same
DNS-rebinding-via-proxy bypass closed for the Magento MCP connector's
requests.Session in _make_request. build_isolated_httpx_client_kwargs()
must disable that whole lookup while still honoring an explicitly
trusted proxy and the SSL_CERT_FILE/SSL_CERT_DIR CA bundle trust_env=False
would otherwise silently stop honoring."""
def test_disables_trust_env(self):
kwargs = build_isolated_httpx_client_kwargs(None)
assert kwargs["trust_env"] is False
def test_omits_proxy_when_none_given(self):
kwargs = build_isolated_httpx_client_kwargs(None)
assert "proxy" not in kwargs
def test_passes_through_an_explicit_proxy(self):
kwargs = build_isolated_httpx_client_kwargs(
"http://trusted-proxy.internal:8080"
)
assert kwargs["proxy"] == "http://trusted-proxy.internal:8080"
assert kwargs["trust_env"] is False
def test_verify_is_a_concrete_ssl_context(self):
kwargs = build_isolated_httpx_client_kwargs(None)
assert isinstance(kwargs["verify"], ssl.SSLContext)
def test_ssl_context_is_built_with_trust_env_true(self, monkeypatch):
# trust_env=False on the client would also silently stop httpx
# from honoring SSL_CERT_FILE/SSL_CERT_DIR for the default CA
# bundle (create_ssl_context() gates that env lookup behind the
# same trust_env flag) -- breaking TLS verification for a host
# behind a private/internal CA. The context must instead be built
# with trust_env=True explicitly, then passed as verify=, which
# httpx honors regardless of the client's own trust_env setting.
sentinel_ctx = object()
captured: dict[str, object] = {}
def _fake_create_ssl_context(**kwargs):
captured.update(kwargs)
return sentinel_ctx
monkeypatch.setattr(
web_content.httpx, "create_ssl_context", _fake_create_ssl_context
)
kwargs = build_isolated_httpx_client_kwargs(None)
assert captured == {"trust_env": True}
assert kwargs["verify"] is sentinel_ctx
@pytest.mark.asyncio
async def test_fetch_constructs_client_with_trust_env_disabled(self):
# End-to-end: WebContentFetcher.fetch() must actually pass these
# kwargs to httpx.AsyncClient, not just have the helper compute them.
captured: dict[str, object] = {}
original_init = httpx.AsyncClient.__init__
def _capture_init(self, *args, **kwargs):
captured.update(kwargs)
return original_init(self, *args, **kwargs)
response = _MockStreamResponse(
body=b"<html><body>hi</body></html>",
headers={"content-type": "text/html"},
)
with (
patch.object(httpx.AsyncClient, "__init__", _capture_init),
patch(
"httpx.AsyncClient.stream", return_value=_MockStreamContext(response)
),
):
await WebContentFetcher().fetch("https://example.com")
assert captured.get("trust_env") is False
assert isinstance(captured.get("verify"), ssl.SSLContext)