Agent Zero AI framework
Fix document query SSRF regression
Alessandrob40874e7c037
diff --git a/plugins/_document_query/helpers/fetch.py b/plugins/_document_query/helpers/fetch.pyindex 65b1a6567..be1afeb90 100644--- a/plugins/_document_query/helpers/fetch.py+++ b/plugins/_document_query/helpers/fetch.py@@ -12,9 +12,8 @@ from pathlib import Pathfrom typing import Awaitable, Callablefrom urllib.parse import urlparse-import aiohttp-from helpers import files+from helpers.network import fetch_public_http_resourceInterventionCallback = Callable[[], Awaitable[None]]@@ -140,53 +139,36 @@ async def _fetch_http(last_error = ""for attempt in range(retries):try:- async with aiohttp.ClientSession(- timeout=aiohttp.ClientTimeout(total=timeout)- ) as session:- async with session.get(uri, allow_redirects=True) as response:- if response.status > 399:- raise ValueError(f"HTTP {response.status}")-- content_length = response.headers.get("content-length")- if content_length and int(content_length) > max_remote_bytes:- size_mb = int(content_length) / 1024 / 1024- raise ValueError(- f"Document exceeds max {max_remote_bytes / 1024 / 1024:.0f}MB: "- f"{size_mb:.2f} MB ({uri})"- )-- chunks: list[bytes] = []- downloaded = 0- async for chunk in response.content.iter_chunked(64 * 1024):- downloaded += len(chunk)- if downloaded > max_remote_bytes:- size_mb = downloaded / 1024 / 1024- raise ValueError(- f"Document exceeds max {max_remote_bytes / 1024 / 1024:.0f}MB: "- f"{size_mb:.2f} MB ({uri})"- )- chunks.append(chunk)- if intervention_callback:- await intervention_callback()-- content_type = response.headers.get("content-type", "")- mimetype, charset = _parse_content_type(content_type)- if not mimetype or mimetype == "application/octet-stream":- mimetype = guessed_mimetype or "application/octet-stream"- if mimetype == "application/octet-stream":- raise ValueError(- f"Unsupported document mimetype '{mimetype}' ({uri})"- )-- return FetchedDocument(- uri=str(response.url),- source_uri=uri,- scheme=response.url.scheme or scheme,- mimetype=mimetype,- encoding=encoding,- charset=charset,- content=b"".join(chunks),- )+ if intervention_callback:+ await intervention_callback()+ resource = await asyncio.to_thread(+ fetch_public_http_resource,+ uri,+ max_bytes=max_remote_bytes,+ timeout=(timeout, timeout),+ )+ if intervention_callback:+ await intervention_callback()++ mimetype = (+ resource.content_type+ or guessed_mimetype+ or "application/octet-stream"+ )+ if mimetype == "application/octet-stream":+ raise ValueError(+ f"Unsupported document mimetype '{mimetype}' ({uri})"+ )++ return FetchedDocument(+ uri=resource.url,+ source_uri=uri,+ scheme=urlparse(resource.url).scheme or scheme,+ mimetype=mimetype,+ encoding=encoding,+ charset=resource.encoding,+ content=resource.content,+ )except Exception as e:last_error = str(e)if attempt < retries - 1:@@ -196,20 +178,6 @@ async def _fetch_http(raise ValueError(f"Document fetch error: {uri} ({last_error})")--def _parse_content_type(value: str) -> tuple[str | None, str | None]:- if not value:- return None, None- parts = [part.strip() for part in value.split(";") if part.strip()]- mimetype = parts[0].lower() if parts else None- charset = None- for part in parts[1:]:- if part.lower().startswith("charset="):- charset = part.split("=", 1)[1].strip("\"'")- break- return mimetype, charset--register_protocol_handler("file", _fetch_file)register_protocol_handler("http", _fetch_http)register_protocol_handler("https", _fetch_http)diff --git a/tests/test_document_query_plugin.py b/tests/test_document_query_plugin.pyindex e9fa25ef3..26bf2a696 100644--- a/tests/test_document_query_plugin.py+++ b/tests/test_document_query_plugin.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import ipaddressimport sysfrom pathlib import Path@@ -11,6 +12,7 @@ ROOT = Path(__file__).resolve().parents[1]if str(ROOT) not in sys.path:sys.path.insert(0, str(ROOT))+from helpers import network as network_helperfrom plugins._document_query import hooks as document_query_hooksfrom plugins._document_query.helpers.fetch import FetchedDocument, fetch_public_resourceimport plugins._document_query.helpers.document_query as document_query_module@@ -113,6 +115,124 @@ def test_fetch_file_detects_mimetype_and_reads_once(tmp_path):assert fetched.text() == "hello\nworld\n"+def test_fetch_http_blocks_non_public_destinations():+ with pytest.raises(ValueError, match="Blocked non-public address"):+ run_async(+ fetch_public_resource(+ "http://127.0.0.1/internal.txt",+ {"fetch_retries": 1},+ )+ )+++def test_fetch_http_blocks_redirects_to_non_public_destinations(monkeypatch):+ source = "https://public.example/start"+ calls = []++ class RedirectResponse:+ status_code = 302+ headers = {"Location": "http://127.0.0.1/internal.txt"}+ encoding = None++ def __enter__(self):+ return self++ def __exit__(self, *_args):+ return False++ class FakeSession:+ trust_env = True++ def get(self, url, **kwargs):+ calls.append((url, kwargs, self.trust_env))+ return RedirectResponse()++ def resolve_host_ips(hostname):+ address = "127.0.0.1" if hostname == "127.0.0.1" else "93.184.216.34"+ return (ipaddress.ip_address(address),)++ monkeypatch.setattr(network_helper, "resolve_host_ips", resolve_host_ips)+ monkeypatch.setattr(network_helper.requests, "Session", FakeSession)++ with pytest.raises(ValueError, match="Blocked non-public address"):+ run_async(fetch_public_resource(source, {"fetch_retries": 1}))++ assert [url for url, _kwargs, _trust_env in calls] == [source]+++def test_fetch_http_preserves_public_redirects_and_request_compatibility(monkeypatch):+ source = "https://public.example/start"+ destination = "https://public.example/report.txt"+ calls = []++ class FakeResponse:+ def __init__(self, status_code, headers, body=b""):+ self.status_code = status_code+ self.headers = headers+ self.encoding = "utf-8"+ self.body = body++ def __enter__(self):+ return self++ def __exit__(self, *_args):+ return False++ def iter_content(self, chunk_size):+ assert chunk_size == 64 * 1024+ yield self.body++ class FakeSession:+ trust_env = True++ def get(self, url, **kwargs):+ calls.append((url, kwargs, self.trust_env))+ if url == source:+ return FakeResponse(302, {"Location": "/report.txt"})+ return FakeResponse(+ 200,+ {+ "Content-Length": "9",+ "Content-Type": "text/plain; charset=utf-8",+ },+ b"public ok",+ )++ monkeypatch.setattr(+ network_helper,+ "resolve_host_ips",+ lambda _hostname: (ipaddress.ip_address("93.184.216.34"),),+ )+ monkeypatch.setattr(network_helper.requests, "Session", FakeSession)+ monkeypatch.setenv("USER_AGENT", "AgentZeroTest")++ fetched = run_async(+ fetch_public_resource(+ source,+ {+ "fetch_retries": 1,+ "fetch_timeout": 2,+ "max_remote_bytes": 1024,+ },+ )+ )++ assert fetched.uri == destination+ assert fetched.mimetype == "text/plain"+ assert fetched.text() == "public ok"+ assert [url for url, _kwargs, _trust_env in calls] == [source, destination]+ assert all(not trust_env for _url, _kwargs, trust_env in calls)+ assert all(+ kwargs["allow_redirects"] is False+ for _url, kwargs, _trust_env in calls+ )+ assert all(kwargs["timeout"] == (2.0, 2.0) for _url, kwargs, _trust_env in calls)+ assert all(+ kwargs["headers"] == {"User-Agent": "AgentZeroTest"}+ for _url, kwargs, _trust_env in calls+ )++def test_parser_registry_prefers_liteparse_for_pdf():parsers = get_parsers_for_mimetype("application/pdf", {"liteparse_enabled": True})