Skip to content

Agent Zero AI framework

Fix document query SSRF regression

Alessandrob40874e7c037
diff --git a/plugins/_document_query/helpers/fetch.py b/plugins/_document_query/helpers/fetch.py
index 65b1a6567..be1afeb90 100644
--- a/plugins/_document_query/helpers/fetch.py
+++ b/plugins/_document_query/helpers/fetch.py
@@ -12,9 +12,8 @@ from pathlib import Path
from typing import Awaitable, Callable
from urllib.parse import urlparse
-import aiohttp
-
from helpers import files
+from helpers.network import fetch_public_http_resource
InterventionCallback = Callable[[], Awaitable[None]]
@@ -140,53 +139,36 @@ async def _fetch_http(
last_error = ""
for attempt in range(retries):
try:
- async with aiohttp.ClientSession(
- timeout=aiohttp.ClientTimeout(total=timeout)
- ) as session:
- async with session.get(uri, allow_redirects=True) as response:
- if response.status > 399:
- raise ValueError(f"HTTP {response.status}")
-
- content_length = response.headers.get("content-length")
- if content_length and int(content_length) > max_remote_bytes:
- size_mb = int(content_length) / 1024 / 1024
- raise ValueError(
- f"Document exceeds max {max_remote_bytes / 1024 / 1024:.0f}MB: "
- f"{size_mb:.2f} MB ({uri})"
- )
-
- chunks: list[bytes] = []
- downloaded = 0
- async for chunk in response.content.iter_chunked(64 * 1024):
- downloaded += len(chunk)
- if downloaded > max_remote_bytes:
- size_mb = downloaded / 1024 / 1024
- raise ValueError(
- f"Document exceeds max {max_remote_bytes / 1024 / 1024:.0f}MB: "
- f"{size_mb:.2f} MB ({uri})"
- )
- chunks.append(chunk)
- if intervention_callback:
- await intervention_callback()
-
- content_type = response.headers.get("content-type", "")
- mimetype, charset = _parse_content_type(content_type)
- if not mimetype or mimetype == "application/octet-stream":
- mimetype = guessed_mimetype or "application/octet-stream"
- if mimetype == "application/octet-stream":
- raise ValueError(
- f"Unsupported document mimetype '{mimetype}' ({uri})"
- )
-
- return FetchedDocument(
- uri=str(response.url),
- source_uri=uri,
- scheme=response.url.scheme or scheme,
- mimetype=mimetype,
- encoding=encoding,
- charset=charset,
- content=b"".join(chunks),
- )
+ if intervention_callback:
+ await intervention_callback()
+ resource = await asyncio.to_thread(
+ fetch_public_http_resource,
+ uri,
+ max_bytes=max_remote_bytes,
+ timeout=(timeout, timeout),
+ )
+ if intervention_callback:
+ await intervention_callback()
+
+ mimetype = (
+ resource.content_type
+ or guessed_mimetype
+ or "application/octet-stream"
+ )
+ if mimetype == "application/octet-stream":
+ raise ValueError(
+ f"Unsupported document mimetype '{mimetype}' ({uri})"
+ )
+
+ return FetchedDocument(
+ uri=resource.url,
+ source_uri=uri,
+ scheme=urlparse(resource.url).scheme or scheme,
+ mimetype=mimetype,
+ encoding=encoding,
+ charset=resource.encoding,
+ content=resource.content,
+ )
except Exception as e:
last_error = str(e)
if attempt < retries - 1:
@@ -196,20 +178,6 @@ async def _fetch_http(
raise ValueError(f"Document fetch error: {uri} ({last_error})")
-
-def _parse_content_type(value: str) -> tuple[str | None, str | None]:
- if not value:
- return None, None
- parts = [part.strip() for part in value.split(";") if part.strip()]
- mimetype = parts[0].lower() if parts else None
- charset = None
- for part in parts[1:]:
- if part.lower().startswith("charset="):
- charset = part.split("=", 1)[1].strip("\"'")
- break
- return mimetype, charset
-
-
register_protocol_handler("file", _fetch_file)
register_protocol_handler("http", _fetch_http)
register_protocol_handler("https", _fetch_http)
diff --git a/tests/test_document_query_plugin.py b/tests/test_document_query_plugin.py
index e9fa25ef3..26bf2a696 100644
--- a/tests/test_document_query_plugin.py
+++ b/tests/test_document_query_plugin.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import ipaddress
import sys
from pathlib import Path
@@ -11,6 +12,7 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
+from helpers import network as network_helper
from plugins._document_query import hooks as document_query_hooks
from plugins._document_query.helpers.fetch import FetchedDocument, fetch_public_resource
import plugins._document_query.helpers.document_query as document_query_module
@@ -113,6 +115,124 @@ def test_fetch_file_detects_mimetype_and_reads_once(tmp_path):
assert fetched.text() == "hello\nworld\n"
+def test_fetch_http_blocks_non_public_destinations():
+ with pytest.raises(ValueError, match="Blocked non-public address"):
+ run_async(
+ fetch_public_resource(
+ "http://127.0.0.1/internal.txt",
+ {"fetch_retries": 1},
+ )
+ )
+
+
+def test_fetch_http_blocks_redirects_to_non_public_destinations(monkeypatch):
+ source = "https://public.example/start"
+ calls = []
+
+ class RedirectResponse:
+ status_code = 302
+ headers = {"Location": "http://127.0.0.1/internal.txt"}
+ encoding = None
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ class FakeSession:
+ trust_env = True
+
+ def get(self, url, **kwargs):
+ calls.append((url, kwargs, self.trust_env))
+ return RedirectResponse()
+
+ def resolve_host_ips(hostname):
+ address = "127.0.0.1" if hostname == "127.0.0.1" else "93.184.216.34"
+ return (ipaddress.ip_address(address),)
+
+ monkeypatch.setattr(network_helper, "resolve_host_ips", resolve_host_ips)
+ monkeypatch.setattr(network_helper.requests, "Session", FakeSession)
+
+ with pytest.raises(ValueError, match="Blocked non-public address"):
+ run_async(fetch_public_resource(source, {"fetch_retries": 1}))
+
+ assert [url for url, _kwargs, _trust_env in calls] == [source]
+
+
+def test_fetch_http_preserves_public_redirects_and_request_compatibility(monkeypatch):
+ source = "https://public.example/start"
+ destination = "https://public.example/report.txt"
+ calls = []
+
+ class FakeResponse:
+ def __init__(self, status_code, headers, body=b""):
+ self.status_code = status_code
+ self.headers = headers
+ self.encoding = "utf-8"
+ self.body = body
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ def iter_content(self, chunk_size):
+ assert chunk_size == 64 * 1024
+ yield self.body
+
+ class FakeSession:
+ trust_env = True
+
+ def get(self, url, **kwargs):
+ calls.append((url, kwargs, self.trust_env))
+ if url == source:
+ return FakeResponse(302, {"Location": "/report.txt"})
+ return FakeResponse(
+ 200,
+ {
+ "Content-Length": "9",
+ "Content-Type": "text/plain; charset=utf-8",
+ },
+ b"public ok",
+ )
+
+ monkeypatch.setattr(
+ network_helper,
+ "resolve_host_ips",
+ lambda _hostname: (ipaddress.ip_address("93.184.216.34"),),
+ )
+ monkeypatch.setattr(network_helper.requests, "Session", FakeSession)
+ monkeypatch.setenv("USER_AGENT", "AgentZeroTest")
+
+ fetched = run_async(
+ fetch_public_resource(
+ source,
+ {
+ "fetch_retries": 1,
+ "fetch_timeout": 2,
+ "max_remote_bytes": 1024,
+ },
+ )
+ )
+
+ assert fetched.uri == destination
+ assert fetched.mimetype == "text/plain"
+ assert fetched.text() == "public ok"
+ assert [url for url, _kwargs, _trust_env in calls] == [source, destination]
+ assert all(not trust_env for _url, _kwargs, trust_env in calls)
+ assert all(
+ kwargs["allow_redirects"] is False
+ for _url, kwargs, _trust_env in calls
+ )
+ assert all(kwargs["timeout"] == (2.0, 2.0) for _url, kwargs, _trust_env in calls)
+ assert all(
+ kwargs["headers"] == {"User-Agent": "AgentZeroTest"}
+ for _url, kwargs, _trust_env in calls
+ )
+
+
def test_parser_registry_prefers_liteparse_for_pdf():
parsers = get_parsers_for_mimetype("application/pdf", {"liteparse_enabled": True})