Agent Zero AI framework
Fix SSRF in document_query remote fetching (CVE-2026-4308)
Alessandro6397acc092a5
diff --git a/helpers/document_query.py b/helpers/document_query.pyindex 1f6925f4d..fe98ee988 100644--- a/helpers/document_query.py+++ b/helpers/document_query.py@@ -1,7 +1,6 @@import mimetypesimport osimport asyncio-import aiohttpimport jsonfrom helpers.vector_db import VectorDB@@ -13,8 +12,6 @@ from urllib.parse import urlparsefrom typing import Callable, Sequence, List, Optional, Tuplefrom datetime import datetime-from langchain_community.document_loaders import AsyncHtmlLoader-from langchain_community.document_loaders.text import TextLoaderfrom langchain_community.document_loaders.pdf import PyMuPDFLoaderfrom langchain_community.document_transformers import MarkdownifyTransformerfrom langchain_community.document_loaders.parsers.images import TesseractBlobParser@@ -24,12 +21,14 @@ from langchain.schema import SystemMessage, HumanMessagefrom helpers.print_style import PrintStylefrom helpers import files, errors+from helpers.network import HttpFetchResult, fetch_public_http_resourcefrom agent import Agentfrom langchain.text_splitter import RecursiveCharacterTextSplitterDEFAULT_SEARCH_THRESHOLD = 0.5+MAX_REMOTE_DOCUMENT_BYTES = 50 * 1024 * 1024class DocumentQueryStore:@@ -450,45 +449,19 @@ class DocumentQueryHelper:scheme = url.scheme or "file"mimetype, encoding = mimetypes.guess_type(document_uri)mimetype = mimetype or "application/octet-stream"+ remote_resource: HttpFetchResult | None = None- if mimetype == "application/octet-stream":- if url.scheme in ["http", "https"]:- response: aiohttp.ClientResponse | None = None- retries = 0- last_error = ""- while not response and retries < 3:- try:- async with aiohttp.ClientSession() as session:- response = await session.head(- document_uri,- timeout=aiohttp.ClientTimeout(total=2.0),- allow_redirects=True,- )- if response.status > 399:- raise Exception(response.status)- break- except Exception as e:- await asyncio.sleep(1)- last_error = str(e)- retries += 1- await self.agent.handle_intervention()-- if not response:- raise ValueError(- f"DocumentQueryHelper::document_get_content: Document fetch error: {document_uri} ({last_error})"- )-- mimetype = response.headers["content-type"]- if "content-length" in response.headers:- content_length = (- float(response.headers["content-length"]) / 1024 / 1024- ) # MB- if content_length > 50.0:- raise ValueError(- f"Document content length exceeds max. 50MB: {content_length} MB ({document_uri})"- )- if mimetype and "; charset=" in mimetype:- mimetype = mimetype.split("; charset=")[0]+ if scheme in ["http", "https"]:+ remote_resource = await asyncio.to_thread(+ fetch_public_http_resource,+ document_uri,+ max_bytes=MAX_REMOTE_DOCUMENT_BYTES,+ )+ if (+ remote_resource.content_type+ and remote_resource.content_type != "application/octet-stream"+ ):+ mimetype = remote_resource.content_typeif scheme == "file":try:@@ -515,16 +488,24 @@ class DocumentQueryHelper:if not exists:await self.agent.handle_intervention()if mimetype.startswith("image/"):- document_content = self.handle_image_document(document_uri, scheme)+ document_content = self.handle_image_document(+ document_uri, scheme, remote_resource=remote_resource+ )elif mimetype == "text/html":- document_content = self.handle_html_document(document_uri, scheme)+ document_content = self.handle_html_document(+ document_uri, scheme, remote_resource=remote_resource+ )elif mimetype.startswith("text/") or mimetype == "application/json":- document_content = self.handle_text_document(document_uri, scheme)+ document_content = self.handle_text_document(+ document_uri, scheme, remote_resource=remote_resource+ )elif mimetype == "application/pdf":- document_content = self.handle_pdf_document(document_uri, scheme)+ document_content = self.handle_pdf_document(+ document_uri, scheme, remote_resource=remote_resource+ )else:document_content = self.handle_unstructured_document(- document_uri, scheme+ document_uri, scheme, remote_resource=remote_resource)if add_to_db:self.progress_callback(f"Indexing document")@@ -550,13 +531,53 @@ class DocumentQueryHelper:)return document_content- def handle_image_document(self, document: str, scheme: str) -> str:- return self.handle_unstructured_document(document, scheme)+ @staticmethod+ def _decode_remote_text(remote_resource: HttpFetchResult) -> str:+ encoding = remote_resource.encoding or "utf-8"+ try:+ return remote_resource.content.decode(encoding)+ except (LookupError, UnicodeDecodeError):+ return remote_resource.content.decode("utf-8", errors="replace")- def handle_html_document(self, document: str, scheme: str) -> str:+ @staticmethod+ def _get_temp_file_suffix(+ document: str, remote_resource: HttpFetchResult | None = None+ ) -> str:+ parsed = urlparse(document)+ _stem, ext = os.path.splitext(parsed.path or document)+ if ext:+ return ext++ if remote_resource and remote_resource.content_type:+ guessed_ext = mimetypes.guess_extension(+ remote_resource.content_type, strict=False+ )+ if guessed_ext:+ return guessed_ext++ return ".bin"++ def handle_image_document(+ self,+ document: str,+ scheme: str,+ remote_resource: HttpFetchResult | None = None,+ ) -> str:+ return self.handle_unstructured_document(+ document, scheme, remote_resource=remote_resource+ )++ def handle_html_document(+ self,+ document: str,+ scheme: str,+ remote_resource: HttpFetchResult | None = None,+ ) -> str:if scheme in ["http", "https"]:- loader = AsyncHtmlLoader(web_path=document)- parts: list[Document] = loader.load()+ if remote_resource is None:+ raise ValueError("Missing prefetched remote HTML content")+ html_content = self._decode_remote_text(remote_resource)+ parts = [Document(page_content=html_content, metadata={"source": document})]elif scheme == "file":# Use RFC file operations instead of TextLoaderfile_content_bytes = files.read_file_bin(document)@@ -573,10 +594,19 @@ class DocumentQueryHelper:])- def handle_text_document(self, document: str, scheme: str) -> str:+ def handle_text_document(+ self,+ document: str,+ scheme: str,+ remote_resource: HttpFetchResult | None = None,+ ) -> str:if scheme in ["http", "https"]:- loader = AsyncHtmlLoader(web_path=document)- elements: list[Document] = loader.load()+ if remote_resource is None:+ raise ValueError("Missing prefetched remote text content")+ file_content = self._decode_remote_text(remote_resource)+ elements = [+ Document(page_content=file_content, metadata={"source": document})+ ]elif scheme == "file":# Use RFC file operations instead of TextLoaderfile_content_bytes = files.read_file_bin(document)@@ -590,7 +620,12 @@ class DocumentQueryHelper:return "\n".join([element.page_content for element in elements])- def handle_pdf_document(self, document: str, scheme: str) -> str:+ def handle_pdf_document(+ self,+ document: str,+ scheme: str,+ remote_resource: HttpFetchResult | None = None,+ ) -> str:temp_file_path = ""if scheme == "file":# Use RFC file operations to read the PDF file as binary@@ -602,17 +637,12 @@ class DocumentQueryHelper:temp_file.write(file_content_bytes)temp_file_path = temp_file.nameelif scheme in ["http", "https"]:- # download the file from the web url to a temporary file using python libraries for downloading- import requestsimport tempfile+ if remote_resource is None:+ raise ValueError("Missing prefetched remote PDF content")with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_file:- response = requests.get(document, timeout=10.0)- if response.status_code != 200:- raise ValueError(- f"DocumentQueryHelper::handle_pdf_document: Failed to download PDF from {document}: {response.status_code}"- )- temp_file.write(response.content)+ temp_file.write(remote_resource.content)temp_file_path = temp_file.nameelse:raise ValueError(f"Unsupported scheme: {scheme}")@@ -658,18 +688,35 @@ class DocumentQueryHelper:finally:os.unlink(temp_file_path)- def handle_unstructured_document(self, document: str, scheme: str) -> str:+ def handle_unstructured_document(+ self,+ document: str,+ scheme: str,+ remote_resource: HttpFetchResult | None = None,+ ) -> str:elements: list[Document] = []if scheme in ["http", "https"]:- # loader = UnstructuredURLLoader(urls=[document], mode="single")- loader = UnstructuredLoader(- web_url=document,- mode="single",- partition_via_api=False,- # chunking_strategy="by_page",- strategy="hi_res",- )- elements = loader.load()+ if remote_resource is None:+ raise ValueError("Missing prefetched remote document content")+ import tempfile++ temp_file_path = ""+ suffix = self._get_temp_file_suffix(document, remote_resource)+ with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:+ temp_file.write(remote_resource.content)+ temp_file_path = temp_file.name++ try:+ loader = UnstructuredLoader(+ file_path=temp_file_path,+ mode="single",+ partition_via_api=False,+ # chunking_strategy="by_page",+ strategy="hi_res",+ )+ elements = loader.load()+ finally:+ os.unlink(temp_file_path)elif scheme == "file":# Use RFC file operations to read the file as binaryfile_content_bytes = files.read_file_bin(document)diff --git a/helpers/network.py b/helpers/network.pyindex aa5a100b4..7cbb6974b 100644--- a/helpers/network.py+++ b/helpers/network.py@@ -1,5 +1,164 @@+from __future__ import annotations++from dataclasses import dataclass+import ipaddressimport socketimport struct+from urllib.parse import urljoin, urlparse++import requests+++SAFE_HTTP_SCHEMES = frozenset({"http", "https"})+DEFAULT_FETCH_TIMEOUT = (3.05, 10.0)+++@dataclass(frozen=True)+class HttpFetchResult:+ url: str+ content: bytes+ content_type: str | None+ encoding: str | None+++class UnsafeUrlError(ValueError):+ """Raised when a remote URL resolves to a non-public destination."""+++def _normalize_content_type(content_type: str | None) -> str | None:+ if not content_type:+ return None+ return content_type.split(";", 1)[0].strip().lower() or None+++def resolve_host_ips(hostname: str) -> tuple[ipaddress._BaseAddress, ...]:+ try:+ results = socket.getaddrinfo(+ hostname,+ None,+ family=socket.AF_UNSPEC,+ type=socket.SOCK_STREAM,+ )+ except socket.gaierror as exc:+ raise UnsafeUrlError(f"Unable to resolve hostname '{hostname}'") from exc++ ips: list[ipaddress._BaseAddress] = []+ seen: set[str] = set()+ for _family, _type, _proto, _canonname, sockaddr in results:+ address = sockaddr[0]+ if "%" in address:+ address = address.split("%", 1)[0]+ ip = ipaddress.ip_address(address)+ key = ip.compressed+ if key in seen:+ continue+ seen.add(key)+ ips.append(ip)++ if not ips:+ raise UnsafeUrlError(f"Hostname '{hostname}' did not resolve to an IP address")++ return tuple(ips)+++def validate_public_http_url(url: str) -> tuple[ipaddress._BaseAddress, ...]:+ parsed = urlparse(url)++ if parsed.scheme not in SAFE_HTTP_SCHEMES:+ raise UnsafeUrlError("Only http:// and https:// URLs are supported")+ if not parsed.hostname:+ raise UnsafeUrlError("URL hostname is required")+ if parsed.username or parsed.password:+ raise UnsafeUrlError("URLs with embedded credentials are not allowed")++ hostname = parsed.hostname.rstrip(".").lower()+ if hostname == "localhost" or hostname.endswith(".localhost"):+ raise UnsafeUrlError(f"Blocked local hostname '{hostname}'")++ ips = resolve_host_ips(hostname)+ blocked = [str(ip) for ip in ips if not ip.is_global]+ if blocked:+ raise UnsafeUrlError(+ f"Blocked non-public address resolution for '{hostname}': {', '.join(blocked)}"+ )++ return ips+++def fetch_public_http_resource(+ url: str,+ *,+ max_bytes: int,+ max_redirects: int = 5,+ timeout: tuple[float, float] = DEFAULT_FETCH_TIMEOUT,+) -> HttpFetchResult:+ current_url = url+ session = requests.Session()+ session.trust_env = False++ for redirect_count in range(max_redirects + 1):+ validate_public_http_url(current_url)++ try:+ with session.get(+ current_url,+ stream=True,+ allow_redirects=False,+ timeout=timeout,+ ) as response:+ if 300 <= response.status_code < 400:+ location = response.headers.get("Location")+ if not location:+ raise ValueError(+ f"Remote URL redirect is missing a Location header: {current_url}"+ )+ if redirect_count >= max_redirects:+ raise ValueError(+ f"Remote URL exceeded redirect limit ({max_redirects}): {url}"+ )+ current_url = urljoin(current_url, location)+ continue++ if response.status_code >= 400:+ raise ValueError(+ f"Remote URL returned HTTP {response.status_code}: {current_url}"+ )++ content_length = response.headers.get("Content-Length")+ if content_length:+ try:+ declared_length = int(content_length)+ except ValueError:+ declared_length = None+ if declared_length is not None and declared_length > max_bytes:+ raise ValueError(+ f"Remote document exceeds max size {max_bytes} bytes: {current_url}"+ )++ body = bytearray()+ for chunk in response.iter_content(chunk_size=64 * 1024):+ if not chunk:+ continue+ body.extend(chunk)+ if len(body) > max_bytes:+ raise ValueError(+ f"Remote document exceeds max size {max_bytes} bytes: {current_url}"+ )++ return HttpFetchResult(+ url=current_url,+ content=bytes(body),+ content_type=_normalize_content_type(+ response.headers.get("Content-Type")+ ),+ encoding=response.encoding,+ )+ except requests.RequestException as exc:+ raise ValueError(+ f"Remote document fetch failed for {current_url}: {exc}"+ ) from exc++ raise ValueError(f"Remote URL exceeded redirect limit ({max_redirects}): {url}")def is_loopback_address(address: str) -> bool: