Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(parse): detect binary URL imports after GET (#2203)
Misaka3d4bcc0a5367
diff --git a/openviking/parse/accessors/http_accessor.py b/openviking/parse/accessors/http_accessor.pyindex 6c9941145..cb0a52dfd 100644--- a/openviking/parse/accessors/http_accessor.py+++ b/openviking/parse/accessors/http_accessor.py@@ -19,11 +19,16 @@ Features:import tempfilefrom enum import Enumfrom pathlib import Path-from typing import Any, Dict, Optional, Tuple, Union+from typing import Any, Dict, Mapping, Optional, Tuple, Unionfrom urllib.parse import unquote, urlparsefrom openviking.parse.base import lazy_importfrom openviking.parse.parsers.constants import CODE_EXTENSIONS+from openviking.parse.parsers.media.constants import (+ AUDIO_EXTENSIONS,+ IMAGE_EXTENSIONS,+ VIDEO_EXTENSIONS,+)from openviking.utils.network_guard import build_httpx_request_validation_hooksfrom openviking_cli.exceptions import PermissionDeniedErrorfrom openviking_cli.utils.logger import get_logger@@ -33,6 +38,17 @@ from .mime_types import MEDIA_TYPE_ALIASES, IANAMediaType, get_preferred_extensilogger = get_logger(__name__)+DOCUMENT_EXTENSIONS = {+ ".doc",+ ".docx",+ ".xls",+ ".xlsx",+ ".xlsm",+ ".pptx",+ ".epub",+ ".zip",+}+class URLType(Enum):"""URL content types for routing to appropriate parsers."""@@ -42,6 +58,10 @@ class URLType(Enum):DOWNLOAD_MD = "download_md" # Markdown file download linkDOWNLOAD_TXT = "download_txt" # Text file download linkDOWNLOAD_HTML = "download_html" # HTML file download link+ DOWNLOAD_IMAGE = "download_image" # Image file download link+ DOWNLOAD_AUDIO = "download_audio" # Audio file download link+ DOWNLOAD_VIDEO = "download_video" # Video file download link+ DOWNLOAD_DOCUMENT = "download_document" # Office/e-book/archive document linkUNKNOWN = "unknown" # Unknown or unsupported type@@ -72,6 +92,10 @@ class URLTypeDetector:".text": URLType.DOWNLOAD_TXT,".html": URLType.DOWNLOAD_HTML,".htm": URLType.DOWNLOAD_HTML,+ **dict.fromkeys(IMAGE_EXTENSIONS, URLType.DOWNLOAD_IMAGE),+ **dict.fromkeys(AUDIO_EXTENSIONS, URLType.DOWNLOAD_AUDIO),+ **dict.fromkeys(VIDEO_EXTENSIONS, URLType.DOWNLOAD_VIDEO),+ **dict.fromkeys(DOCUMENT_EXTENSIONS, URLType.DOWNLOAD_DOCUMENT),}# === IANA Media Type to URL type mapping ===@@ -95,6 +119,20 @@ class URLTypeDetector:# Plain text"text/plain": URLType.DOWNLOAD_TXT,"text/*": URLType.DOWNLOAD_TXT,+ # Media files. Some signed object-storage URLs reject HEAD, but for+ # extensionless URLs that do return headers this routes to media parsers.+ "image/*": URLType.DOWNLOAD_IMAGE,+ "audio/*": URLType.DOWNLOAD_AUDIO,+ "video/*": URLType.DOWNLOAD_VIDEO,+ # Document formats supported by ParserRegistry. The final extension+ # still comes from URL path, Content-Disposition, or IANA media type.+ "application/msword": URLType.DOWNLOAD_DOCUMENT,+ "application/vnd.ms-excel": URLType.DOWNLOAD_DOCUMENT,+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document": URLType.DOWNLOAD_DOCUMENT,+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": URLType.DOWNLOAD_DOCUMENT,+ "application/vnd.openxmlformats-officedocument.presentationml.presentation": URLType.DOWNLOAD_DOCUMENT,+ "application/epub+zip": URLType.DOWNLOAD_DOCUMENT,+ "application/zip": URLType.DOWNLOAD_DOCUMENT,}# URLType to file extension mapping@@ -104,6 +142,10 @@ class URLTypeDetector:URLType.DOWNLOAD_MD: ".md",URLType.DOWNLOAD_TXT: ".txt",URLType.DOWNLOAD_HTML: ".html",+ URLType.DOWNLOAD_IMAGE: ".png",+ URLType.DOWNLOAD_AUDIO: ".mp3",+ URLType.DOWNLOAD_VIDEO: ".mp4",+ URLType.DOWNLOAD_DOCUMENT: ".zip",URLType.UNKNOWN: ".html",}@@ -166,32 +208,15 @@ class URLTypeDetector:async with httpx.AsyncClient(**client_kwargs) as client:response = await client.head(url)- # Get raw headers for debugging- content_type_raw = response.headers.get("content-type", "")- content_disposition = response.headers.get("content-disposition", "")-- meta["content_type_raw"] = content_type_raw- meta["content_disposition_raw"] = content_dispositionmeta["status_code"] = response.status_code+ if not (200 <= response.status_code < 300):+ meta["head_status_skipped"] = response.status_code+ raise RuntimeError(f"HEAD returned {response.status_code}; headers not trusted")- # === Step 2a: Check Content-Disposition for filename (RFC 6266) ===- filename_from_disposition = self._extract_filename_from_disposition(- content_disposition- )- if filename_from_disposition:- meta["filename_from_disposition"] = filename_from_disposition- filename_lower = filename_from_disposition.lower()- for ext, url_type in self.EXTENSION_MAP.items():- if filename_lower.endswith(ext):- meta["detected_by"] = "content_disposition"- meta["extension"] = ext- return url_type, meta-- # === Step 2b: Check Content-Type (RFC 6838) ===- if content_type_raw:- url_type = self._detect_from_media_type(content_type_raw, meta)- if url_type != URLType.UNKNOWN:- return url_type, meta+ # === Step 2a/2b: Check Content-Disposition and Content-Type ===+ url_type = self.detect_from_headers(response.headers, meta)+ if url_type != URLType.UNKNOWN:+ return url_type, metaexcept PermissionDeniedError:raise@@ -203,7 +228,58 @@ class URLTypeDetector:meta["detected_by"] = "default"return URLType.WEBPAGE, meta- def _detect_from_media_type(self, content_type: str, meta: Dict[str, Any]) -> URLType:+ def detect_from_headers(+ self,+ headers: Mapping[str, str],+ meta: Dict[str, Any],+ detected_by_prefix: str = "",+ ) -> URLType:+ """+ Detect URL type from HTTP response headers.++ Args:+ headers: Response headers from HEAD or GET.+ meta: Metadata dict to update.+ detected_by_prefix: Optional prefix for meta["detected_by"], e.g. "get_".++ Returns:+ Detected URLType, or URLType.UNKNOWN if no header matched.+ """+ content_type_raw = headers.get("content-type", "")+ content_disposition = headers.get("content-disposition", "")++ meta["content_type_raw"] = content_type_raw+ meta["content_disposition_raw"] = content_disposition++ # Check Content-Disposition for filename (RFC 6266)+ filename_from_disposition = self._extract_filename_from_disposition(content_disposition)+ if filename_from_disposition:+ meta["filename_from_disposition"] = filename_from_disposition+ filename_lower = filename_from_disposition.lower()+ for ext, url_type in self.EXTENSION_MAP.items():+ if filename_lower.endswith(ext):+ meta["detected_by"] = f"{detected_by_prefix}content_disposition"+ meta["extension"] = ext+ return url_type++ # Check Content-Type (RFC 6838)+ if content_type_raw:+ url_type = self._detect_from_media_type(+ content_type_raw,+ meta,+ detected_by_prefix=detected_by_prefix,+ )+ if url_type != URLType.UNKNOWN:+ return url_type++ return URLType.UNKNOWN++ def _detect_from_media_type(+ self,+ content_type: str,+ meta: Dict[str, Any],+ detected_by_prefix: str = "",+ ) -> URLType:"""Detect URL type from IANA media type.@@ -240,17 +316,17 @@ class URLTypeDetector:if media_type.suffix:media_type_with_suffix = f"{media_type_key}+{media_type.suffix}"if media_type_with_suffix in self.MEDIA_TYPE_MAP:- meta["detected_by"] = "media_type_suffix"+ meta["detected_by"] = f"{detected_by_prefix}media_type_suffix"return self.MEDIA_TYPE_MAP[media_type_with_suffix]if media_type_key in self.MEDIA_TYPE_MAP:- meta["detected_by"] = "media_type"+ meta["detected_by"] = f"{detected_by_prefix}media_type"return self.MEDIA_TYPE_MAP[media_type_key]# Check for wildcard matchesfor pattern, url_type in self.MEDIA_TYPE_MAP.items():if media_type.matches(pattern):- meta["detected_by"] = "media_type_pattern"+ meta["detected_by"] = f"{detected_by_prefix}media_type_pattern"meta["media_type_pattern"] = patternreturn url_type@@ -432,20 +508,7 @@ class HTTPAccessor(DataAccessor):request_validator=request_validator,)- # Determine file extension using IANA standards- ext = self._determine_file_extension(url, url_type, detect_meta)-- # Create temp file- temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=ext)- temp_path = temp_file.name- temp_file.close()-- # Get original filename from URL or Content-Disposition- original_filename = detect_meta.get("filename_from_disposition")- if not original_filename:- original_filename = self._extract_filename_from_url(url)-- meta = {**detect_meta, "extension": ext, "original_filename": original_filename}+ temp_path: Optional[str] = Nonetry:# Download content@@ -484,20 +547,190 @@ class HTTPAccessor(DataAccessor):user_msg = "HTTP request failed: unexpected error."raise RuntimeError(f"{user_msg} URL: {url}. Details: {e}") from e+ meta = self._finalize_download_metadata(+ url=url,+ initial_url_type=url_type,+ initial_meta=detect_meta,+ response_headers=response.headers,+ content=response.content,+ )+ url_type = meta["resolved_url_type"]+ ext = meta["extension"]++ # Create temp file after GET headers/content have had a chance+ # to refine detection. This avoids routing extensionless signed+ # URLs as HTML when HEAD is rejected by object storage.+ temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=ext)+ temp_path = temp_file.name+ temp_file.close()+# Write to temp filePath(temp_path).write_bytes(response.content)return temp_path, url_type, metaexcept Exception:# Clean up on error- try:- p = Path(temp_path)- if p.exists():- p.unlink(missing_ok=True)- except Exception:- pass+ if temp_path:+ try:+ p = Path(temp_path)+ if p.exists():+ p.unlink(missing_ok=True)+ except Exception:+ passraise+ def _finalize_download_metadata(+ self,+ url: str,+ initial_url_type: URLType,+ initial_meta: Dict[str, Any],+ response_headers: Mapping[str, str],+ content: bytes,+ ) -> Dict[str, Any]:+ """Resolve final URL type, extension, and filename using GET data."""+ meta = dict(initial_meta)+ url_type = initial_url_type++ get_meta: Dict[str, Any] = {}+ get_url_type = self._url_detector.detect_from_headers(+ response_headers,+ get_meta,+ detected_by_prefix="get_",+ )+ if get_url_type != URLType.UNKNOWN and self._should_refine_url_type(url_type, get_url_type):+ url_type = get_url_type+ meta.update(get_meta)+ meta["refined_by_get_headers"] = True+ else:+ meta.setdefault("get_content_type_raw", response_headers.get("content-type", ""))+ meta.setdefault(+ "get_content_disposition_raw",+ response_headers.get("content-disposition", ""),+ )+ if get_meta.get("filename_from_disposition"):+ meta.setdefault(+ "get_filename_from_disposition",+ get_meta["filename_from_disposition"],+ )++ magic_url_type, magic_ext = self._detect_from_magic_bytes(content)+ if magic_url_type != URLType.UNKNOWN and self._should_use_magic_bytes(+ url_type, initial_meta+ ):+ url_type = magic_url_type+ meta["detected_by"] = "magic_bytes"+ meta["magic_extension"] = magic_ext+ meta["refined_by_magic_bytes"] = True++ meta["resolved_url_type"] = url_type+ if meta.get("refined_by_magic_bytes") and magic_ext:+ ext = magic_ext+ else:+ ext = self._determine_file_extension(url, url_type, meta)++ original_filename = meta.get("filename_from_disposition")+ if not original_filename:+ original_filename = self._extract_filename_from_url(url)+ if original_filename == "download" and ext and ext != ".html":+ original_filename = f"download{ext}"++ meta.update({"extension": ext, "original_filename": original_filename})+ return meta++ @staticmethod+ def _should_refine_url_type(current: URLType, candidate: URLType) -> bool:+ """Only replace ambiguous/default webpage guesses with file types."""+ if candidate in (URLType.UNKNOWN, URLType.WEBPAGE):+ return False+ if current in (URLType.UNKNOWN, URLType.WEBPAGE):+ return True+ return current == candidate++ @staticmethod+ def _should_use_magic_bytes(current: URLType, initial_meta: Dict[str, Any]) -> bool:+ """Use binary signatures for ambiguous detections, but not explicit URL extensions."""+ if initial_meta.get("detected_by") == "extension":+ return False+ return current in {+ URLType.UNKNOWN,+ URLType.WEBPAGE,+ URLType.DOWNLOAD_TXT,+ URLType.DOWNLOAD_HTML,+ }++ @staticmethod+ def _detect_from_magic_bytes(content: bytes) -> Tuple[URLType, Optional[str]]:+ """Detect common binary types when headers are unavailable or generic."""+ sample = content[:16]+ if sample.startswith(b"\x89PNG\r\n\x1a\n"):+ return URLType.DOWNLOAD_IMAGE, ".png"+ if sample.startswith(b"\xff\xd8\xff"):+ return URLType.DOWNLOAD_IMAGE, ".jpg"+ if sample.startswith(b"GIF87a") or sample.startswith(b"GIF89a"):+ return URLType.DOWNLOAD_IMAGE, ".gif"+ if sample.startswith(b"BM"):+ return URLType.DOWNLOAD_IMAGE, ".bmp"+ if sample.startswith(b"RIFF") and len(sample) >= 12:+ riff_type = sample[8:12]+ if riff_type == b"WEBP":+ return URLType.DOWNLOAD_IMAGE, ".webp"+ if riff_type == b"WAVE":+ return URLType.DOWNLOAD_AUDIO, ".wav"+ if riff_type == b"AVI ":+ return URLType.DOWNLOAD_VIDEO, ".avi"+ if sample.startswith(b"ID3") or sample.startswith(b"\xff\xfb"):+ return URLType.DOWNLOAD_AUDIO, ".mp3"+ if len(sample) >= 12 and sample[4:8] == b"ftyp":+ brand = sample[8:12].lower()+ if brand in {b"qt ", b"moov"}:+ return URLType.DOWNLOAD_VIDEO, ".mov"+ return URLType.DOWNLOAD_VIDEO, ".mp4"+ if sample.startswith(b"\x1f\x8b"):+ return URLType.UNKNOWN, ".gz"+ svg_sample = content[:512].lstrip()+ if svg_sample.startswith(b"<svg") or (+ svg_sample.startswith(b"<?xml") and b"<svg" in svg_sample+ ):+ return URLType.DOWNLOAD_IMAGE, ".svg"+ if sample.startswith(b"%PDF-"):+ return URLType.DOWNLOAD_PDF, ".pdf"+ if sample.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):+ return URLType.DOWNLOAD_DOCUMENT, ".doc"+ if (+ sample.startswith(b"PK\x03\x04")+ or sample.startswith(b"PK\x05\x06")+ or sample.startswith(b"PK\x07\x08")+ ):+ return URLType.DOWNLOAD_DOCUMENT, HTTPAccessor._detect_zip_based_extension(content)+ return URLType.UNKNOWN, None++ @staticmethod+ def _detect_zip_based_extension(content: bytes) -> str:+ """Differentiate Office/EPUB/ZIP files that all share the ZIP signature."""+ try:+ from io import BytesIO+ from zipfile import ZipFile++ with ZipFile(BytesIO(content)) as zf:+ names = set(zf.namelist())+ if "word/document.xml" in names or any(name.startswith("word/") for name in names):+ return ".docx"+ if "xl/workbook.xml" in names or any(name.startswith("xl/") for name in names):+ return ".xlsx"+ if "ppt/presentation.xml" in names or any(+ name.startswith("ppt/") for name in names+ ):+ return ".pptx"+ if "mimetype" in names:+ try:+ if zf.read("mimetype", 64) == b"application/epub+zip":+ return ".epub"+ except Exception:+ pass+ except Exception:+ return ".zip"+ return ".zip"+def _determine_file_extension(self,url: str,diff --git a/tests/parse/test_url_filename_preservation.py b/tests/parse/test_url_filename_preservation.pyindex 42a62ac66..e7cc16dc1 100644--- a/tests/parse/test_url_filename_preservation.py+++ b/tests/parse/test_url_filename_preservation.py@@ -9,6 +9,7 @@ Verifies fix for https://github.com/volcengine/OpenViking/issues/251:import pytest+from openviking.parse.accessors.http_accessor import HTTPAccessorfrom openviking.parse.parsers.html import HTMLParser, URLType, URLTypeDetector@@ -118,3 +119,267 @@ class TestURLTypeDetectorCodeExtensions:url = "https://example.com/page.html"url_type, meta = await self.detector.detect(url)assert url_type == URLType.DOWNLOAD_HTML++ @pytest.mark.asyncio+ async def test_signed_png_extension_detected_without_head(self):+ url = (+ "https://example.com/assets/image.png?"+ "OSSAccessKeyId=key&Expires=1779533415&Signature=abc%3D"+ )+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_IMAGE+ assert meta["detected_by"] == "extension"++ @pytest.mark.asyncio+ async def test_signed_docx_extension_detected_without_head(self):+ url = (+ "https://example.com/docs/report.docx?"+ "OSSAccessKeyId=key&Expires=1779533415&Signature=abc%3D"+ )+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_DOCUMENT+ assert meta["detected_by"] == "extension"++ @pytest.mark.asyncio+ async def test_signed_doc_extension_detected_without_head(self):+ url = (+ "https://example.com/docs/report.doc?"+ "OSSAccessKeyId=key&Expires=1779533415&Signature=abc%3D"+ )+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_DOCUMENT+ assert meta["detected_by"] == "extension"+++class TestHTTPAccessorGetFallback:+ """Test GET header/content fallback for signed URLs that reject HEAD."""++ @pytest.mark.asyncio+ async def test_get_content_disposition_refines_extensionless_docx_url(self, monkeypatch):+ _patch_httpx_client(+ monkeypatch,+ headers={+ "content-disposition": 'attachment; filename="report.docx"',+ "content-type": "application/octet-stream",+ },+ content=_zip_bytes({"word/document.xml": b"<w:document />"}),+ )++ accessor = HTTPAccessor()+ temp_path, url_type, meta = await accessor._download_url(+ "https://example.com/download?id=1"+ )++ assert url_type == URLType.DOWNLOAD_DOCUMENT+ assert temp_path.endswith(".docx")+ assert meta["original_filename"] == "report.docx"+ assert meta["detected_by"] == "get_content_disposition"+ assert meta["refined_by_get_headers"] is True++ @pytest.mark.asyncio+ async def test_get_content_type_refines_extensionless_png_url(self, monkeypatch):+ _patch_httpx_client(+ monkeypatch,+ headers={"content-type": "image/png"},+ content=b"\x89PNG\r\n\x1a\nimage",+ )++ accessor = HTTPAccessor()+ temp_path, url_type, meta = await accessor._download_url(+ "https://example.com/download?id=2"+ )++ assert url_type == URLType.DOWNLOAD_IMAGE+ assert temp_path.endswith(".png")+ assert meta["original_filename"] == "download.png"+ assert meta["detected_by"] == "get_media_type_pattern"+ assert meta["refined_by_get_headers"] is True++ @pytest.mark.asyncio+ async def test_magic_bytes_refines_generic_extensionless_legacy_doc_url(self, monkeypatch):+ _patch_httpx_client(+ monkeypatch,+ headers={"content-type": "application/octet-stream"},+ content=b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1legacy-doc",+ )++ accessor = HTTPAccessor()+ temp_path, url_type, meta = await accessor._download_url(+ "https://example.com/download?id=3"+ )++ assert url_type == URLType.DOWNLOAD_DOCUMENT+ assert temp_path.endswith(".doc")+ assert meta["original_filename"] == "download.doc"+ assert meta["detected_by"] == "magic_bytes"+ assert meta["refined_by_magic_bytes"] is True++ @pytest.mark.asyncio+ async def test_magic_bytes_distinguishes_extensionless_xlsx_url(self, monkeypatch):+ _patch_httpx_client(+ monkeypatch,+ headers={"content-type": "application/octet-stream"},+ content=_zip_bytes({"xl/workbook.xml": b"<workbook />"}),+ )++ accessor = HTTPAccessor()+ temp_path, url_type, meta = await accessor._download_url(+ "https://example.com/download?id=4"+ )++ assert url_type == URLType.DOWNLOAD_DOCUMENT+ assert temp_path.endswith(".xlsx")+ assert meta["original_filename"] == "download.xlsx"++ @pytest.mark.asyncio+ async def test_magic_bytes_distinguishes_extensionless_zip_url(self, monkeypatch):+ _patch_httpx_client(+ monkeypatch,+ headers={"content-type": "application/octet-stream"},+ content=_zip_bytes({"files/readme.txt": b"hello"}),+ )++ accessor = HTTPAccessor()+ temp_path, url_type, meta = await accessor._download_url(+ "https://example.com/download?id=5"+ )++ assert url_type == URLType.DOWNLOAD_DOCUMENT+ assert temp_path.endswith(".zip")+ assert meta["original_filename"] == "download.zip"++ @pytest.mark.asyncio+ async def test_non_2xx_head_headers_do_not_pollute_get_detection(self, monkeypatch):+ _patch_httpx_client(+ monkeypatch,+ headers={"content-type": "image/png"},+ content=b"\x89PNG\r\n\x1a\nimage",+ head_headers={"content-type": "application/xml"},+ head_status_code=403,+ fail_head=False,+ )++ accessor = HTTPAccessor()+ temp_path, url_type, meta = await accessor._download_url(+ "https://example.com/download?id=head-403"+ )++ assert url_type == URLType.DOWNLOAD_IMAGE+ assert temp_path.endswith(".png")+ assert meta["head_status_skipped"] == 403+ assert meta["detected_by"] == "get_media_type_pattern"+ assert meta["content_type_raw"] != "application/xml"++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "content, expected_url_type, expected_ext",+ [+ (b"BM" + b"\x00" * 14, URLType.DOWNLOAD_IMAGE, ".bmp"),+ (b"RIFF" + b"\x00" * 4 + b"WEBP" + b"data", URLType.DOWNLOAD_IMAGE, ".webp"),+ (b"RIFF" + b"\x00" * 4 + b"WAVE" + b"data", URLType.DOWNLOAD_AUDIO, ".wav"),+ (b"RIFF" + b"\x00" * 4 + b"AVI " + b"data", URLType.DOWNLOAD_VIDEO, ".avi"),+ (b"ID3" + b"\x00" * 13, URLType.DOWNLOAD_AUDIO, ".mp3"),+ (b"\x00\x00\x00\x18ftypisom" + b"\x00" * 4, URLType.DOWNLOAD_VIDEO, ".mp4"),+ (b"\x00\x00\x00\x14ftypqt " + b"\x00" * 4, URLType.DOWNLOAD_VIDEO, ".mov"),+ (b'<?xml version="1.0"?><svg></svg>', URLType.DOWNLOAD_IMAGE, ".svg"),+ ],+ )+ async def test_magic_bytes_detect_common_media_types(+ self, monkeypatch, content, expected_url_type, expected_ext+ ):+ _patch_httpx_client(+ monkeypatch,+ headers={"content-type": "application/octet-stream"},+ content=content,+ )++ accessor = HTTPAccessor()+ temp_path, url_type, meta = await accessor._download_url(+ "https://example.com/download?id=media"+ )++ assert url_type == expected_url_type+ assert temp_path.endswith(expected_ext)+ assert meta["original_filename"] == f"download{expected_ext}"+ assert meta["detected_by"] == "magic_bytes"++ @pytest.mark.asyncio+ async def test_magic_bytes_recognizes_gzip_without_routing_to_document(self, monkeypatch):+ _patch_httpx_client(+ monkeypatch,+ headers={"content-type": "application/octet-stream"},+ content=b"\x1f\x8b\x08\x00gzip",+ )++ accessor = HTTPAccessor()+ temp_path, url_type, meta = await accessor._download_url(+ "https://example.com/download?id=gzip"+ )++ assert url_type == URLType.WEBPAGE+ assert temp_path.endswith(".html")+ assert meta["resolved_url_type"] == URLType.WEBPAGE+ assert meta["detected_by"] == "default"++ def test_zip_detection_falls_back_to_zip_on_parse_error(self):+ assert HTTPAccessor._detect_zip_based_extension(b"PK\x03\x04not-a-valid-zip") == ".zip"++ def test_document_default_extension_is_zip(self):+ assert URLTypeDetector().get_extension_for_type(URLType.DOWNLOAD_DOCUMENT) == ".zip"+++def _patch_httpx_client(+ monkeypatch,+ headers,+ content: bytes,+ *,+ head_headers=None,+ head_status_code: int = 200,+ fail_head: bool = True,+) -> None:+ """Patch httpx.AsyncClient used by HTTPAccessor for HEAD and GET."""++ class FakeResponse:+ def __init__(self, response_headers, response_content, status_code: int = 200):+ self.status_code = status_code+ self.headers = response_headers+ self.content = response_content++ def raise_for_status(self):+ return None++ response_headers = headers+ response_head_headers = head_headers if head_headers is not None else headers++ class FakeAsyncClient:+ def __init__(self, **_kwargs):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, *_args):+ return False++ async def head(self, _url):+ if fail_head:+ raise RuntimeError("SignatureDoesNotMatch")+ return FakeResponse(response_head_headers, b"", head_status_code)++ async def get(self, _url, headers=None):+ return FakeResponse(response_headers, content)++ import httpx++ monkeypatch.setattr(httpx, "AsyncClient", FakeAsyncClient)+++def _zip_bytes(files):+ from io import BytesIO+ from zipfile import ZipFile++ buffer = BytesIO()+ with ZipFile(buffer, "w") as zf:+ for name, content in files.items():+ zf.writestr(name, content)+ return buffer.getvalue()