Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
HTTP URL Accessor.
Fetches HTTP/HTTPS URLs and makes them available as local files.
This is the DataAccessor layer extracted from HTMLParser.
Features:
- Downloads web pages to local HTML files
- Downloads files (PDF, Markdown, etc.) to local files
- Supports GitHub/GitLab blob to raw URL conversion
- Follows redirects
- Network guard integration
- Detailed error classification (network, timeout, auth, etc.)
- IANA Media Type (MIME) based content detection for URLs without file extensions
"""
import socket
import tempfile
from enum import Enum
from pathlib import Path
from typing import Any, Dict, Mapping, Optional, Tuple, Union
from urllib.parse import unquote, urlparse
from openviking.parse.base import lazy_import
from openviking.parse.network import (
ParserNetworkContext,
ParserNetworkOperation,
create_parser_httpx_client,
)
from openviking.parse.parsers.constants import CODE_EXTENSIONS
from openviking.parse.parsers.media.constants import (
AUDIO_EXTENSIONS,
IMAGE_EXTENSIONS,
VIDEO_EXTENSIONS,
)
from openviking.parse.temp import get_parser_download_temp_dir
from openviking.utils import is_code_hosting_blob_url
from openviking.utils.exceptions import error_code_from_http_status
from openviking.utils.network_guard import build_httpx_request_validation_hooks
from openviking_cli.exceptions import (
DeadlineExceededError,
InvalidArgumentError,
OpenVikingError,
PermissionDeniedError,
UnavailableError,
)
from openviking_cli.utils.logger import get_logger
from .base import DataAccessor, LocalResource, SourceType
from .mime_types import MEDIA_TYPE_ALIASES, IANAMediaType, get_preferred_extension
logger = get_logger(__name__)
DOCUMENT_EXTENSIONS = {
".doc",
".docx",
".xls",
".xlsx",
".xlsm",
".pptx",
".epub",
".zip",
}
class URLType(Enum):
"""URL content types for routing to appropriate parsers."""
WEBPAGE = "webpage" # HTML webpage to parse
DOWNLOAD_PDF = "download_pdf" # PDF file download link
DOWNLOAD_MD = "download_md" # Markdown file download link
DOWNLOAD_TXT = "download_txt" # Text file download link
DOWNLOAD_HTML = "download_html" # HTML file download link
DOWNLOAD_IMAGE = "download_image" # Image file download link
DOWNLOAD_AUDIO = "download_audio" # Audio file download link
DOWNLOAD_VIDEO = "download_video" # Video file download link
DOWNLOAD_DOCUMENT = "download_document" # Office/e-book/archive document link
UNKNOWN = "unknown" # Unknown or unsupported type
class URLTypeDetector:
"""
Detector for URL content types.
Uses IANA Media Type (MIME) standards for robust content detection:
1. Check file extension (fast path)
2. Check Content-Disposition header for filename (most reliable)
3. Check Content-Type header (IANA standard media types)
4. Fall back to default behavior
References:
- RFC 6838: Media Type Specifications and Registration Procedures
- RFC 6266: Use of the Content-Disposition Header Field in HTTP
"""
# === Extension to URL type mapping ===
# CODE_EXTENSIONS spread comes first so explicit entries below override
# (e.g., .html/.htm -> DOWNLOAD_HTML instead of DOWNLOAD_TXT)
EXTENSION_MAP: Dict[str, URLType] = {
**dict.fromkeys(CODE_EXTENSIONS, URLType.DOWNLOAD_TXT),
".pdf": URLType.DOWNLOAD_PDF,
".md": URLType.DOWNLOAD_MD,
".markdown": URLType.DOWNLOAD_MD,
".txt": URLType.DOWNLOAD_TXT,
".text": URLType.DOWNLOAD_TXT,
".html": URLType.DOWNLOAD_HTML,
".htm": URLType.DOWNLOAD_HTML,
**dict.fromkeys(IMAGE_EXTENSIONS, URLType.DOWNLOAD_IMAGE),
**dict.fromkeys(AUDIO_EXTENSIONS, URLType.DOWNLOAD_AUDIO),
**dict.fromkeys(VIDEO_EXTENSIONS, URLType.DOWNLOAD_VIDEO),
**dict.fromkeys(DOCUMENT_EXTENSIONS, URLType.DOWNLOAD_DOCUMENT),
}
# === IANA Media Type to URL type mapping ===
# Maps IANA registered media types to our internal URLType
# Patterns can be:
# - Exact match: "application/pdf"
# - Wildcard: "text/*"
# - Type only: "image" (treated as "image/*")
# NOTE: .html/.htm extensions are mapped to DOWNLOAD_HTML via EXTENSION_MAP,
# while text/html Content-Type is mapped to WEBPAGE here for URLs
# without extensions (like https://example.com/page)
MEDIA_TYPE_MAP: Dict[str, URLType] = {
# PDF
"application/pdf": URLType.DOWNLOAD_PDF,
# Markdown
"text/markdown": URLType.DOWNLOAD_MD,
"text/x-markdown": URLType.DOWNLOAD_MD,
# HTML/webpage (for URLs without .html extension)
"text/html": URLType.WEBPAGE,
"application/xhtml+xml": URLType.WEBPAGE,
# Plain text
"text/plain": URLType.DOWNLOAD_TXT,
"text/*": URLType.DOWNLOAD_TXT,
# Media files. Some signed object-storage URLs reject HEAD, but for
# extensionless URLs that do return headers this routes to media parsers.
"image/*": URLType.DOWNLOAD_IMAGE,
"audio/*": URLType.DOWNLOAD_AUDIO,
"video/*": URLType.DOWNLOAD_VIDEO,
# Document formats supported by ParserRegistry. The final extension
# still comes from URL path, Content-Disposition, or IANA media type.
"application/msword": URLType.DOWNLOAD_DOCUMENT,
"application/vnd.ms-excel": URLType.DOWNLOAD_DOCUMENT,
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": URLType.DOWNLOAD_DOCUMENT,
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": URLType.DOWNLOAD_DOCUMENT,
"application/vnd.openxmlformats-officedocument.presentationml.presentation": URLType.DOWNLOAD_DOCUMENT,
"application/epub+zip": URLType.DOWNLOAD_DOCUMENT,
"application/zip": URLType.DOWNLOAD_DOCUMENT,
}
# URLType to file extension mapping
URL_TYPE_TO_EXT: Dict[URLType, str] = {
URLType.WEBPAGE: ".html",
URLType.DOWNLOAD_PDF: ".pdf",
URLType.DOWNLOAD_MD: ".md",
URLType.DOWNLOAD_TXT: ".txt",
URLType.DOWNLOAD_HTML: ".html",
URLType.DOWNLOAD_IMAGE: ".png",
URLType.DOWNLOAD_AUDIO: ".mp3",
URLType.DOWNLOAD_VIDEO: ".mp4",
URLType.DOWNLOAD_DOCUMENT: ".zip",
URLType.UNKNOWN: ".html",
}
def __init__(self, timeout: float = 10.0):
"""Initialize URL type detector."""
self.timeout = timeout
async def detect(
self,
url: str,
timeout: Optional[float] = None,
request_validator=None,
headers: Optional[Mapping[str, str]] = None,
) -> Tuple[URLType, Dict[str, Any]]:
"""
Detect URL content type using IANA standards.
Detection order (most reliable to least reliable):
1. File extension from URL path (if valid and recognized)
2. Filename from Content-Disposition header (RFC 6266)
3. IANA Media Type from Content-Type header (RFC 6838)
4. Default to WEBPAGE
Args:
url: URL to detect
timeout: HTTP request timeout in seconds (optional, overrides detector's default)
request_validator: Optional network request validator
Returns:
(URLType, metadata dict with detection details)
"""
meta = {
"url": url,
"detected_by": "unknown",
}
parsed = urlparse(url)
path_lower = parsed.path.lower()
valid_extensions = set(self.EXTENSION_MAP.keys())
# === Step 1: Check extension from URL path ===
path_ext = Path(path_lower).suffix
if path_ext and path_ext in valid_extensions:
for ext, url_type in self.EXTENSION_MAP.items():
if path_lower.endswith(ext):
meta["detected_by"] = "extension"
meta["extension"] = ext
return url_type, meta
# === Step 2: Send HEAD request for headers ===
try:
httpx = lazy_import("httpx")
client_kwargs = {
"timeout": timeout if timeout is not None else self.timeout,
"follow_redirects": True,
}
event_hooks = build_httpx_request_validation_hooks(request_validator)
if event_hooks:
client_kwargs["event_hooks"] = event_hooks
client_kwargs["trust_env"] = False
client = create_parser_httpx_client(
ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, url),
client_cls=httpx.AsyncClient,
**client_kwargs,
)
async with client as active_client:
response = await active_client.head(url, headers=headers)
meta["status_code"] = response.status_code
if not (200 <= response.status_code < 300):
meta["head_status_skipped"] = response.status_code
raise RuntimeError(f"HEAD returned {response.status_code}; headers not trusted")
# === Step 2a/2b: Check Content-Disposition and Content-Type ===
url_type = self.detect_from_headers(response.headers, meta)
if url_type != URLType.UNKNOWN:
return url_type, meta
except PermissionDeniedError:
raise
except Exception as e:
meta["detection_error"] = str(e)
logger.debug(f"[URLTypeDetector] HEAD request failed: {e}, falling back to default")
# === Step 3: Default behavior ===
meta["detected_by"] = "default"
return URLType.WEBPAGE, meta
def detect_from_headers(
self,
headers: Mapping[str, str],
meta: Dict[str, Any],
detected_by_prefix: str = "",
) -> URLType:
"""
Detect URL type from HTTP response headers.
Args:
headers: Response headers from HEAD or GET.
meta: Metadata dict to update.
detected_by_prefix: Optional prefix for meta["detected_by"], e.g. "get_".
Returns:
Detected URLType, or URLType.UNKNOWN if no header matched.
"""
content_type_raw = headers.get("content-type", "")
content_disposition = headers.get("content-disposition", "")
meta["content_type_raw"] = content_type_raw
meta["content_disposition_raw"] = content_disposition
# Check Content-Disposition for filename (RFC 6266)
filename_from_disposition = self._extract_filename_from_disposition(content_disposition)
if filename_from_disposition:
meta["filename_from_disposition"] = filename_from_disposition
filename_lower = filename_from_disposition.lower()
for ext, url_type in self.EXTENSION_MAP.items():
if filename_lower.endswith(ext):
meta["detected_by"] = f"{detected_by_prefix}content_disposition"
meta["extension"] = ext
return url_type
# Check Content-Type (RFC 6838)
if content_type_raw:
url_type = self._detect_from_media_type(
content_type_raw,
meta,
detected_by_prefix=detected_by_prefix,
)
if url_type != URLType.UNKNOWN:
return url_type
return URLType.UNKNOWN
def _detect_from_media_type(
self,
content_type: str,
meta: Dict[str, Any],
detected_by_prefix: str = "",
) -> URLType:
"""
Detect URL type from IANA media type.
Args:
content_type: Content-Type header value
meta: Metadata dict to update
Returns:
Detected URLType, or URLType.UNKNOWN if no match
"""
# Normalize and parse according to IANA standards
media_type_str = content_type.lower().strip()
# Handle common aliases
media_type_token = media_type_str.split(";", 1)[0].strip()
if media_type_token in MEDIA_TYPE_ALIASES:
meta["media_type_alias"] = media_type_token
media_type_str = (
MEDIA_TYPE_ALIASES[media_type_token] + media_type_str[len(media_type_token) :]
)
# Parse into structured IANAMediaType
try:
media_type = IANAMediaType.parse(media_type_str)
meta["media_type"] = str(media_type)
meta["media_type_type"] = media_type.type
meta["media_type_subtype"] = media_type.subtype
if media_type.suffix:
meta["media_type_suffix"] = media_type.suffix
except Exception as e:
logger.debug(f"[URLTypeDetector] Failed to parse media type: {e}")
meta["media_type_parse_error"] = str(e)
return URLType.UNKNOWN
# Check for exact match first
media_type_key = f"{media_type.type}/{media_type.subtype}"
if media_type.suffix:
media_type_with_suffix = f"{media_type_key}+{media_type.suffix}"
if media_type_with_suffix in self.MEDIA_TYPE_MAP:
meta["detected_by"] = f"{detected_by_prefix}media_type_suffix"
return self.MEDIA_TYPE_MAP[media_type_with_suffix]
if media_type_key in self.MEDIA_TYPE_MAP:
meta["detected_by"] = f"{detected_by_prefix}media_type"
return self.MEDIA_TYPE_MAP[media_type_key]
# Check for wildcard matches
for pattern, url_type in self.MEDIA_TYPE_MAP.items():
if media_type.matches(pattern):
meta["detected_by"] = f"{detected_by_prefix}media_type_pattern"
meta["media_type_pattern"] = pattern
return url_type
return URLType.UNKNOWN
@staticmethod
def _extract_filename_from_disposition(content_disposition: str) -> Optional[str]:
"""
Extract filename from Content-Disposition header per RFC 6266.
Handles formats:
- inline; filename="2601.00014v1.pdf"
- attachment; filename=document.pdf
- attachment; filename*=UTF-8''encoded.pdf
- attachment; filename="foo.pdf"; size=12345
Args:
content_disposition: Content-Disposition header value
Returns:
Extracted filename, or None if not found
"""
if not content_disposition:
return None
import re
content_disposition = content_disposition.strip()
# Try filename*=UTF-8''... format first (RFC 5987)
utf8_match = re.search(r"filename\*=UTF-8''([^;]+)", content_disposition, re.I)
if utf8_match:
from urllib.parse import unquote
return unquote(utf8_match.group(1))
# Try filename="..." format (quoted-string)
quoted_match = re.search(r'filename="([^"]+)"', content_disposition, re.I)
if quoted_match:
return quoted_match.group(1)
# Try filename=... format (token)
simple_match = re.search(r"filename=([^;]+)", content_disposition, re.I)
if simple_match:
return simple_match.group(1).strip()
return None
def get_extension_for_type(self, url_type: URLType) -> str:
"""Get file extension for URL type."""
return self.URL_TYPE_TO_EXT.get(url_type, ".html")
class HTTPAccessor(DataAccessor):
"""
Accessor for HTTP/HTTPS URLs.
Features:
- Downloads web pages to local HTML files
- Downloads files (PDF, Markdown, etc.) to local files
- Supports GitHub/GitLab blob to raw URL conversion
- Follows redirects
- Network guard integration
- Detailed error classification (network, timeout, auth, etc.)
- IANA Media Type based detection for URLs without extensions
"""
PRIORITY = 50 # Lower than GitAccessor, higher than fallback
DEFAULT_USER_AGENT = (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
)
def __init__(
self,
timeout: float = 30.0,
user_agent: Optional[str] = None,
):
"""Initialize HTTP accessor."""
self.timeout = timeout
self.user_agent = user_agent or self.DEFAULT_USER_AGENT
self._url_detector = URLTypeDetector(timeout=min(timeout, 10.0))
@property
def priority(self) -> int:
return self.PRIORITY
def can_handle(self, source: Union[str, Path], **kwargs) -> bool:
"""
Check if this accessor can handle the source.
Handles any HTTP/HTTPS URL.
NOTE: GitAccessor and FeishuAccessor have higher priority
and will be checked first for their specific URL types.
"""
source_str = str(source)
return source_str.startswith(("http://", "https://"))
async def access(self, source: Union[str, Path], **kwargs) -> LocalResource:
"""
Fetch the HTTP URL to a local file or directory.
Args:
source: HTTP/HTTPS URL
**kwargs: Additional arguments (request_validator, etc.)
Returns:
LocalResource pointing to the downloaded file or crawled web directory
"""
source_str = str(source)
request_validator = kwargs.get("request_validator")
tos_signature = kwargs.get("tos_signature")
tos_access = kwargs.get("tos_access")
# Download the URL
download_kwargs = {"request_validator": request_validator}
if tos_signature is not None:
download_kwargs["tos_signature"] = tos_signature
if tos_access is not None:
download_kwargs["tos_access"] = tos_access
temp_path, url_type, meta = await self._download_url(source_str, **download_kwargs)
# Both an extensionless text/html page (WEBPAGE) and an explicit
# ``.html``/``.htm`` URL (DOWNLOAD_HTML) are webpages the user may want
# to crawl: route both through WebImporter so ``depth``/``max_pages``
# apply. A plain single-page import is just the ``depth=0`` case.
#
# Exception: a code-hosting single-file URL (GitHub/GitLab ``blob`` or
# GitHub ``raw``) is semantically one file, not a site. ``_download_url``
# already rewrote it to raw and fetched the file, so keep it on the
# single-file path instead of crawling the hosting UI shell.
if url_type in (URLType.WEBPAGE, URLType.DOWNLOAD_HTML) and not is_code_hosting_blob_url(
source_str
):
from openviking.parse.accessors.web_importer import (
WebImporter,
parse_web_import_options,
)
try:
options = parse_web_import_options(kwargs)
result = await WebImporter().import_to_directory(
root_url=source_str,
options=options,
request_validator=request_validator,
)
finally:
Path(temp_path).unlink(missing_ok=True)
meta.update(result.meta)
meta.update(
{
"url": source_str,
"downloaded": True,
"url_type": URLType.WEBPAGE.value,
}
)
return LocalResource(
path=result.path,
source_type=SourceType.HTTP,
original_source=source_str,
meta=meta,
is_temporary=True,
)
# Build metadata
meta.update(
{
"url": source_str,
"downloaded": True,
"url_type": url_type.value,
}
)
return LocalResource(
path=Path(temp_path),
source_type=SourceType.HTTP,
original_source=source_str,
meta=meta,
is_temporary=True,
)
@staticmethod
def _extract_filename_from_url(url: str) -> str:
"""
Extract and URL-decode the original filename from a URL.
Args:
url: URL to extract filename from
Returns:
Decoded filename (e.g., "schemas.py" from ".../schemas.py")
Falls back to "download" if no filename can be extracted.
"""
parsed = urlparse(url)
# URL-decode path to handle encoded characters (e.g., %E7%99%BE -> Chinese chars)
decoded_path = unquote(parsed.path)
basename = Path(decoded_path).name
return basename if basename else "download"
async def _download_url(
self,
url: str,
request_validator=None,
tos_signature: Optional[str] = None,
tos_access: Optional[str] = None,
) -> Tuple[str, URLType, Dict[str, Any]]:
"""
Download URL content to a temporary file.
Args:
url: URL to download
request_validator: Optional network request validator
Returns:
Tuple of (path to temporary file, URLType, metadata dict)
"""
httpx = lazy_import("httpx")
# Convert GitHub/GitLab blob URLs to raw
url = self._convert_to_raw_url(url)
# Detect URL type first to get proper extension
url_type, detect_meta = await self._url_detector.detect(
url,
request_validator=request_validator,
headers=self._request_headers(tos_signature, tos_access),
)
temp_path: Optional[str] = None
try:
# Download content
client_kwargs = {
"timeout": self.timeout,
"follow_redirects": True,
}
event_hooks = build_httpx_request_validation_hooks(request_validator)
if event_hooks:
client_kwargs["event_hooks"] = event_hooks
client_kwargs["trust_env"] = False
client = create_parser_httpx_client(
ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, url),
client_cls=httpx.AsyncClient,
**client_kwargs,
)
async with client as active_client:
headers = self._request_headers(tos_signature, tos_access)
try:
response = await active_client.get(url, headers=headers)
response.raise_for_status()
except httpx.ConnectError as e:
cause: BaseException | None = e
while cause is not None:
if isinstance(cause, socket.gaierror) and cause.errno == socket.EAI_NONAME:
raise InvalidArgumentError(
f"Source URL host does not exist: {url}", details={"source": url}
) from e
cause = cause.__cause__ or cause.__context__
raise UnavailableError(
"source URL", reason=f"Could not connect to {url}: {e}"
) from e
except httpx.TimeoutException as e:
raise DeadlineExceededError(f"Fetch source URL {url}", self.timeout) from e
except httpx.HTTPStatusError as e:
status_code = e.response.status_code
raise OpenVikingError(
f"Source URL returned HTTP {status_code}: {url}",
code=error_code_from_http_status(status_code),
details={"source": url, "upstream_status_code": status_code},
) from e
except httpx.InvalidURL as e:
raise InvalidArgumentError(f"Invalid source URL: {url}") from e
except httpx.RequestError as e:
raise UnavailableError("source URL", reason=str(e)) from e
meta = self._finalize_download_metadata(
url=url,
initial_url_type=url_type,
initial_meta=detect_meta,
response_headers=response.headers,
content=response.content,
)
url_type = meta["resolved_url_type"]
ext = meta["extension"]
# Create temp file after GET headers/content have had a chance
# to refine detection. This avoids routing extensionless signed
# URLs as HTML when HEAD is rejected by object storage.
temp_file = tempfile.NamedTemporaryFile(
delete=False,
suffix=ext,
dir=get_parser_download_temp_dir(),
)
temp_path = temp_file.name
temp_file.close()
# Write to temp file
Path(temp_path).write_bytes(response.content)
return temp_path, url_type, meta
except Exception:
# Clean up on error
if temp_path:
try:
p = Path(temp_path)
if p.exists():
p.unlink(missing_ok=True)
except Exception:
pass
raise
def _request_headers(
self,
tos_signature: Optional[str],
tos_access: Optional[str],
) -> Dict[str, str]:
headers = {"User-Agent": self.user_agent}
if tos_signature is not None and tos_access is not None:
raise ValueError("tos_signature and tos_access cannot both be provided")
if tos_signature is not None:
headers["X-Tos-Signature"] = tos_signature
elif tos_access is not None:
headers["X-Tos-Access"] = tos_access
return headers
def _finalize_download_metadata(
self,
url: str,
initial_url_type: URLType,
initial_meta: Dict[str, Any],
response_headers: Mapping[str, str],
content: bytes,
) -> Dict[str, Any]:
"""Resolve final URL type, extension, and filename using GET data."""
meta = dict(initial_meta)
url_type = initial_url_type
get_meta: Dict[str, Any] = {}
get_url_type = self._url_detector.detect_from_headers(
response_headers,
get_meta,
detected_by_prefix="get_",
)
if get_url_type != URLType.UNKNOWN and self._should_refine_url_type(
url_type,
get_url_type,
):
url_type = get_url_type
meta.update(get_meta)
meta["refined_by_get_headers"] = True
else:
meta.setdefault("get_content_type_raw", response_headers.get("content-type", ""))
meta.setdefault(
"get_content_disposition_raw",
response_headers.get("content-disposition", ""),
)
if get_meta.get("filename_from_disposition"):
meta.setdefault(
"get_filename_from_disposition",
get_meta["filename_from_disposition"],
)
magic_url_type, magic_ext = self._detect_from_magic_bytes(content)
if magic_url_type != URLType.UNKNOWN and self._should_use_magic_bytes(
url_type, initial_meta
):
url_type = magic_url_type
meta["detected_by"] = "magic_bytes"
meta["magic_extension"] = magic_ext
meta["refined_by_magic_bytes"] = True
meta["resolved_url_type"] = url_type
if meta.get("refined_by_magic_bytes") and magic_ext:
ext = magic_ext
else:
ext = self._determine_file_extension(url, url_type, meta)
original_filename = meta.get("filename_from_disposition")
if not original_filename:
original_filename = self._extract_filename_from_url(url)
if original_filename == "download" and ext and ext != ".html":
original_filename = f"download{ext}"
meta.update({"extension": ext, "original_filename": original_filename})
return meta
@staticmethod
def _should_refine_url_type(
current: URLType,
candidate: URLType,
) -> bool:
"""Only replace ambiguous/default webpage guesses with file types."""
if candidate in (URLType.UNKNOWN, URLType.WEBPAGE):
return False
if current in (URLType.UNKNOWN, URLType.WEBPAGE):
return True
return current == candidate
@staticmethod
def _should_use_magic_bytes(current: URLType, initial_meta: Dict[str, Any]) -> bool:
"""Use binary signatures for ambiguous detections, but not explicit URL extensions."""
if initial_meta.get("detected_by") == "extension":
return False
return current in {
URLType.UNKNOWN,
URLType.WEBPAGE,
URLType.DOWNLOAD_TXT,
URLType.DOWNLOAD_HTML,
}
@staticmethod
def _detect_from_magic_bytes(content: bytes) -> Tuple[URLType, Optional[str]]:
"""Detect common binary types when headers are unavailable or generic."""
sample = content[:16]
if sample.startswith(b"\x89PNG\r\n\x1a\n"):
return URLType.DOWNLOAD_IMAGE, ".png"
if sample.startswith(b"\xff\xd8\xff"):
return URLType.DOWNLOAD_IMAGE, ".jpg"
if sample.startswith(b"GIF87a") or sample.startswith(b"GIF89a"):
return URLType.DOWNLOAD_IMAGE, ".gif"
if sample.startswith(b"BM"):
return URLType.DOWNLOAD_IMAGE, ".bmp"
if sample.startswith(b"RIFF") and len(sample) >= 12:
riff_type = sample[8:12]
if riff_type == b"WEBP":
return URLType.DOWNLOAD_IMAGE, ".webp"
if riff_type == b"WAVE":
return URLType.DOWNLOAD_AUDIO, ".wav"
if riff_type == b"AVI ":
return URLType.DOWNLOAD_VIDEO, ".avi"
if sample.startswith(b"ID3") or sample.startswith(b"\xff\xfb"):
return URLType.DOWNLOAD_AUDIO, ".mp3"
if sample.startswith(b"\x0b\x77"):
return URLType.DOWNLOAD_AUDIO, ".ac3"
if len(sample) >= 12 and sample[4:8] == b"ftyp":
brand = sample[8:12].lower()
if brand in {b"qt ", b"moov"}:
return URLType.DOWNLOAD_VIDEO, ".mov"
return URLType.DOWNLOAD_VIDEO, ".mp4"
if sample.startswith(b"\x1f\x8b"):
return URLType.UNKNOWN, ".gz"
svg_sample = content[:512].lstrip()
if svg_sample.startswith(b"<svg") or (
svg_sample.startswith(b"<?xml") and b"<svg" in svg_sample
):
return URLType.DOWNLOAD_IMAGE, ".svg"
if sample.startswith(b"%PDF-"):
return URLType.DOWNLOAD_PDF, ".pdf"
if sample.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):
return URLType.DOWNLOAD_DOCUMENT, ".doc"
if (
sample.startswith(b"PK\x03\x04")
or sample.startswith(b"PK\x05\x06")
or sample.startswith(b"PK\x07\x08")
):
return URLType.DOWNLOAD_DOCUMENT, HTTPAccessor._detect_zip_based_extension(content)
return URLType.UNKNOWN, None
@staticmethod
def _detect_zip_based_extension(content: bytes) -> str:
"""Differentiate Office/EPUB/ZIP files that all share the ZIP signature."""
try:
from io import BytesIO
from zipfile import ZipFile
with ZipFile(BytesIO(content)) as zf:
names = set(zf.namelist())
if "word/document.xml" in names or any(name.startswith("word/") for name in names):
return ".docx"
if "xl/workbook.xml" in names or any(name.startswith("xl/") for name in names):
return ".xlsx"
if "ppt/presentation.xml" in names or any(
name.startswith("ppt/") for name in names
):
return ".pptx"
if "mimetype" in names:
try:
if zf.read("mimetype", 64) == b"application/epub+zip":
return ".epub"
except Exception:
pass
except Exception:
return ".zip"
return ".zip"
def _determine_file_extension(
self,
url: str,
url_type: URLType,
detect_meta: Dict[str, Any],
) -> str:
"""
Determine appropriate file extension using multiple strategies.
Strategy order (most reliable first):
1. Extension from Content-Disposition filename
2. Extension from URL path (if valid)
3. Use IANA media type mapping
4. Use URL type based extension
Args:
url: Original URL
url_type: Detected URL type
detect_meta: Detection metadata
Returns:
File extension including dot (e.g., ".pdf")
"""
valid_extensions = set(URLTypeDetector.EXTENSION_MAP.keys())
# 1. Try extension from Content-Disposition filename
filename_from_disposition = detect_meta.get("filename_from_disposition")
if filename_from_disposition:
ext = Path(filename_from_disposition.lower()).suffix
if ext and ext in valid_extensions:
return ext
# 2. Try extension from URL path (if valid)
parsed = urlparse(url)
decoded_path = unquote(parsed.path)
ext = Path(decoded_path).suffix
if ext and ext.lower() in valid_extensions:
return ext.lower()
# 3. Try IANA media type to extension mapping
media_type_str = detect_meta.get("media_type") or detect_meta.get("content_type_raw")
if media_type_str:
iana_ext = get_preferred_extension(media_type_str)
if iana_ext:
return iana_ext
# 4. Fall back to URL type based extension
return self._url_detector.get_extension_for_type(url_type)
def _convert_to_raw_url(self, url: str) -> str:
"""Convert GitHub/GitLab blob URL to raw URL."""
parsed = urlparse(url)
try:
from openviking_cli.utils.config import get_openviking_config
config = get_openviking_config()
# NOTE: github_domains/gitlab_domains are in CodeConfig, not HTMLConfig
github_domains = config.code.github_domains
gitlab_domains = config.code.gitlab_domains
github_raw_domain = config.code.github_raw_domain
if parsed.netloc in github_domains:
path_parts = parsed.path.strip("/").split("/")
if len(path_parts) >= 4 and path_parts[2] == "blob":
# Remove 'blob'
new_path = "/".join(path_parts[:2] + path_parts[3:])
return f"https://{github_raw_domain}/{new_path}"
if parsed.netloc in gitlab_domains and "/blob/" in parsed.path:
return url.replace("/blob/", "/raw/")
except Exception as e:
logger.debug(
f"[HTTPAccessor] Failed to convert blob URL to raw: {e}, "
f"falling back to original URL: {url}"
)
return url