Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(parse): normalize MIME aliases with parameters (#3393)

Wu JiaCheng061359a2f54b
diff --git a/openviking/parse/accessors/http_accessor.py b/openviking/parse/accessors/http_accessor.py
index dab3dac03..7fe50ed58 100644
--- a/openviking/parse/accessors/http_accessor.py
+++ b/openviking/parse/accessors/http_accessor.py
@@ -295,9 +295,12 @@ class URLTypeDetector:
media_type_str = content_type.lower().strip()
# Handle common aliases
- if media_type_str in MEDIA_TYPE_ALIASES:
- meta["media_type_alias"] = media_type_str
- media_type_str = MEDIA_TYPE_ALIASES[media_type_str]
+ media_type_token = media_type_str.split(";", 1)[0].strip()
+ if media_type_token in MEDIA_TYPE_ALIASES:
+ meta["media_type_alias"] = media_type_token
+ media_type_str = (
+ MEDIA_TYPE_ALIASES[media_type_token] + media_type_str[len(media_type_token) :]
+ )
# Parse into structured IANAMediaType
try:
diff --git a/openviking/parse/accessors/mime_types.py b/openviking/parse/accessors/mime_types.py
index 29cce1b03..270f0aa42 100644
--- a/openviking/parse/accessors/mime_types.py
+++ b/openviking/parse/accessors/mime_types.py
@@ -183,6 +183,12 @@ MEDIA_TYPE_ALIASES: Dict[str, str] = {
"video/mp4": "video/mp4", # Keep as is (standard)
}
+
+def _normalize_media_type(media_type: str) -> str:
+ """Return the media type token without parameters."""
+ return media_type.lower().strip().split(";", 1)[0].strip()
+
+
# Comprehensive IANA media type to file extension mapping
# Based on IANA registry and common usage
IANA_MEDIA_TYPE_TO_EXTENSION: Dict[str, List[str]] = {
@@ -317,16 +323,12 @@ def get_preferred_extension(media_type: str) -> Optional[str]:
return None
# Normalize and parse
- media_type = media_type.lower().strip()
+ media_type = _normalize_media_type(media_type)
# Handle aliases first
if media_type in MEDIA_TYPE_ALIASES:
media_type = MEDIA_TYPE_ALIASES[media_type]
- # Strip parameters
- if ";" in media_type:
- media_type = media_type.split(";", 1)[0].strip()
-
# Exact match
if media_type in IANA_MEDIA_TYPE_TO_EXTENSION:
exts = IANA_MEDIA_TYPE_TO_EXTENSION[media_type]
@@ -379,14 +381,10 @@ def get_all_extensions(media_type: str) -> List[str]:
if not media_type:
return []
- media_type = media_type.lower().strip()
+ media_type = _normalize_media_type(media_type)
# Handle aliases first
if media_type in MEDIA_TYPE_ALIASES:
media_type = MEDIA_TYPE_ALIASES[media_type]
- # Strip parameters
- if ";" in media_type:
- media_type = media_type.split(";", 1)[0].strip()
-
return IANA_MEDIA_TYPE_TO_EXTENSION.get(media_type, [])
diff --git a/tests/parse/test_url_filename_preservation.py b/tests/parse/test_url_filename_preservation.py
index a4b49cd70..4088636c4 100644
--- a/tests/parse/test_url_filename_preservation.py
+++ b/tests/parse/test_url_filename_preservation.py
@@ -10,7 +10,7 @@ Verifies fix for https://github.com/volcengine/OpenViking/issues/251:
import pytest
from openviking.parse.accessors.http_accessor import HTTPAccessor, URLType, URLTypeDetector
-from openviking.parse.accessors.mime_types import get_preferred_extension
+from openviking.parse.accessors.mime_types import get_all_extensions, get_preferred_extension
class TestExtractFilenameFromUrl:
@@ -173,6 +173,19 @@ class TestHTTPAccessorGetFallback:
def test_audio_ac3_mime_type_preserves_extension(self):
assert get_preferred_extension("audio/ac3") == ".ac3"
+ def test_content_type_alias_with_parameters_preserves_extension(self):
+ assert get_preferred_extension("image/jpg; profile=display-p3") == ".jpg"
+ assert get_all_extensions("image/jpg; profile=display-p3") == [".jpg", ".jpeg"]
+
+ def test_content_type_alias_with_parameters_routes_to_pdf(self):
+ detector = URLTypeDetector()
+ metadata = {}
+
+ url_type = detector._detect_from_media_type("application/x-pdf; charset=binary", metadata)
+
+ assert url_type == URLType.DOWNLOAD_PDF
+ assert metadata["media_type_alias"] == "application/x-pdf"
+
@pytest.mark.asyncio
async def test_ac3_url_with_generic_content_type_stays_audio(self, monkeypatch):
_patch_httpx_client(