Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(parse): normalize MIME aliases with parameters (#3393)
Wu JiaCheng061359a2f54b
diff --git a/openviking/parse/accessors/http_accessor.py b/openviking/parse/accessors/http_accessor.pyindex dab3dac03..7fe50ed58 100644--- a/openviking/parse/accessors/http_accessor.py+++ b/openviking/parse/accessors/http_accessor.py@@ -295,9 +295,12 @@ class URLTypeDetector:media_type_str = content_type.lower().strip()# Handle common aliases- if media_type_str in MEDIA_TYPE_ALIASES:- meta["media_type_alias"] = media_type_str- media_type_str = MEDIA_TYPE_ALIASES[media_type_str]+ media_type_token = media_type_str.split(";", 1)[0].strip()+ if media_type_token in MEDIA_TYPE_ALIASES:+ meta["media_type_alias"] = media_type_token+ media_type_str = (+ MEDIA_TYPE_ALIASES[media_type_token] + media_type_str[len(media_type_token) :]+ )# Parse into structured IANAMediaTypetry:diff --git a/openviking/parse/accessors/mime_types.py b/openviking/parse/accessors/mime_types.pyindex 29cce1b03..270f0aa42 100644--- a/openviking/parse/accessors/mime_types.py+++ b/openviking/parse/accessors/mime_types.py@@ -183,6 +183,12 @@ MEDIA_TYPE_ALIASES: Dict[str, str] = {"video/mp4": "video/mp4", # Keep as is (standard)}++def _normalize_media_type(media_type: str) -> str:+ """Return the media type token without parameters."""+ return media_type.lower().strip().split(";", 1)[0].strip()++# Comprehensive IANA media type to file extension mapping# Based on IANA registry and common usageIANA_MEDIA_TYPE_TO_EXTENSION: Dict[str, List[str]] = {@@ -317,16 +323,12 @@ def get_preferred_extension(media_type: str) -> Optional[str]:return None# Normalize and parse- media_type = media_type.lower().strip()+ media_type = _normalize_media_type(media_type)# Handle aliases firstif media_type in MEDIA_TYPE_ALIASES:media_type = MEDIA_TYPE_ALIASES[media_type]- # Strip parameters- if ";" in media_type:- media_type = media_type.split(";", 1)[0].strip()-# Exact matchif media_type in IANA_MEDIA_TYPE_TO_EXTENSION:exts = IANA_MEDIA_TYPE_TO_EXTENSION[media_type]@@ -379,14 +381,10 @@ def get_all_extensions(media_type: str) -> List[str]:if not media_type:return []- media_type = media_type.lower().strip()+ media_type = _normalize_media_type(media_type)# Handle aliases firstif media_type in MEDIA_TYPE_ALIASES:media_type = MEDIA_TYPE_ALIASES[media_type]- # Strip parameters- if ";" in media_type:- media_type = media_type.split(";", 1)[0].strip()-return IANA_MEDIA_TYPE_TO_EXTENSION.get(media_type, [])diff --git a/tests/parse/test_url_filename_preservation.py b/tests/parse/test_url_filename_preservation.pyindex a4b49cd70..4088636c4 100644--- a/tests/parse/test_url_filename_preservation.py+++ b/tests/parse/test_url_filename_preservation.py@@ -10,7 +10,7 @@ Verifies fix for https://github.com/volcengine/OpenViking/issues/251:import pytestfrom openviking.parse.accessors.http_accessor import HTTPAccessor, URLType, URLTypeDetector-from openviking.parse.accessors.mime_types import get_preferred_extension+from openviking.parse.accessors.mime_types import get_all_extensions, get_preferred_extensionclass TestExtractFilenameFromUrl:@@ -173,6 +173,19 @@ class TestHTTPAccessorGetFallback:def test_audio_ac3_mime_type_preserves_extension(self):assert get_preferred_extension("audio/ac3") == ".ac3"+ def test_content_type_alias_with_parameters_preserves_extension(self):+ assert get_preferred_extension("image/jpg; profile=display-p3") == ".jpg"+ assert get_all_extensions("image/jpg; profile=display-p3") == [".jpg", ".jpeg"]++ def test_content_type_alias_with_parameters_routes_to_pdf(self):+ detector = URLTypeDetector()+ metadata = {}++ url_type = detector._detect_from_media_type("application/x-pdf; charset=binary", metadata)++ assert url_type == URLType.DOWNLOAD_PDF+ assert metadata["media_type_alias"] == "application/x-pdf"+@pytest.mark.asyncioasync def test_ac3_url_with_generic_content_type_stays_audio(self, monkeypatch):_patch_httpx_client(