Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(parse): preserve original filename and extension when importing from URL (#619)
Matt Van Horn45915aeaf914
diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.pyindex 48ae4fa7b..deaf0a86e 100644--- a/openviking/parse/parsers/html.py+++ b/openviking/parse/parsers/html.py@@ -18,7 +18,7 @@ import timefrom enum import Enumfrom pathlib import Pathfrom typing import Any, Dict, List, Optional, Tuple, Union-from urllib.parse import urlparse+from urllib.parse import unquote, urlparsefrom openviking.parse.base import (NodeType,@@ -28,6 +28,7 @@ from openviking.parse.base import (lazy_import,)from openviking.parse.parsers.base_parser import BaseParser+from openviking.parse.parsers.constants import CODE_EXTENSIONSfrom openviking_cli.utils.config import get_openviking_config@@ -53,7 +54,10 @@ class URLTypeDetector:"""# Extension to URL type mapping+ # CODE_EXTENSIONS spread comes first so explicit entries below override+ # (e.g., .html/.htm -> DOWNLOAD_HTML instead of DOWNLOAD_TXT)EXTENSION_MAP = {+ **dict.fromkeys(CODE_EXTENSIONS, URLType.DOWNLOAD_TXT),".pdf": URLType.DOWNLOAD_PDF,".md": URLType.DOWNLOAD_MD,".markdown": URLType.DOWNLOAD_MD,@@ -345,6 +349,24 @@ class HTMLParser(BaseParser):warnings=[f"Failed to fetch webpage: {e}"],)+ @staticmethod+ def _extract_filename_from_url(url: str) -> str:+ """+ Extract and URL-decode the original filename from a URL.++ Args:+ url: URL to extract filename from++ Returns:+ Decoded filename (e.g., "schemas.py" from ".../schemas.py")+ Falls back to "download" if no filename can be extracted.+ """+ parsed = urlparse(url)+ # URL-decode path to handle encoded characters (e.g., %E7%99%BE -> Chinese chars)+ decoded_path = unquote(parsed.path)+ basename = Path(decoded_path).name+ return basename if basename else "download"+async def _handle_download_link(self, url: str, file_type: str, start_time: float, meta: Dict[str, Any], **kwargs) -> ParseResult:@@ -365,22 +387,30 @@ class HTMLParser(BaseParser):# Download to temporary filetemp_path = await self._download_file(url)+ # Extract original filename from URL for use as source_path,+ # so parsers use it instead of the temp file name.+ original_filename = self._extract_filename_from_url(url)+# Get appropriate parserif file_type == "pdf":from openviking.parse.parsers.pdf import PDFParserparser = PDFParser()- result = await parser.parse(temp_path)+ result = await parser.parse(temp_path, resource_name=Path(original_filename).stem)elif file_type == "markdown":from openviking.parse.parsers.markdown import MarkdownParserparser = MarkdownParser()- result = await parser.parse(temp_path)+ content = Path(temp_path).read_text(encoding="utf-8")+ result = await parser.parse_content(+ content, source_path=original_filename, **kwargs+ )elif file_type == "text":- from openviking.parse.parsers.text import TextParser-- parser = TextParser()- result = await parser.parse(temp_path)+ # For text/code files, preserve the original filename and extension.+ # Read the downloaded content and save it with the original name+ # instead of routing through TextParser->MarkdownParser which+ # would rename it to .md and split it into sections.+ result = await self._save_downloaded_text(temp_path, original_filename, start_time)elif file_type == "html":# Parse downloaded HTML locallyreturn await self._parse_local_file(Path(temp_path), start_time, **kwargs)@@ -510,6 +540,57 @@ class HTMLParser(BaseParser):return url+ async def _save_downloaded_text(+ self, temp_path: str, original_filename: str, start_time: float+ ) -> ParseResult:+ """+ Save a downloaded text/code file preserving its original filename and extension.++ Instead of routing through TextParser -> MarkdownParser (which renames to .md+ and splits into sections), this saves the file directly into a VikingFS temp+ directory with its original name.++ Args:+ temp_path: Path to the downloaded temporary file+ original_filename: Original filename from URL (e.g., "schemas.py")+ start_time: Parse start timestamp++ Returns:+ ParseResult with temp_dir_path set+ """+ from openviking.storage.viking_fs import get_viking_fs++ content = Path(temp_path).read_text(encoding="utf-8")+ doc_name = Path(original_filename).stem++ viking_fs = get_viking_fs()+ temp_uri = viking_fs.create_temp_uri()+ await viking_fs.mkdir(temp_uri)++ # Create document root directory (TreeBuilder expects exactly one dir)+ root_dir = f"{temp_uri}/{doc_name}"+ await viking_fs.mkdir(root_dir)++ # Save with original filename (preserving extension)+ file_uri = f"{root_dir}/{original_filename}"+ await viking_fs.write_file(file_uri, content)++ root = ResourceNode(+ type=NodeType.ROOT,+ title=doc_name,+ level=0,+ )++ result = create_parse_result(+ root=root,+ source_path=original_filename,+ source_format="text",+ parser_name="HTMLParser",+ parse_time=time.time() - start_time,+ )+ result.temp_dir_path = temp_uri+ return result+async def _download_file(self, url: str) -> str:"""Download file from URL to temporary location.@@ -527,9 +608,10 @@ class HTMLParser(BaseParser):url = self._convert_to_raw_url(url)- # Determine file extension from URL+ # Determine file extension from URL (decode first to handle encoded paths)parsed = urlparse(url)- ext = Path(parsed.path).suffix or ".tmp"+ decoded_path = unquote(parsed.path)+ ext = Path(decoded_path).suffix or ".tmp"# Create temp filetemp_file = tempfile.NamedTemporaryFile(delete=False, suffix=ext)diff --git a/tests/parse/test_url_filename_preservation.py b/tests/parse/test_url_filename_preservation.pynew file mode 100644index 000000000..42a62ac66--- /dev/null+++ b/tests/parse/test_url_filename_preservation.py@@ -0,0 +1,120 @@+"""Tests for URL filename preservation when importing resources via URL.++Verifies fix for https://github.com/volcengine/OpenViking/issues/251:+- Original filename preserved (not temp file name)+- File extension preserved (.py stays .py, not converted to .md)+- URL-encoded characters decoded properly+- Code file extensions routed to download, not webpage parse+"""++import pytest++from openviking.parse.parsers.html import HTMLParser, URLType, URLTypeDetector+++class TestExtractFilenameFromUrl:+ """Test HTMLParser._extract_filename_from_url."""++ def test_simple_filename(self):+ url = "https://example.com/path/to/schemas.py"+ assert HTMLParser._extract_filename_from_url(url) == "schemas.py"++ def test_url_encoded_path(self):+ url = "https://example.com/%E7%99%BE%E5%BA%A64/src/baidu_search/schemas.py"+ assert HTMLParser._extract_filename_from_url(url) == "schemas.py"++ def test_url_encoded_filename(self):+ url = "https://example.com/path/%E6%96%87%E4%BB%B6.py"+ assert HTMLParser._extract_filename_from_url(url) == "\u6587\u4ef6.py"++ def test_query_params_ignored(self):+ url = "https://example.com/file.py?version=2&token=abc"+ assert HTMLParser._extract_filename_from_url(url) == "file.py"++ def test_no_filename_fallback(self):+ url = "https://example.com/"+ assert HTMLParser._extract_filename_from_url(url) == "download"++ def test_cos_url(self):+ url = (+ "https://cos.ap-beijing.myqcloud.com/bucket/"+ "%E7%99%BE%E5%BA%A64/src/baidu_search/schemas.py"+ )+ assert HTMLParser._extract_filename_from_url(url) == "schemas.py"++ def test_markdown_extension(self):+ url = "https://example.com/docs/README.md"+ assert HTMLParser._extract_filename_from_url(url) == "README.md"++ def test_no_extension(self):+ url = "https://example.com/path/Makefile"+ assert HTMLParser._extract_filename_from_url(url) == "Makefile"+++class TestURLTypeDetectorCodeExtensions:+ """Test that code file extensions are routed to DOWNLOAD_TXT, not WEBPAGE."""++ def setup_method(self):+ self.detector = URLTypeDetector()++ @pytest.mark.asyncio+ async def test_py_extension_detected(self):+ url = "https://example.com/path/schemas.py"+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_TXT+ assert meta["detected_by"] == "extension"++ @pytest.mark.asyncio+ async def test_js_extension_detected(self):+ url = "https://example.com/path/index.js"+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_TXT++ @pytest.mark.asyncio+ async def test_yaml_extension_detected(self):+ url = "https://example.com/config.yaml"+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_TXT++ @pytest.mark.asyncio+ async def test_json_extension_detected(self):+ url = "https://example.com/data.json"+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_TXT++ @pytest.mark.asyncio+ async def test_go_extension_detected(self):+ url = "https://example.com/main.go"+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_TXT++ @pytest.mark.asyncio+ async def test_rs_extension_detected(self):+ url = "https://example.com/lib.rs"+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_TXT++ @pytest.mark.asyncio+ async def test_url_encoded_py_extension(self):+ url = "https://example.com/%E7%99%BE%E5%BA%A64/src/schemas.py"+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_TXT++ @pytest.mark.asyncio+ async def test_md_still_routes_to_markdown(self):+ url = "https://example.com/README.md"+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_MD++ @pytest.mark.asyncio+ async def test_pdf_still_routes_to_pdf(self):+ url = "https://example.com/paper.pdf"+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_PDF++ @pytest.mark.asyncio+ async def test_html_still_routes_to_download_html(self):+ """Ensure .html overrides CODE_EXTENSIONS mapping to DOWNLOAD_TXT."""+ url = "https://example.com/page.html"+ url_type, meta = await self.detector.detect(url)+ assert url_type == URLType.DOWNLOAD_HTML