Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(parser): hash & shorten filenames that exceed filesystem limit (#205)
Ethon361cad61cb31
diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.pyindex 28e47885f..34711ef0c 100644--- a/openviking/parse/parsers/html.py+++ b/openviking/parse/parsers/html.py@@ -11,6 +11,7 @@ Unified parser that handles:Preserves natural document hierarchy and filters out navigation/ads."""+import hashlibimport reimport tempfileimport time@@ -599,8 +600,14 @@ class HTMLParser(BaseParser):return result- def _sanitize_for_path(self, text: str) -> str:- """Sanitize text for use in file path."""+ def _sanitize_for_path(self, text: str, max_length: int = 50) -> str:+ """Sanitize text for use in file path, hash & shorten if too long."""safe = re.sub(r"[^\w\u4e00-\u9fff\s-]", "", text)safe = re.sub(r"\s+", "_", safe)- return safe.strip("_")[:50] or "section"+ safe = safe.strip("_")+ if not safe:+ return "section"+ if len(safe) > max_length:+ hash_suffix = hashlib.sha256(text.encode()).hexdigest()[:8]+ return f"{safe[: max_length - 9]}_{hash_suffix}"+ return safediff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.pyindex e6ddbe3ad..bcb4f48ae 100644--- a/openviking/parse/parsers/markdown.py+++ b/openviking/parse/parsers/markdown.py@@ -17,6 +17,7 @@ The parser handles scenarios:5. Oversized sections without subsections → split by paragraphs"""+import hashlibimport reimport timefrom pathlib import Path@@ -333,10 +334,16 @@ class MarkdownParser(BaseParser):return parts if parts else [content]- def _sanitize_for_path(self, text: str) -> str:+ def _sanitize_for_path(self, text: str, max_length: int = 50) -> str:safe = re.sub(r"[^\w\u4e00-\u9fff\s-]", "", text)safe = re.sub(r"\s+", "_", safe)- return safe.strip("_")[:50] or "section"+ safe = safe.strip("_")+ if not safe:+ return "section"+ if len(safe) > max_length:+ hash_suffix = hashlib.sha256(text.encode()).hexdigest()[:8]+ return f"{safe[: max_length - 9]}_{hash_suffix}"+ return safe# ========== New Parsing Logic (v5.0) ==========@@ -570,25 +577,32 @@ class MarkdownParser(BaseParser):Smart merged filename generation, limited to MAX_MERGED_FILENAME_LENGTH characters.Strategy:- - Single section: Use directly (truncated to MAX_MERGED_FILENAME_LENGTH chars)+ - Single section: Use directly (truncated with hash if needed)- Multiple sections: {first_section}_{count}more (e.g., Intro_3more)- Total length strictly limited: MAX_MERGED_FILENAME_LENGTH characters+ - Hash suffix ensures uniqueness when truncation occurs"""if not sections:return "merged"names = [n for n, _, _ in sections]count = len(names)+ max_len = self.MAX_MERGED_FILENAME_LENGTHif count == 1:- name = names[0][: self.MAX_MERGED_FILENAME_LENGTH]+ name = names[0]else:suffix = f"_{count}more"- max_first_len = self.MAX_MERGED_FILENAME_LENGTH - len(suffix)+ max_first_len = max_len - len(suffix)first_name = names[0][: max(max_first_len, 1)]name = f"{first_name}{suffix}"- name = name[: self.MAX_MERGED_FILENAME_LENGTH].strip("_")+ if len(name) > max_len:+ full_key = "_".join(names)+ hash_suffix = hashlib.sha256(full_key.encode()).hexdigest()[:8]+ name = f"{name[: max_len - 9]}_{hash_suffix}"++ name = name.strip("_")return name or "merged"async def _save_merged(diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex a8e52c72f..611ede50e 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -13,6 +13,7 @@ Responsibilities:"""import asyncio+import hashlibimport jsonfrom dataclasses import dataclass, fieldfrom datetime import datetime@@ -646,12 +647,31 @@ class VikingFS:# ========== URI Conversion ==========+ # Maximum bytes for a single filename component (filesystem limit is typically 255)+ _MAX_FILENAME_BYTES = 255++ @staticmethod+ def _shorten_component(component: str, max_bytes: int = 255) -> str:+ """Shorten a path component if its UTF-8 encoding exceeds max_bytes."""+ if len(component.encode("utf-8")) <= max_bytes:+ return component+ hash_suffix = hashlib.sha256(component.encode("utf-8")).hexdigest()[:8]+ # Trim to fit within max_bytes after adding hash suffix+ prefix = component+ target = max_bytes - len(f"_{hash_suffix}".encode("utf-8"))+ while len(prefix.encode("utf-8")) > target and prefix:+ prefix = prefix[:-1]+ return f"{prefix}_{hash_suffix}"+def _uri_to_path(self, uri: str) -> str:"""viking://user/memories/preferences/test -> /local/user/memories/preferences/test"""remainder = uri[len("viking://") :].strip("/")if not remainder:return "/local"- return f"/local/{remainder}"+ # Ensure each path component does not exceed filesystem filename limit+ parts = remainder.split("/")+ safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]+ return f"/local/{'/'.join(safe_parts)}"def _path_to_uri(self, path: str) -> str:"""/local/user/memories/preferences -> viking://user/memories/preferences"""diff --git a/openviking_cli/utils/downloader.py b/openviking_cli/utils/downloader.pyindex ff263f813..d9a73efa3 100644--- a/openviking_cli/utils/downloader.py+++ b/openviking_cli/utils/downloader.py@@ -7,6 +7,7 @@ Provides basic file download functionality.For URL content parsing, use HTMLParser instead."""+import hashlibimport refrom pathlib import Pathfrom typing import Optional, Tuple@@ -92,8 +93,8 @@ async def download_file(return None, str(e)-def _generate_filename(url: str) -> str:- """Generate filename from URL."""+def _generate_filename(url: str, max_length: int = 50) -> str:+ """Generate filename from URL, hash & shorten if too long."""from urllib.parse import urlparseparsed = urlparse(url)@@ -103,7 +104,17 @@ def _generate_filename(url: str) -> str:name = Path(path).stemname = re.sub(r"[^a-zA-Z0-9_\-\u4e00-\u9fff]", "_", name)name = re.sub(r"_+", "_", name)- return name[:50] if name else "download"+ if not name:+ return "download"+ if len(name) > max_length:+ hash_suffix = hashlib.sha256(url.encode()).hexdigest()[:8]+ return f"{name[: max_length - 9]}_{hash_suffix}"+ return namehost = parsed.netloc.replace(".", "_")- return host[:50] if host else "download"+ if not host:+ return "download"+ if len(host) > max_length:+ hash_suffix = hashlib.sha256(url.encode()).hexdigest()[:8]+ return f"{host[: max_length - 9]}_{hash_suffix}"+ return hostdiff --git a/tests/parse/test_filename_safety.py b/tests/parse/test_filename_safety.pynew file mode 100644index 000000000..9c81cc2e2--- /dev/null+++ b/tests/parse/test_filename_safety.py@@ -0,0 +1,185 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""Tests for filename safety: hash & shorten when names are too long (issue #171)."""++import hashlib+++class TestSanitizeForPath:+ """Test _sanitize_for_path in MarkdownParser and HTMLParser."""++ def _make_md_parser(self):+ from openviking.parse.parsers.markdown import MarkdownParser++ return MarkdownParser()++ def test_short_text_unchanged(self):+ parser = self._make_md_parser()+ assert parser._sanitize_for_path("Hello World") == "Hello_World"++ def test_empty_text_returns_section(self):+ parser = self._make_md_parser()+ assert parser._sanitize_for_path("") == "section"+ assert parser._sanitize_for_path("!!!") == "section"++ def test_special_chars_removed(self):+ parser = self._make_md_parser()+ result = parser._sanitize_for_path("Hello, World! (test)")+ assert result == "Hello_World_test"++ def test_chinese_preserved(self):+ parser = self._make_md_parser()+ result = parser._sanitize_for_path("你好世界")+ assert result == "你好世界"++ def test_long_text_truncated_with_hash(self):+ parser = self._make_md_parser()+ long_text = "a" * 100+ result = parser._sanitize_for_path(long_text)+ assert len(result) <= 50+ expected_hash = hashlib.sha256(long_text.encode()).hexdigest()[:8]+ assert result.endswith(f"_{expected_hash}")++ def test_exact_boundary_not_hashed(self):+ parser = self._make_md_parser()+ text = "a" * 50+ result = parser._sanitize_for_path(text)+ assert result == text+ assert len(result) == 50++ def test_one_over_boundary_hashed(self):+ parser = self._make_md_parser()+ text = "a" * 51+ result = parser._sanitize_for_path(text)+ assert len(result) <= 50+ assert "_" in result # has hash suffix++ def test_custom_max_length(self):+ parser = self._make_md_parser()+ text = "a" * 30+ result = parser._sanitize_for_path(text, max_length=20)+ assert len(result) <= 20+ expected_hash = hashlib.sha256(text.encode()).hexdigest()[:8]+ assert result.endswith(f"_{expected_hash}")++ def test_shell_comment_heading(self):+ """Simulate shell script comments being treated as markdown headings."""+ parser = self._make_md_parser()+ heading = "Usage: curl -fsSL https://raw.githubusercontent.com/volcengine/openviking/refs/tags/[email protected]/crates/ov_cli/install.sh | bash"+ result = parser._sanitize_for_path(heading)+ assert len(result) <= 50+++class TestGenerateMergedFilename:+ """Test _generate_merged_filename in MarkdownParser."""++ def _make_md_parser(self):+ from openviking.parse.parsers.markdown import MarkdownParser++ return MarkdownParser()++ def test_single_short_section(self):+ parser = self._make_md_parser()+ result = parser._generate_merged_filename([("intro", "content", 100)])+ assert result == "intro"++ def test_multiple_sections(self):+ parser = self._make_md_parser()+ sections = [("intro", "c1", 10), ("body", "c2", 20), ("end", "c3", 30)]+ result = parser._generate_merged_filename(sections)+ assert "3more" in result+ assert len(result) <= 32++ def test_empty_sections(self):+ parser = self._make_md_parser()+ assert parser._generate_merged_filename([]) == "merged"++ def test_long_single_name_hashed(self):+ parser = self._make_md_parser()+ long_name = "a" * 100+ result = parser._generate_merged_filename([(long_name, "content", 50)])+ assert len(result) <= 32++ def test_result_never_exceeds_limit(self):+ parser = self._make_md_parser()+ # Create many sections with long names+ sec_list = [(f"very_long_section_name_{i}", f"content_{i}", 10) for i in range(20)]+ result = parser._generate_merged_filename(sec_list)+ assert len(result) <= 32+++class TestShortenComponent:+ """Test VikingFS._shorten_component."""++ def test_short_component_unchanged(self):+ from openviking.storage.viking_fs import VikingFS++ assert VikingFS._shorten_component("hello") == "hello"++ def test_long_component_shortened(self):+ from openviking.storage.viking_fs import VikingFS++ long_name = "a" * 300+ result = VikingFS._shorten_component(long_name)+ assert len(result.encode("utf-8")) <= 255++ def test_exact_255_bytes_unchanged(self):+ from openviking.storage.viking_fs import VikingFS++ name = "a" * 255+ assert VikingFS._shorten_component(name) == name++ def test_256_bytes_shortened(self):+ from openviking.storage.viking_fs import VikingFS++ name = "a" * 256+ result = VikingFS._shorten_component(name)+ assert len(result.encode("utf-8")) <= 255+ expected_hash = hashlib.sha256(name.encode("utf-8")).hexdigest()[:8]+ assert result.endswith(f"_{expected_hash}")++ def test_unicode_multibyte_handling(self):+ from openviking.storage.viking_fs import VikingFS++ # Chinese chars are 3 bytes each in UTF-8+ name = "你" * 100 # 300 bytes+ result = VikingFS._shorten_component(name)+ assert len(result.encode("utf-8")) <= 255++ def test_realistic_long_filename(self):+ """Simulate the exact bug from issue #171."""+ from openviking.storage.viking_fs import VikingFS++ long_filename = (+ "tmp5vacylnx_OpenViking_CLI_Installer_Usage_curl_-fsSL_"+ "httpsrawgithubusercontentcomvolce_Example_curl_-fsSL_"+ "httpsrawgithubusercontentcomvol_Skip_checksum_"+ "SKIP_CHECKSUM1_curl_-fsSL_bash_Colors_for_output_"+ "Detect_platform_and_architecture_Get_latest_release_info_"+ "Download_and_extract_binary"+ )+ result = VikingFS._shorten_component(long_filename)+ assert len(result.encode("utf-8")) <= 255+++class TestDownloaderGenerateFilename:+ """Test _generate_filename in downloader."""++ def test_short_url(self):+ from openviking_cli.utils.downloader import _generate_filename++ result = _generate_filename("https://example.com/file.pdf")+ assert result == "file"++ def test_long_path_url(self):+ from openviking_cli.utils.downloader import _generate_filename++ url = "https://example.com/" + "a" * 200 + ".pdf"+ result = _generate_filename(url)+ assert len(result) <= 50++ def test_host_only_url(self):+ from openviking_cli.utils.downloader import _generate_filename++ result = _generate_filename("https://example.com/")+ assert result == "example_com"