Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(parser): hash & shorten filenames that exceed filesystem limit (#205)

Ethon361cad61cb31
diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.py
index 28e47885f..34711ef0c 100644
--- a/openviking/parse/parsers/html.py
+++ b/openviking/parse/parsers/html.py
@@ -11,6 +11,7 @@ Unified parser that handles:
Preserves natural document hierarchy and filters out navigation/ads.
"""
+import hashlib
import re
import tempfile
import time
@@ -599,8 +600,14 @@ class HTMLParser(BaseParser):
return result
- def _sanitize_for_path(self, text: str) -> str:
- """Sanitize text for use in file path."""
+ def _sanitize_for_path(self, text: str, max_length: int = 50) -> str:
+ """Sanitize text for use in file path, hash & shorten if too long."""
safe = re.sub(r"[^\w\u4e00-\u9fff\s-]", "", text)
safe = re.sub(r"\s+", "_", safe)
- return safe.strip("_")[:50] or "section"
+ safe = safe.strip("_")
+ if not safe:
+ return "section"
+ if len(safe) > max_length:
+ hash_suffix = hashlib.sha256(text.encode()).hexdigest()[:8]
+ return f"{safe[: max_length - 9]}_{hash_suffix}"
+ return safe
diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.py
index e6ddbe3ad..bcb4f48ae 100644
--- a/openviking/parse/parsers/markdown.py
+++ b/openviking/parse/parsers/markdown.py
@@ -17,6 +17,7 @@ The parser handles scenarios:
5. Oversized sections without subsections → split by paragraphs
"""
+import hashlib
import re
import time
from pathlib import Path
@@ -333,10 +334,16 @@ class MarkdownParser(BaseParser):
return parts if parts else [content]
- def _sanitize_for_path(self, text: str) -> str:
+ def _sanitize_for_path(self, text: str, max_length: int = 50) -> str:
safe = re.sub(r"[^\w\u4e00-\u9fff\s-]", "", text)
safe = re.sub(r"\s+", "_", safe)
- return safe.strip("_")[:50] or "section"
+ safe = safe.strip("_")
+ if not safe:
+ return "section"
+ if len(safe) > max_length:
+ hash_suffix = hashlib.sha256(text.encode()).hexdigest()[:8]
+ return f"{safe[: max_length - 9]}_{hash_suffix}"
+ return safe
# ========== New Parsing Logic (v5.0) ==========
@@ -570,25 +577,32 @@ class MarkdownParser(BaseParser):
Smart merged filename generation, limited to MAX_MERGED_FILENAME_LENGTH characters.
Strategy:
- - Single section: Use directly (truncated to MAX_MERGED_FILENAME_LENGTH chars)
+ - Single section: Use directly (truncated with hash if needed)
- Multiple sections: {first_section}_{count}more (e.g., Intro_3more)
- Total length strictly limited: MAX_MERGED_FILENAME_LENGTH characters
+ - Hash suffix ensures uniqueness when truncation occurs
"""
if not sections:
return "merged"
names = [n for n, _, _ in sections]
count = len(names)
+ max_len = self.MAX_MERGED_FILENAME_LENGTH
if count == 1:
- name = names[0][: self.MAX_MERGED_FILENAME_LENGTH]
+ name = names[0]
else:
suffix = f"_{count}more"
- max_first_len = self.MAX_MERGED_FILENAME_LENGTH - len(suffix)
+ max_first_len = max_len - len(suffix)
first_name = names[0][: max(max_first_len, 1)]
name = f"{first_name}{suffix}"
- name = name[: self.MAX_MERGED_FILENAME_LENGTH].strip("_")
+ if len(name) > max_len:
+ full_key = "_".join(names)
+ hash_suffix = hashlib.sha256(full_key.encode()).hexdigest()[:8]
+ name = f"{name[: max_len - 9]}_{hash_suffix}"
+
+ name = name.strip("_")
return name or "merged"
async def _save_merged(
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index a8e52c72f..611ede50e 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -13,6 +13,7 @@ Responsibilities:
"""
import asyncio
+import hashlib
import json
from dataclasses import dataclass, field
from datetime import datetime
@@ -646,12 +647,31 @@ class VikingFS:
# ========== URI Conversion ==========
+ # Maximum bytes for a single filename component (filesystem limit is typically 255)
+ _MAX_FILENAME_BYTES = 255
+
+ @staticmethod
+ def _shorten_component(component: str, max_bytes: int = 255) -> str:
+ """Shorten a path component if its UTF-8 encoding exceeds max_bytes."""
+ if len(component.encode("utf-8")) <= max_bytes:
+ return component
+ hash_suffix = hashlib.sha256(component.encode("utf-8")).hexdigest()[:8]
+ # Trim to fit within max_bytes after adding hash suffix
+ prefix = component
+ target = max_bytes - len(f"_{hash_suffix}".encode("utf-8"))
+ while len(prefix.encode("utf-8")) > target and prefix:
+ prefix = prefix[:-1]
+ return f"{prefix}_{hash_suffix}"
+
def _uri_to_path(self, uri: str) -> str:
"""viking://user/memories/preferences/test -> /local/user/memories/preferences/test"""
remainder = uri[len("viking://") :].strip("/")
if not remainder:
return "/local"
- return f"/local/{remainder}"
+ # Ensure each path component does not exceed filesystem filename limit
+ parts = remainder.split("/")
+ safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]
+ return f"/local/{'/'.join(safe_parts)}"
def _path_to_uri(self, path: str) -> str:
"""/local/user/memories/preferences -> viking://user/memories/preferences"""
diff --git a/openviking_cli/utils/downloader.py b/openviking_cli/utils/downloader.py
index ff263f813..d9a73efa3 100644
--- a/openviking_cli/utils/downloader.py
+++ b/openviking_cli/utils/downloader.py
@@ -7,6 +7,7 @@ Provides basic file download functionality.
For URL content parsing, use HTMLParser instead.
"""
+import hashlib
import re
from pathlib import Path
from typing import Optional, Tuple
@@ -92,8 +93,8 @@ async def download_file(
return None, str(e)
-def _generate_filename(url: str) -> str:
- """Generate filename from URL."""
+def _generate_filename(url: str, max_length: int = 50) -> str:
+ """Generate filename from URL, hash & shorten if too long."""
from urllib.parse import urlparse
parsed = urlparse(url)
@@ -103,7 +104,17 @@ def _generate_filename(url: str) -> str:
name = Path(path).stem
name = re.sub(r"[^a-zA-Z0-9_\-\u4e00-\u9fff]", "_", name)
name = re.sub(r"_+", "_", name)
- return name[:50] if name else "download"
+ if not name:
+ return "download"
+ if len(name) > max_length:
+ hash_suffix = hashlib.sha256(url.encode()).hexdigest()[:8]
+ return f"{name[: max_length - 9]}_{hash_suffix}"
+ return name
host = parsed.netloc.replace(".", "_")
- return host[:50] if host else "download"
+ if not host:
+ return "download"
+ if len(host) > max_length:
+ hash_suffix = hashlib.sha256(url.encode()).hexdigest()[:8]
+ return f"{host[: max_length - 9]}_{hash_suffix}"
+ return host
diff --git a/tests/parse/test_filename_safety.py b/tests/parse/test_filename_safety.py
new file mode 100644
index 000000000..9c81cc2e2
--- /dev/null
+++ b/tests/parse/test_filename_safety.py
@@ -0,0 +1,185 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Tests for filename safety: hash & shorten when names are too long (issue #171)."""
+
+import hashlib
+
+
+class TestSanitizeForPath:
+ """Test _sanitize_for_path in MarkdownParser and HTMLParser."""
+
+ def _make_md_parser(self):
+ from openviking.parse.parsers.markdown import MarkdownParser
+
+ return MarkdownParser()
+
+ def test_short_text_unchanged(self):
+ parser = self._make_md_parser()
+ assert parser._sanitize_for_path("Hello World") == "Hello_World"
+
+ def test_empty_text_returns_section(self):
+ parser = self._make_md_parser()
+ assert parser._sanitize_for_path("") == "section"
+ assert parser._sanitize_for_path("!!!") == "section"
+
+ def test_special_chars_removed(self):
+ parser = self._make_md_parser()
+ result = parser._sanitize_for_path("Hello, World! (test)")
+ assert result == "Hello_World_test"
+
+ def test_chinese_preserved(self):
+ parser = self._make_md_parser()
+ result = parser._sanitize_for_path("你好世界")
+ assert result == "你好世界"
+
+ def test_long_text_truncated_with_hash(self):
+ parser = self._make_md_parser()
+ long_text = "a" * 100
+ result = parser._sanitize_for_path(long_text)
+ assert len(result) <= 50
+ expected_hash = hashlib.sha256(long_text.encode()).hexdigest()[:8]
+ assert result.endswith(f"_{expected_hash}")
+
+ def test_exact_boundary_not_hashed(self):
+ parser = self._make_md_parser()
+ text = "a" * 50
+ result = parser._sanitize_for_path(text)
+ assert result == text
+ assert len(result) == 50
+
+ def test_one_over_boundary_hashed(self):
+ parser = self._make_md_parser()
+ text = "a" * 51
+ result = parser._sanitize_for_path(text)
+ assert len(result) <= 50
+ assert "_" in result # has hash suffix
+
+ def test_custom_max_length(self):
+ parser = self._make_md_parser()
+ text = "a" * 30
+ result = parser._sanitize_for_path(text, max_length=20)
+ assert len(result) <= 20
+ expected_hash = hashlib.sha256(text.encode()).hexdigest()[:8]
+ assert result.endswith(f"_{expected_hash}")
+
+ def test_shell_comment_heading(self):
+ """Simulate shell script comments being treated as markdown headings."""
+ parser = self._make_md_parser()
+ heading = "Usage: curl -fsSL https://raw.githubusercontent.com/volcengine/openviking/refs/tags/[email protected]/crates/ov_cli/install.sh | bash"
+ result = parser._sanitize_for_path(heading)
+ assert len(result) <= 50
+
+
+class TestGenerateMergedFilename:
+ """Test _generate_merged_filename in MarkdownParser."""
+
+ def _make_md_parser(self):
+ from openviking.parse.parsers.markdown import MarkdownParser
+
+ return MarkdownParser()
+
+ def test_single_short_section(self):
+ parser = self._make_md_parser()
+ result = parser._generate_merged_filename([("intro", "content", 100)])
+ assert result == "intro"
+
+ def test_multiple_sections(self):
+ parser = self._make_md_parser()
+ sections = [("intro", "c1", 10), ("body", "c2", 20), ("end", "c3", 30)]
+ result = parser._generate_merged_filename(sections)
+ assert "3more" in result
+ assert len(result) <= 32
+
+ def test_empty_sections(self):
+ parser = self._make_md_parser()
+ assert parser._generate_merged_filename([]) == "merged"
+
+ def test_long_single_name_hashed(self):
+ parser = self._make_md_parser()
+ long_name = "a" * 100
+ result = parser._generate_merged_filename([(long_name, "content", 50)])
+ assert len(result) <= 32
+
+ def test_result_never_exceeds_limit(self):
+ parser = self._make_md_parser()
+ # Create many sections with long names
+ sec_list = [(f"very_long_section_name_{i}", f"content_{i}", 10) for i in range(20)]
+ result = parser._generate_merged_filename(sec_list)
+ assert len(result) <= 32
+
+
+class TestShortenComponent:
+ """Test VikingFS._shorten_component."""
+
+ def test_short_component_unchanged(self):
+ from openviking.storage.viking_fs import VikingFS
+
+ assert VikingFS._shorten_component("hello") == "hello"
+
+ def test_long_component_shortened(self):
+ from openviking.storage.viking_fs import VikingFS
+
+ long_name = "a" * 300
+ result = VikingFS._shorten_component(long_name)
+ assert len(result.encode("utf-8")) <= 255
+
+ def test_exact_255_bytes_unchanged(self):
+ from openviking.storage.viking_fs import VikingFS
+
+ name = "a" * 255
+ assert VikingFS._shorten_component(name) == name
+
+ def test_256_bytes_shortened(self):
+ from openviking.storage.viking_fs import VikingFS
+
+ name = "a" * 256
+ result = VikingFS._shorten_component(name)
+ assert len(result.encode("utf-8")) <= 255
+ expected_hash = hashlib.sha256(name.encode("utf-8")).hexdigest()[:8]
+ assert result.endswith(f"_{expected_hash}")
+
+ def test_unicode_multibyte_handling(self):
+ from openviking.storage.viking_fs import VikingFS
+
+ # Chinese chars are 3 bytes each in UTF-8
+ name = "你" * 100 # 300 bytes
+ result = VikingFS._shorten_component(name)
+ assert len(result.encode("utf-8")) <= 255
+
+ def test_realistic_long_filename(self):
+ """Simulate the exact bug from issue #171."""
+ from openviking.storage.viking_fs import VikingFS
+
+ long_filename = (
+ "tmp5vacylnx_OpenViking_CLI_Installer_Usage_curl_-fsSL_"
+ "httpsrawgithubusercontentcomvolce_Example_curl_-fsSL_"
+ "httpsrawgithubusercontentcomvol_Skip_checksum_"
+ "SKIP_CHECKSUM1_curl_-fsSL_bash_Colors_for_output_"
+ "Detect_platform_and_architecture_Get_latest_release_info_"
+ "Download_and_extract_binary"
+ )
+ result = VikingFS._shorten_component(long_filename)
+ assert len(result.encode("utf-8")) <= 255
+
+
+class TestDownloaderGenerateFilename:
+ """Test _generate_filename in downloader."""
+
+ def test_short_url(self):
+ from openviking_cli.utils.downloader import _generate_filename
+
+ result = _generate_filename("https://example.com/file.pdf")
+ assert result == "file"
+
+ def test_long_path_url(self):
+ from openviking_cli.utils.downloader import _generate_filename
+
+ url = "https://example.com/" + "a" * 200 + ".pdf"
+ result = _generate_filename(url)
+ assert len(result) <= 50
+
+ def test_host_only_url(self):
+ from openviking_cli.utils.downloader import _generate_filename
+
+ result = _generate_filename("https://example.com/")
+ assert result == "example_com"