Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: optimize markdown slicing, and fix a rust CLI bug for compability (#1919)
MaojiaShenged298cd6a9bf
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rsindex 02adfc78e..9e7da0819 100644--- a/crates/ov_cli/src/client.rs+++ b/crates/ov_cli/src/client.rs@@ -410,7 +410,10 @@ impl HttpClient {) -> Result<serde_json::Value> {let path_obj = Path::new(path);- // Determine effective parent and create_parent flag+ // Determine effective parent and create_parent flag.+ // Only send create_parent when the user explicitly selected+ // --parent-auto-create, so older servers that do not support the+ // field still accept the request.let (effective_parent, create_parent) = match (parent, parent_auto_create) {(Some(p), None) => (Some(p), false),(None, Some(p)) => (Some(p), true),@@ -418,6 +421,16 @@ impl HttpClient {(Some(_), Some(_)) => unreachable!("handled in cli"),};+ let build_body = |base: serde_json::Value| {+ let mut body = base;+ if create_parent {+ body.as_object_mut()+ .expect("add_resource request body must be an object")+ .insert("create_parent".to_string(), serde_json::Value::Bool(true));+ }+ body+ };+if path_obj.exists() {if path_obj.is_dir() {let source_name = path_obj@@ -435,12 +448,11 @@ impl HttpClient {self.upload_temp_file(zip_file.path()).await?};- let body = serde_json::json!({+ let body = build_body(serde_json::json!({"temp_file_id": temp_file_id,"source_name": source_name,"to": to,"parent": effective_parent,- "create_parent": create_parent,"reason": reason,"instruction": instruction,"wait": wait,@@ -451,7 +463,7 @@ impl HttpClient {"exclude": exclude,"directly_upload_media": directly_upload_media,"watch_interval": watch_interval,- });+ }));let dynamic_timeout = TimeoutConfig::for_resource_processing().calculate(zip_file.path())?;self.base.post_with_timeout("/api/v1/resources", &body, dynamic_timeout).await@@ -466,12 +478,11 @@ impl HttpClient {self.upload_temp_file(path_obj).await?};- let body = serde_json::json!({+ let body = build_body(serde_json::json!({"temp_file_id": temp_file_id,"source_name": source_name,"to": to,"parent": effective_parent,- "create_parent": create_parent,"reason": reason,"instruction": instruction,"wait": wait,@@ -482,16 +493,15 @@ impl HttpClient {"exclude": exclude,"directly_upload_media": directly_upload_media,"watch_interval": watch_interval,- });+ }));let dynamic_timeout = TimeoutConfig::for_resource_processing().calculate(path_obj)?;self.base.post_with_timeout("/api/v1/resources", &body, dynamic_timeout).await} else {- let body = serde_json::json!({+ let body = build_body(serde_json::json!({"path": path,"to": to,"parent": effective_parent,- "create_parent": create_parent,"reason": reason,"instruction": instruction,"wait": wait,@@ -502,16 +512,15 @@ impl HttpClient {"exclude": exclude,"directly_upload_media": directly_upload_media,"watch_interval": watch_interval,- });+ }));self.post("/api/v1/resources", &body).await}} else {- let body = serde_json::json!({+ let body = build_body(serde_json::json!({"path": path,"to": to,"parent": effective_parent,- "create_parent": create_parent,"reason": reason,"instruction": instruction,"wait": wait,@@ -522,7 +531,7 @@ impl HttpClient {"exclude": exclude,"directly_upload_media": directly_upload_media,"watch_interval": watch_interval,- });+ }));self.post("/api/v1/resources", &body).await}diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.pyindex 447053a3c..a59c28a68 100644--- a/openviking/parse/parsers/markdown.py+++ b/openviking/parse/parsers/markdown.py@@ -86,7 +86,7 @@ class MarkdownParser(BaseParser):"""# Configuration constants- DEFAULT_MAX_SECTION_SIZE = 1024 # Maximum tokens per section+ DEFAULT_MAX_SECTION_SIZE = 2048 # Maximum tokens per sectionDEFAULT_MIN_SECTION_TOKENS = 512 # Minimum tokens to create a separate sectionMAX_MERGED_FILENAME_LENGTH = 32 # Maximum length for merged section filenames@@ -372,10 +372,16 @@ class MarkdownParser(BaseParser):para_tokens = self._estimate_token_count(para)para_len = len(para)- # Single paragraph too long (by tokens or chars): force split by characters+ # Single paragraph too long (by tokens or chars): force split by characters.+ # If the already accumulated prefix is very short, merge it into this+ # oversized paragraph first so we do not create a low-value tiny chunk+ # like `section_1.md` that only contains the heading/introduction.if para_tokens > max_size or para_len > max_chars:if current:- parts.append(current.strip())+ if current_tokens < self.DEFAULT_MIN_SECTION_TOKENS:+ para = current + "\n\n" + para+ else:+ parts.append(current.strip())current = ""current_tokens = 0for i in range(0, len(para), max_chars):@@ -391,7 +397,13 @@ class MarkdownParser(BaseParser):current_tokens += para_tokensif current.strip():- parts.append(current.strip())+ # Avoid emitting a tiny trailing chunk when all earlier content has+ # already been split out (for example, a huge paragraph followed by+ # a short "no data" tail). Fold the tail back into the previous part.+ if parts and current_tokens < self.DEFAULT_MIN_SECTION_TOKENS:+ parts[-1] = f"{parts[-1]}\n\n{current.strip()}".strip()+ else:+ parts.append(current.strip())return parts if parts else [content]@@ -525,35 +537,117 @@ class MarkdownParser(BaseParser):]pending = []+ buffered_section = None++ async def flush_buffered() -> None:+ nonlocal buffered_section+ if buffered_section is not None:+ await self._save_section(+ content,+ headings,+ parent_dir,+ buffered_section,+ max_size,+ min_size,+ )+ buffered_section = None+for sec in expanded:name, tokens, content_text = sec["name"], sec["tokens"], sec["content"]has_children = sec["has_children"]# Handle small sectionsif tokens < min_size:- pending = await self._try_add_to_pending(- viking_fs, parent_dir, pending, (name, content_text, tokens), max_size- )+ if pending and sum(t for _, _, t in pending) + tokens > max_size:+ await flush_buffered()+ await self._save_merged(viking_fs, parent_dir, pending)+ pending = []+ pending.append((name, content_text, tokens))continue- # Try merge with pending- if pending and self._can_merge(pending, tokens, max_size, has_children):- pending.append((name, content_text, tokens))+ if pending:+ await flush_buffered()++ # Try merge with pending+ if self._can_merge(pending, tokens, max_size, has_children):+ pending.append((name, content_text, tokens))+ await self._save_merged(viking_fs, parent_dir, pending)+ pending = []+ continue++ # Avoid flushing a single tiny section as a standalone low-value file.+ if self._should_merge_pending_into_next(pending):+ sec = self._merge_pending_into_next_section(pending, sec)+ pending = []+ else:+ await self._save_merged(viking_fs, parent_dir, pending)+ pending = []+ else:+ await flush_buffered()++ buffered_section = sec++ if pending:+ # No next section exists. Fold a single tiny pending section back into+ # the previous saved candidate instead of emitting a standalone file.+ if buffered_section is not None and self._should_merge_pending_into_next(pending):+ buffered_section = self._merge_pending_into_previous_section(+ buffered_section, pending+ )+ pending = []+ else:+ await flush_buffered()await self._save_merged(viking_fs, parent_dir, pending)pending = []- continue-- # Save pending and process current section- pending = await self._flush_pending(viking_fs, parent_dir, pending)- await self._save_section(content, headings, parent_dir, sec, max_size, min_size)- # Save remaining pending- await self._flush_pending(viking_fs, parent_dir, pending)+ await flush_buffered()def _can_merge(self, pending: List, tokens: int, max_size: int, has_children: bool) -> bool:"""Check if section can merge with pending."""return sum(t for _, _, t in pending) + tokens <= max_size and not has_children+ def _should_merge_pending_into_next(self, pending: List[Tuple[str, str, int]]) -> bool:+ """Prefer folding a single tiny pending section into the next section."""+ return len(pending) == 1 and pending[0][2] <= self.DEFAULT_MIN_SECTION_TOKENS++ def _merge_pending_into_next_section(+ self, pending: List[Tuple[str, str, int]], section: Dict[str, Any]+ ) -> Dict[str, Any]:+ """Attach a tiny pending section to the following section."""+ _, pending_content, _ = pending[0]+ merged = dict(section)+ merged["content"] = f"{pending_content}\n\n{section['content']}".strip()+ merged["tokens"] = self._estimate_token_count(merged["content"])++ if merged.get("has_children"):+ direct_content = section.get("direct_content", "").strip()+ merged["direct_content"] = (+ f"{pending_content}\n\n{direct_content}".strip()+ if direct_content+ else pending_content+ )++ return merged++ def _merge_pending_into_previous_section(+ self, section: Dict[str, Any], pending: List[Tuple[str, str, int]]+ ) -> Dict[str, Any]:+ """Attach a tiny trailing pending section back into the previous section."""+ _, pending_content, _ = pending[0]+ merged = dict(section)+ merged["content"] = f"{section['content']}\n\n{pending_content}".strip()+ merged["tokens"] = self._estimate_token_count(merged["content"])++ if merged.get("has_children"):+ direct_content = section.get("direct_content", "").strip()+ merged["direct_content"] = (+ f"{direct_content}\n\n{pending_content}".strip()+ if direct_content+ else pending_content+ )++ return merged+async def _try_add_to_pending(self, viking_fs, parent_dir: str, pending: List, item: Tuple, max_size: int) -> List:diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex 01373954a..861afce1e 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -1123,16 +1123,17 @@ class VikingFS:if name in [".", ".."]:continuerel_path = f"{current_rel}/{name}" if current_rel else name+ is_dir = entry.get("isDir", False)new_entry = {"uri": self._path_to_uri(f"{current_path}/{name}", ctx=ctx),- "size": entry.get("size", 0),- "isDir": entry.get("isDir", False),+ "size": 0 if is_dir else entry.get("size", 0),+ "isDir": is_dir,"modTime": format_simplified(parse_iso_datetime(entry.get("modTime", "")), now),}new_entry["rel_path"] = rel_pathif not self._is_accessible(new_entry["uri"], real_ctx):continue- if entry.get("isDir"):+ if is_dir:all_entries.append(new_entry)await _walk(f"{current_path}/{name}", rel_path, current_depth + 1)elif not name.startswith("."):@@ -2212,15 +2213,16 @@ class VikingFS:parsed_time = parse_iso_datetime(raw_time)elif isinstance(entry.get("mtime"), (int, float)):parsed_time = datetime.fromtimestamp(entry["mtime"], tz=timezone.utc)+ is_dir = entry.get("isDir", False)new_entry = {"uri": self._path_to_uri(f"{path}/{name}", ctx=ctx),- "size": entry.get("size", 0),- "isDir": entry.get("isDir", False),+ "size": 0 if is_dir else entry.get("size", 0),+ "isDir": is_dir,"modTime": format_simplified(parsed_time, now),}if not self._is_accessible(new_entry["uri"], real_ctx):continue- if entry.get("isDir"):+ if is_dir:all_entries.append(new_entry)elif not name.startswith("."):all_entries.append(new_entry)diff --git a/openviking_cli/utils/config/parser_config.py b/openviking_cli/utils/config/parser_config.pyindex dcf424f9b..e4589671e 100644--- a/openviking_cli/utils/config/parser_config.py+++ b/openviking_cli/utils/config/parser_config.py@@ -37,7 +37,7 @@ class ParserConfig:encoding: str = "utf-8"# Smart splitting configuration- max_section_size: int = 1000 # Maximum tokens per section before splitting+ max_section_size: int = 2048 # Maximum tokens per section before splittingsection_size_flexibility: float = 0.3 # Allow 30% overflow to maintain coherencemax_section_chars: int = (6000 # Hard character limit per section (guards against token estimation errors)diff --git a/tests/parse/test_filename_safety.py b/tests/parse/test_filename_safety.pydeleted file mode 100644index aecd4d62a..000000000--- a/tests/parse/test_filename_safety.py+++ /dev/null@@ -1,209 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-"""Tests for filename safety: hash & shorten when names are too long (issue #171)."""--import hashlib-from unittest.mock import AsyncMock, patch--import pytest---class TestSanitizeForPath:- """Test _sanitize_for_path in MarkdownParser and HTMLParser."""-- def _make_md_parser(self):- from openviking.parse.parsers.markdown import MarkdownParser-- return MarkdownParser()-- def test_short_text_unchanged(self):- parser = self._make_md_parser()- assert parser._sanitize_for_path("Hello World") == "Hello_World"-- def test_empty_text_returns_section(self):- parser = self._make_md_parser()- assert parser._sanitize_for_path("") == "section"- assert parser._sanitize_for_path("!!!") == "section"-- def test_special_chars_removed(self):- parser = self._make_md_parser()- result = parser._sanitize_for_path("Hello, World! (test)")- assert result == "Hello_World_test"-- def test_chinese_preserved(self):- parser = self._make_md_parser()- result = parser._sanitize_for_path("你好世界")- assert result == "你好世界"-- def test_long_text_truncated_with_hash(self):- parser = self._make_md_parser()- long_text = "a" * 100- result = parser._sanitize_for_path(long_text)- assert len(result) <= 50- expected_hash = hashlib.sha256(long_text.encode()).hexdigest()[:8]- assert result.endswith(f"_{expected_hash}")-- def test_exact_boundary_not_hashed(self):- parser = self._make_md_parser()- text = "a" * 50- result = parser._sanitize_for_path(text)- assert result == text- assert len(result) == 50-- def test_one_over_boundary_hashed(self):- parser = self._make_md_parser()- text = "a" * 51- result = parser._sanitize_for_path(text)- assert len(result) <= 50- assert "_" in result # has hash suffix-- def test_custom_max_length(self):- parser = self._make_md_parser()- text = "a" * 30- result = parser._sanitize_for_path(text, max_length=20)- assert len(result) <= 20- expected_hash = hashlib.sha256(text.encode()).hexdigest()[:8]- assert result.endswith(f"_{expected_hash}")-- def test_shell_comment_heading(self):- """Simulate shell script comments being treated as markdown headings."""- parser = self._make_md_parser()- heading = "Usage: curl -fsSL https://raw.githubusercontent.com/volcengine/openviking/refs/tags/[email protected]/crates/ov_cli/install.sh | bash"- result = parser._sanitize_for_path(heading)- assert len(result) <= 50---class TestGenerateMergedFilename:- """Test _generate_merged_filename in MarkdownParser."""-- def _make_md_parser(self):- from openviking.parse.parsers.markdown import MarkdownParser-- return MarkdownParser()-- def test_single_short_section(self):- parser = self._make_md_parser()- result = parser._generate_merged_filename([("intro", "content", 100)])- assert result == "intro"-- def test_multiple_sections(self):- parser = self._make_md_parser()- sections = [("intro", "c1", 10), ("body", "c2", 20), ("end", "c3", 30)]- result = parser._generate_merged_filename(sections)- assert "3more" in result- assert len(result) <= 32-- def test_empty_sections(self):- parser = self._make_md_parser()- assert parser._generate_merged_filename([]) == "merged"-- def test_long_single_name_hashed(self):- parser = self._make_md_parser()- long_name = "a" * 100- result = parser._generate_merged_filename([(long_name, "content", 50)])- assert len(result) <= 32-- def test_result_never_exceeds_limit(self):- parser = self._make_md_parser()- # Create many sections with long names- sec_list = [(f"very_long_section_name_{i}", f"content_{i}", 10) for i in range(20)]- result = parser._generate_merged_filename(sec_list)- assert len(result) <= 32---async def test_markdown_resource_name_is_not_stemmed_again():- from openviking.parse.parsers.markdown import MarkdownParser-- parser = MarkdownParser()- mock_vfs = type("MockVikingFS", (), {})()- mock_vfs.create_temp_uri = lambda: "viking://temp/test"- mock_vfs.mkdir = AsyncMock()- mock_vfs.write_file = AsyncMock()-- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):- result = await parser.parse_content(- "short body",- source_path="/tmp/upload.yaml",- resource_name="access.dns.md",- )-- assert result.root.title == "access.dns.md"- mock_vfs.mkdir.assert_any_await("viking://temp/test/access.dns.md")---class TestShortenComponent:- """Test VikingFS._shorten_component."""-- def test_short_component_unchanged(self):- from openviking.storage.viking_fs import VikingFS-- assert VikingFS._shorten_component("hello") == "hello"-- def test_long_component_shortened(self):- from openviking.storage.viking_fs import VikingFS-- long_name = "a" * 300- result = VikingFS._shorten_component(long_name)- assert len(result.encode("utf-8")) <= 255-- def test_exact_255_bytes_unchanged(self):- from openviking.storage.viking_fs import VikingFS-- name = "a" * 255- assert VikingFS._shorten_component(name) == name-- def test_256_bytes_shortened(self):- from openviking.storage.viking_fs import VikingFS-- name = "a" * 256- result = VikingFS._shorten_component(name)- assert len(result.encode("utf-8")) <= 255- expected_hash = hashlib.sha256(name.encode("utf-8")).hexdigest()[:8]- assert result.endswith(f"_{expected_hash}")-- def test_unicode_multibyte_handling(self):- from openviking.storage.viking_fs import VikingFS-- # Chinese chars are 3 bytes each in UTF-8- name = "你" * 100 # 300 bytes- result = VikingFS._shorten_component(name)- assert len(result.encode("utf-8")) <= 255-- def test_realistic_long_filename(self):- """Simulate the exact bug from issue #171."""- from openviking.storage.viking_fs import VikingFS-- long_filename = (- "tmp5vacylnx_OpenViking_CLI_Installer_Usage_curl_-fsSL_"- "httpsrawgithubusercontentcomvolce_Example_curl_-fsSL_"- "httpsrawgithubusercontentcomvol_Skip_checksum_"- "SKIP_CHECKSUM1_curl_-fsSL_bash_Colors_for_output_"- "Detect_platform_and_architecture_Get_latest_release_info_"- "Download_and_extract_binary"- )- result = VikingFS._shorten_component(long_filename)- assert len(result.encode("utf-8")) <= 255---class TestDownloaderGenerateFilename:- """Test _generate_filename in downloader."""-- def test_short_url(self):- from openviking_cli.utils.downloader import _generate_filename-- result = _generate_filename("https://example.com/file.pdf")- assert result == "file"-- def test_long_path_url(self):- from openviking_cli.utils.downloader import _generate_filename-- url = "https://example.com/" + "a" * 200 + ".pdf"- result = _generate_filename(url)- assert len(result) <= 50-- def test_host_only_url(self):- from openviking_cli.utils.downloader import _generate_filename-- result = _generate_filename("https://example.com/")- assert result == "example_com"diff --git a/tests/parse/test_markdown_char_limit.py b/tests/parse/test_markdown_char_limit.pydeleted file mode 100644index 7e7d0aaf0..000000000--- a/tests/parse/test_markdown_char_limit.py+++ /dev/null@@ -1,362 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-"""Tests for MarkdownParser hard character limit enforcement (max_section_chars)."""--from unittest.mock import AsyncMock, MagicMock, patch--import pytest--from openviking.parse.parsers.markdown import MarkdownParser-from openviking_cli.utils.config.parser_config import (- FeishuConfig,- ParserConfig,- load_parser_configs_from_dict,-)--# ----------------------------------------------------------------------------# ParserConfig-# ------------------------------------------------------------------------------class TestParserConfigMaxSectionChars:- def test_default_value(self):- config = ParserConfig()- assert config.max_section_chars == 6000-- def test_custom_value(self):- config = ParserConfig(max_section_chars=3000)- assert config.max_section_chars == 3000-- def test_from_dict(self):- config = ParserConfig.from_dict({"max_section_chars": 2000})- assert config.max_section_chars == 2000-- def test_from_dict_missing_key_uses_default(self):- config = ParserConfig.from_dict({})- assert config.max_section_chars == 6000-- def test_from_dict_rejects_unknown_key(self):- with pytest.raises(ValueError, match="max_section_chars"):- ParserConfig.from_dict({"max_section_chras": 2000})-- def test_load_parser_configs_rejects_unknown_parser_section(self):- with pytest.raises(ValueError, match="markdown"):- load_parser_configs_from_dict({"markdwon": {}})-- def test_validate_rejects_zero(self):- config = ParserConfig(max_section_chars=0)- with pytest.raises(ValueError, match="max_section_chars"):- config.validate()-- def test_validate_rejects_negative(self):- config = ParserConfig(max_section_chars=-1)- with pytest.raises(ValueError, match="max_section_chars"):- config.validate()-- def test_validate_accepts_positive(self):- config = ParserConfig(max_section_chars=1)- config.validate() # must not raise---class TestFeishuConfigValidation:- def test_validate_accepts_defaults(self):- config = FeishuConfig()- config.validate()-- @pytest.mark.parametrize(- ("field", "value", "message"),- [- ("domain", "", "domain"),- ("max_rows_per_sheet", 0, "max_rows_per_sheet"),- ("max_rows_per_sheet", -1, "max_rows_per_sheet"),- ("max_records_per_table", 0, "max_records_per_table"),- ("max_records_per_table", -1, "max_records_per_table"),- ("request_timeout", 0, "request_timeout"),- ("request_timeout", -0.1, "request_timeout"),- ],- )- def test_validate_rejects_invalid_values(self, field, value, message):- config = FeishuConfig(**{field: value})- with pytest.raises(ValueError, match=message):- config.validate()---# ----------------------------------------------------------------------------# _smart_split_content — unit tests (no VikingFS needed)-# ------------------------------------------------------------------------------class TestSmartSplitContentCharLimit:- def _make_parser(self, max_section_chars: int = 100) -> MarkdownParser:- """Parser with a tight char limit for easy testing."""- config = ParserConfig(max_section_size=1000, max_section_chars=max_section_chars)- return MarkdownParser(config=config)-- def test_short_content_returned_as_single_part(self):- parser = self._make_parser(max_section_chars=500)- content = "Hello world. " * 10 # 130 chars, well within limit- parts = parser._smart_split_content(content, max_size=1000)- assert len(parts) == 1-- def test_paragraph_exceeding_char_limit_is_force_split(self):- """A single paragraph longer than max_section_chars must be split."""- parser = self._make_parser(max_section_chars=50)- # One paragraph of 200 chars — no \n\n so it stays as one para- long_para = "x" * 200- parts = parser._smart_split_content(long_para, max_size=1000)- # Each part must be <= max_section_chars- for part in parts:- assert len(part) <= 50- # All content is preserved- assert "".join(p.strip() for p in parts) == long_para-- def test_accumulated_chunks_respect_char_limit(self):- """Multiple small paragraphs that together exceed max_chars are split."""- parser = self._make_parser(max_section_chars=80)- # Each paragraph is 40 chars; two together (40+2+40=82) exceed the limit- para = "y" * 40- content = f"{para}\n\n{para}\n\n{para}"- parts = parser._smart_split_content(content, max_size=1000)- for part in parts:- assert len(part) <= 80-- def test_char_limit_splits_even_when_token_estimate_is_small(self):- """Content within token limit but exceeding char limit must still be split."""- # Use a very large max_section_chars to NOT trigger char splitting- parser_no_limit = self._make_parser(max_section_chars=10000)- # Use a tight max_section_chars to force splitting- parser_with_limit = self._make_parser(max_section_chars=50)-- content = "a" * 120 # single paragraph, ~36 estimated tokens (well under 1000)-- parts_no_limit = parser_no_limit._smart_split_content(content, max_size=1000)- parts_with_limit = parser_with_limit._smart_split_content(content, max_size=1000)-- # Without char limit: one part (token estimate is fine)- assert len(parts_no_limit) == 1- # With char limit: must be split into multiple parts- assert len(parts_with_limit) > 1- for part in parts_with_limit:- assert len(part) <= 50-- def test_empty_paragraphs_handled(self):- parser = self._make_parser(max_section_chars=200)- content = "line1\n\n\n\nline2"- parts = parser._smart_split_content(content, max_size=1000)- assert all(p.strip() for p in parts) # no blank-only parts-- def test_content_entirely_preserved_after_split(self):- """No characters should be lost during splitting."""- parser = self._make_parser(max_section_chars=60)- # Build content where each paragraph is 50 chars- para = "p" * 50- content = "\n\n".join([para] * 5)- parts = parser._smart_split_content(content, max_size=1000)- # Reconstruct: strip whitespace added by joining- total_chars = sum(len(p.replace(" ", "").replace("\n", "")) for p in parts)- original_chars = len(content.replace(" ", "").replace("\n", ""))- assert total_chars == original_chars---# ----------------------------------------------------------------------------# _save_section — char limit gates (async, VikingFS mocked)-# ------------------------------------------------------------------------------class TestSaveSectionCharLimit:- def _make_parser(self, max_section_chars: int = 100) -> MarkdownParser:- config = ParserConfig(max_section_size=1000, max_section_chars=max_section_chars)- return MarkdownParser(config=config)-- def _mock_viking_fs(self):- vfs = MagicMock()- vfs.write_file = AsyncMock()- vfs.mkdir = AsyncMock()- return vfs-- @pytest.mark.asyncio- async def test_section_within_both_limits_saved_as_file(self):- """Section under token AND char limit → single .md file."""- parser = self._make_parser(max_section_chars=500)- mock_vfs = self._mock_viking_fs()-- section = {- "name": "intro",- "tokens": 50,- "content": "Short content",- "has_children": False,- }-- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):- await parser._save_section("", [], "viking://tmp/root", section, 1000, 512)-- mock_vfs.write_file.assert_called_once()- call_path = mock_vfs.write_file.call_args[0][0]- assert call_path.endswith("intro.md")-- @pytest.mark.asyncio- async def test_section_exceeding_char_limit_is_split_even_if_tokens_ok(self):- """Section within token limit but over char limit must NOT be written as single file."""- parser = self._make_parser(max_section_chars=50)- mock_vfs = self._mock_viking_fs()-- long_content = "word " * 30 # 150 chars, ~14 estimated tokens (under 1000)- section = {- "name": "big_section",- "tokens": 14,- "content": long_content,- "has_children": False,- }-- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):- await parser._save_section("", [], "viking://tmp/root", section, 1000, 512)-- # Must have written at least one file- mock_vfs.write_file.assert_called()- # Every written chunk must respect the char limit- for call in mock_vfs.write_file.call_args_list:- written_content = call[0][1]- assert len(written_content) <= 50, (- f"Written content exceeds char limit: {len(written_content)} chars"- )- # A directory should have been created for the split- mock_vfs.mkdir.assert_called()---# ----------------------------------------------------------------------------# _save_merged — char limit enforcement (async, VikingFS mocked)-# ------------------------------------------------------------------------------class TestSaveMergedCharLimit:- def _make_parser(self, max_section_chars: int) -> MarkdownParser:- config = ParserConfig(max_section_size=1000, max_section_chars=max_section_chars)- return MarkdownParser(config=config)-- def _mock_viking_fs(self):- vfs = MagicMock()- vfs.write_file = AsyncMock()- return vfs-- @pytest.mark.asyncio- async def test_merged_within_char_limit_saved_as_single_file(self):- parser = self._make_parser(max_section_chars=500)- mock_vfs = self._mock_viking_fs()-- sections = [("s1", "hello", 5), ("s2", "world", 5)]- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):- await parser._save_merged(mock_vfs, "viking://tmp/root", sections)-- mock_vfs.write_file.assert_called_once()- path, content = mock_vfs.write_file.call_args[0]- assert path.endswith(".md")- assert "hello" in content and "world" in content-- @pytest.mark.asyncio- async def test_merged_exceeding_char_limit_is_split_into_multiple_files(self):- """When joined content exceeds max_section_chars, _save_merged must split it."""- parser = self._make_parser(max_section_chars=60)- mock_vfs = self._mock_viking_fs()-- # Each section is 40 chars; joined = 40 + "\n\n" + 40 = 82 chars > 60- sections = [("a", "A" * 40, 12), ("b", "B" * 40, 12)]- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):- await parser._save_merged(mock_vfs, "viking://tmp/root", sections)-- # Should have written multiple files- assert mock_vfs.write_file.call_count > 1- for call in mock_vfs.write_file.call_args_list:- written_content = call[0][1]- assert len(written_content) <= 60, (- f"Merged split part exceeds char limit: {len(written_content)} chars"- )-- @pytest.mark.asyncio- async def test_many_small_sections_merged_correctly_split(self):- """Many token-small but char-large sections that accumulate past the limit."""- parser = self._make_parser(max_section_chars=100)- mock_vfs = self._mock_viking_fs()-- # 10 sections × 30 chars = 300 chars + separators >> 100 chars- sections = [(f"s{i}", "z" * 30, 9) for i in range(10)]- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):- await parser._save_merged(mock_vfs, "viking://tmp/root", sections)-- # Every written part must respect the char limit- mock_vfs.write_file.assert_called()- for call in mock_vfs.write_file.call_args_list:- written_content = call[0][1]- assert len(written_content) <= 100, (- f"Part exceeds char limit: {len(written_content)} chars"- )---# ----------------------------------------------------------------------------# _parse_and_create_structure — small-document char limit (async, VikingFS mocked)-# ------------------------------------------------------------------------------class TestParseAndCreateStructureCharLimit:- def _make_parser(self, max_section_size: int = 1000, max_section_chars: int = 100):- config = ParserConfig(- max_section_size=max_section_size, max_section_chars=max_section_chars- )- return MarkdownParser(config=config)-- def _mock_viking_fs(self):- vfs = MagicMock()- vfs.write_file = AsyncMock()- vfs.mkdir = AsyncMock()- return vfs-- @pytest.mark.asyncio- async def test_small_doc_within_both_limits_saved_as_single_file(self):- parser = self._make_parser(max_section_size=1000, max_section_chars=500)- mock_vfs = self._mock_viking_fs()- content = "# Title\n\nShort body." # well under both limits-- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):- await parser._parse_and_create_structure(content, [], "viking://tmp/root")-- mock_vfs.write_file.assert_called_once()- written_content = mock_vfs.write_file.call_args[0][1]- assert written_content == content-- @pytest.mark.asyncio- async def test_small_doc_exceeding_char_limit_is_not_saved_as_single_file(self):- """Even if token estimate is tiny, a doc over max_section_chars must be split."""- parser = self._make_parser(max_section_size=1000, max_section_chars=30)- mock_vfs = self._mock_viking_fs()-- # 200 chars but NO headings → no sections, falls back to paragraph split- content = "a" * 200-- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):- await parser._parse_and_create_structure(content, [], "viking://tmp/root")-- # Must have been split: each written chunk ≤ max_section_chars- assert mock_vfs.write_file.call_count > 1- for call in mock_vfs.write_file.call_args_list:- written = call[0][1]- assert len(written) <= 30-- @pytest.mark.asyncio- async def test_doc_with_headings_exceeding_char_limit_is_not_saved_as_single_file(self):- """Small-document fast-path with headings: char limit must still be enforced."""- parser = self._make_parser(max_section_size=1000, max_section_chars=50)- mock_vfs = self._mock_viking_fs()-- # 200 chars WITH a heading — exercises the char-check at the- # "estimated_tokens <= max_size AND len(content) <= max_chars" guard- content = "# Heading\n\n" + "b" * 189 # total > 50 chars, token estimate << 1000-- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):- headings = parser._find_headings(content)- await parser._parse_and_create_structure(content, headings, "viking://tmp/root")-- # Must not have been saved as a single file equal to the full content- for call in mock_vfs.write_file.call_args_list:- written = call[0][1]- assert len(written) <= 50, (- f"Single file written with {len(written)} chars, exceeds limit of 50"- )diff --git a/tests/server/test_api_filesystem.py b/tests/server/test_api_filesystem.pydeleted file mode 100644index de98979c4..000000000--- a/tests/server/test_api_filesystem.py+++ /dev/null@@ -1,253 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0--"""Tests for filesystem endpoints: ls, tree, stat, mkdir, rm, mv."""--import httpx--from openviking.pyagfs.exceptions import AGFSHTTPError-from openviking.storage.errors import ResourceBusyError---async def test_ls_root(client: httpx.AsyncClient):- resp = await client.get("/api/v1/fs/ls", params={"uri": "viking://"})- assert resp.status_code == 200- body = resp.json()- assert body["status"] == "ok"- assert isinstance(body["result"], list)---async def test_ls_simple(client: httpx.AsyncClient):- resp = await client.get(- "/api/v1/fs/ls",- params={"uri": "viking://", "simple": True},- )- assert resp.status_code == 200- body = resp.json()- assert body["status"] == "ok"- assert isinstance(body["result"], list)- # Each item must be a non-empty URI string (fixes #218)- for item in body["result"]:- assert isinstance(item, str)- assert item.startswith("viking://")---async def test_ls_simple_agent_output(client: httpx.AsyncClient):- """Ensure --simple with output=agent returns URI strings, not empty."""- resp = await client.get(- "/api/v1/fs/ls",- params={"uri": "viking://", "simple": True, "output": "agent"},- )- assert resp.status_code == 200- body = resp.json()- assert body["status"] == "ok"- assert isinstance(body["result"], list)- for item in body["result"]:- assert isinstance(item, str)- assert item.startswith("viking://")---async def test_mkdir_and_ls(client: httpx.AsyncClient):- resp = await client.post(- "/api/v1/fs/mkdir",- json={"uri": "viking://resources/test_dir/"},- )- assert resp.status_code == 200- assert resp.json()["status"] == "ok"-- resp = await client.get(- "/api/v1/fs/ls",- params={"uri": "viking://resources/"},- )- assert resp.status_code == 200---async def test_mkdir_with_description_initializes_abstract_and_enqueues_l0(- client: httpx.AsyncClient,- monkeypatch,-):- seen = {}-- async def _fake_vectorize_directory_meta(**kwargs):- seen.update(kwargs)-- monkeypatch.setattr(- "openviking.service.fs_service.vectorize_directory_meta",- _fake_vectorize_directory_meta,- )-- uri = "viking://resources/described_dir/"- description = "Directory for API docs"- resp = await client.post(- "/api/v1/fs/mkdir",- json={"uri": uri, "description": description},- )- assert resp.status_code == 200- assert resp.json()["status"] == "ok"-- abstract_resp = await client.get(- "/api/v1/content/abstract",- params={"uri": uri},- )- assert abstract_resp.status_code == 200- assert abstract_resp.json()["result"] == description- assert seen["uri"] == "viking://resources/described_dir"- assert seen["abstract"] == description- assert seen["overview"] == ""- assert seen["context_type"] == "resource"- assert seen["include_overview"] is False- assert seen["ctx"] is not None---async def test_tree(client: httpx.AsyncClient):- resp = await client.get("/api/v1/fs/tree", params={"uri": "viking://"})- assert resp.status_code == 200- body = resp.json()- assert body["status"] == "ok"---async def test_stat_not_found(client: httpx.AsyncClient):- resp = await client.get(- "/api/v1/fs/stat",- params={"uri": "viking://resources/nonexistent/xyz"},- )- assert resp.status_code == 404- body = resp.json()- assert body["status"] == "error"---async def test_rm_directory_without_recursive_returns_failed_precondition(- client: httpx.AsyncClient,-):- await client.post(- "/api/v1/fs/mkdir",- json={"uri": "viking://resources/rm_dir_without_recursive/"},- )- resp = await client.request(- "DELETE",- "/api/v1/fs",- params={"uri": "viking://resources/rm_dir_without_recursive"},- )- assert resp.status_code == 412- body = resp.json()- assert body["status"] == "error"- assert body["error"]["code"] == "FAILED_PRECONDITION"---async def test_rm_missing_uri_is_idempotent(client: httpx.AsyncClient):- missing_uri = "viking://resources/definitely-missing-rm-target"-- resp = await client.request(- "DELETE",- "/api/v1/fs",- params={"uri": missing_uri, "recursive": True},- )-- assert resp.status_code == 200- body = resp.json()- assert body["status"] == "ok"- assert body["result"] == {"uri": missing_uri}---async def test_rm_unsupported_scheme_returns_invalid_uri(client: httpx.AsyncClient):- resp = await client.request(- "DELETE",- "/api/v1/fs",- params={"uri": "s3://bucket/missing", "recursive": True},- )-- assert resp.status_code == 400- body = resp.json()- assert body["status"] == "error"- assert body["error"]["code"] == "INVALID_URI"- assert "unsupported URI scheme 's3'" in body["error"]["message"]---async def test_ls_invalid_scope_hides_internal_scopes(client: httpx.AsyncClient):- resp = await client.get("/api/v1/fs/ls", params={"uri": "ssd"})-- assert resp.status_code == 400- body = resp.json()- message = body["error"]["message"]- assert body["status"] == "error"- assert body["error"]["code"] == "INVALID_URI"- assert "resources" in message- assert "temp" not in message- assert "queue" not in message- assert "frozenset" not in message---async def test_resource_ops(client_with_resource):- """Test stat, ls_recursive, mv, rm on a single shared resource."""- import uuid-- client, uri = client_with_resource-- # stat- resp = await client.get("/api/v1/fs/stat", params={"uri": uri})- assert resp.status_code == 200- assert resp.json()["status"] == "ok"-- # ls recursive- resp = await client.get(- "/api/v1/fs/ls",- params={"uri": "viking://", "recursive": True},- )- assert resp.status_code == 200- body = resp.json()- assert body["status"] == "ok"- assert isinstance(body["result"], list)-- # mv- unique = uuid.uuid4().hex[:8]- new_uri = uri.rstrip("/") + f"_mv_{unique}/"- resp = await client.post(- "/api/v1/fs/mv",- json={"from_uri": uri, "to_uri": new_uri},- )- assert resp.status_code == 200- assert resp.json()["status"] == "ok"-- # rm (on the moved uri)- resp = await client.request("DELETE", "/api/v1/fs", params={"uri": new_uri, "recursive": True})- assert resp.status_code == 200- assert resp.json()["status"] == "ok"---async def test_rm_maps_resource_busy_to_conflict(client, service, monkeypatch):- async def fake_rm(uri, ctx=None, recursive=False):- raise ResourceBusyError(f"Resource is being processed: {uri}")-- monkeypatch.setattr(service.fs, "rm", fake_rm)- resp = await client.request(- "DELETE",- "/api/v1/fs",- params={"uri": "viking://resources/locked", "recursive": True},- )- assert resp.status_code == 409- body = resp.json()- assert body["status"] == "error"- assert body["error"]["code"] == "CONFLICT"---async def test_rm_agfs_internal_error_does_not_look_successful(client, service, monkeypatch):- class FakeAGFS:- def __init__(self, wrapped):- self._wrapped = wrapped-- def stat(self, path):- raise AGFSHTTPError("Internal server error", 500)-- def __getattr__(self, name):- return getattr(self._wrapped, name)-- monkeypatch.setattr(service.fs._viking_fs, "agfs", FakeAGFS(service.fs._viking_fs.agfs))- resp = await client.request(- "DELETE",- "/api/v1/fs",- params={"uri": "viking://resources/delete-failure", "recursive": True},- )- assert resp.status_code == 503- body = resp.json()- assert body["status"] == "error"- assert body["error"]["code"] == "UNAVAILABLE"