Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: optimize markdown slicing, and fix a rust CLI bug for compability (#1919)

MaojiaShenged298cd6a9bf
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rs
index 02adfc78e..9e7da0819 100644
--- a/crates/ov_cli/src/client.rs
+++ b/crates/ov_cli/src/client.rs
@@ -410,7 +410,10 @@ impl HttpClient {
) -> Result<serde_json::Value> {
let path_obj = Path::new(path);
- // Determine effective parent and create_parent flag
+ // Determine effective parent and create_parent flag.
+ // Only send create_parent when the user explicitly selected
+ // --parent-auto-create, so older servers that do not support the
+ // field still accept the request.
let (effective_parent, create_parent) = match (parent, parent_auto_create) {
(Some(p), None) => (Some(p), false),
(None, Some(p)) => (Some(p), true),
@@ -418,6 +421,16 @@ impl HttpClient {
(Some(_), Some(_)) => unreachable!("handled in cli"),
};
+ let build_body = |base: serde_json::Value| {
+ let mut body = base;
+ if create_parent {
+ body.as_object_mut()
+ .expect("add_resource request body must be an object")
+ .insert("create_parent".to_string(), serde_json::Value::Bool(true));
+ }
+ body
+ };
+
if path_obj.exists() {
if path_obj.is_dir() {
let source_name = path_obj
@@ -435,12 +448,11 @@ impl HttpClient {
self.upload_temp_file(zip_file.path()).await?
};
- let body = serde_json::json!({
+ let body = build_body(serde_json::json!({
"temp_file_id": temp_file_id,
"source_name": source_name,
"to": to,
"parent": effective_parent,
- "create_parent": create_parent,
"reason": reason,
"instruction": instruction,
"wait": wait,
@@ -451,7 +463,7 @@ impl HttpClient {
"exclude": exclude,
"directly_upload_media": directly_upload_media,
"watch_interval": watch_interval,
- });
+ }));
let dynamic_timeout = TimeoutConfig::for_resource_processing().calculate(zip_file.path())?;
self.base.post_with_timeout("/api/v1/resources", &body, dynamic_timeout).await
@@ -466,12 +478,11 @@ impl HttpClient {
self.upload_temp_file(path_obj).await?
};
- let body = serde_json::json!({
+ let body = build_body(serde_json::json!({
"temp_file_id": temp_file_id,
"source_name": source_name,
"to": to,
"parent": effective_parent,
- "create_parent": create_parent,
"reason": reason,
"instruction": instruction,
"wait": wait,
@@ -482,16 +493,15 @@ impl HttpClient {
"exclude": exclude,
"directly_upload_media": directly_upload_media,
"watch_interval": watch_interval,
- });
+ }));
let dynamic_timeout = TimeoutConfig::for_resource_processing().calculate(path_obj)?;
self.base.post_with_timeout("/api/v1/resources", &body, dynamic_timeout).await
} else {
- let body = serde_json::json!({
+ let body = build_body(serde_json::json!({
"path": path,
"to": to,
"parent": effective_parent,
- "create_parent": create_parent,
"reason": reason,
"instruction": instruction,
"wait": wait,
@@ -502,16 +512,15 @@ impl HttpClient {
"exclude": exclude,
"directly_upload_media": directly_upload_media,
"watch_interval": watch_interval,
- });
+ }));
self.post("/api/v1/resources", &body).await
}
} else {
- let body = serde_json::json!({
+ let body = build_body(serde_json::json!({
"path": path,
"to": to,
"parent": effective_parent,
- "create_parent": create_parent,
"reason": reason,
"instruction": instruction,
"wait": wait,
@@ -522,7 +531,7 @@ impl HttpClient {
"exclude": exclude,
"directly_upload_media": directly_upload_media,
"watch_interval": watch_interval,
- });
+ }));
self.post("/api/v1/resources", &body).await
}
diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.py
index 447053a3c..a59c28a68 100644
--- a/openviking/parse/parsers/markdown.py
+++ b/openviking/parse/parsers/markdown.py
@@ -86,7 +86,7 @@ class MarkdownParser(BaseParser):
"""
# Configuration constants
- DEFAULT_MAX_SECTION_SIZE = 1024 # Maximum tokens per section
+ DEFAULT_MAX_SECTION_SIZE = 2048 # Maximum tokens per section
DEFAULT_MIN_SECTION_TOKENS = 512 # Minimum tokens to create a separate section
MAX_MERGED_FILENAME_LENGTH = 32 # Maximum length for merged section filenames
@@ -372,10 +372,16 @@ class MarkdownParser(BaseParser):
para_tokens = self._estimate_token_count(para)
para_len = len(para)
- # Single paragraph too long (by tokens or chars): force split by characters
+ # Single paragraph too long (by tokens or chars): force split by characters.
+ # If the already accumulated prefix is very short, merge it into this
+ # oversized paragraph first so we do not create a low-value tiny chunk
+ # like `section_1.md` that only contains the heading/introduction.
if para_tokens > max_size or para_len > max_chars:
if current:
- parts.append(current.strip())
+ if current_tokens < self.DEFAULT_MIN_SECTION_TOKENS:
+ para = current + "\n\n" + para
+ else:
+ parts.append(current.strip())
current = ""
current_tokens = 0
for i in range(0, len(para), max_chars):
@@ -391,7 +397,13 @@ class MarkdownParser(BaseParser):
current_tokens += para_tokens
if current.strip():
- parts.append(current.strip())
+ # Avoid emitting a tiny trailing chunk when all earlier content has
+ # already been split out (for example, a huge paragraph followed by
+ # a short "no data" tail). Fold the tail back into the previous part.
+ if parts and current_tokens < self.DEFAULT_MIN_SECTION_TOKENS:
+ parts[-1] = f"{parts[-1]}\n\n{current.strip()}".strip()
+ else:
+ parts.append(current.strip())
return parts if parts else [content]
@@ -525,35 +537,117 @@ class MarkdownParser(BaseParser):
]
pending = []
+ buffered_section = None
+
+ async def flush_buffered() -> None:
+ nonlocal buffered_section
+ if buffered_section is not None:
+ await self._save_section(
+ content,
+ headings,
+ parent_dir,
+ buffered_section,
+ max_size,
+ min_size,
+ )
+ buffered_section = None
+
for sec in expanded:
name, tokens, content_text = sec["name"], sec["tokens"], sec["content"]
has_children = sec["has_children"]
# Handle small sections
if tokens < min_size:
- pending = await self._try_add_to_pending(
- viking_fs, parent_dir, pending, (name, content_text, tokens), max_size
- )
+ if pending and sum(t for _, _, t in pending) + tokens > max_size:
+ await flush_buffered()
+ await self._save_merged(viking_fs, parent_dir, pending)
+ pending = []
+ pending.append((name, content_text, tokens))
continue
- # Try merge with pending
- if pending and self._can_merge(pending, tokens, max_size, has_children):
- pending.append((name, content_text, tokens))
+ if pending:
+ await flush_buffered()
+
+ # Try merge with pending
+ if self._can_merge(pending, tokens, max_size, has_children):
+ pending.append((name, content_text, tokens))
+ await self._save_merged(viking_fs, parent_dir, pending)
+ pending = []
+ continue
+
+ # Avoid flushing a single tiny section as a standalone low-value file.
+ if self._should_merge_pending_into_next(pending):
+ sec = self._merge_pending_into_next_section(pending, sec)
+ pending = []
+ else:
+ await self._save_merged(viking_fs, parent_dir, pending)
+ pending = []
+ else:
+ await flush_buffered()
+
+ buffered_section = sec
+
+ if pending:
+ # No next section exists. Fold a single tiny pending section back into
+ # the previous saved candidate instead of emitting a standalone file.
+ if buffered_section is not None and self._should_merge_pending_into_next(pending):
+ buffered_section = self._merge_pending_into_previous_section(
+ buffered_section, pending
+ )
+ pending = []
+ else:
+ await flush_buffered()
await self._save_merged(viking_fs, parent_dir, pending)
pending = []
- continue
-
- # Save pending and process current section
- pending = await self._flush_pending(viking_fs, parent_dir, pending)
- await self._save_section(content, headings, parent_dir, sec, max_size, min_size)
- # Save remaining pending
- await self._flush_pending(viking_fs, parent_dir, pending)
+ await flush_buffered()
def _can_merge(self, pending: List, tokens: int, max_size: int, has_children: bool) -> bool:
"""Check if section can merge with pending."""
return sum(t for _, _, t in pending) + tokens <= max_size and not has_children
+ def _should_merge_pending_into_next(self, pending: List[Tuple[str, str, int]]) -> bool:
+ """Prefer folding a single tiny pending section into the next section."""
+ return len(pending) == 1 and pending[0][2] <= self.DEFAULT_MIN_SECTION_TOKENS
+
+ def _merge_pending_into_next_section(
+ self, pending: List[Tuple[str, str, int]], section: Dict[str, Any]
+ ) -> Dict[str, Any]:
+ """Attach a tiny pending section to the following section."""
+ _, pending_content, _ = pending[0]
+ merged = dict(section)
+ merged["content"] = f"{pending_content}\n\n{section['content']}".strip()
+ merged["tokens"] = self._estimate_token_count(merged["content"])
+
+ if merged.get("has_children"):
+ direct_content = section.get("direct_content", "").strip()
+ merged["direct_content"] = (
+ f"{pending_content}\n\n{direct_content}".strip()
+ if direct_content
+ else pending_content
+ )
+
+ return merged
+
+ def _merge_pending_into_previous_section(
+ self, section: Dict[str, Any], pending: List[Tuple[str, str, int]]
+ ) -> Dict[str, Any]:
+ """Attach a tiny trailing pending section back into the previous section."""
+ _, pending_content, _ = pending[0]
+ merged = dict(section)
+ merged["content"] = f"{section['content']}\n\n{pending_content}".strip()
+ merged["tokens"] = self._estimate_token_count(merged["content"])
+
+ if merged.get("has_children"):
+ direct_content = section.get("direct_content", "").strip()
+ merged["direct_content"] = (
+ f"{direct_content}\n\n{pending_content}".strip()
+ if direct_content
+ else pending_content
+ )
+
+ return merged
+
async def _try_add_to_pending(
self, viking_fs, parent_dir: str, pending: List, item: Tuple, max_size: int
) -> List:
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 01373954a..861afce1e 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -1123,16 +1123,17 @@ class VikingFS:
if name in [".", ".."]:
continue
rel_path = f"{current_rel}/{name}" if current_rel else name
+ is_dir = entry.get("isDir", False)
new_entry = {
"uri": self._path_to_uri(f"{current_path}/{name}", ctx=ctx),
- "size": entry.get("size", 0),
- "isDir": entry.get("isDir", False),
+ "size": 0 if is_dir else entry.get("size", 0),
+ "isDir": is_dir,
"modTime": format_simplified(parse_iso_datetime(entry.get("modTime", "")), now),
}
new_entry["rel_path"] = rel_path
if not self._is_accessible(new_entry["uri"], real_ctx):
continue
- if entry.get("isDir"):
+ if is_dir:
all_entries.append(new_entry)
await _walk(f"{current_path}/{name}", rel_path, current_depth + 1)
elif not name.startswith("."):
@@ -2212,15 +2213,16 @@ class VikingFS:
parsed_time = parse_iso_datetime(raw_time)
elif isinstance(entry.get("mtime"), (int, float)):
parsed_time = datetime.fromtimestamp(entry["mtime"], tz=timezone.utc)
+ is_dir = entry.get("isDir", False)
new_entry = {
"uri": self._path_to_uri(f"{path}/{name}", ctx=ctx),
- "size": entry.get("size", 0),
- "isDir": entry.get("isDir", False),
+ "size": 0 if is_dir else entry.get("size", 0),
+ "isDir": is_dir,
"modTime": format_simplified(parsed_time, now),
}
if not self._is_accessible(new_entry["uri"], real_ctx):
continue
- if entry.get("isDir"):
+ if is_dir:
all_entries.append(new_entry)
elif not name.startswith("."):
all_entries.append(new_entry)
diff --git a/openviking_cli/utils/config/parser_config.py b/openviking_cli/utils/config/parser_config.py
index dcf424f9b..e4589671e 100644
--- a/openviking_cli/utils/config/parser_config.py
+++ b/openviking_cli/utils/config/parser_config.py
@@ -37,7 +37,7 @@ class ParserConfig:
encoding: str = "utf-8"
# Smart splitting configuration
- max_section_size: int = 1000 # Maximum tokens per section before splitting
+ max_section_size: int = 2048 # Maximum tokens per section before splitting
section_size_flexibility: float = 0.3 # Allow 30% overflow to maintain coherence
max_section_chars: int = (
6000 # Hard character limit per section (guards against token estimation errors)
diff --git a/tests/parse/test_filename_safety.py b/tests/parse/test_filename_safety.py
deleted file mode 100644
index aecd4d62a..000000000
--- a/tests/parse/test_filename_safety.py
+++ /dev/null
@@ -1,209 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""Tests for filename safety: hash & shorten when names are too long (issue #171)."""
-
-import hashlib
-from unittest.mock import AsyncMock, patch
-
-import pytest
-
-
-class TestSanitizeForPath:
- """Test _sanitize_for_path in MarkdownParser and HTMLParser."""
-
- def _make_md_parser(self):
- from openviking.parse.parsers.markdown import MarkdownParser
-
- return MarkdownParser()
-
- def test_short_text_unchanged(self):
- parser = self._make_md_parser()
- assert parser._sanitize_for_path("Hello World") == "Hello_World"
-
- def test_empty_text_returns_section(self):
- parser = self._make_md_parser()
- assert parser._sanitize_for_path("") == "section"
- assert parser._sanitize_for_path("!!!") == "section"
-
- def test_special_chars_removed(self):
- parser = self._make_md_parser()
- result = parser._sanitize_for_path("Hello, World! (test)")
- assert result == "Hello_World_test"
-
- def test_chinese_preserved(self):
- parser = self._make_md_parser()
- result = parser._sanitize_for_path("你好世界")
- assert result == "你好世界"
-
- def test_long_text_truncated_with_hash(self):
- parser = self._make_md_parser()
- long_text = "a" * 100
- result = parser._sanitize_for_path(long_text)
- assert len(result) <= 50
- expected_hash = hashlib.sha256(long_text.encode()).hexdigest()[:8]
- assert result.endswith(f"_{expected_hash}")
-
- def test_exact_boundary_not_hashed(self):
- parser = self._make_md_parser()
- text = "a" * 50
- result = parser._sanitize_for_path(text)
- assert result == text
- assert len(result) == 50
-
- def test_one_over_boundary_hashed(self):
- parser = self._make_md_parser()
- text = "a" * 51
- result = parser._sanitize_for_path(text)
- assert len(result) <= 50
- assert "_" in result # has hash suffix
-
- def test_custom_max_length(self):
- parser = self._make_md_parser()
- text = "a" * 30
- result = parser._sanitize_for_path(text, max_length=20)
- assert len(result) <= 20
- expected_hash = hashlib.sha256(text.encode()).hexdigest()[:8]
- assert result.endswith(f"_{expected_hash}")
-
- def test_shell_comment_heading(self):
- """Simulate shell script comments being treated as markdown headings."""
- parser = self._make_md_parser()
- heading = "Usage: curl -fsSL https://raw.githubusercontent.com/volcengine/openviking/refs/tags/[email protected]/crates/ov_cli/install.sh | bash"
- result = parser._sanitize_for_path(heading)
- assert len(result) <= 50
-
-
-class TestGenerateMergedFilename:
- """Test _generate_merged_filename in MarkdownParser."""
-
- def _make_md_parser(self):
- from openviking.parse.parsers.markdown import MarkdownParser
-
- return MarkdownParser()
-
- def test_single_short_section(self):
- parser = self._make_md_parser()
- result = parser._generate_merged_filename([("intro", "content", 100)])
- assert result == "intro"
-
- def test_multiple_sections(self):
- parser = self._make_md_parser()
- sections = [("intro", "c1", 10), ("body", "c2", 20), ("end", "c3", 30)]
- result = parser._generate_merged_filename(sections)
- assert "3more" in result
- assert len(result) <= 32
-
- def test_empty_sections(self):
- parser = self._make_md_parser()
- assert parser._generate_merged_filename([]) == "merged"
-
- def test_long_single_name_hashed(self):
- parser = self._make_md_parser()
- long_name = "a" * 100
- result = parser._generate_merged_filename([(long_name, "content", 50)])
- assert len(result) <= 32
-
- def test_result_never_exceeds_limit(self):
- parser = self._make_md_parser()
- # Create many sections with long names
- sec_list = [(f"very_long_section_name_{i}", f"content_{i}", 10) for i in range(20)]
- result = parser._generate_merged_filename(sec_list)
- assert len(result) <= 32
-
-
-async def test_markdown_resource_name_is_not_stemmed_again():
- from openviking.parse.parsers.markdown import MarkdownParser
-
- parser = MarkdownParser()
- mock_vfs = type("MockVikingFS", (), {})()
- mock_vfs.create_temp_uri = lambda: "viking://temp/test"
- mock_vfs.mkdir = AsyncMock()
- mock_vfs.write_file = AsyncMock()
-
- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):
- result = await parser.parse_content(
- "short body",
- source_path="/tmp/upload.yaml",
- resource_name="access.dns.md",
- )
-
- assert result.root.title == "access.dns.md"
- mock_vfs.mkdir.assert_any_await("viking://temp/test/access.dns.md")
-
-
-class TestShortenComponent:
- """Test VikingFS._shorten_component."""
-
- def test_short_component_unchanged(self):
- from openviking.storage.viking_fs import VikingFS
-
- assert VikingFS._shorten_component("hello") == "hello"
-
- def test_long_component_shortened(self):
- from openviking.storage.viking_fs import VikingFS
-
- long_name = "a" * 300
- result = VikingFS._shorten_component(long_name)
- assert len(result.encode("utf-8")) <= 255
-
- def test_exact_255_bytes_unchanged(self):
- from openviking.storage.viking_fs import VikingFS
-
- name = "a" * 255
- assert VikingFS._shorten_component(name) == name
-
- def test_256_bytes_shortened(self):
- from openviking.storage.viking_fs import VikingFS
-
- name = "a" * 256
- result = VikingFS._shorten_component(name)
- assert len(result.encode("utf-8")) <= 255
- expected_hash = hashlib.sha256(name.encode("utf-8")).hexdigest()[:8]
- assert result.endswith(f"_{expected_hash}")
-
- def test_unicode_multibyte_handling(self):
- from openviking.storage.viking_fs import VikingFS
-
- # Chinese chars are 3 bytes each in UTF-8
- name = "你" * 100 # 300 bytes
- result = VikingFS._shorten_component(name)
- assert len(result.encode("utf-8")) <= 255
-
- def test_realistic_long_filename(self):
- """Simulate the exact bug from issue #171."""
- from openviking.storage.viking_fs import VikingFS
-
- long_filename = (
- "tmp5vacylnx_OpenViking_CLI_Installer_Usage_curl_-fsSL_"
- "httpsrawgithubusercontentcomvolce_Example_curl_-fsSL_"
- "httpsrawgithubusercontentcomvol_Skip_checksum_"
- "SKIP_CHECKSUM1_curl_-fsSL_bash_Colors_for_output_"
- "Detect_platform_and_architecture_Get_latest_release_info_"
- "Download_and_extract_binary"
- )
- result = VikingFS._shorten_component(long_filename)
- assert len(result.encode("utf-8")) <= 255
-
-
-class TestDownloaderGenerateFilename:
- """Test _generate_filename in downloader."""
-
- def test_short_url(self):
- from openviking_cli.utils.downloader import _generate_filename
-
- result = _generate_filename("https://example.com/file.pdf")
- assert result == "file"
-
- def test_long_path_url(self):
- from openviking_cli.utils.downloader import _generate_filename
-
- url = "https://example.com/" + "a" * 200 + ".pdf"
- result = _generate_filename(url)
- assert len(result) <= 50
-
- def test_host_only_url(self):
- from openviking_cli.utils.downloader import _generate_filename
-
- result = _generate_filename("https://example.com/")
- assert result == "example_com"
diff --git a/tests/parse/test_markdown_char_limit.py b/tests/parse/test_markdown_char_limit.py
deleted file mode 100644
index 7e7d0aaf0..000000000
--- a/tests/parse/test_markdown_char_limit.py
+++ /dev/null
@@ -1,362 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""Tests for MarkdownParser hard character limit enforcement (max_section_chars)."""
-
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-from openviking.parse.parsers.markdown import MarkdownParser
-from openviking_cli.utils.config.parser_config import (
- FeishuConfig,
- ParserConfig,
- load_parser_configs_from_dict,
-)
-
-# ---------------------------------------------------------------------------
-# ParserConfig
-# ---------------------------------------------------------------------------
-
-
-class TestParserConfigMaxSectionChars:
- def test_default_value(self):
- config = ParserConfig()
- assert config.max_section_chars == 6000
-
- def test_custom_value(self):
- config = ParserConfig(max_section_chars=3000)
- assert config.max_section_chars == 3000
-
- def test_from_dict(self):
- config = ParserConfig.from_dict({"max_section_chars": 2000})
- assert config.max_section_chars == 2000
-
- def test_from_dict_missing_key_uses_default(self):
- config = ParserConfig.from_dict({})
- assert config.max_section_chars == 6000
-
- def test_from_dict_rejects_unknown_key(self):
- with pytest.raises(ValueError, match="max_section_chars"):
- ParserConfig.from_dict({"max_section_chras": 2000})
-
- def test_load_parser_configs_rejects_unknown_parser_section(self):
- with pytest.raises(ValueError, match="markdown"):
- load_parser_configs_from_dict({"markdwon": {}})
-
- def test_validate_rejects_zero(self):
- config = ParserConfig(max_section_chars=0)
- with pytest.raises(ValueError, match="max_section_chars"):
- config.validate()
-
- def test_validate_rejects_negative(self):
- config = ParserConfig(max_section_chars=-1)
- with pytest.raises(ValueError, match="max_section_chars"):
- config.validate()
-
- def test_validate_accepts_positive(self):
- config = ParserConfig(max_section_chars=1)
- config.validate() # must not raise
-
-
-class TestFeishuConfigValidation:
- def test_validate_accepts_defaults(self):
- config = FeishuConfig()
- config.validate()
-
- @pytest.mark.parametrize(
- ("field", "value", "message"),
- [
- ("domain", "", "domain"),
- ("max_rows_per_sheet", 0, "max_rows_per_sheet"),
- ("max_rows_per_sheet", -1, "max_rows_per_sheet"),
- ("max_records_per_table", 0, "max_records_per_table"),
- ("max_records_per_table", -1, "max_records_per_table"),
- ("request_timeout", 0, "request_timeout"),
- ("request_timeout", -0.1, "request_timeout"),
- ],
- )
- def test_validate_rejects_invalid_values(self, field, value, message):
- config = FeishuConfig(**{field: value})
- with pytest.raises(ValueError, match=message):
- config.validate()
-
-
-# ---------------------------------------------------------------------------
-# _smart_split_content — unit tests (no VikingFS needed)
-# ---------------------------------------------------------------------------
-
-
-class TestSmartSplitContentCharLimit:
- def _make_parser(self, max_section_chars: int = 100) -> MarkdownParser:
- """Parser with a tight char limit for easy testing."""
- config = ParserConfig(max_section_size=1000, max_section_chars=max_section_chars)
- return MarkdownParser(config=config)
-
- def test_short_content_returned_as_single_part(self):
- parser = self._make_parser(max_section_chars=500)
- content = "Hello world. " * 10 # 130 chars, well within limit
- parts = parser._smart_split_content(content, max_size=1000)
- assert len(parts) == 1
-
- def test_paragraph_exceeding_char_limit_is_force_split(self):
- """A single paragraph longer than max_section_chars must be split."""
- parser = self._make_parser(max_section_chars=50)
- # One paragraph of 200 chars — no \n\n so it stays as one para
- long_para = "x" * 200
- parts = parser._smart_split_content(long_para, max_size=1000)
- # Each part must be <= max_section_chars
- for part in parts:
- assert len(part) <= 50
- # All content is preserved
- assert "".join(p.strip() for p in parts) == long_para
-
- def test_accumulated_chunks_respect_char_limit(self):
- """Multiple small paragraphs that together exceed max_chars are split."""
- parser = self._make_parser(max_section_chars=80)
- # Each paragraph is 40 chars; two together (40+2+40=82) exceed the limit
- para = "y" * 40
- content = f"{para}\n\n{para}\n\n{para}"
- parts = parser._smart_split_content(content, max_size=1000)
- for part in parts:
- assert len(part) <= 80
-
- def test_char_limit_splits_even_when_token_estimate_is_small(self):
- """Content within token limit but exceeding char limit must still be split."""
- # Use a very large max_section_chars to NOT trigger char splitting
- parser_no_limit = self._make_parser(max_section_chars=10000)
- # Use a tight max_section_chars to force splitting
- parser_with_limit = self._make_parser(max_section_chars=50)
-
- content = "a" * 120 # single paragraph, ~36 estimated tokens (well under 1000)
-
- parts_no_limit = parser_no_limit._smart_split_content(content, max_size=1000)
- parts_with_limit = parser_with_limit._smart_split_content(content, max_size=1000)
-
- # Without char limit: one part (token estimate is fine)
- assert len(parts_no_limit) == 1
- # With char limit: must be split into multiple parts
- assert len(parts_with_limit) > 1
- for part in parts_with_limit:
- assert len(part) <= 50
-
- def test_empty_paragraphs_handled(self):
- parser = self._make_parser(max_section_chars=200)
- content = "line1\n\n\n\nline2"
- parts = parser._smart_split_content(content, max_size=1000)
- assert all(p.strip() for p in parts) # no blank-only parts
-
- def test_content_entirely_preserved_after_split(self):
- """No characters should be lost during splitting."""
- parser = self._make_parser(max_section_chars=60)
- # Build content where each paragraph is 50 chars
- para = "p" * 50
- content = "\n\n".join([para] * 5)
- parts = parser._smart_split_content(content, max_size=1000)
- # Reconstruct: strip whitespace added by joining
- total_chars = sum(len(p.replace(" ", "").replace("\n", "")) for p in parts)
- original_chars = len(content.replace(" ", "").replace("\n", ""))
- assert total_chars == original_chars
-
-
-# ---------------------------------------------------------------------------
-# _save_section — char limit gates (async, VikingFS mocked)
-# ---------------------------------------------------------------------------
-
-
-class TestSaveSectionCharLimit:
- def _make_parser(self, max_section_chars: int = 100) -> MarkdownParser:
- config = ParserConfig(max_section_size=1000, max_section_chars=max_section_chars)
- return MarkdownParser(config=config)
-
- def _mock_viking_fs(self):
- vfs = MagicMock()
- vfs.write_file = AsyncMock()
- vfs.mkdir = AsyncMock()
- return vfs
-
- @pytest.mark.asyncio
- async def test_section_within_both_limits_saved_as_file(self):
- """Section under token AND char limit → single .md file."""
- parser = self._make_parser(max_section_chars=500)
- mock_vfs = self._mock_viking_fs()
-
- section = {
- "name": "intro",
- "tokens": 50,
- "content": "Short content",
- "has_children": False,
- }
-
- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):
- await parser._save_section("", [], "viking://tmp/root", section, 1000, 512)
-
- mock_vfs.write_file.assert_called_once()
- call_path = mock_vfs.write_file.call_args[0][0]
- assert call_path.endswith("intro.md")
-
- @pytest.mark.asyncio
- async def test_section_exceeding_char_limit_is_split_even_if_tokens_ok(self):
- """Section within token limit but over char limit must NOT be written as single file."""
- parser = self._make_parser(max_section_chars=50)
- mock_vfs = self._mock_viking_fs()
-
- long_content = "word " * 30 # 150 chars, ~14 estimated tokens (under 1000)
- section = {
- "name": "big_section",
- "tokens": 14,
- "content": long_content,
- "has_children": False,
- }
-
- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):
- await parser._save_section("", [], "viking://tmp/root", section, 1000, 512)
-
- # Must have written at least one file
- mock_vfs.write_file.assert_called()
- # Every written chunk must respect the char limit
- for call in mock_vfs.write_file.call_args_list:
- written_content = call[0][1]
- assert len(written_content) <= 50, (
- f"Written content exceeds char limit: {len(written_content)} chars"
- )
- # A directory should have been created for the split
- mock_vfs.mkdir.assert_called()
-
-
-# ---------------------------------------------------------------------------
-# _save_merged — char limit enforcement (async, VikingFS mocked)
-# ---------------------------------------------------------------------------
-
-
-class TestSaveMergedCharLimit:
- def _make_parser(self, max_section_chars: int) -> MarkdownParser:
- config = ParserConfig(max_section_size=1000, max_section_chars=max_section_chars)
- return MarkdownParser(config=config)
-
- def _mock_viking_fs(self):
- vfs = MagicMock()
- vfs.write_file = AsyncMock()
- return vfs
-
- @pytest.mark.asyncio
- async def test_merged_within_char_limit_saved_as_single_file(self):
- parser = self._make_parser(max_section_chars=500)
- mock_vfs = self._mock_viking_fs()
-
- sections = [("s1", "hello", 5), ("s2", "world", 5)]
- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):
- await parser._save_merged(mock_vfs, "viking://tmp/root", sections)
-
- mock_vfs.write_file.assert_called_once()
- path, content = mock_vfs.write_file.call_args[0]
- assert path.endswith(".md")
- assert "hello" in content and "world" in content
-
- @pytest.mark.asyncio
- async def test_merged_exceeding_char_limit_is_split_into_multiple_files(self):
- """When joined content exceeds max_section_chars, _save_merged must split it."""
- parser = self._make_parser(max_section_chars=60)
- mock_vfs = self._mock_viking_fs()
-
- # Each section is 40 chars; joined = 40 + "\n\n" + 40 = 82 chars > 60
- sections = [("a", "A" * 40, 12), ("b", "B" * 40, 12)]
- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):
- await parser._save_merged(mock_vfs, "viking://tmp/root", sections)
-
- # Should have written multiple files
- assert mock_vfs.write_file.call_count > 1
- for call in mock_vfs.write_file.call_args_list:
- written_content = call[0][1]
- assert len(written_content) <= 60, (
- f"Merged split part exceeds char limit: {len(written_content)} chars"
- )
-
- @pytest.mark.asyncio
- async def test_many_small_sections_merged_correctly_split(self):
- """Many token-small but char-large sections that accumulate past the limit."""
- parser = self._make_parser(max_section_chars=100)
- mock_vfs = self._mock_viking_fs()
-
- # 10 sections × 30 chars = 300 chars + separators >> 100 chars
- sections = [(f"s{i}", "z" * 30, 9) for i in range(10)]
- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):
- await parser._save_merged(mock_vfs, "viking://tmp/root", sections)
-
- # Every written part must respect the char limit
- mock_vfs.write_file.assert_called()
- for call in mock_vfs.write_file.call_args_list:
- written_content = call[0][1]
- assert len(written_content) <= 100, (
- f"Part exceeds char limit: {len(written_content)} chars"
- )
-
-
-# ---------------------------------------------------------------------------
-# _parse_and_create_structure — small-document char limit (async, VikingFS mocked)
-# ---------------------------------------------------------------------------
-
-
-class TestParseAndCreateStructureCharLimit:
- def _make_parser(self, max_section_size: int = 1000, max_section_chars: int = 100):
- config = ParserConfig(
- max_section_size=max_section_size, max_section_chars=max_section_chars
- )
- return MarkdownParser(config=config)
-
- def _mock_viking_fs(self):
- vfs = MagicMock()
- vfs.write_file = AsyncMock()
- vfs.mkdir = AsyncMock()
- return vfs
-
- @pytest.mark.asyncio
- async def test_small_doc_within_both_limits_saved_as_single_file(self):
- parser = self._make_parser(max_section_size=1000, max_section_chars=500)
- mock_vfs = self._mock_viking_fs()
- content = "# Title\n\nShort body." # well under both limits
-
- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):
- await parser._parse_and_create_structure(content, [], "viking://tmp/root")
-
- mock_vfs.write_file.assert_called_once()
- written_content = mock_vfs.write_file.call_args[0][1]
- assert written_content == content
-
- @pytest.mark.asyncio
- async def test_small_doc_exceeding_char_limit_is_not_saved_as_single_file(self):
- """Even if token estimate is tiny, a doc over max_section_chars must be split."""
- parser = self._make_parser(max_section_size=1000, max_section_chars=30)
- mock_vfs = self._mock_viking_fs()
-
- # 200 chars but NO headings → no sections, falls back to paragraph split
- content = "a" * 200
-
- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):
- await parser._parse_and_create_structure(content, [], "viking://tmp/root")
-
- # Must have been split: each written chunk ≤ max_section_chars
- assert mock_vfs.write_file.call_count > 1
- for call in mock_vfs.write_file.call_args_list:
- written = call[0][1]
- assert len(written) <= 30
-
- @pytest.mark.asyncio
- async def test_doc_with_headings_exceeding_char_limit_is_not_saved_as_single_file(self):
- """Small-document fast-path with headings: char limit must still be enforced."""
- parser = self._make_parser(max_section_size=1000, max_section_chars=50)
- mock_vfs = self._mock_viking_fs()
-
- # 200 chars WITH a heading — exercises the char-check at the
- # "estimated_tokens <= max_size AND len(content) <= max_chars" guard
- content = "# Heading\n\n" + "b" * 189 # total > 50 chars, token estimate << 1000
-
- with patch.object(parser, "_get_viking_fs", return_value=mock_vfs):
- headings = parser._find_headings(content)
- await parser._parse_and_create_structure(content, headings, "viking://tmp/root")
-
- # Must not have been saved as a single file equal to the full content
- for call in mock_vfs.write_file.call_args_list:
- written = call[0][1]
- assert len(written) <= 50, (
- f"Single file written with {len(written)} chars, exceeds limit of 50"
- )
diff --git a/tests/server/test_api_filesystem.py b/tests/server/test_api_filesystem.py
deleted file mode 100644
index de98979c4..000000000
--- a/tests/server/test_api_filesystem.py
+++ /dev/null
@@ -1,253 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-
-"""Tests for filesystem endpoints: ls, tree, stat, mkdir, rm, mv."""
-
-import httpx
-
-from openviking.pyagfs.exceptions import AGFSHTTPError
-from openviking.storage.errors import ResourceBusyError
-
-
-async def test_ls_root(client: httpx.AsyncClient):
- resp = await client.get("/api/v1/fs/ls", params={"uri": "viking://"})
- assert resp.status_code == 200
- body = resp.json()
- assert body["status"] == "ok"
- assert isinstance(body["result"], list)
-
-
-async def test_ls_simple(client: httpx.AsyncClient):
- resp = await client.get(
- "/api/v1/fs/ls",
- params={"uri": "viking://", "simple": True},
- )
- assert resp.status_code == 200
- body = resp.json()
- assert body["status"] == "ok"
- assert isinstance(body["result"], list)
- # Each item must be a non-empty URI string (fixes #218)
- for item in body["result"]:
- assert isinstance(item, str)
- assert item.startswith("viking://")
-
-
-async def test_ls_simple_agent_output(client: httpx.AsyncClient):
- """Ensure --simple with output=agent returns URI strings, not empty."""
- resp = await client.get(
- "/api/v1/fs/ls",
- params={"uri": "viking://", "simple": True, "output": "agent"},
- )
- assert resp.status_code == 200
- body = resp.json()
- assert body["status"] == "ok"
- assert isinstance(body["result"], list)
- for item in body["result"]:
- assert isinstance(item, str)
- assert item.startswith("viking://")
-
-
-async def test_mkdir_and_ls(client: httpx.AsyncClient):
- resp = await client.post(
- "/api/v1/fs/mkdir",
- json={"uri": "viking://resources/test_dir/"},
- )
- assert resp.status_code == 200
- assert resp.json()["status"] == "ok"
-
- resp = await client.get(
- "/api/v1/fs/ls",
- params={"uri": "viking://resources/"},
- )
- assert resp.status_code == 200
-
-
-async def test_mkdir_with_description_initializes_abstract_and_enqueues_l0(
- client: httpx.AsyncClient,
- monkeypatch,
-):
- seen = {}
-
- async def _fake_vectorize_directory_meta(**kwargs):
- seen.update(kwargs)
-
- monkeypatch.setattr(
- "openviking.service.fs_service.vectorize_directory_meta",
- _fake_vectorize_directory_meta,
- )
-
- uri = "viking://resources/described_dir/"
- description = "Directory for API docs"
- resp = await client.post(
- "/api/v1/fs/mkdir",
- json={"uri": uri, "description": description},
- )
- assert resp.status_code == 200
- assert resp.json()["status"] == "ok"
-
- abstract_resp = await client.get(
- "/api/v1/content/abstract",
- params={"uri": uri},
- )
- assert abstract_resp.status_code == 200
- assert abstract_resp.json()["result"] == description
- assert seen["uri"] == "viking://resources/described_dir"
- assert seen["abstract"] == description
- assert seen["overview"] == ""
- assert seen["context_type"] == "resource"
- assert seen["include_overview"] is False
- assert seen["ctx"] is not None
-
-
-async def test_tree(client: httpx.AsyncClient):
- resp = await client.get("/api/v1/fs/tree", params={"uri": "viking://"})
- assert resp.status_code == 200
- body = resp.json()
- assert body["status"] == "ok"
-
-
-async def test_stat_not_found(client: httpx.AsyncClient):
- resp = await client.get(
- "/api/v1/fs/stat",
- params={"uri": "viking://resources/nonexistent/xyz"},
- )
- assert resp.status_code == 404
- body = resp.json()
- assert body["status"] == "error"
-
-
-async def test_rm_directory_without_recursive_returns_failed_precondition(
- client: httpx.AsyncClient,
-):
- await client.post(
- "/api/v1/fs/mkdir",
- json={"uri": "viking://resources/rm_dir_without_recursive/"},
- )
- resp = await client.request(
- "DELETE",
- "/api/v1/fs",
- params={"uri": "viking://resources/rm_dir_without_recursive"},
- )
- assert resp.status_code == 412
- body = resp.json()
- assert body["status"] == "error"
- assert body["error"]["code"] == "FAILED_PRECONDITION"
-
-
-async def test_rm_missing_uri_is_idempotent(client: httpx.AsyncClient):
- missing_uri = "viking://resources/definitely-missing-rm-target"
-
- resp = await client.request(
- "DELETE",
- "/api/v1/fs",
- params={"uri": missing_uri, "recursive": True},
- )
-
- assert resp.status_code == 200
- body = resp.json()
- assert body["status"] == "ok"
- assert body["result"] == {"uri": missing_uri}
-
-
-async def test_rm_unsupported_scheme_returns_invalid_uri(client: httpx.AsyncClient):
- resp = await client.request(
- "DELETE",
- "/api/v1/fs",
- params={"uri": "s3://bucket/missing", "recursive": True},
- )
-
- assert resp.status_code == 400
- body = resp.json()
- assert body["status"] == "error"
- assert body["error"]["code"] == "INVALID_URI"
- assert "unsupported URI scheme 's3'" in body["error"]["message"]
-
-
-async def test_ls_invalid_scope_hides_internal_scopes(client: httpx.AsyncClient):
- resp = await client.get("/api/v1/fs/ls", params={"uri": "ssd"})
-
- assert resp.status_code == 400
- body = resp.json()
- message = body["error"]["message"]
- assert body["status"] == "error"
- assert body["error"]["code"] == "INVALID_URI"
- assert "resources" in message
- assert "temp" not in message
- assert "queue" not in message
- assert "frozenset" not in message
-
-
-async def test_resource_ops(client_with_resource):
- """Test stat, ls_recursive, mv, rm on a single shared resource."""
- import uuid
-
- client, uri = client_with_resource
-
- # stat
- resp = await client.get("/api/v1/fs/stat", params={"uri": uri})
- assert resp.status_code == 200
- assert resp.json()["status"] == "ok"
-
- # ls recursive
- resp = await client.get(
- "/api/v1/fs/ls",
- params={"uri": "viking://", "recursive": True},
- )
- assert resp.status_code == 200
- body = resp.json()
- assert body["status"] == "ok"
- assert isinstance(body["result"], list)
-
- # mv
- unique = uuid.uuid4().hex[:8]
- new_uri = uri.rstrip("/") + f"_mv_{unique}/"
- resp = await client.post(
- "/api/v1/fs/mv",
- json={"from_uri": uri, "to_uri": new_uri},
- )
- assert resp.status_code == 200
- assert resp.json()["status"] == "ok"
-
- # rm (on the moved uri)
- resp = await client.request("DELETE", "/api/v1/fs", params={"uri": new_uri, "recursive": True})
- assert resp.status_code == 200
- assert resp.json()["status"] == "ok"
-
-
-async def test_rm_maps_resource_busy_to_conflict(client, service, monkeypatch):
- async def fake_rm(uri, ctx=None, recursive=False):
- raise ResourceBusyError(f"Resource is being processed: {uri}")
-
- monkeypatch.setattr(service.fs, "rm", fake_rm)
- resp = await client.request(
- "DELETE",
- "/api/v1/fs",
- params={"uri": "viking://resources/locked", "recursive": True},
- )
- assert resp.status_code == 409
- body = resp.json()
- assert body["status"] == "error"
- assert body["error"]["code"] == "CONFLICT"
-
-
-async def test_rm_agfs_internal_error_does_not_look_successful(client, service, monkeypatch):
- class FakeAGFS:
- def __init__(self, wrapped):
- self._wrapped = wrapped
-
- def stat(self, path):
- raise AGFSHTTPError("Internal server error", 500)
-
- def __getattr__(self, name):
- return getattr(self._wrapped, name)
-
- monkeypatch.setattr(service.fs._viking_fs, "agfs", FakeAGFS(service.fs._viking_fs.agfs))
- resp = await client.request(
- "DELETE",
- "/api/v1/fs",
- params={"uri": "viking://resources/delete-failure", "recursive": True},
- )
- assert resp.status_code == 503
- body = resp.json()
- assert body["status"] == "error"
- assert body["error"]["code"] == "UNAVAILABLE"