Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(markdown): keep force-split chunks within the token budget (#3672)

bianbiandashen3087f943a2dd
diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.py
index b3615b50b..2000f4ef3 100644
--- a/openviking/parse/parsers/markdown.py
+++ b/openviking/parse/parsers/markdown.py
@@ -155,6 +155,9 @@ class MarkdownParser(BaseParser):
# Configuration constants
DEFAULT_MAX_SECTION_SIZE = 2048 # Maximum tokens per section
DEFAULT_MIN_SECTION_TOKENS = 512 # Minimum tokens to create a separate section
+ # Worst-case token density used by _estimate_token_count (CJK ~0.7 tok/char).
+ # Used to bound force-split chunk size so it also respects the token budget.
+ MAX_TOKENS_PER_CHAR = 0.7
MAX_MERGED_FILENAME_LENGTH = 32 # Maximum length for merged section filenames
# Image validation constants
@@ -646,8 +649,14 @@ class MarkdownParser(BaseParser):
parts.append(current.strip())
current = ""
current_tokens = 0
- for i in range(0, len(para), max_chars):
- parts.append(para[i : i + max_chars].strip())
+ # Step by the smaller of the char limit and the token-safe
+ # width, so a paragraph that is under max_chars but over
+ # max_size tokens (e.g. dense CJK text) is still split enough
+ # to keep every chunk within the token budget.
+ token_safe_chars = max(1, int(max_size / self.MAX_TOKENS_PER_CHAR))
+ step = min(max_chars, token_safe_chars)
+ for i in range(0, len(para), step):
+ parts.append(para[i : i + step].strip())
elif (
current_tokens + para_tokens > max_size or len(current) + len(para) + 2 > max_chars
) and current:
@@ -1673,4 +1682,4 @@ class MarkdownParser(BaseParser):
# This provides better coverage for multilingual documents
cjk_chars = len(re.findall(r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]", content))
other_chars = len(re.findall(r"[^\s]", content)) - cjk_chars
- return int(cjk_chars * 0.7 + other_chars * 0.3)
+ return int(cjk_chars * self.MAX_TOKENS_PER_CHAR + other_chars * 0.3)
diff --git a/tests/parse/test_markdown_split_token_budget.py b/tests/parse/test_markdown_split_token_budget.py
new file mode 100644
index 000000000..3749301ee
--- /dev/null
+++ b/tests/parse/test_markdown_split_token_budget.py
@@ -0,0 +1,27 @@
+"""Regression test: _smart_split_content must respect the token budget.
+
+The force-split branch previously stepped through an oversized paragraph by
+``max_chars`` only. A paragraph that is under the character limit but over the
+token budget (e.g. dense CJK text, weighted ~0.7 token/char) was therefore
+emitted as a single chunk that exceeded ``max_size`` tokens.
+"""
+
+from openviking.parse.parsers.markdown import MarkdownParser
+
+
+def test_smart_split_enforces_token_budget_for_cjk_paragraph():
+ parser = MarkdownParser()
+ max_size = 2048
+ # 5000 CJK chars ~= 3500 estimated tokens (> max_size) but < the default
+ # char limit, so the char-only force-split produced one over-budget chunk.
+ paragraph = "\u4e2d" * 5000
+ parts = parser._smart_split_content(paragraph, max_size=max_size)
+ worst = max(parser._estimate_token_count(p) for p in parts)
+ assert worst <= max_size, f"chunk has {worst} tokens, exceeds max_size={max_size}"
+
+
+def test_smart_split_preserves_content():
+ parser = MarkdownParser()
+ paragraph = "\u4e2d" * 5000
+ parts = parser._smart_split_content(paragraph, max_size=2048)
+ assert "".join(parts) == paragraph