Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(markdown): keep force-split chunks within the token budget (#3672)
bianbiandashen3087f943a2dd
diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.pyindex b3615b50b..2000f4ef3 100644--- a/openviking/parse/parsers/markdown.py+++ b/openviking/parse/parsers/markdown.py@@ -155,6 +155,9 @@ class MarkdownParser(BaseParser):# Configuration constantsDEFAULT_MAX_SECTION_SIZE = 2048 # Maximum tokens per sectionDEFAULT_MIN_SECTION_TOKENS = 512 # Minimum tokens to create a separate section+ # Worst-case token density used by _estimate_token_count (CJK ~0.7 tok/char).+ # Used to bound force-split chunk size so it also respects the token budget.+ MAX_TOKENS_PER_CHAR = 0.7MAX_MERGED_FILENAME_LENGTH = 32 # Maximum length for merged section filenames# Image validation constants@@ -646,8 +649,14 @@ class MarkdownParser(BaseParser):parts.append(current.strip())current = ""current_tokens = 0- for i in range(0, len(para), max_chars):- parts.append(para[i : i + max_chars].strip())+ # Step by the smaller of the char limit and the token-safe+ # width, so a paragraph that is under max_chars but over+ # max_size tokens (e.g. dense CJK text) is still split enough+ # to keep every chunk within the token budget.+ token_safe_chars = max(1, int(max_size / self.MAX_TOKENS_PER_CHAR))+ step = min(max_chars, token_safe_chars)+ for i in range(0, len(para), step):+ parts.append(para[i : i + step].strip())elif (current_tokens + para_tokens > max_size or len(current) + len(para) + 2 > max_chars) and current:@@ -1673,4 +1682,4 @@ class MarkdownParser(BaseParser):# This provides better coverage for multilingual documentscjk_chars = len(re.findall(r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]", content))other_chars = len(re.findall(r"[^\s]", content)) - cjk_chars- return int(cjk_chars * 0.7 + other_chars * 0.3)+ return int(cjk_chars * self.MAX_TOKENS_PER_CHAR + other_chars * 0.3)diff --git a/tests/parse/test_markdown_split_token_budget.py b/tests/parse/test_markdown_split_token_budget.pynew file mode 100644index 000000000..3749301ee--- /dev/null+++ b/tests/parse/test_markdown_split_token_budget.py@@ -0,0 +1,27 @@+"""Regression test: _smart_split_content must respect the token budget.++The force-split branch previously stepped through an oversized paragraph by+``max_chars`` only. A paragraph that is under the character limit but over the+token budget (e.g. dense CJK text, weighted ~0.7 token/char) was therefore+emitted as a single chunk that exceeded ``max_size`` tokens.+"""++from openviking.parse.parsers.markdown import MarkdownParser+++def test_smart_split_enforces_token_budget_for_cjk_paragraph():+ parser = MarkdownParser()+ max_size = 2048+ # 5000 CJK chars ~= 3500 estimated tokens (> max_size) but < the default+ # char limit, so the char-only force-split produced one over-budget chunk.+ paragraph = "\u4e2d" * 5000+ parts = parser._smart_split_content(paragraph, max_size=max_size)+ worst = max(parser._estimate_token_count(p) for p in parts)+ assert worst <= max_size, f"chunk has {worst} tokens, exceeds max_size={max_size}"+++def test_smart_split_preserves_content():+ parser = MarkdownParser()+ paragraph = "\u4e2d" * 5000+ parts = parser._smart_split_content(paragraph, max_size=2048)+ assert "".join(parts) == paragraph