Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
"""Native section limits use characters, independently of legacy token settings."""
import pytest
from openviking.parse.parsers.markdown import MarkdownParser
from openviking_cli.utils.config.parser_config import ParserConfig
@pytest.mark.parametrize("character", ["中", "a", "🙂"])
def test_smart_split_character_budget_preserves_content(character):
parser = MarkdownParser()
paragraph = character * 5000
parts = parser._smart_split_content(paragraph, max_size=2000)
assert [len(part) for part in parts] == [2000, 2000, 1000]
assert "".join(parts) == paragraph
def test_short_tail_does_not_overflow_character_budget():
parser = MarkdownParser()
parts = parser._smart_split_content("中" * 4000 + "\n\ntail", max_size=2000)
assert all(len(part) <= 2000 for part in parts)
assert parts[-1] == "tail"
@pytest.mark.asyncio
@pytest.mark.parametrize("with_headings", [False, True])
async def test_layout_uses_character_budget_ignoring_legacy_tokens(with_headings):
parser = MarkdownParser(ParserConfig(max_section_size=1))
content = ("# Title\n\n" if with_headings else "") + "中" * 1990
layout = await parser._compute_layout(content, "viking://temp/test", resource_name="doc")
writes = [op for op in layout.ops if op.kind == "write"]
assert len(writes) == 1
assert writes[0].content == content
@pytest.mark.asyncio
async def test_heading_merges_stay_within_character_budget():
parser = MarkdownParser()
content = "\n\n".join(f"## Heading {i}\n\n" + "文" * 750 for i in range(12))
layout = await parser._compute_layout(content, "viking://temp/test", resource_name="doc")
writes = [op for op in layout.ops if op.kind == "write"]
assert all(len(op.content) <= 2000 for op in writes)
assert sum(op.content.count("文") for op in writes) == 9000
@pytest.mark.parametrize("leading_row", ["", "| short |\n"])
def test_oversized_table_row_remains_intact_with_header(leading_row):
parser = MarkdownParser()
row = "| " + "文" * 2500 + " |"
header = "| Column |\n| --- |"
content = header + "\n" + leading_row + row
parts = parser._smart_split_content(content, max_size=2000)
assert header + "\n" + row in parts
assert header not in parts
assert sum(part.count(row) for part in parts) == 1
def test_table_header_does_not_push_ordinary_rows_over_budget():
parser = MarkdownParser()
rows = ["| " + "文" * 1950 + " |" for _ in range(2)]
content = "| " + "标题" * 40 + " |\n| --- |\n" + "\n".join(rows)
parts = parser._smart_split_content(content, max_size=2000)
assert all(len(part) <= 2000 for part in parts)
assert sum(part.count("文") for part in parts) == 3900
assert all(any(row in part for part in parts) for row in rows)
@pytest.mark.parametrize("header_size", [1990, 2100])
def test_oversized_header_and_row_stay_together_without_duplicate_header(header_size):
parser = MarkdownParser()
header = "| " + "H" * header_size + " | X |\n| --- | --- |"
row = "| " + "文" * 2500 + " | Y |"
content = header + "\n" + row
assert parser._smart_split_content(content, max_size=2000) == [content]