Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Generate resource L0 summaries from overview prompt (#2890)
agentfdcfa0ef1069
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex dd5d13e9d..2006d9875 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -812,8 +812,7 @@ class SemanticDagExecutor:overview = await self._processor._generate_overview(dir_uri, file_summaries, children_abstracts)- abstract = self._processor._extract_abstract_from_overview(overview)- overview, abstract = self._processor._enforce_size_limits(overview, abstract)+ overview, abstract = self._processor._normalize_overview_generation(overview)# Write directly, protected by the outer semantic lock.try:diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex e5b6723de..9232071ce 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -3,6 +3,7 @@"""SemanticProcessor: Processes messages from SemanticQueue, generates .abstract.md and .overview.md."""import asyncio+import reimport threadingfrom contextlib import nullcontextfrom dataclasses import dataclass, field@@ -652,11 +653,10 @@ class SemanticProcessor(DequeueHandlerBase):for file_path, summary in zip(file_paths, file_summaries, strict=False)if file_path in paths_to_vectorize and summary is not None]- overview = await self._generate_overview(+ generated_content = await self._generate_overview(dir_uri, completed_summaries, [], llm_sem=llm_sem)- abstract = self._extract_abstract_from_overview(overview)- overview, abstract = self._enforce_size_limits(overview, abstract)+ overview, abstract = self._normalize_overview_generation(generated_content)try:wrote_semantics = await self._write_memory_directory_semantics(@@ -1155,8 +1155,49 @@ class SemanticProcessor(DequeueHandlerBase):else:return await self._generate_text_summary(file_path, file_name, llm_sem, ctx=ctx)+ def _replace_index_references(+ self, generated_content: str, file_index_map: Dict[int, str]+ ) -> str:+ def replace_index(match):+ idx = int(match.group(1))+ return file_index_map.get(idx, match.group(0))++ return re.sub(r"\[(\d+)\]", replace_index, generated_content)++ def _truncate_generated_text(self, text: str, max_chars: int) -> str:+ if max_chars <= 0 or len(text) <= max_chars:+ return text++ if max_chars <= 3:+ return text[:max_chars]++ first_sentence_end = None+ last_sentence_end_within_limit = None+ for sentence_end_match in re.finditer(+ r"\.(?!\d)(?=\s|$)|[!?](?=\s|$)|[。?!]", text+ ):+ sentence_end = sentence_end_match.end()+ if first_sentence_end is None:+ first_sentence_end = sentence_end+ if sentence_end <= max_chars:+ last_sentence_end_within_limit = sentence_end+ elif last_sentence_end_within_limit is not None:+ break++ if last_sentence_end_within_limit is not None:+ return text[:last_sentence_end_within_limit].strip()+ if first_sentence_end is not None:+ return text[:first_sentence_end].strip()++ candidate = text[: max_chars - 3].rstrip()+ word_boundary = candidate.rfind(" ")+ if word_boundary > 0:+ return candidate[:word_boundary].rstrip() + "..."++ return candidate + "..."+def _extract_abstract_from_overview(self, overview_content: str) -> str:- """Extract abstract from overview.md."""+ """Extract an abstract from the Markdown overview brief description."""lines = overview_content.split("\n")# Skip header lines (starting with #)@@ -1178,13 +1219,19 @@ class SemanticProcessor(DequeueHandlerBase):return "\n".join(content_lines).strip()+ def _normalize_overview_generation(self, generated_content: str) -> Tuple[str, str]:+ """Convert raw Markdown overview output into final L1 overview and L0 abstract."""+ overview = generated_content+ abstract = self._extract_abstract_from_overview(generated_content)+ return self._enforce_size_limits(overview, abstract)+def _enforce_size_limits(self, overview: str, abstract: str) -> Tuple[str, str]:"""Enforce max size limits on overview and abstract."""semantic = get_openviking_config().semanticif len(overview) > semantic.overview_max_chars:- overview = overview[: semantic.overview_max_chars]+ overview = self._truncate_generated_text(overview, semantic.overview_max_chars)if len(abstract) > semantic.abstract_max_chars:- abstract = abstract[: semantic.abstract_max_chars - 3] + "..."+ abstract = self._truncate_generated_text(abstract, semantic.abstract_max_chars)return overview, abstractdef _parse_overview_md(self, overview_content: str) -> Dict[str, str]:@@ -1247,7 +1294,7 @@ class SemanticProcessor(DequeueHandlerBase):children_abstracts: List[Dict[str, str]],llm_sem: Optional[asyncio.Semaphore] = None,) -> str:- """Generate directory's .overview.md (L1).+ """Generate raw directory overview model output.For small directories, generates a single overview from all file summaries.For large directories that would exceed the prompt budget, splits file@@ -1260,7 +1307,7 @@ class SemanticProcessor(DequeueHandlerBase):children_abstracts: Subdirectory summary listReturns:- Overview content+ Markdown overview content generated by the model."""config = get_openviking_config()@@ -1359,9 +1406,8 @@ class SemanticProcessor(DequeueHandlerBase):output_language: str = "en",) -> str:"""Generate overview from a single prompt (small directories)."""- import re-- vlm = get_openviking_config().vlm+ config = get_openviking_config()+ vlm = config.vlmtry:prompt = render_prompt(@@ -1377,12 +1423,7 @@ class SemanticProcessor(DequeueHandlerBase):with bind_telemetry_stage("resource_summarize"):overview = await vlm.get_completion_async(prompt)- # Post-process: replace [number] with actual file name- def replace_index(match):- idx = int(match.group(1))- return file_index_map.get(idx, match.group(0))-- overview = re.sub(r"\[(\d+)\]", replace_index, overview)+ overview = self._replace_index_references(overview, file_index_map)return overview.strip()@@ -1407,10 +1448,9 @@ class SemanticProcessor(DequeueHandlerBase):Splits file summaries into batches, generates a partial overview perbatch, then merges all partials into a final overview."""- import re-- vlm = get_openviking_config().vlm- semantic = get_openviking_config().semantic+ config = get_openviking_config()+ vlm = config.vlm+ semantic = config.semanticbatch_size = semantic.overview_batch_sizedir_name = dir_uri.split("/")[-1]@@ -1459,19 +1499,12 @@ class SemanticProcessor(DequeueHandlerBase):)batch_prompts.append((batch_idx, prompt, batch_index_map))- def make_replacer(idx_map):- def replacer(match):- idx = int(match.group(1))- return idx_map.get(idx, match.group(0))-- return replacer-async def _run_batch(batch_idx: int, prompt: str, batch_index_map: Dict[int, str]) -> None:try:async with llm_sem:with bind_telemetry_stage("resource_summarize"):partial = await vlm.get_completion_async(prompt)- partial = re.sub(r"\[(\d+)\]", make_replacer(batch_index_map), partial)+ partial = self._replace_index_references(partial, batch_index_map)partial_overviews[batch_idx] = partial.strip()except Exception as e:logger.warning(@@ -1503,6 +1536,7 @@ class SemanticProcessor(DequeueHandlerBase):)with bind_telemetry_stage("resource_summarize"):overview = await vlm.get_completion_async(prompt)+ overview = self._replace_index_references(overview, file_index_map)return overview.strip()except Exception as e:logger.error(diff --git a/tests/storage/test_semantic_dag_incremental.py b/tests/storage/test_semantic_dag_incremental.pyindex 60d165599..ab69653b8 100644--- a/tests/storage/test_semantic_dag_incremental.py+++ b/tests/storage/test_semantic_dag_incremental.py@@ -86,11 +86,8 @@ class _FakeProcessor:lines.append(f"- {name}: {summary}")return "\n".join(lines)- def _extract_abstract_from_overview(self, overview):- return "abstract"-- def _enforce_size_limits(self, overview, abstract):- return overview, abstract+ def _normalize_overview_generation(self, overview):+ return overview, "abstract"async def _sync_topdown_recursive(self, root_uri, target_uri, ctx=None, file_change_status=None, lock=Nonediff --git a/tests/storage/test_semantic_dag_skip_files.py b/tests/storage/test_semantic_dag_skip_files.pyindex 95e0568fa..69448e97d 100644--- a/tests/storage/test_semantic_dag_skip_files.py+++ b/tests/storage/test_semantic_dag_skip_files.py@@ -54,11 +54,8 @@ class _FakeProcessor:async def _generate_overview(self, dir_uri, file_summaries, children_abstracts):return "overview"- def _extract_abstract_from_overview(self, overview):- return "abstract"-- def _enforce_size_limits(self, overview, abstract):- return overview, abstract+ def _normalize_overview_generation(self, overview):+ return overview, "abstract"async def _vectorize_directory(self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=Nonediff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.pyindex 19e3974cc..31b057edf 100644--- a/tests/storage/test_semantic_dag_stats.py+++ b/tests/storage/test_semantic_dag_stats.py@@ -16,7 +16,8 @@ class _FakeVikingFS:self._tree = treeself.writes = []- async def ls(self, uri, ctx=None):+ async def ls(self, uri, node_limit=None, ctx=None):+ del node_limitreturn self._tree.get(uri, [])async def write_file(self, path, content, ctx=None):@@ -37,11 +38,8 @@ class _FakeProcessor:async def _generate_overview(self, dir_uri, file_summaries, children_abstracts):return "overview"- def _extract_abstract_from_overview(self, overview):- return "abstract"-- def _enforce_size_limits(self, overview, abstract):- return overview, abstract+ def _normalize_overview_generation(self, overview):+ return overview, "abstract"async def _vectorize_directory(self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=Nonediff --git a/tests/storage/test_semantic_processor_l0_l1.py b/tests/storage/test_semantic_processor_l0_l1.pynew file mode 100644index 000000000..16c2c165d--- /dev/null+++ b/tests/storage/test_semantic_processor_l0_l1.py@@ -0,0 +1,153 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++from types import SimpleNamespace++from openviking.storage.queuefs import semantic_processor as semantic_processor_module+from openviking.storage.queuefs.semantic_processor import SemanticProcessor+++def _patch_semantic_limits(monkeypatch, *, abstract_max_chars=256, overview_max_chars=4000):+ config = SimpleNamespace(+ semantic=SimpleNamespace(+ abstract_max_chars=abstract_max_chars,+ overview_max_chars=overview_max_chars,+ )+ )+ monkeypatch.setattr(semantic_processor_module, "get_openviking_config", lambda: config)+++def test_markdown_overview_uses_brief_description_as_abstract(monkeypatch):+ _patch_semantic_limits(monkeypatch)+ processor = SemanticProcessor()+ generated = (+ "# README\n\n"+ "This brief description is the retrieval abstract.\n\n"+ "## Quick Navigation\n\n"+ "- Read README.md"+ )++ overview, abstract = processor._normalize_overview_generation(generated)++ assert overview == generated+ assert abstract == "This brief description is the retrieval abstract."+++def test_markdown_overview_extracts_multiline_brief_description(monkeypatch):+ _patch_semantic_limits(monkeypatch)+ processor = SemanticProcessor()+ generated = (+ "# README\n\n"+ "This is the first abstract line.\n"+ "This is the second abstract line.\n\n"+ "## Quick Navigation\n\n"+ "- Read README.md"+ )++ overview, abstract = processor._normalize_overview_generation(generated)++ assert overview == generated+ assert abstract == "This is the first abstract line.\nThis is the second abstract line."+++def test_index_references_are_replaced_inside_markdown_overview(monkeypatch):+ _patch_semantic_limits(monkeypatch)+ processor = SemanticProcessor()+ generated = "# README\n\nUse [1] to get started."++ replaced = processor._replace_index_references(generated, {1: "README.md"})++ assert replaced == "# README\n\nUse README.md to get started."+++def test_abstract_truncation_prefers_complete_sentence(monkeypatch):+ _patch_semantic_limits(monkeypatch, abstract_max_chars=80)+ processor = SemanticProcessor()+ abstract = (+ "This is a complete sentence. "+ "This second sentence contains onboarding material that would be cut."+ )++ overview, abstract = processor._enforce_size_limits("# README\n\nBody", abstract)++ assert overview == "# README\n\nBody"+ assert abstract == "This is a complete sentence."+++def test_abstract_truncation_keeps_first_sentence_even_over_limit(monkeypatch):+ _patch_semantic_limits(monkeypatch, abstract_max_chars=80)+ processor = SemanticProcessor()+ first_sentence = (+ "This directory is a timestamped media storage container for a single MP4 video "+ "file, organized to preserve the exact capture or creation time of its contents."+ )+ abstract = f"{first_sentence} This second sentence should be omitted."++ _, abstract = processor._enforce_size_limits("# video\n\nBody", abstract)++ assert abstract == first_sentence+++def test_overview_truncation_prefers_complete_sentence(monkeypatch):+ _patch_semantic_limits(monkeypatch, overview_max_chars=45)+ processor = SemanticProcessor()+ overview = (+ "# README\n\n"+ "This is a complete sentence. "+ "This second sentence would be cut in the middle."+ )++ overview, abstract = processor._enforce_size_limits(overview, "abstract")++ assert overview == "# README\n\nThis is a complete sentence."+ assert abstract == "abstract"+++def test_overview_truncation_keeps_last_complete_sentence_within_limit(monkeypatch):+ _patch_semantic_limits(monkeypatch, overview_max_chars=57)+ processor = SemanticProcessor()+ overview = (+ "# README\n\n"+ "First sentence. "+ "Second sentence. "+ "Third sentence should be omitted."+ )++ overview, abstract = processor._enforce_size_limits(overview, "abstract")++ assert overview == "# README\n\nFirst sentence. Second sentence."+ assert abstract == "abstract"+++def test_truncation_keeps_multiple_short_sentences_within_limit(monkeypatch):+ _patch_semantic_limits(monkeypatch, abstract_max_chars=10)+ processor = SemanticProcessor()++ _, abstract = processor._enforce_size_limits("# README\n\nBody", "A. B. C. D.E.")++ assert abstract == "A. B. C."+++def test_abstract_truncation_does_not_treat_decimal_point_as_sentence_end_without_period(+ monkeypatch,+):+ _patch_semantic_limits(monkeypatch, abstract_max_chars=24)+ processor = SemanticProcessor()+ abstract = "This covers version 3.14 compatibility checks for onboarding"++ _, abstract = processor._enforce_size_limits("# README\n\nBody", abstract)++ assert abstract == "This covers version..."+++def test_abstract_truncation_accepts_sentence_period_after_number(monkeypatch):+ _patch_semantic_limits(monkeypatch, abstract_max_chars=70)+ processor = SemanticProcessor()+ abstract = (+ "This import check was generated at 16:55. "+ "This second sentence would otherwise be truncated midstream."+ )++ _, abstract = processor._enforce_size_limits("# README\n\nBody", abstract)++ assert abstract == "This import check was generated at 16:55."diff --git a/tests/storage/test_semantic_processor_language.py b/tests/storage/test_semantic_processor_language.pyindex d06909f97..3dda0ef2d 100644--- a/tests/storage/test_semantic_processor_language.py+++ b/tests/storage/test_semantic_processor_language.py@@ -208,6 +208,9 @@ class TestOverviewGenerationFlow:},)assert f"Output Language: {lang}" in prompt+ assert "Output in Markdown format" in prompt+ assert "Brief Description" in prompt+ assert "abstract_max_chars" not in promptdef test_overview_generation_prompt_preserves_repository_hierarchy(self):prompt = render_prompt(@@ -234,7 +237,6 @@ class TestOverviewGenerationFlow:in prompt)-class LanguageAwareMockVLM:"""语言感知的 MockVLM,根据 prompt 中的 Output Language 返回对应语言的响应。"""diff --git a/tests/storage/test_semantic_queue_memory_dedupe.py b/tests/storage/test_semantic_queue_memory_dedupe.pyindex cebfba885..d8a810cda 100644--- a/tests/storage/test_semantic_queue_memory_dedupe.py+++ b/tests/storage/test_semantic_queue_memory_dedupe.py@@ -145,8 +145,8 @@ class _FakeVikingFS:class _FakeMemoryDirFS:- async def ls(self, uri, ctx=None):- del uri, ctx+ async def ls(self, uri, node_limit=None, ctx=None):+ del uri, node_limit, ctxreturn [{"name": "first.md", "isDir": False},{"name": "second.md", "isDir": False},@@ -233,11 +233,10 @@ async def test_memory_directory_summarizes_all_uncached_files(monkeypatch):)monkeypatch.setattr(processor, "_generate_single_file_summary", generate_file_summary)monkeypatch.setattr(processor, "_generate_overview", generate_overview)- monkeypatch.setattr(processor, "_extract_abstract_from_overview", lambda overview: "abstract")monkeypatch.setattr(processor,- "_enforce_size_limits",- lambda overview, abstract: (overview, abstract),+ "_normalize_overview_generation",+ lambda overview: (overview, "abstract"),)monkeypatch.setattr(processor, "_write_memory_directory_semantics", write_semantics)@@ -285,11 +284,10 @@ async def test_memory_directory_vectorizes_changed_files_with_generated_summary()monkeypatch.setattr(processor, "_generate_single_file_summary", generate_file_summary)monkeypatch.setattr(processor, "_generate_overview", generate_overview)- monkeypatch.setattr(processor, "_extract_abstract_from_overview", lambda overview: "abstract")monkeypatch.setattr(processor,- "_enforce_size_limits",- lambda overview, abstract: (overview, abstract),+ "_normalize_overview_generation",+ lambda overview: (overview, "abstract"),)monkeypatch.setattr(processor, "_write_memory_directory_semantics", write_semantics)monkeypatch.setattr(processor, "_vectorize_single_file", vectorize_single_file)