Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Generate resource L0 summaries from overview prompt (#2890)

agentfdcfa0ef1069
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py
index dd5d13e9d..2006d9875 100644
--- a/openviking/storage/queuefs/semantic_dag.py
+++ b/openviking/storage/queuefs/semantic_dag.py
@@ -812,8 +812,7 @@ class SemanticDagExecutor:
overview = await self._processor._generate_overview(
dir_uri, file_summaries, children_abstracts
)
- abstract = self._processor._extract_abstract_from_overview(overview)
- overview, abstract = self._processor._enforce_size_limits(overview, abstract)
+ overview, abstract = self._processor._normalize_overview_generation(overview)
# Write directly, protected by the outer semantic lock.
try:
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index e5b6723de..9232071ce 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -3,6 +3,7 @@
"""SemanticProcessor: Processes messages from SemanticQueue, generates .abstract.md and .overview.md."""
import asyncio
+import re
import threading
from contextlib import nullcontext
from dataclasses import dataclass, field
@@ -652,11 +653,10 @@ class SemanticProcessor(DequeueHandlerBase):
for file_path, summary in zip(file_paths, file_summaries, strict=False)
if file_path in paths_to_vectorize and summary is not None
]
- overview = await self._generate_overview(
+ generated_content = await self._generate_overview(
dir_uri, completed_summaries, [], llm_sem=llm_sem
)
- abstract = self._extract_abstract_from_overview(overview)
- overview, abstract = self._enforce_size_limits(overview, abstract)
+ overview, abstract = self._normalize_overview_generation(generated_content)
try:
wrote_semantics = await self._write_memory_directory_semantics(
@@ -1155,8 +1155,49 @@ class SemanticProcessor(DequeueHandlerBase):
else:
return await self._generate_text_summary(file_path, file_name, llm_sem, ctx=ctx)
+ def _replace_index_references(
+ self, generated_content: str, file_index_map: Dict[int, str]
+ ) -> str:
+ def replace_index(match):
+ idx = int(match.group(1))
+ return file_index_map.get(idx, match.group(0))
+
+ return re.sub(r"\[(\d+)\]", replace_index, generated_content)
+
+ def _truncate_generated_text(self, text: str, max_chars: int) -> str:
+ if max_chars <= 0 or len(text) <= max_chars:
+ return text
+
+ if max_chars <= 3:
+ return text[:max_chars]
+
+ first_sentence_end = None
+ last_sentence_end_within_limit = None
+ for sentence_end_match in re.finditer(
+ r"\.(?!\d)(?=\s|$)|[!?](?=\s|$)|[。?!]", text
+ ):
+ sentence_end = sentence_end_match.end()
+ if first_sentence_end is None:
+ first_sentence_end = sentence_end
+ if sentence_end <= max_chars:
+ last_sentence_end_within_limit = sentence_end
+ elif last_sentence_end_within_limit is not None:
+ break
+
+ if last_sentence_end_within_limit is not None:
+ return text[:last_sentence_end_within_limit].strip()
+ if first_sentence_end is not None:
+ return text[:first_sentence_end].strip()
+
+ candidate = text[: max_chars - 3].rstrip()
+ word_boundary = candidate.rfind(" ")
+ if word_boundary > 0:
+ return candidate[:word_boundary].rstrip() + "..."
+
+ return candidate + "..."
+
def _extract_abstract_from_overview(self, overview_content: str) -> str:
- """Extract abstract from overview.md."""
+ """Extract an abstract from the Markdown overview brief description."""
lines = overview_content.split("\n")
# Skip header lines (starting with #)
@@ -1178,13 +1219,19 @@ class SemanticProcessor(DequeueHandlerBase):
return "\n".join(content_lines).strip()
+ def _normalize_overview_generation(self, generated_content: str) -> Tuple[str, str]:
+ """Convert raw Markdown overview output into final L1 overview and L0 abstract."""
+ overview = generated_content
+ abstract = self._extract_abstract_from_overview(generated_content)
+ return self._enforce_size_limits(overview, abstract)
+
def _enforce_size_limits(self, overview: str, abstract: str) -> Tuple[str, str]:
"""Enforce max size limits on overview and abstract."""
semantic = get_openviking_config().semantic
if len(overview) > semantic.overview_max_chars:
- overview = overview[: semantic.overview_max_chars]
+ overview = self._truncate_generated_text(overview, semantic.overview_max_chars)
if len(abstract) > semantic.abstract_max_chars:
- abstract = abstract[: semantic.abstract_max_chars - 3] + "..."
+ abstract = self._truncate_generated_text(abstract, semantic.abstract_max_chars)
return overview, abstract
def _parse_overview_md(self, overview_content: str) -> Dict[str, str]:
@@ -1247,7 +1294,7 @@ class SemanticProcessor(DequeueHandlerBase):
children_abstracts: List[Dict[str, str]],
llm_sem: Optional[asyncio.Semaphore] = None,
) -> str:
- """Generate directory's .overview.md (L1).
+ """Generate raw directory overview model output.
For small directories, generates a single overview from all file summaries.
For large directories that would exceed the prompt budget, splits file
@@ -1260,7 +1307,7 @@ class SemanticProcessor(DequeueHandlerBase):
children_abstracts: Subdirectory summary list
Returns:
- Overview content
+ Markdown overview content generated by the model.
"""
config = get_openviking_config()
@@ -1359,9 +1406,8 @@ class SemanticProcessor(DequeueHandlerBase):
output_language: str = "en",
) -> str:
"""Generate overview from a single prompt (small directories)."""
- import re
-
- vlm = get_openviking_config().vlm
+ config = get_openviking_config()
+ vlm = config.vlm
try:
prompt = render_prompt(
@@ -1377,12 +1423,7 @@ class SemanticProcessor(DequeueHandlerBase):
with bind_telemetry_stage("resource_summarize"):
overview = await vlm.get_completion_async(prompt)
- # Post-process: replace [number] with actual file name
- def replace_index(match):
- idx = int(match.group(1))
- return file_index_map.get(idx, match.group(0))
-
- overview = re.sub(r"\[(\d+)\]", replace_index, overview)
+ overview = self._replace_index_references(overview, file_index_map)
return overview.strip()
@@ -1407,10 +1448,9 @@ class SemanticProcessor(DequeueHandlerBase):
Splits file summaries into batches, generates a partial overview per
batch, then merges all partials into a final overview.
"""
- import re
-
- vlm = get_openviking_config().vlm
- semantic = get_openviking_config().semantic
+ config = get_openviking_config()
+ vlm = config.vlm
+ semantic = config.semantic
batch_size = semantic.overview_batch_size
dir_name = dir_uri.split("/")[-1]
@@ -1459,19 +1499,12 @@ class SemanticProcessor(DequeueHandlerBase):
)
batch_prompts.append((batch_idx, prompt, batch_index_map))
- def make_replacer(idx_map):
- def replacer(match):
- idx = int(match.group(1))
- return idx_map.get(idx, match.group(0))
-
- return replacer
-
async def _run_batch(batch_idx: int, prompt: str, batch_index_map: Dict[int, str]) -> None:
try:
async with llm_sem:
with bind_telemetry_stage("resource_summarize"):
partial = await vlm.get_completion_async(prompt)
- partial = re.sub(r"\[(\d+)\]", make_replacer(batch_index_map), partial)
+ partial = self._replace_index_references(partial, batch_index_map)
partial_overviews[batch_idx] = partial.strip()
except Exception as e:
logger.warning(
@@ -1503,6 +1536,7 @@ class SemanticProcessor(DequeueHandlerBase):
)
with bind_telemetry_stage("resource_summarize"):
overview = await vlm.get_completion_async(prompt)
+ overview = self._replace_index_references(overview, file_index_map)
return overview.strip()
except Exception as e:
logger.error(
diff --git a/tests/storage/test_semantic_dag_incremental.py b/tests/storage/test_semantic_dag_incremental.py
index 60d165599..ab69653b8 100644
--- a/tests/storage/test_semantic_dag_incremental.py
+++ b/tests/storage/test_semantic_dag_incremental.py
@@ -86,11 +86,8 @@ class _FakeProcessor:
lines.append(f"- {name}: {summary}")
return "\n".join(lines)
- def _extract_abstract_from_overview(self, overview):
- return "abstract"
-
- def _enforce_size_limits(self, overview, abstract):
- return overview, abstract
+ def _normalize_overview_generation(self, overview):
+ return overview, "abstract"
async def _sync_topdown_recursive(
self, root_uri, target_uri, ctx=None, file_change_status=None, lock=None
diff --git a/tests/storage/test_semantic_dag_skip_files.py b/tests/storage/test_semantic_dag_skip_files.py
index 95e0568fa..69448e97d 100644
--- a/tests/storage/test_semantic_dag_skip_files.py
+++ b/tests/storage/test_semantic_dag_skip_files.py
@@ -54,11 +54,8 @@ class _FakeProcessor:
async def _generate_overview(self, dir_uri, file_summaries, children_abstracts):
return "overview"
- def _extract_abstract_from_overview(self, overview):
- return "abstract"
-
- def _enforce_size_limits(self, overview, abstract):
- return overview, abstract
+ def _normalize_overview_generation(self, overview):
+ return overview, "abstract"
async def _vectorize_directory(
self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None
diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.py
index 19e3974cc..31b057edf 100644
--- a/tests/storage/test_semantic_dag_stats.py
+++ b/tests/storage/test_semantic_dag_stats.py
@@ -16,7 +16,8 @@ class _FakeVikingFS:
self._tree = tree
self.writes = []
- async def ls(self, uri, ctx=None):
+ async def ls(self, uri, node_limit=None, ctx=None):
+ del node_limit
return self._tree.get(uri, [])
async def write_file(self, path, content, ctx=None):
@@ -37,11 +38,8 @@ class _FakeProcessor:
async def _generate_overview(self, dir_uri, file_summaries, children_abstracts):
return "overview"
- def _extract_abstract_from_overview(self, overview):
- return "abstract"
-
- def _enforce_size_limits(self, overview, abstract):
- return overview, abstract
+ def _normalize_overview_generation(self, overview):
+ return overview, "abstract"
async def _vectorize_directory(
self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None
diff --git a/tests/storage/test_semantic_processor_l0_l1.py b/tests/storage/test_semantic_processor_l0_l1.py
new file mode 100644
index 000000000..16c2c165d
--- /dev/null
+++ b/tests/storage/test_semantic_processor_l0_l1.py
@@ -0,0 +1,153 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+from types import SimpleNamespace
+
+from openviking.storage.queuefs import semantic_processor as semantic_processor_module
+from openviking.storage.queuefs.semantic_processor import SemanticProcessor
+
+
+def _patch_semantic_limits(monkeypatch, *, abstract_max_chars=256, overview_max_chars=4000):
+ config = SimpleNamespace(
+ semantic=SimpleNamespace(
+ abstract_max_chars=abstract_max_chars,
+ overview_max_chars=overview_max_chars,
+ )
+ )
+ monkeypatch.setattr(semantic_processor_module, "get_openviking_config", lambda: config)
+
+
+def test_markdown_overview_uses_brief_description_as_abstract(monkeypatch):
+ _patch_semantic_limits(monkeypatch)
+ processor = SemanticProcessor()
+ generated = (
+ "# README\n\n"
+ "This brief description is the retrieval abstract.\n\n"
+ "## Quick Navigation\n\n"
+ "- Read README.md"
+ )
+
+ overview, abstract = processor._normalize_overview_generation(generated)
+
+ assert overview == generated
+ assert abstract == "This brief description is the retrieval abstract."
+
+
+def test_markdown_overview_extracts_multiline_brief_description(monkeypatch):
+ _patch_semantic_limits(monkeypatch)
+ processor = SemanticProcessor()
+ generated = (
+ "# README\n\n"
+ "This is the first abstract line.\n"
+ "This is the second abstract line.\n\n"
+ "## Quick Navigation\n\n"
+ "- Read README.md"
+ )
+
+ overview, abstract = processor._normalize_overview_generation(generated)
+
+ assert overview == generated
+ assert abstract == "This is the first abstract line.\nThis is the second abstract line."
+
+
+def test_index_references_are_replaced_inside_markdown_overview(monkeypatch):
+ _patch_semantic_limits(monkeypatch)
+ processor = SemanticProcessor()
+ generated = "# README\n\nUse [1] to get started."
+
+ replaced = processor._replace_index_references(generated, {1: "README.md"})
+
+ assert replaced == "# README\n\nUse README.md to get started."
+
+
+def test_abstract_truncation_prefers_complete_sentence(monkeypatch):
+ _patch_semantic_limits(monkeypatch, abstract_max_chars=80)
+ processor = SemanticProcessor()
+ abstract = (
+ "This is a complete sentence. "
+ "This second sentence contains onboarding material that would be cut."
+ )
+
+ overview, abstract = processor._enforce_size_limits("# README\n\nBody", abstract)
+
+ assert overview == "# README\n\nBody"
+ assert abstract == "This is a complete sentence."
+
+
+def test_abstract_truncation_keeps_first_sentence_even_over_limit(monkeypatch):
+ _patch_semantic_limits(monkeypatch, abstract_max_chars=80)
+ processor = SemanticProcessor()
+ first_sentence = (
+ "This directory is a timestamped media storage container for a single MP4 video "
+ "file, organized to preserve the exact capture or creation time of its contents."
+ )
+ abstract = f"{first_sentence} This second sentence should be omitted."
+
+ _, abstract = processor._enforce_size_limits("# video\n\nBody", abstract)
+
+ assert abstract == first_sentence
+
+
+def test_overview_truncation_prefers_complete_sentence(monkeypatch):
+ _patch_semantic_limits(monkeypatch, overview_max_chars=45)
+ processor = SemanticProcessor()
+ overview = (
+ "# README\n\n"
+ "This is a complete sentence. "
+ "This second sentence would be cut in the middle."
+ )
+
+ overview, abstract = processor._enforce_size_limits(overview, "abstract")
+
+ assert overview == "# README\n\nThis is a complete sentence."
+ assert abstract == "abstract"
+
+
+def test_overview_truncation_keeps_last_complete_sentence_within_limit(monkeypatch):
+ _patch_semantic_limits(monkeypatch, overview_max_chars=57)
+ processor = SemanticProcessor()
+ overview = (
+ "# README\n\n"
+ "First sentence. "
+ "Second sentence. "
+ "Third sentence should be omitted."
+ )
+
+ overview, abstract = processor._enforce_size_limits(overview, "abstract")
+
+ assert overview == "# README\n\nFirst sentence. Second sentence."
+ assert abstract == "abstract"
+
+
+def test_truncation_keeps_multiple_short_sentences_within_limit(monkeypatch):
+ _patch_semantic_limits(monkeypatch, abstract_max_chars=10)
+ processor = SemanticProcessor()
+
+ _, abstract = processor._enforce_size_limits("# README\n\nBody", "A. B. C. D.E.")
+
+ assert abstract == "A. B. C."
+
+
+def test_abstract_truncation_does_not_treat_decimal_point_as_sentence_end_without_period(
+ monkeypatch,
+):
+ _patch_semantic_limits(monkeypatch, abstract_max_chars=24)
+ processor = SemanticProcessor()
+ abstract = "This covers version 3.14 compatibility checks for onboarding"
+
+ _, abstract = processor._enforce_size_limits("# README\n\nBody", abstract)
+
+ assert abstract == "This covers version..."
+
+
+def test_abstract_truncation_accepts_sentence_period_after_number(monkeypatch):
+ _patch_semantic_limits(monkeypatch, abstract_max_chars=70)
+ processor = SemanticProcessor()
+ abstract = (
+ "This import check was generated at 16:55. "
+ "This second sentence would otherwise be truncated midstream."
+ )
+
+ _, abstract = processor._enforce_size_limits("# README\n\nBody", abstract)
+
+ assert abstract == "This import check was generated at 16:55."
diff --git a/tests/storage/test_semantic_processor_language.py b/tests/storage/test_semantic_processor_language.py
index d06909f97..3dda0ef2d 100644
--- a/tests/storage/test_semantic_processor_language.py
+++ b/tests/storage/test_semantic_processor_language.py
@@ -208,6 +208,9 @@ class TestOverviewGenerationFlow:
},
)
assert f"Output Language: {lang}" in prompt
+ assert "Output in Markdown format" in prompt
+ assert "Brief Description" in prompt
+ assert "abstract_max_chars" not in prompt
def test_overview_generation_prompt_preserves_repository_hierarchy(self):
prompt = render_prompt(
@@ -234,7 +237,6 @@ class TestOverviewGenerationFlow:
in prompt
)
-
class LanguageAwareMockVLM:
"""语言感知的 MockVLM,根据 prompt 中的 Output Language 返回对应语言的响应。"""
diff --git a/tests/storage/test_semantic_queue_memory_dedupe.py b/tests/storage/test_semantic_queue_memory_dedupe.py
index cebfba885..d8a810cda 100644
--- a/tests/storage/test_semantic_queue_memory_dedupe.py
+++ b/tests/storage/test_semantic_queue_memory_dedupe.py
@@ -145,8 +145,8 @@ class _FakeVikingFS:
class _FakeMemoryDirFS:
- async def ls(self, uri, ctx=None):
- del uri, ctx
+ async def ls(self, uri, node_limit=None, ctx=None):
+ del uri, node_limit, ctx
return [
{"name": "first.md", "isDir": False},
{"name": "second.md", "isDir": False},
@@ -233,11 +233,10 @@ async def test_memory_directory_summarizes_all_uncached_files(monkeypatch):
)
monkeypatch.setattr(processor, "_generate_single_file_summary", generate_file_summary)
monkeypatch.setattr(processor, "_generate_overview", generate_overview)
- monkeypatch.setattr(processor, "_extract_abstract_from_overview", lambda overview: "abstract")
monkeypatch.setattr(
processor,
- "_enforce_size_limits",
- lambda overview, abstract: (overview, abstract),
+ "_normalize_overview_generation",
+ lambda overview: (overview, "abstract"),
)
monkeypatch.setattr(processor, "_write_memory_directory_semantics", write_semantics)
@@ -285,11 +284,10 @@ async def test_memory_directory_vectorizes_changed_files_with_generated_summary(
)
monkeypatch.setattr(processor, "_generate_single_file_summary", generate_file_summary)
monkeypatch.setattr(processor, "_generate_overview", generate_overview)
- monkeypatch.setattr(processor, "_extract_abstract_from_overview", lambda overview: "abstract")
monkeypatch.setattr(
processor,
- "_enforce_size_limits",
- lambda overview, abstract: (overview, abstract),
+ "_normalize_overview_generation",
+ lambda overview: (overview, "abstract"),
)
monkeypatch.setattr(processor, "_write_memory_directory_semantics", write_semantics)
monkeypatch.setattr(processor, "_vectorize_single_file", vectorize_single_file)