Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: batch child directory overview summaries (#3154)

huangruitengb0ea896a68b1
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 2afe6dd74..82e68af5d 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -1354,13 +1354,15 @@ class SemanticProcessor(DequeueHandlerBase):
# Budget guard: check if prompt would be oversized
estimated_size = len(file_summaries_str) + len(children_abstracts_str)
over_budget = estimated_size > semantic.max_overview_prompt_chars
- many_files = len(file_summaries) > semantic.overview_batch_size
+ entry_count = len(file_summaries) + len(children_abstracts)
+ many_entries = entry_count > semantic.overview_batch_size
- if over_budget and many_files:
- # Many files, oversized prompt → batch and merge
+ if over_budget and many_entries:
+ # Many entries, oversized prompt → batch and merge
logger.info(
f"Overview prompt for {dir_uri} exceeds budget "
- f"({estimated_size} chars, {len(file_summaries)} files). "
+ f"({estimated_size} chars, {len(file_summaries)} files, "
+ f"{len(children_abstracts)} subdirectories). "
f"Splitting into batches of {semantic.overview_batch_size}."
)
overview = await self._batched_generate_overview(
@@ -1450,10 +1452,10 @@ class SemanticProcessor(DequeueHandlerBase):
llm_sem: Optional[asyncio.Semaphore] = None,
output_language: str = "en",
) -> str:
- """Generate overview by batching file summaries and merging.
+ """Generate overview by batching file and subdirectory summaries.
- Splits file summaries into batches, generates a partial overview per
- batch, then merges all partials into a final overview.
+ Splits both input kinds into batches, generates a partial overview per
+ batch, then merges the partials without repeating the raw inputs.
"""
config = get_openviking_config()
vlm = config.vlm
@@ -1461,46 +1463,36 @@ class SemanticProcessor(DequeueHandlerBase):
batch_size = semantic.overview_batch_size
dir_name = dir_uri.split("/")[-1]
- # Split file summaries into batches
- batches = [
- file_summaries[i : i + batch_size] for i in range(0, len(file_summaries), batch_size)
+ work_items = [("file", index, item) for index, item in enumerate(file_summaries, 1)] + [
+ ("child", None, item) for item in children_abstracts
]
+ batches = [work_items[i : i + batch_size] for i in range(0, len(work_items), batch_size)]
logger.info(f"Generating overview for {dir_uri} in {len(batches)} batches")
- # Build children abstracts string (used in first batch + merge)
- children_abstracts_str = (
- "\n".join(f"- {item['name']}/: {item['abstract']}" for item in children_abstracts)
- if children_abstracts
- else "None"
- )
-
- # Generate partial overview per batch concurrently using global file indices
+ # Generate partial overviews concurrently using global file indices.
if llm_sem is None:
llm_sem = asyncio.Semaphore(self.max_concurrent_llm)
partial_overviews = [None] * len(batches)
- global_offset = 0
batch_prompts: List[Tuple[int, str, Dict[int, str]]] = []
for batch_idx, batch in enumerate(batches):
- # Build per-batch index map using global offsets
batch_lines = []
+ child_lines = []
batch_index_map = {}
- for local_idx, item in enumerate(batch):
- global_idx = global_offset + local_idx + 1
- batch_index_map[global_idx] = item["name"]
- batch_lines.append(f"[{global_idx}] {item['name']}: {item['summary']}")
- batch_str = "\n".join(batch_lines)
- global_offset += len(batch)
-
- # Include children abstracts in the first batch
- children_str = children_abstracts_str if batch_idx == 0 else "None"
+ for entry_kind, global_idx, item in batch:
+ if entry_kind == "file":
+ assert global_idx is not None
+ batch_index_map[global_idx] = item["name"]
+ batch_lines.append(f"[{global_idx}] {item['name']}: {item['summary']}")
+ else:
+ child_lines.append(f"- {item['name']}/: {item['abstract']}")
prompt = render_prompt(
"semantic.overview_generation",
{
"dir_name": dir_name,
- "file_summaries": batch_str,
- "children_abstracts": children_str,
+ "file_summaries": "\n".join(batch_lines) or "None",
+ "children_abstracts": "\n".join(child_lines) or "None",
"output_language": output_language,
},
)
@@ -1529,7 +1521,7 @@ class SemanticProcessor(DequeueHandlerBase):
if len(partial_overviews) == 1:
return partial_overviews[0]
- # Merge partials into a final overview (include children for context)
+ # Merge partials only; each child abstract is already represented once.
combined = "\n\n---\n\n".join(partial_overviews)
try:
prompt = render_prompt(
@@ -1537,7 +1529,7 @@ class SemanticProcessor(DequeueHandlerBase):
{
"dir_name": dir_name,
"file_summaries": combined,
- "children_abstracts": children_abstracts_str,
+ "children_abstracts": "None",
"output_language": output_language,
},
)
diff --git a/tests/storage/test_semantic_processor_overview_batching.py b/tests/storage/test_semantic_processor_overview_batching.py
new file mode 100644
index 000000000..3af2139ad
--- /dev/null
+++ b/tests/storage/test_semantic_processor_overview_batching.py
@@ -0,0 +1,61 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+from types import SimpleNamespace
+
+import pytest
+
+from openviking.storage.queuefs import semantic_processor as semantic_processor_module
+from openviking.storage.queuefs.semantic_processor import SemanticProcessor
+
+
+class RecordingVLM:
+ def __init__(self):
+ self.prompts = []
+
+ def is_available(self):
+ return True
+
+ async def get_completion_async(self, prompt):
+ self.prompts.append(prompt)
+ return f"overview-{len(self.prompts)}"
+
+
+async def test_children_only_oversized_overview_is_batched(monkeypatch):
+ vlm = RecordingVLM()
+ config = SimpleNamespace(
+ vlm=vlm,
+ semantic=SimpleNamespace(
+ max_overview_prompt_chars=20,
+ overview_batch_size=2,
+ ),
+ output_language_override="en",
+ )
+ monkeypatch.setattr(
+ semantic_processor_module,
+ "get_openviking_config",
+ lambda: config,
+ )
+ monkeypatch.setattr(
+ semantic_processor_module,
+ "render_prompt",
+ lambda _name, values: (
+ f"files={values['file_summaries']}|children={values['children_abstracts']}"
+ ),
+ )
+ children = [{"name": f"child-{index}", "abstract": "x" * 20} for index in range(3)]
+
+ overview = await SemanticProcessor()._generate_overview(
+ "viking://resources/root",
+ file_summaries=[],
+ children_abstracts=children,
+ )
+
+ assert overview == "overview-3"
+ assert len(vlm.prompts) == 3
+ assert "child-0" in vlm.prompts[0]
+ assert "child-1" in vlm.prompts[0]
+ assert "child-2" not in vlm.prompts[0]
+ assert "child-2" in vlm.prompts[1]
+ assert all(f"child-{index}" not in vlm.prompts[2] for index in range(3))