Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(semantic): use Markdown links instead of [N] indices in L1 overview (#4251)
Maojia Sheng8ab07b1e4c16
diff --git a/openviking/prompts/templates/semantic/overview_generation.yaml b/openviking/prompts/templates/semantic/overview_generation.yamlindex dc0f1ab62..c4f3a31ca 100644--- a/openviking/prompts/templates/semantic/overview_generation.yaml+++ b/openviking/prompts/templates/semantic/overview_generation.yaml@@ -48,7 +48,12 @@ template: |[Files and Their Summaries in Directory]{{ file_summaries }}- Note: Files are numbered as [1], [2], [3], etc.+ Note: Each entry above carries a compact link placeholder in the form+ "(link: viking://input_sample_fN)" for files and "(link: viking://input_sample_cN)"+ for subdirectories. When you reference an entry, write a normal Markdown link+ using its placeholder as the URL, e.g. [display title](viking://input_sample_f1).+ Use a natural display title (typically the entry's name); do not print the raw+ placeholder text and do not repeat the name outside the link.Some entries may be code structure skeletons (showing imports, classes, functions)rather than prose summaries — treat them as structural descriptions of the file.@@ -84,13 +89,13 @@ template: |4. **{{ "快速导航" if output_language == "zh-CN" else "Quick Navigation" }}** (H2): Decision Tree style- Guide with "What do you want to learn?" or "What do you want to do?"- Use → arrow to point to specific files or directories- - **Use file number references**: such as [1], [2], [3]+ - **Link each entry as a Markdown link** using its placeholder, e.g. [display title](viking://input_sample_f1)- Concise keyword descriptions- When sampled, introduce the list with a note that these are representative entries and the full directory contains more5. **{{ "详细说明" if output_language == "zh-CN" else "Detailed Description" }}** (H2):- When sampled, begin with a one-sentence note stating that the following subsections cover only the sampled entries (X of Y), not the full directory.- - One H3 subsection for each file/subdirectory+ - One H3 subsection for each file/subdirectory; the H3 heading is a Markdown link to the entry using its placeholder, e.g. ### [display title](viking://input_sample_f1)- Use the file summaries or subdirectory summaries provided above as description contentTotal length: 400-800 wordsdiff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex b8110e24e..829fb25b7 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -1060,14 +1060,32 @@ class SemanticProcessor(DequeueHandlerBase):else:return await self._generate_text_summary(file_path, file_name, llm_sem, ctx=ctx)- def _replace_index_references(- self, generated_content: str, file_index_map: Dict[int, str]+ def _child_summary_line(+ self,+ dir_uri: str,+ idx: int,+ item: Dict[str, str],+ link_map: Dict[str, str],+ ) -> str:+ """Render a subdirectory summary line with a collision-free link placeholder."""+ placeholder = f"viking://input_sample_c{idx}"+ link_map[placeholder] = f"{dir_uri}/{item['name']}"+ return f"- {item['name']}/ (link: {placeholder}): {item['abstract']}"++ def _replace_link_references(+ self, generated_content: str, link_map: Dict[str, str]) -> str:- def replace_index(match):- idx = int(match.group(1))- return file_index_map.get(idx, match.group(0))+ """Resolve link placeholders (viking://input_sample_fN / cN) to real URIs.++ The model is fed compact, collision-free placeholders and asked to emit+ Markdown links against them; here we map each placeholder back to the+ entry's real URI. Unknown placeholders are left untouched.+ """++ def replace_link(match):+ return link_map.get(match.group(0), match.group(0))- return re.sub(r"\[(\d+)\]", replace_index, generated_content)+ return re.sub(r"viking://input_sample_[fc]\d+", replace_link, generated_content)def _truncate_generated_text(self, text: str, max_chars: int) -> str:if max_chars <= 0 or len(text) <= max_chars:@@ -1253,17 +1271,27 @@ class SemanticProcessor(DequeueHandlerBase):"entries; generalize cautiously and do not treat them as exhaustive.")- # Build file index mapping and summary string- file_index_map = {}+ # Build link placeholder mapping and summary string.+ # The model is fed collision-free placeholders (viking://input_sample_fN)+ # and asked to emit Markdown links; placeholders are resolved to real+ # URIs in post-processing. This avoids the fragility of bare [N] indices+ # and of asking the model to reproduce long/CJK URIs verbatim.+ link_map: Dict[str, str] = {}file_summaries_lines = []for idx, item in enumerate(file_summaries, 1):- file_index_map[idx] = item["name"]- file_summaries_lines.append(f"[{idx}] {item['name']}: {item['summary']}")+ placeholder = f"viking://input_sample_f{idx}"+ link_map[placeholder] = f"{dir_uri}/{item['name']}"+ file_summaries_lines.append(+ f"- {item['name']} (link: {placeholder}): {item['summary']}"+ )file_summaries_str = "\n".join(file_summaries_lines) if file_summaries_lines else "None"# Build subdirectory summary stringchildren_abstracts_str = (- "\n".join(f"- {item['name']}/: {item['abstract']}" for item in children_abstracts)+ "\n".join(+ self._child_summary_line(dir_uri, idx, item, link_map)+ for idx, item in enumerate(children_abstracts, 1)+ )if children_abstractselse "None")@@ -1295,7 +1323,7 @@ class SemanticProcessor(DequeueHandlerBase):dir_uri,file_summaries,children_abstracts,- file_index_map,+ link_map,llm_sem=llm_sem,output_language=output_language,directory_coverage=directory_coverage,@@ -1313,13 +1341,15 @@ class SemanticProcessor(DequeueHandlerBase):truncated_lines = []for idx, item in enumerate(file_summaries, 1):summary = item["summary"][:per_file]- truncated_lines.append(f"[{idx}] {item['name']}: {summary}")+ truncated_lines.append(+ f"- {item['name']} (link: viking://input_sample_f{idx}): {summary}"+ )file_summaries_str = "\n".join(truncated_lines)overview = await self._single_generate_overview(dir_uri,file_summaries_str,children_abstracts_str,- file_index_map,+ link_map,output_language=output_language,directory_coverage=directory_coverage,)@@ -1328,7 +1358,7 @@ class SemanticProcessor(DequeueHandlerBase):dir_uri,file_summaries_str,children_abstracts_str,- file_index_map,+ link_map,output_language=output_language,directory_coverage=directory_coverage,)@@ -1340,7 +1370,7 @@ class SemanticProcessor(DequeueHandlerBase):dir_uri: str,file_summaries_str: str,children_abstracts_str: str,- file_index_map: Dict[int, str],+ link_map: Dict[str, str],output_language: str = "en",directory_coverage: str = "",) -> str:@@ -1363,7 +1393,7 @@ class SemanticProcessor(DequeueHandlerBase):with bind_telemetry_stage("resource_summarize"):overview = await vlm.get_completion_async(prompt)- overview = self._replace_index_references(overview, file_index_map)+ overview = self._replace_link_references(overview, link_map)return overview.strip()@@ -1379,7 +1409,7 @@ class SemanticProcessor(DequeueHandlerBase):dir_uri: str,file_summaries: List[Dict[str, str]],children_abstracts: List[Dict[str, str]],- file_index_map: Dict[int, str],+ link_map: Dict[str, str],llm_sem: Optional[asyncio.Semaphore] = None,output_language: str = "en",directory_coverage: str = "",@@ -1396,28 +1426,35 @@ class SemanticProcessor(DequeueHandlerBase):dir_name = dir_uri.split("/")[-1]work_items = [("file", index, item) for index, item in enumerate(file_summaries, 1)] + [- ("child", None, item) for item in children_abstracts+ ("child", index, item) for index, item in enumerate(children_abstracts, 1)]batches = [work_items[i : i + batch_size] for i in range(0, len(work_items), batch_size)]logger.info(f"Generating overview for {dir_uri} in {len(batches)} batches")- # Generate partial overviews concurrently using global file indices.+ # Generate partial overviews concurrently using global link placeholders.if llm_sem is None:llm_sem = asyncio.Semaphore(self.max_concurrent_llm)partial_overviews = [None] * len(batches)- batch_prompts: List[Tuple[int, str, Dict[int, str]]] = []+ batch_prompts: List[Tuple[int, str, Dict[str, str]]] = []for batch_idx, batch in enumerate(batches):batch_lines = []child_lines = []- batch_index_map = {}+ batch_link_map: Dict[str, str] = {}for entry_kind, global_idx, item in batch:+ assert global_idx is not Noneif entry_kind == "file":- assert global_idx is not None- batch_index_map[global_idx] = item["name"]- batch_lines.append(f"[{global_idx}] {item['name']}: {item['summary']}")+ placeholder = f"viking://input_sample_f{global_idx}"+ batch_link_map[placeholder] = link_map.get(+ placeholder, f"{dir_uri}/{item['name']}"+ )+ batch_lines.append(+ f"- {item['name']} (link: {placeholder}): {item['summary']}"+ )else:- child_lines.append(f"- {item['name']}/: {item['abstract']}")+ child_lines.append(+ self._child_summary_line(dir_uri, global_idx, item, batch_link_map)+ )prompt = render_prompt("semantic.overview_generation",@@ -1429,14 +1466,16 @@ class SemanticProcessor(DequeueHandlerBase):"directory_coverage": directory_coverage,},)- batch_prompts.append((batch_idx, prompt, batch_index_map))+ batch_prompts.append((batch_idx, prompt, batch_link_map))- async def _run_batch(batch_idx: int, prompt: str, batch_index_map: Dict[int, str]) -> None:+ async def _run_batch(+ batch_idx: int, prompt: str, batch_link_map: Dict[str, str]+ ) -> None:try:async with llm_sem:with bind_telemetry_stage("resource_summarize"):partial = await vlm.get_completion_async(prompt)- partial = self._replace_index_references(partial, batch_index_map)+ partial = self._replace_link_references(partial, batch_link_map)partial_overviews[batch_idx] = partial.strip()except Exception as e:logger.warning(@@ -1455,6 +1494,8 @@ class SemanticProcessor(DequeueHandlerBase):return partial_overviews[0]# Merge partials only; each child abstract is already represented once.+ # Placeholders were already resolved to real URIs in each partial, so no+ # further link substitution is needed after the merge.combined = "\n\n---\n\n".join(partial_overviews)try:prompt = render_prompt(@@ -1469,7 +1510,6 @@ class SemanticProcessor(DequeueHandlerBase):)with bind_telemetry_stage("resource_summarize"):overview = await vlm.get_completion_async(prompt)- overview = self._replace_index_references(overview, file_index_map)return overview.strip()except Exception as e:logger.error(diff --git a/tests/storage/test_semantic_processor_l0_l1.py b/tests/storage/test_semantic_processor_l0_l1.pyindex abc58fe8f..bbbb43e02 100644--- a/tests/storage/test_semantic_processor_l0_l1.py+++ b/tests/storage/test_semantic_processor_l0_l1.py@@ -85,14 +85,18 @@ def test_directory_coverage_section_is_excluded_from_abstract(monkeypatch):)-def test_index_references_are_replaced_inside_markdown_overview(monkeypatch):+def test_link_references_are_replaced_inside_markdown_overview(monkeypatch):_patch_semantic_limits(monkeypatch)processor = SemanticProcessor()- generated = "# README\n\nUse [1] to get started."+ generated = "# README\n\nSee [README](viking://input_sample_f1) to get started."- replaced = processor._replace_index_references(generated, {1: "README.md"})+ replaced = processor._replace_link_references(+ generated, {"viking://input_sample_f1": "viking://resources/docs/README.md"}+ )- assert replaced == "# README\n\nUse README.md to get started."+ assert replaced == (+ "# README\n\nSee [README](viking://resources/docs/README.md) to get started."+ )def test_abstract_truncation_prefers_complete_sentence(monkeypatch):