Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(semantic): recover malformed sidecars without failing reads (#5013)
Qin Haojief098476d969d
diff --git a/openviking/retrieve/context_assembler/tiers.py b/openviking/retrieve/context_assembler/tiers.pyindex 20cdd562c..0c4c05a9d 100644--- a/openviking/retrieve/context_assembler/tiers.py+++ b/openviking/retrieve/context_assembler/tiers.py@@ -26,7 +26,7 @@ from openviking.retrieve.context_assembler.params import (Tier,)from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils-from openviking.storage.abstract_overview import body_for_preview+from openviking.storage.abstract_overview import AbstractOverviewFormatError, body_for_previewOVERVIEW_HEADING_LIMIT = 24OVERVIEW_PARAGRAPH_CHARS = 400@@ -110,7 +110,10 @@ def overview_from_content(candidate: Candidate, content: str) -> str:return ""if candidate.is_directory:# The cached content already is the directory's own overview sidecar.- return body_for_preview(content).strip()+ try:+ return body_for_preview(content).strip()+ except AbstractOverviewFormatError:+ return ""if _is_code_uri(candidate.base_uri):extraction = extract_skeleton_result(filename_from_uri(candidate.base_uri), content)if extraction.text:diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.pyindex fbce14750..65bc2ed1f 100644--- a/openviking/retrieve/hierarchical_retriever.py+++ b/openviking/retrieve/hierarchical_retriever.py@@ -22,7 +22,7 @@ from openviking.models.rerank import RerankClientfrom openviking.retrieve.memory_lifecycle import hotness_scorefrom openviking.retrieve.retrieval_stats import get_stats_collectorfrom openviking.server.identity import RequestContext-from openviking.storage.abstract_overview import body_for_preview+from openviking.storage.abstract_overview import AbstractOverviewFormatError, body_for_previewfrom openviking.storage.expr import FilterExprfrom openviking.storage.vikingdb_manager import VikingDBManager, VikingDBManagerProxyfrom openviking.telemetry import get_current_telemetry@@ -635,7 +635,11 @@ class HierarchicalRetriever:# the public find/search preview contract body-only at its final# conversion boundary. L2 user Markdown is intentionally left# untouched, including ordinary YAML frontmatter.- abstract = body_for_preview(abstract)+ try:+ abstract = body_for_preview(abstract)+ except AbstractOverviewFormatError as exc:+ logger.warning("Malformed sidecar in retrieval result %s: %s", display_uri, exc)+ abstract = ""results.append(MatchedContext(diff --git a/openviking/storage/abstract_overview.py b/openviking/storage/abstract_overview.pyindex 3b73b11c4..cdc823d8f 100644--- a/openviking/storage/abstract_overview.py+++ b/openviking/storage/abstract_overview.py@@ -425,7 +425,11 @@ async def _read_existing_document(viking_fs: Any, uri: str, ctx: Optional[RequestContext]) -> Optional[AbstractOverviewDocument]:raw = await _raw_if_exists(viking_fs, uri, ctx)- return parse_abstract_overview(raw) if raw is not None else None+ try:+ return parse_abstract_overview(raw) if raw is not None else None+ except AbstractOverviewFormatError as exc:+ logger.warning("[Semantic] Ignoring malformed sidecar %s: %s", uri, exc)+ return Noneasync def write_abstract_overview(diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex c30f47286..0ddccfbd4 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -698,8 +698,6 @@ class SemanticDagExecutor:if file_name in existing_summaries:return {"name": file_name, "summary": existing_summaries[file_name]}- except AbstractOverviewFormatError:- raiseexcept Exception as e:logger.debug(f"Failed to read existing summary from overview.md for {file_path}: {e}")@@ -754,8 +752,6 @@ class SemanticDagExecutor:overview = await self._viking_fs.read_file(f"{target_path}/.overview.md", ctx=self._ctx)abstract = await self._viking_fs.read_file(f"{target_path}/.abstract.md", ctx=self._ctx)return body_for_preview(overview), body_for_preview(abstract)- except AbstractOverviewFormatError:- raiseexcept Exception:return None, None@@ -797,11 +793,6 @@ class SemanticDagExecutor:summary_dict = await self._processor._generate_single_file_summary(file_path, llm_sem=self._llm_sem, ctx=self._ctx)- except AbstractOverviewFormatError:- # A generated sidecar that opted into OKF must never be treated as- # an empty file summary; doing so would silently feed metadata or- # corrupted YAML into a later regeneration.- raiseexcept Exception as e:logger.warning(f"Failed to generate summary for {file_path}: {e}")summary_dict = {"name": file_name, "summary": ""}@@ -1028,9 +1019,11 @@ class SemanticDagExecutor:)if not children_changed:- need_vectorize = Falseoverview, abstract = await self._read_existing_overview_abstract(dir_uri)should_write = overview is None or abstract is None+ # Rebuilt sidecars must also replace their stale vectors.+ need_vectorize = should_write+ children_changed = should_writeif should_write and (overview is None or abstract is None):async with node.lock:file_summaries = self._finalize_file_summaries(node)diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 045431b5b..633b83bc7 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -35,7 +35,6 @@ from openviking.prompts import render_promptfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.task_work_index import detach_task_contextfrom openviking.storage.abstract_overview import (- AbstractOverviewFormatError,AbstractOverviewWriteResult,body_for_preview,deterministic_sample,@@ -655,8 +654,6 @@ class SemanticProcessor(DequeueHandlerBase):logger.info(f"Parsed {len(existing_summaries)} existing summaries from overview.md")- except AbstractOverviewFormatError:- raiseexcept Exception as e:logger.debug(f"No existing overview.md found for {dir_uri}: {e}")@@ -1145,6 +1142,8 @@ class SemanticProcessor(DequeueHandlerBase):in_header = Truefor line in lines:+ if line.strip() == "---":+ continueif in_header and line.startswith("#"):continueelif in_header and line.strip():diff --git a/openviking/storage/viking_fs/_semantic.py b/openviking/storage/viking_fs/_semantic.pyindex ee4aab40c..1681ae4e0 100644--- a/openviking/storage/viking_fs/_semantic.py+++ b/openviking/storage/viking_fs/_semantic.py@@ -9,7 +9,11 @@ from openviking.core.context import ContextLevelfrom openviking.core.retrieval_targets import resolve_retrieval_targetsfrom openviking.server.error_mapping import is_not_found_error, map_exceptionfrom openviking.server.identity import RequestContext-from openviking.storage.abstract_overview import body_for_preview, render_abstract_overview+from openviking.storage.abstract_overview import (+ AbstractOverviewFormatError,+ body_for_preview,+ render_abstract_overview,+)from openviking.storage.acl import AclActionfrom openviking.storage.viking_fs._base import (_ensure_filter_present,@@ -42,15 +46,16 @@ class _SemanticMixin:file_path = f"{path}/.abstract.md"try:content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))+ return body_for_preview(self._decode_bytes(content_bytes))+ except AbstractOverviewFormatError as exc:+ logger.warning("Malformed directory abstract for %s: %s", uri, exc)except Exception as exc:if not is_not_found_error(exc):mapped = map_exception(exc, resource=uri)if mapped is not None:raise mapped from excraise- return f"# {uri} [Directory abstract is not ready]"-- return body_for_preview(self._decode_bytes(content_bytes))+ return f"# {uri} [Directory abstract is not ready]"async def _read_abstract_for_known_dir(self,@@ -173,16 +178,16 @@ class _SemanticMixin:file_path = f"{path}/.overview.md"try:content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))+ return body_for_preview(self._decode_bytes(content_bytes))+ except AbstractOverviewFormatError as exc:+ logger.warning("Malformed directory overview for %s: %s", uri, exc)except Exception as exc:if not is_not_found_error(exc):mapped = map_exception(exc, resource=uri)if mapped is not None:raise mapped from excraise- # Fallback to default if .overview.md doesn't exist- return f"# {uri}\n\n[Directory overview is not ready]"-- return body_for_preview(self._decode_bytes(content_bytes))+ return f"# {uri}\n\n[Directory overview is not ready]"async def find(self,diff --git a/tests/retrieve/test_hierarchical_retriever_rerank.py b/tests/retrieve/test_hierarchical_retriever_rerank.pyindex d1c44acd2..56509e3c6 100644--- a/tests/retrieve/test_hierarchical_retriever_rerank.py+++ b/tests/retrieve/test_hierarchical_retriever_rerank.py@@ -697,13 +697,20 @@ async def test_convert_to_matched_contexts_defaults_tags_and_body_previews():),),_result("viking://resources/demo.md", 0.8, level=2, abstract=markdown),+ _result(+ "viking://resources/malformed",+ 0.7,+ level=int(ContextLevel.ABSTRACT),+ abstract="---\n",+ ),],ctx=_ctx(),)- assert [item.search_tags for item in result] == [[], [], []]+ assert [item.search_tags for item in result] == [[], [], [], []]assert [item.abstract for item in result] == ["Visible abstract.","# Visible overview",markdown,+ "",]diff --git a/tests/server/test_api_content.py b/tests/server/test_api_content.pyindex b291fe3e7..9b6a903b4 100644--- a/tests/server/test_api_content.py+++ b/tests/server/test_api_content.py@@ -171,20 +171,28 @@ async def test_read_internal_scope_uri_returns_invalid_uri(client, uri: str):assert "frozenset" not in body["error"]["message"]-async def test_abstract_content(client_with_resource):+async def test_abstract_content(client_with_resource, service):client, uri = client_with_resource+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)+ await service.viking_fs.write_file(f"{uri}/.abstract.md", "---\n", ctx=ctx)+resp = await client.get("/api/v1/content/abstract", params={"uri": uri})assert resp.status_code == 200body = resp.json()assert body["status"] == "ok"+ assert body["result"] == f"# {uri} [Directory abstract is not ready]"-async def test_overview_content(client_with_resource):+async def test_overview_content(client_with_resource, service):client, uri = client_with_resource+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)+ await service.viking_fs.write_file(f"{uri}/.overview.md", "---\n", ctx=ctx)+resp = await client.get("/api/v1/content/overview", params={"uri": uri})assert resp.status_code == 200body = resp.json()assert body["status"] == "ok"+ assert body["result"] == f"# {uri}\n\n[Directory overview is not ready]"async def test_abstract_file_uri_returns_failed_precondition(client_with_resource):diff --git a/tests/server/test_api_content_write.py b/tests/server/test_api_content_write.pyindex 69570e654..d1834806a 100644--- a/tests/server/test_api_content_write.py+++ b/tests/server/test_api_content_write.py@@ -77,6 +77,13 @@ async def test_write_allows_existing_semantic_sidecar_but_rejects_relations(clie)assert overview_resp.status_code == 200+ malformed_resp = await client.post(+ "/api/v1/content/write",+ json={"uri": f"{uri}/.overview.md", "content": "---\n"},+ )+ assert malformed_resp.status_code == 400+ assert malformed_resp.json()["error"]["code"] == "INVALID_ARGUMENT"+relations_resp = await client.post("/api/v1/content/write",json={"uri": f"{uri}/.relations.json", "content": "new content"},diff --git a/tests/storage/test_abstract_overview_freshness.py b/tests/storage/test_abstract_overview_freshness.pyindex 62eed2678..70a0d5c6e 100644--- a/tests/storage/test_abstract_overview_freshness.py+++ b/tests/storage/test_abstract_overview_freshness.py@@ -141,7 +141,7 @@ async def test_concurrent_pending_marks_do_not_overwrite_each_other():async def test_partial_sidecar_baseline_refreshes_immediately():dir_uri = "viking://resources/wide"files = _files(dir_uri, pending=3)- files.pop(f"{dir_uri}/.abstract.md")+ files[f"{dir_uri}/.abstract.md"] = "---\n"fs = _FakeFS(files)decision = await plan_abstract_overview_refresh(diff --git a/tests/storage/test_semantic_dag_incremental.py b/tests/storage/test_semantic_dag_incremental.pyindex 5e21a705d..898873b14 100644--- a/tests/storage/test_semantic_dag_incremental.py+++ b/tests/storage/test_semantic_dag_incremental.py@@ -37,7 +37,7 @@ class _FakeVikingFS:async def ls(self, uri, node_limit=None, ctx=None):return self._tree.get(self._norm(uri), [])- async def stat(self, uri, ctx=None):+ async def stat(self, uri, ctx=None, skip_count=False):content = self._file_contents.get(self._norm(uri), "")return {"size": len(content)}@@ -165,7 +165,8 @@ class _FakeProcessor:@pytest.mark.asyncio-async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypatch):[email protected]("sidecar_state", ["valid", "malformed_overview", "malformed_abstract"])+async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypatch, sidecar_state):root_uri = "viking://resources/root"tree = {root_uri: [@@ -193,6 +194,9 @@ async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypaf"{root_uri}/.abstract.md": "old-abstract",},)+ if sidecar_state != "valid":+ filename = ".overview.md" if sidecar_state == "malformed_overview" else ".abstract.md"+ fake_fs._file_contents[f"{root_uri}/{filename}"] = "---\n"monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_openviking_config",@@ -208,17 +212,26 @@ async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypactx=ctx,incremental_update=True,target_uri=root_uri,- changes={"modified": [f"{root_uri}/a.txt"]},+ changes={"modified": [f"{root_uri}/a.txt"]} if sidecar_state == "valid" else {},)await executor.run(root_uri)- assert processor.summarized_files == [f"{root_uri}/a.txt"]- assert processor.vectorized_files == [f"{root_uri}/a.txt"]+ expected_files = {+ "valid": [f"{root_uri}/a.txt"],+ "malformed_overview": [f"{root_uri}/a.txt", f"{root_uri}/b.txt"],+ "malformed_abstract": [],+ }[sidecar_state]+ assert processor.summarized_files == expected_files+ assert processor.vectorized_files == expected_files+ assert processor.vectorized_dirs == [root_uri]assert processor.sync_calls == []overview = parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.overview.md"]).body- assert "- a.txt: summary" in overview- assert "- b.txt: old-b" in overview+ expected_a = "old-a" if sidecar_state == "malformed_abstract" else "summary"+ expected_b = "summary" if sidecar_state == "malformed_overview" else "old-b"+ assert f"- a.txt: {expected_a}" in overview+ assert f"- b.txt: {expected_b}" in overview+ assert parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.abstract.md"]).body.strip()@pytest.mark.asynciodiff --git a/tests/storage/test_semantic_processor_l0_l1.py b/tests/storage/test_semantic_processor_l0_l1.pyindex b6d02fe2a..ea9dac4d7 100644--- a/tests/storage/test_semantic_processor_l0_l1.py+++ b/tests/storage/test_semantic_processor_l0_l1.py@@ -93,6 +93,7 @@ def test_markdown_overview_uses_brief_description_as_abstract(monkeypatch):processor = SemanticProcessor()generated = ("# README\n\n"+ "---\n\n""This brief description is the retrieval abstract.\n\n""## Quick Navigation\n\n""- Read README.md"