Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(semantic): recover malformed sidecars without failing reads (#5013)

Qin Haojief098476d969d
diff --git a/openviking/retrieve/context_assembler/tiers.py b/openviking/retrieve/context_assembler/tiers.py
index 20cdd562c..0c4c05a9d 100644
--- a/openviking/retrieve/context_assembler/tiers.py
+++ b/openviking/retrieve/context_assembler/tiers.py
@@ -26,7 +26,7 @@ from openviking.retrieve.context_assembler.params import (
Tier,
)
from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils
-from openviking.storage.abstract_overview import body_for_preview
+from openviking.storage.abstract_overview import AbstractOverviewFormatError, body_for_preview
OVERVIEW_HEADING_LIMIT = 24
OVERVIEW_PARAGRAPH_CHARS = 400
@@ -110,7 +110,10 @@ def overview_from_content(candidate: Candidate, content: str) -> str:
return ""
if candidate.is_directory:
# The cached content already is the directory's own overview sidecar.
- return body_for_preview(content).strip()
+ try:
+ return body_for_preview(content).strip()
+ except AbstractOverviewFormatError:
+ return ""
if _is_code_uri(candidate.base_uri):
extraction = extract_skeleton_result(filename_from_uri(candidate.base_uri), content)
if extraction.text:
diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.py
index fbce14750..65bc2ed1f 100644
--- a/openviking/retrieve/hierarchical_retriever.py
+++ b/openviking/retrieve/hierarchical_retriever.py
@@ -22,7 +22,7 @@ from openviking.models.rerank import RerankClient
from openviking.retrieve.memory_lifecycle import hotness_score
from openviking.retrieve.retrieval_stats import get_stats_collector
from openviking.server.identity import RequestContext
-from openviking.storage.abstract_overview import body_for_preview
+from openviking.storage.abstract_overview import AbstractOverviewFormatError, body_for_preview
from openviking.storage.expr import FilterExpr
from openviking.storage.vikingdb_manager import VikingDBManager, VikingDBManagerProxy
from openviking.telemetry import get_current_telemetry
@@ -635,7 +635,11 @@ class HierarchicalRetriever:
# the public find/search preview contract body-only at its final
# conversion boundary. L2 user Markdown is intentionally left
# untouched, including ordinary YAML frontmatter.
- abstract = body_for_preview(abstract)
+ try:
+ abstract = body_for_preview(abstract)
+ except AbstractOverviewFormatError as exc:
+ logger.warning("Malformed sidecar in retrieval result %s: %s", display_uri, exc)
+ abstract = ""
results.append(
MatchedContext(
diff --git a/openviking/storage/abstract_overview.py b/openviking/storage/abstract_overview.py
index 3b73b11c4..cdc823d8f 100644
--- a/openviking/storage/abstract_overview.py
+++ b/openviking/storage/abstract_overview.py
@@ -425,7 +425,11 @@ async def _read_existing_document(
viking_fs: Any, uri: str, ctx: Optional[RequestContext]
) -> Optional[AbstractOverviewDocument]:
raw = await _raw_if_exists(viking_fs, uri, ctx)
- return parse_abstract_overview(raw) if raw is not None else None
+ try:
+ return parse_abstract_overview(raw) if raw is not None else None
+ except AbstractOverviewFormatError as exc:
+ logger.warning("[Semantic] Ignoring malformed sidecar %s: %s", uri, exc)
+ return None
async def write_abstract_overview(
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py
index c30f47286..0ddccfbd4 100644
--- a/openviking/storage/queuefs/semantic_dag.py
+++ b/openviking/storage/queuefs/semantic_dag.py
@@ -698,8 +698,6 @@ class SemanticDagExecutor:
if file_name in existing_summaries:
return {"name": file_name, "summary": existing_summaries[file_name]}
- except AbstractOverviewFormatError:
- raise
except Exception as e:
logger.debug(f"Failed to read existing summary from overview.md for {file_path}: {e}")
@@ -754,8 +752,6 @@ class SemanticDagExecutor:
overview = await self._viking_fs.read_file(f"{target_path}/.overview.md", ctx=self._ctx)
abstract = await self._viking_fs.read_file(f"{target_path}/.abstract.md", ctx=self._ctx)
return body_for_preview(overview), body_for_preview(abstract)
- except AbstractOverviewFormatError:
- raise
except Exception:
return None, None
@@ -797,11 +793,6 @@ class SemanticDagExecutor:
summary_dict = await self._processor._generate_single_file_summary(
file_path, llm_sem=self._llm_sem, ctx=self._ctx
)
- except AbstractOverviewFormatError:
- # A generated sidecar that opted into OKF must never be treated as
- # an empty file summary; doing so would silently feed metadata or
- # corrupted YAML into a later regeneration.
- raise
except Exception as e:
logger.warning(f"Failed to generate summary for {file_path}: {e}")
summary_dict = {"name": file_name, "summary": ""}
@@ -1028,9 +1019,11 @@ class SemanticDagExecutor:
)
if not children_changed:
- need_vectorize = False
overview, abstract = await self._read_existing_overview_abstract(dir_uri)
should_write = overview is None or abstract is None
+ # Rebuilt sidecars must also replace their stale vectors.
+ need_vectorize = should_write
+ children_changed = should_write
if should_write and (overview is None or abstract is None):
async with node.lock:
file_summaries = self._finalize_file_summaries(node)
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 045431b5b..633b83bc7 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -35,7 +35,6 @@ from openviking.prompts import render_prompt
from openviking.server.identity import RequestContext, Role
from openviking.service.task_work_index import detach_task_context
from openviking.storage.abstract_overview import (
- AbstractOverviewFormatError,
AbstractOverviewWriteResult,
body_for_preview,
deterministic_sample,
@@ -655,8 +654,6 @@ class SemanticProcessor(DequeueHandlerBase):
logger.info(
f"Parsed {len(existing_summaries)} existing summaries from overview.md"
)
- except AbstractOverviewFormatError:
- raise
except Exception as e:
logger.debug(f"No existing overview.md found for {dir_uri}: {e}")
@@ -1145,6 +1142,8 @@ class SemanticProcessor(DequeueHandlerBase):
in_header = True
for line in lines:
+ if line.strip() == "---":
+ continue
if in_header and line.startswith("#"):
continue
elif in_header and line.strip():
diff --git a/openviking/storage/viking_fs/_semantic.py b/openviking/storage/viking_fs/_semantic.py
index ee4aab40c..1681ae4e0 100644
--- a/openviking/storage/viking_fs/_semantic.py
+++ b/openviking/storage/viking_fs/_semantic.py
@@ -9,7 +9,11 @@ from openviking.core.context import ContextLevel
from openviking.core.retrieval_targets import resolve_retrieval_targets
from openviking.server.error_mapping import is_not_found_error, map_exception
from openviking.server.identity import RequestContext
-from openviking.storage.abstract_overview import body_for_preview, render_abstract_overview
+from openviking.storage.abstract_overview import (
+ AbstractOverviewFormatError,
+ body_for_preview,
+ render_abstract_overview,
+)
from openviking.storage.acl import AclAction
from openviking.storage.viking_fs._base import (
_ensure_filter_present,
@@ -42,15 +46,16 @@ class _SemanticMixin:
file_path = f"{path}/.abstract.md"
try:
content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))
+ return body_for_preview(self._decode_bytes(content_bytes))
+ except AbstractOverviewFormatError as exc:
+ logger.warning("Malformed directory abstract for %s: %s", uri, exc)
except Exception as exc:
if not is_not_found_error(exc):
mapped = map_exception(exc, resource=uri)
if mapped is not None:
raise mapped from exc
raise
- return f"# {uri} [Directory abstract is not ready]"
-
- return body_for_preview(self._decode_bytes(content_bytes))
+ return f"# {uri} [Directory abstract is not ready]"
async def _read_abstract_for_known_dir(
self,
@@ -173,16 +178,16 @@ class _SemanticMixin:
file_path = f"{path}/.overview.md"
try:
content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))
+ return body_for_preview(self._decode_bytes(content_bytes))
+ except AbstractOverviewFormatError as exc:
+ logger.warning("Malformed directory overview for %s: %s", uri, exc)
except Exception as exc:
if not is_not_found_error(exc):
mapped = map_exception(exc, resource=uri)
if mapped is not None:
raise mapped from exc
raise
- # Fallback to default if .overview.md doesn't exist
- return f"# {uri}\n\n[Directory overview is not ready]"
-
- return body_for_preview(self._decode_bytes(content_bytes))
+ return f"# {uri}\n\n[Directory overview is not ready]"
async def find(
self,
diff --git a/tests/retrieve/test_hierarchical_retriever_rerank.py b/tests/retrieve/test_hierarchical_retriever_rerank.py
index d1c44acd2..56509e3c6 100644
--- a/tests/retrieve/test_hierarchical_retriever_rerank.py
+++ b/tests/retrieve/test_hierarchical_retriever_rerank.py
@@ -697,13 +697,20 @@ async def test_convert_to_matched_contexts_defaults_tags_and_body_previews():
),
),
_result("viking://resources/demo.md", 0.8, level=2, abstract=markdown),
+ _result(
+ "viking://resources/malformed",
+ 0.7,
+ level=int(ContextLevel.ABSTRACT),
+ abstract="---\n",
+ ),
],
ctx=_ctx(),
)
- assert [item.search_tags for item in result] == [[], [], []]
+ assert [item.search_tags for item in result] == [[], [], [], []]
assert [item.abstract for item in result] == [
"Visible abstract.",
"# Visible overview",
markdown,
+ "",
]
diff --git a/tests/server/test_api_content.py b/tests/server/test_api_content.py
index b291fe3e7..9b6a903b4 100644
--- a/tests/server/test_api_content.py
+++ b/tests/server/test_api_content.py
@@ -171,20 +171,28 @@ async def test_read_internal_scope_uri_returns_invalid_uri(client, uri: str):
assert "frozenset" not in body["error"]["message"]
-async def test_abstract_content(client_with_resource):
+async def test_abstract_content(client_with_resource, service):
client, uri = client_with_resource
+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
+ await service.viking_fs.write_file(f"{uri}/.abstract.md", "---\n", ctx=ctx)
+
resp = await client.get("/api/v1/content/abstract", params={"uri": uri})
assert resp.status_code == 200
body = resp.json()
assert body["status"] == "ok"
+ assert body["result"] == f"# {uri} [Directory abstract is not ready]"
-async def test_overview_content(client_with_resource):
+async def test_overview_content(client_with_resource, service):
client, uri = client_with_resource
+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
+ await service.viking_fs.write_file(f"{uri}/.overview.md", "---\n", ctx=ctx)
+
resp = await client.get("/api/v1/content/overview", params={"uri": uri})
assert resp.status_code == 200
body = resp.json()
assert body["status"] == "ok"
+ assert body["result"] == f"# {uri}\n\n[Directory overview is not ready]"
async def test_abstract_file_uri_returns_failed_precondition(client_with_resource):
diff --git a/tests/server/test_api_content_write.py b/tests/server/test_api_content_write.py
index 69570e654..d1834806a 100644
--- a/tests/server/test_api_content_write.py
+++ b/tests/server/test_api_content_write.py
@@ -77,6 +77,13 @@ async def test_write_allows_existing_semantic_sidecar_but_rejects_relations(clie
)
assert overview_resp.status_code == 200
+ malformed_resp = await client.post(
+ "/api/v1/content/write",
+ json={"uri": f"{uri}/.overview.md", "content": "---\n"},
+ )
+ assert malformed_resp.status_code == 400
+ assert malformed_resp.json()["error"]["code"] == "INVALID_ARGUMENT"
+
relations_resp = await client.post(
"/api/v1/content/write",
json={"uri": f"{uri}/.relations.json", "content": "new content"},
diff --git a/tests/storage/test_abstract_overview_freshness.py b/tests/storage/test_abstract_overview_freshness.py
index 62eed2678..70a0d5c6e 100644
--- a/tests/storage/test_abstract_overview_freshness.py
+++ b/tests/storage/test_abstract_overview_freshness.py
@@ -141,7 +141,7 @@ async def test_concurrent_pending_marks_do_not_overwrite_each_other():
async def test_partial_sidecar_baseline_refreshes_immediately():
dir_uri = "viking://resources/wide"
files = _files(dir_uri, pending=3)
- files.pop(f"{dir_uri}/.abstract.md")
+ files[f"{dir_uri}/.abstract.md"] = "---\n"
fs = _FakeFS(files)
decision = await plan_abstract_overview_refresh(
diff --git a/tests/storage/test_semantic_dag_incremental.py b/tests/storage/test_semantic_dag_incremental.py
index 5e21a705d..898873b14 100644
--- a/tests/storage/test_semantic_dag_incremental.py
+++ b/tests/storage/test_semantic_dag_incremental.py
@@ -37,7 +37,7 @@ class _FakeVikingFS:
async def ls(self, uri, node_limit=None, ctx=None):
return self._tree.get(self._norm(uri), [])
- async def stat(self, uri, ctx=None):
+ async def stat(self, uri, ctx=None, skip_count=False):
content = self._file_contents.get(self._norm(uri), "")
return {"size": len(content)}
@@ -165,7 +165,8 @@ class _FakeProcessor:
@pytest.mark.asyncio
-async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypatch):
[email protected]("sidecar_state", ["valid", "malformed_overview", "malformed_abstract"])
+async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypatch, sidecar_state):
root_uri = "viking://resources/root"
tree = {
root_uri: [
@@ -193,6 +194,9 @@ async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypa
f"{root_uri}/.abstract.md": "old-abstract",
},
)
+ if sidecar_state != "valid":
+ filename = ".overview.md" if sidecar_state == "malformed_overview" else ".abstract.md"
+ fake_fs._file_contents[f"{root_uri}/{filename}"] = "---\n"
monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
monkeypatch.setattr(
"openviking.storage.queuefs.semantic_dag.get_openviking_config",
@@ -208,17 +212,26 @@ async def test_direct_incremental_update_uses_changes_without_temp_sync(monkeypa
ctx=ctx,
incremental_update=True,
target_uri=root_uri,
- changes={"modified": [f"{root_uri}/a.txt"]},
+ changes={"modified": [f"{root_uri}/a.txt"]} if sidecar_state == "valid" else {},
)
await executor.run(root_uri)
- assert processor.summarized_files == [f"{root_uri}/a.txt"]
- assert processor.vectorized_files == [f"{root_uri}/a.txt"]
+ expected_files = {
+ "valid": [f"{root_uri}/a.txt"],
+ "malformed_overview": [f"{root_uri}/a.txt", f"{root_uri}/b.txt"],
+ "malformed_abstract": [],
+ }[sidecar_state]
+ assert processor.summarized_files == expected_files
+ assert processor.vectorized_files == expected_files
+ assert processor.vectorized_dirs == [root_uri]
assert processor.sync_calls == []
overview = parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.overview.md"]).body
- assert "- a.txt: summary" in overview
- assert "- b.txt: old-b" in overview
+ expected_a = "old-a" if sidecar_state == "malformed_abstract" else "summary"
+ expected_b = "summary" if sidecar_state == "malformed_overview" else "old-b"
+ assert f"- a.txt: {expected_a}" in overview
+ assert f"- b.txt: {expected_b}" in overview
+ assert parse_abstract_overview(fake_fs._file_contents[f"{root_uri}/.abstract.md"]).body.strip()
@pytest.mark.asyncio
diff --git a/tests/storage/test_semantic_processor_l0_l1.py b/tests/storage/test_semantic_processor_l0_l1.py
index b6d02fe2a..ea9dac4d7 100644
--- a/tests/storage/test_semantic_processor_l0_l1.py
+++ b/tests/storage/test_semantic_processor_l0_l1.py
@@ -93,6 +93,7 @@ def test_markdown_overview_uses_brief_description_as_abstract(monkeypatch):
processor = SemanticProcessor()
generated = (
"# README\n\n"
+ "---\n\n"
"This brief description is the retrieval abstract.\n\n"
"## Quick Navigation\n\n"
"- Read README.md"