Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(core): harden privacy and background failure handling (#3548)
Hao Zheaf914ca27b36
diff --git a/openviking/async_client.py b/openviking/async_client.pyindex 1d98c40af..7b09a6a6f 100644--- a/openviking/async_client.py+++ b/openviking/async_client.py@@ -729,7 +729,7 @@ class AsyncOpenViking:simple = kwargs.get("simple", False)output = kwargs.get("output", "original")abs_limit = kwargs.get("abs_limit", 256)- show_all_hidden = kwargs.get("show_all_hidden", True)+ show_all_hidden = kwargs.get("show_all_hidden", False)node_limit = kwargs.get("node_limit", 1000)sort_by = kwargs.get("sort_by")sort_order = kwargs.get("sort_order", "asc")@@ -791,7 +791,7 @@ class AsyncOpenViking:await self._ensure_initialized()output = kwargs.get("output", "original")abs_limit = kwargs.get("abs_limit", 128)- show_all_hidden = kwargs.get("show_all_hidden", True)+ show_all_hidden = kwargs.get("show_all_hidden", False)node_limit = kwargs.get("node_limit", 1000)return await self._client.tree(uri,diff --git a/openviking/client/local.py b/openviking/client/local.pyindex a9baba1f3..8a50c8ff6 100644--- a/openviking/client/local.py+++ b/openviking/client/local.py@@ -633,7 +633,12 @@ class LocalClient(BaseClient):offset: Starting line number (0-indexed). Default 0.limit: Number of lines to read. -1 means read to end. Default -1."""- return await self._service.fs.read(uri, ctx=self._ctx, offset=offset, limit=limit)+ return await self._service.fs.read_visible(+ uri,+ ctx=self._ctx,+ offset=offset,+ limit=limit,+ )async def read_raw(self, uri: str, offset: int = 0, limit: int = -1) -> str:"""Read raw file content, including hidden MEMORY_FIELDS metadata."""diff --git a/openviking/observability/http_observability_middleware.py b/openviking/observability/http_observability_middleware.pyindex 2ab88ba8b..48536cbc6 100644--- a/openviking/observability/http_observability_middleware.py+++ b/openviking/observability/http_observability_middleware.py@@ -824,7 +824,6 @@ def create_http_observability_middleware() -> Callable[[Request, Callable], Resp# Extract request informationraw_path = str(request.url.path)- raw_query = request.url.query or Noneroute_template = _get_route_template(request)request_id = request.state.request_id@@ -834,7 +833,6 @@ def create_http_observability_middleware() -> Callable[[Request, Callable], Resphttp_route=route_template,request_id=request_id,url_path=raw_path,- url_query=raw_query,url_scheme=request.url.scheme,http_host=request.url.netloc,source_type=request.headers.get("x-source-type"),diff --git a/openviking/server/body_dump_middleware.py b/openviking/server/body_dump_middleware.pyindex 28f308023..f27f78ea0 100644--- a/openviking/server/body_dump_middleware.py+++ b/openviking/server/body_dump_middleware.py@@ -25,6 +25,7 @@ except ImportError: # pragma: no cover - OTel optional_SKIP_CONTENT_TYPE_PREFIXES = ("multipart/form-data","application/octet-stream",+ "application/zip","text/event-stream","audio/","video/",diff --git a/openviking/server/mcp_endpoint.py b/openviking/server/mcp_endpoint.pyindex 0c7073aa4..712a4ff9f 100644--- a/openviking/server/mcp_endpoint.py+++ b/openviking/server/mcp_endpoint.py@@ -363,7 +363,7 @@ async def read(uris: str | list[str]) -> str:async def _read_one(uri: str) -> str:async with semaphore:try:- body = await service.fs.read(uri, ctx=ctx)+ body = await service.fs.read_visible(uri, ctx=ctx)if isinstance(body, str) and body.strip():return bodyexcept Exception:diff --git a/openviking/server/routers/content.py b/openviking/server/routers/content.pyindex ceb44d260..fc6a7f97e 100644--- a/openviking/server/routers/content.py+++ b/openviking/server/routers/content.py@@ -150,7 +150,10 @@ async def read(service = get_service()uri = resolve_path_variables(uri)try:- result = await service.fs.read(uri, ctx=_ctx, offset=offset, limit=limit)+ if raw:+ result = await service.fs.read(uri, ctx=_ctx, offset=offset, limit=limit)+ else:+ result = await service.fs.read_visible(uri, ctx=_ctx, offset=offset, limit=limit)except AGFSNotFoundError:raise NotFoundError(uri, "file")except AGFSClientError as e:@@ -159,21 +162,6 @@ async def read(raise mapped from eraise- if not raw:- # 清理MEMORY_FIELDS隐藏注释(v2记忆加工过程中的临时内部数据,不暴露给外部用户)- if isinstance(result, bytes):- text = result.decode("utf-8")- elif isinstance(result, str):- text = result- else:- text = None-- if text:- from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils-- mf = MemoryFileUtils.read(text)- result = mf.content-return Response(status="ok", result=result)diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.pyindex 17d628848..2814e39b3 100644--- a/openviking/service/fs_service.py+++ b/openviking/service/fs_service.py@@ -8,7 +8,7 @@ Provides file system operations: ls, mkdir, rm, mv, tree, stat, read, abstract,from typing import TYPE_CHECKING, Any, Dict, List, Optional-from openviking.core.namespace import context_type_for_uri+from openviking.core.namespace import classify_uri, context_type_for_urifrom openviking.core.uri_validation import validate_optional_viking_uri, validate_viking_urifrom openviking.privacy import (UserPrivacyConfigService,@@ -18,6 +18,7 @@ from openviking.privacy import (from openviking.resource.watch_storage import is_watch_task_control_urifrom openviking.server.identity import RequestContextfrom openviking.session.memory.memory_updater import MemoryUpdater+from openviking.session.memory.utils.content_visibility import visible_contentfrom openviking.storage.content_write import ContentWriteCoordinatorfrom openviking.storage.queuefs import SemanticMsg, get_queue_managerfrom openviking.storage.queuefs.semantic_msg import build_semantic_coalesce_key@@ -31,6 +32,21 @@ from openviking_cli.utils import VikingURI, get_loggerlogger = get_logger(__name__)++def _may_include_memory_content(uri: str) -> bool:+ """Return whether a public subtree read can contain memory files."""+ classification = classify_uri(uri)+ if classification.is_memory:+ return True+ if classification.content_index is not None:+ return False+ return not classification.parts or classification.scope in {"user", "agent"}+++def _visible_grep_content(content: str, uri: str) -> str:+ return visible_content(content, uri=uri)++if TYPE_CHECKING:from openviking.resource.watch_manager import WatchManagerfrom openviking.resource.watch_scheduler import WatchScheduler@@ -413,9 +429,7 @@ class FSService:rollback_resource=lambda: viking_fs.mv(to_uri, from_uri, ctx=ctx),)- async def _sync_watch_after_rm(- self, uri: str, *, account_id: str, context_type: str- ) -> None:+ async def _sync_watch_after_rm(self, uri: str, *, account_id: str, context_type: str) -> None:if context_type != "resource":returnif is_watch_task_control_uri(uri):@@ -493,6 +507,20 @@ class FSService:sliced = lines[offset:] if limit == -1 else lines[offset : offset + limit]return "".join(sliced)+ async def read_visible(+ self,+ uri: str,+ ctx: RequestContext,+ offset: int = 0,+ limit: int = -1,+ ) -> str:+ """Read public content, hiding reserved metadata from memory files."""+ uri = validate_viking_uri(uri)+ if not classify_uri(uri).is_memory:+ return await self.read(uri, ctx=ctx, offset=offset, limit=limit)+ content = await self.read(uri, ctx=ctx)+ return visible_content(content, uri=uri, offset=offset, limit=limit)+async def abstract(self, uri: str, ctx: RequestContext) -> str:"""Read L0 abstract (.abstract.md)."""viking_fs = self._ensure_initialized()@@ -519,15 +547,16 @@ class FSService:viking_fs = self._ensure_initialized()uri = validate_viking_uri(uri)exclude_uri = validate_optional_viking_uri(exclude_uri, field_name="exclude_uri") or None- return await viking_fs.grep(- uri,- pattern,- exclude_uri=exclude_uri,- case_insensitive=case_insensitive,- node_limit=node_limit,- level_limit=level_limit,- ctx=ctx,- )+ kwargs = {+ "exclude_uri": exclude_uri,+ "case_insensitive": case_insensitive,+ "node_limit": node_limit,+ "level_limit": level_limit,+ "ctx": ctx,+ }+ if _may_include_memory_content(uri):+ kwargs["content_transform"] = _visible_grep_content+ return await viking_fs.grep(uri, pattern, **kwargs)async def glob(self,@@ -622,9 +651,7 @@ class FSService:) -> Dict[str, Any]:"""Forward to VikingFS.commit. See viking_fs.commit for semantics."""viking_fs = self._ensure_initialized()- validated = (- [validate_viking_uri(p) for p in paths] if paths is not None else None- )+ validated = [validate_viking_uri(p) for p in paths] if paths is not None else Nonereturn await viking_fs.commit(message=message,paths=validated,@@ -721,9 +748,7 @@ class FSService:viking_fs = self._ensure_initialized()return await viking_fs.get_gitignore(ctx=ctx)- async def set_gitignore(- self, *, content: str, ctx: RequestContext- ) -> None:+ async def set_gitignore(self, *, content: str, ctx: RequestContext) -> None:"""Forward to VikingFS.set_gitignore. Writes the account .ovgitignorecontrol file (validates the size limit)."""viking_fs = self._ensure_initialized()diff --git a/openviking/session/memory/utils/content_visibility.py b/openviking/session/memory/utils/content_visibility.pynew file mode 100644index 000000000..a73b4aff3--- /dev/null+++ b/openviking/session/memory/utils/content_visibility.py@@ -0,0 +1,30 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Public-content projection for persisted memory files."""++from openviking.core.namespace import classify_uri+from openviking.session.memory.utils.memory_fields import strip_memory_fields_trailer+++def visible_content(+ raw_content: str,+ *,+ uri: str,+ offset: int = 0,+ limit: int = -1,+) -> str:+ """Hide a memory file's reserved metadata trailer, then apply line slicing.++ MEMORY_FIELDS is a reserved trailer only in memory namespaces. Resource and+ skill files may legitimately contain similarly shaped HTML comments, so+ their content is returned verbatim apart from the requested line slice.+ """+ content = raw_content+ if classify_uri(uri).is_memory:+ content = strip_memory_fields_trailer(content)++ if offset == 0 and limit == -1:+ return content+ lines = content.splitlines(keepends=True)+ sliced = lines[offset:] if limit == -1 else lines[offset : offset + limit]+ return "".join(sliced)diff --git a/openviking/session/memory/utils/memory_fields.py b/openviking/session/memory/utils/memory_fields.pynew file mode 100644index 000000000..1f336b4d7--- /dev/null+++ b/openviking/session/memory/utils/memory_fields.py@@ -0,0 +1,27 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Shared grammar for reserved ``MEMORY_FIELDS`` comments."""++from __future__ import annotations++import re+from typing import Match, Optional++MEMORY_FIELDS_COMMENT_RE = re.compile(r"<!--\s*MEMORY_FIELDS\b(?P<fields>[\s\S]*?)\s*-->")+++def find_memory_fields_trailer(content: str) -> Optional[Match[str]]:+ """Return the reserved comment when it is the final content element."""+ trailer = None+ for match in MEMORY_FIELDS_COMMENT_RE.finditer(content):+ if not content[match.end() :].strip():+ trailer = match+ return trailer+++def strip_memory_fields_trailer(content: str) -> str:+ """Remove all trailing reserved comments and their whitespace separators."""+ visible = content+ while match := find_memory_fields_trailer(visible):+ visible = visible[: match.start()].rstrip()+ return visiblediff --git a/openviking/session/memory/utils/messages.py b/openviking/session/memory/utils/messages.pyindex 657e85b37..b3ccb1a26 100644--- a/openviking/session/memory/utils/messages.py+++ b/openviking/session/memory/utils/messages.py@@ -5,11 +5,11 @@ Message formatting and memory file parsing utilities."""import json-import refrom typing import Any, Dict, Listimport json_repair+from openviking.session.memory.utils.memory_fields import MEMORY_FIELDS_COMMENT_REfrom openviking.telemetry import tracerfrom openviking_cli.utils import get_logger@@ -92,16 +92,12 @@ def parse_memory_file_with_fields(content: str) -> Dict[str, Any]:if not content:return {"content": ""}- # Pattern to match: <!-- MEMORY_FIELDS ... -->- # Matches multi-line JSON inside the comment- pattern = r"<!--\s*MEMORY_FIELDS\s*([\s\S]*?)\s*-->"-- match = re.search(pattern, content)+ match = MEMORY_FIELDS_COMMENT_RE.search(content)result = {}if match:- fields_json_str = match.group(1).strip()+ fields_json_str = match.group("fields").strip()if fields_json_str:try:fields = json_repair.loads(fields_json_str)@@ -114,7 +110,7 @@ def parse_memory_file_with_fields(content: str) -> Dict[str, Any]:tracer.warning(f"Failed to parse MEMORY_FIELDS JSON: {e}")# Remove the comment from content- content_without_comment = re.sub(pattern, "", content).strip()+ content_without_comment = MEMORY_FIELDS_COMMENT_RE.sub("", content).strip()result["content"] = content_without_commentreturn resultdiff --git a/openviking/session/session.py b/openviking/session/session.pyindex dd8fc71ed..c7a3383de 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -3798,7 +3798,29 @@ class Session:combined: List[Message] = []completed_memory_steps: Dict[str, set[str]] = {}for state in replay_states:- combined.extend(await self._read_archive_messages(state.archive_uri))+ # A terminally-failed earlier archive can have a missing or corrupt+ # messages.jsonl (legacy "no messages" data, or #3417's own+ # archive_read terminal path). Tolerate that exactly like+ # _get_uncovered_archive_messages does, otherwise every subsequent+ # commit's Phase 2 re-raises here and stays permanently poisoned.+ # Real storage failures still propagate.+ try:+ replayed = await self._read_archive_messages(state.archive_uri)+ except _ArchiveMessagesCorruptError:+ logger.warning(+ "Skipping failed archive %s in Phase-2 replay: messages.jsonl is corrupt",+ state.archive_uri,+ )+ continue+ except Exception as exc:+ if not _is_storage_not_found(exc):+ raise+ logger.warning(+ "Skipping failed archive %s in Phase-2 replay: messages.jsonl is missing",+ state.archive_uri,+ )+ continue+ combined.extend(replayed)marker = state.failedself._merge_completed_memory_steps(completed_memory_steps,diff --git a/openviking/storage/queuefs/embedding_tracker.py b/openviking/storage/queuefs/embedding_tracker.pyindex 2da33ecec..a993184c2 100644--- a/openviking/storage/queuefs/embedding_tracker.py+++ b/openviking/storage/queuefs/embedding_tracker.py@@ -200,3 +200,27 @@ class EmbeddingTaskTracker:if record_to_finalize is not None:await self._run_on_complete(semantic_msg_id, record_to_finalize)return remaining++ async def abort(self, semantic_msg_id: str) -> bool:+ """Retire one tracking attempt without running its completion callback.++ Late decrements for an aborted attempt are ignored because its record no+ longer exists. Callers that may retry a logical semantic message must+ register each attempt under a unique tracking ID.++ Returns:+ True when an active record was removed, otherwise False.+ """+ with self._lock:+ record = self._tasks.pop(semantic_msg_id, None)++ if record is None:+ return False++ logger.warning(+ "Aborted embedding tracker for SemanticMsg %s with %s/%s tasks remaining",+ semantic_msg_id,+ record.remaining,+ record.total,+ )+ return Truediff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex d2f8617aa..d2590e87d 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -6,6 +6,7 @@ import asyncioimport threadingfrom dataclasses import dataclass, fieldfrom typing import Any, ClassVar, Dict, List, Optional, Set+from uuid import uuid4from weakref import WeakKeyDictionaryfrom openviking.server.identity import RequestContext@@ -217,6 +218,15 @@ class SemanticDagExecutor:self._pending_vectorize_tasks: List[VectorizeTask] = []self._pending_vectorize_work = 0self._vectorize_done: Optional[asyncio.Event] = None+ self._vectorize_failure: Optional[Exception] = None+ self._vectorize_dispatch_complete = False+ self._embedding_tracker_done: Optional[asyncio.Event] = None+ self._embedding_tracker = None+ # Retries preserve the logical SemanticMsg ID, so each execution needs+ # a distinct tracker key to quarantine late decrements from old attempts.+ self._embedding_tracker_id = f"{self._semantic_msg_id or 'semantic'}:{uuid4()}"+ self._vectorize_finalized = False+ self._vectorize_finalize_lock = asyncio.Lock()self._vectorize_lock = asyncio.Lock()self._file_change_status: Dict[str, bool] = {}self._dir_change_status: Dict[str, bool] = {}@@ -237,12 +247,6 @@ class SemanticDagExecutor:raise self._failure# Mark semantic done after downstream vectorization finishes.- async def wrapped_on_complete() -> None:- if self._telemetry_id and self._semantic_msg_id:- get_request_wait_tracker().mark_semantic_done(- self._telemetry_id, self._semantic_msg_id- )-async with self._vectorize_lock:task_count = self._vectorize_task_counttasks = list(self._pending_vectorize_tasks)@@ -250,24 +254,47 @@ class SemanticDagExecutor:if task_count > 0:from .embedding_tracker import EmbeddingTaskTracker+ embedding_tracker_done = asyncio.Event()+ self._embedding_tracker_done = embedding_tracker_donetracker = EmbeddingTaskTracker.get_instance()+ self._embedding_tracker = trackerawait tracker.register(- semantic_msg_id=self._semantic_msg_id,+ semantic_msg_id=self._embedding_tracker_id,total_count=task_count,- on_complete=wrapped_on_complete,- metadata={"uri": root_uri},+ on_complete=self._on_embedding_tasks_complete,+ metadata={+ "uri": root_uri,+ "semantic_msg_id": self._semantic_msg_id,+ },)await self._dispatch_vectorize_tasks(tasks)+ self._vectorize_dispatch_complete = True++ if embedding_tracker_done.is_set():+ await self._finalize_vectorization()+ if self._vectorize_failure is not None:+ raise self._vectorize_failureelse:- # No vectorize tasks — release lock immediately (via wrapped callback)+ # No vectorize tasks — mark semantic work done immediately.try:- await wrapped_on_complete()+ await self._finalize_vectorization()except Exception as e:logger.error(f"Error in on_complete callback: {e}", exc_info=True)except BaseException:self._closed = Trueawait self._active_work_idle.wait()+ if self._embedding_tracker is not None:+ try:+ # Pending embedding messages may still finish, but their+ # attempt-specific decrements become harmless after abort.+ await self._embedding_tracker.abort(self._embedding_tracker_id)+ except Exception:+ logger.warning(+ "Failed to abort embedding tracker %s",+ self._embedding_tracker_id,+ exc_info=True,+ )raisefinally:self._closed = True@@ -416,11 +443,29 @@ class SemanticDagExecutor:await self._run_vectorize_task(task)except Exception as exc:logger.error("Vectorization dispatch task failed: %s", exc, exc_info=True)+ if self._vectorize_failure is None:+ self._vectorize_failure = excfinally:self._pending_vectorize_work = max(0, self._pending_vectorize_work - 1)if self._pending_vectorize_work == 0 and self._vectorize_done:self._vectorize_done.set()+ async def _on_embedding_tasks_complete(self) -> None:+ if self._embedding_tracker_done is not None:+ self._embedding_tracker_done.set()+ if self._vectorize_dispatch_complete:+ await self._finalize_vectorization()++ async def _finalize_vectorization(self) -> None:+ async with self._vectorize_finalize_lock:+ if self._vectorize_finalized:+ return+ if self._vectorize_failure is None and self._telemetry_id and self._semantic_msg_id:+ get_request_wait_tracker().mark_semantic_done(+ self._telemetry_id, self._semantic_msg_id+ )+ self._vectorize_finalized = True+async def _run_vectorize_task(self, task: VectorizeTask) -> None:if task.task_type == "file":await self._processor._vectorize_single_file(@@ -715,7 +760,7 @@ class SemanticDagExecutor:uri=file_path,context_type=self._context_type,ctx=self._ctx,- semantic_msg_id=self._semantic_msg_id,+ semantic_msg_id=self._embedding_tracker_id,file_path=file_path,summary_dict=summary_dict,parent_uri=parent_uri,@@ -862,7 +907,7 @@ class SemanticDagExecutor:uri=dir_uri,context_type=self._context_type,ctx=self._ctx,- semantic_msg_id=self._semantic_msg_id,+ semantic_msg_id=self._embedding_tracker_id,abstract=abstract,overview=overview,ingest_options=self._ingest_options,diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex 3d9ae81a2..edae4b4ec 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -22,7 +22,7 @@ import timefrom contextlib import contextmanagerfrom dataclasses import dataclass, fieldfrom datetime import datetime, timezone-from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypeVar, Union+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, TypeVar, Unionfrom openviking.core.context import ContextLevelfrom openviking.core.namespace import (@@ -1031,6 +1031,7 @@ class VikingFS:node_limit: Optional[int] = None,level_limit: int = 10,ctx: Optional[RequestContext] = None,+ content_transform: Optional[Callable[[str, str], str]] = None,) -> Dict:"""Content search by pattern or keywords.@@ -1049,6 +1050,7 @@ class VikingFS:node_limit: Maximum number of results to returnlevel_limit: Maximum depth level to traverse (default: 10)ctx: Request context+ content_transform: Optional projection applied before regex matching.Internal bm25 recall limit is auto-adapted from node_limit asmin(node_limit * 5, 100000); when node_limit is unset, use 100000.@@ -1070,8 +1072,12 @@ class VikingFS:self.grep_config.switch_to_remote_threshold if self.grep_config else 10000)- resolved_engine = await self._resolve_grep_engine(- engine, uri, ctx, switch_to_remote_threshold+ # A projection must run before matching. The remote BM25 index contains+ # persisted raw content, so it cannot safely recall projected results.+ resolved_engine = (+ "fs"+ if content_transform is not None+ else await self._resolve_grep_engine(engine, uri, ctx, switch_to_remote_threshold))if resolved_engine == "fs":@@ -1083,6 +1089,7 @@ class VikingFS:node_limit=node_limit,level_limit=level_limit,ctx=ctx,+ content_transform=content_transform,)else: # "vikingdb_then_fs"return await self._grep_vikingdb_then_fs(@@ -1188,21 +1195,30 @@ class VikingFS:return countasync def _grep_fs(- self, uri, pattern, exclude_uri, case_insensitive, node_limit, level_limit, ctx+ self,+ uri,+ pattern,+ exclude_uri,+ case_insensitive,+ node_limit,+ level_limit,+ ctx,+ content_transform=None,):"""Filesystem grep path: prefer native agfs grep and fall back if unavailable."""- try:- return await self._grep_with_agfs(- uri=uri,- pattern=pattern,- exclude_uri=exclude_uri,- case_insensitive=case_insensitive,- node_limit=node_limit,- level_limit=level_limit,- ctx=ctx,- )- except (AttributeError, AGFSNotSupportedError, NotImplementedError) as e:- logger.debug(f"agfs grep unavailable, falling back to VikingFS implementation: {e}")+ if content_transform is None:+ try:+ return await self._grep_with_agfs(+ uri=uri,+ pattern=pattern,+ exclude_uri=exclude_uri,+ case_insensitive=case_insensitive,+ node_limit=node_limit,+ level_limit=level_limit,+ ctx=ctx,+ )+ except (AttributeError, AGFSNotSupportedError, NotImplementedError) as e:+ logger.debug(f"agfs grep unavailable, falling back to VikingFS implementation: {e}")return await self._grep_encrypted(uri=uri,@@ -1212,6 +1228,7 @@ class VikingFS:node_limit=node_limit,level_limit=level_limit,ctx=ctx,+ content_transform=content_transform,)async def _grep_vikingdb_then_fs(@@ -1449,6 +1466,7 @@ class VikingFS:node_limit: Optional[int] = None,level_limit: int = 10,ctx: Optional[RequestContext] = None,+ content_transform: Optional[Callable[[str, str], str]] = None,) -> Dict:"""Grep implementation for encrypted files.@@ -1484,6 +1502,7 @@ class VikingFS:compiled_pattern=compiled_pattern,node_limit=node_limit,ctx=ctx,+ content_transform=content_transform,)return {@@ -1555,13 +1574,20 @@ class VikingFS:compiled_pattern: re.Pattern,node_limit: Optional[int],ctx: Optional[RequestContext] = None,+ content_transform: Optional[Callable[[str, str], str]] = None,) -> tuple[List[Dict[str, Any]], int]:results: List[Dict[str, Any]] = []files_scanned = 0for start in range(0, len(file_uris), _DEFAULT_GREP_FILE_CONCURRENCY):batch_uris = file_uris[start : start + _DEFAULT_GREP_FILE_CONCURRENCY]batch_jobs = [- self._grep_single_file(entry_uri, compiled_pattern, ctx) for entry_uri in batch_uris+ self._grep_single_file(+ entry_uri,+ compiled_pattern,+ ctx,+ content_transform=content_transform,+ )+ for entry_uri in batch_uris]batch_results = await asyncio.gather(*batch_jobs)for matches, scanned_count in batch_results:@@ -1578,11 +1604,14 @@ class VikingFS:entry_uri: str,compiled_pattern: re.Pattern,ctx: Optional[RequestContext] = None,+ content_transform: Optional[Callable[[str, str], str]] = None,) -> tuple[List[Dict[str, Any]], int]:try:content = await self.read(entry_uri, ctx=ctx)if isinstance(content, bytes):content = content.decode("utf-8", errors="replace")+ if content_transform is not None:+ content = content_transform(content, entry_uri)matches: List[Dict[str, Any]] = []lines = content.split("\n")diff --git a/openviking/sync_client.py b/openviking/sync_client.pyindex 657b2ebb3..a2a060443 100644--- a/openviking/sync_client.py+++ b/openviking/sync_client.py@@ -509,6 +509,10 @@ class SyncOpenViking:"""Read file"""return run_async(self._async_client.read(uri, offset=offset, limit=limit))+ def read_raw(self, uri: str, offset: int = 0, limit: int = -1) -> str:+ """Read raw file content, including hidden MEMORY_FIELDS metadata."""+ return run_async(self._async_client.read_raw(uri, offset=offset, limit=limit))+def write(self,uri: str,@@ -579,15 +583,22 @@ class SyncOpenViking:def import_ovpack(self,file_path: str,- target: str,+ parent: Optional[str] = None,on_conflict: Optional[str] = None,vector_mode: Optional[str] = None,+ *,+ target: Optional[str] = None,) -> str:"""Import .ovpack file (triggers vectorization by default)"""+ if parent is not None and target is not None:+ raise ValueError("parent cannot be used with legacy target")+ parent = parent if parent is not None else target+ if parent is None:+ raise TypeError("parent or legacy target is required")return run_async(self._async_client.import_ovpack(file_path,- target,+ parent,on_conflict=on_conflict,vector_mode=vector_mode,)@@ -634,6 +645,14 @@ class SyncOpenViking:"""Wait for all async operations to complete"""return run_async(self._async_client.wait_processed(timeout))+ def build_index(self, resource_uris: Union[str, List[str]], **kwargs) -> Dict[str, Any]:+ """Manually trigger index building for resources."""+ return run_async(self._async_client.build_index(resource_uris, **kwargs))++ def summarize(self, resource_uris: Union[str, List[str]], **kwargs) -> Dict[str, Any]:+ """Manually trigger summarization for resources."""+ return run_async(self._async_client.summarize(resource_uris, **kwargs))+def grep(self,uri: str,diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex 4d3beb1c4..978cd75f3 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -342,6 +342,7 @@ async def vectorize_directory_meta("""enqueued = 0expected = 2 if include_overview else 1+ first_enqueue_error: Optional[Exception] = Nonetry:if not ctx:logger.warning("No context provided for vectorization")@@ -395,6 +396,7 @@ async def vectorize_directory_meta(f"Failed to enqueue directory L0 (abstract) for vectorization: {uri}: {e}",exc_info=True,)+ first_enqueue_error = eif include_overview:# Vectorize L1: .overview.md (overview)@@ -432,6 +434,10 @@ async def vectorize_directory_meta(f"Failed to enqueue directory L1 (overview) for vectorization: {uri}: {e}",exc_info=True,)+ if first_enqueue_error is None:+ first_enqueue_error = e+ if first_enqueue_error is not None:+ raise first_enqueue_errorexcept Exception as e:logger.error(f"Failed to vectorize directory metadata for {uri}: {e}",@@ -607,6 +613,7 @@ async def vectorize_file(registered_wait_root[1],f"Failed to enqueue file vector for {file_path}: {e}",)+ raisefinally:if not enqueued:await _decrement_embedding_tracker(semantic_msg_id, 1)@@ -688,3 +695,4 @@ async def index_resource(except Exception as e:logger.error(f"Failed to scan directory {uri} for indexing: {e}")+ raisediff --git a/tests/client/test_ls_forwarding.py b/tests/client/test_ls_forwarding.pyindex 101eea1b0..b920d8e93 100644--- a/tests/client/test_ls_forwarding.py+++ b/tests/client/test_ls_forwarding.py@@ -27,8 +27,24 @@ async def test_async_openviking_ls_forwards_ordering_and_limit_options():simple=False,output="original",abs_limit=256,- show_all_hidden=True,+ show_all_hidden=False,node_limit=200,sort_by="mtime",sort_order="desc",)+++async def test_async_openviking_tree_hides_internal_files_by_default():+ client = object.__new__(AsyncOpenViking)+ client._ensure_initialized = AsyncMock()+ client._client = SimpleNamespace(tree=AsyncMock(return_value={}))++ await client.tree("viking://session")++ client._client.tree.assert_awaited_once_with(+ "viking://session",+ output="original",+ abs_limit=128,+ show_all_hidden=False,+ node_limit=1000,+ )diff --git a/tests/client/test_rebuild_clients.py b/tests/client/test_rebuild_clients.pyindex ecd32761d..da80aaf68 100644--- a/tests/client/test_rebuild_clients.py+++ b/tests/client/test_rebuild_clients.py@@ -204,6 +204,78 @@ async def test_local_client_reindex_forwards_to_service():)+async def test_local_client_read_uses_public_content_projection():+ client = object.__new__(LocalClient)+ client._ctx = object()+ client._service = SimpleNamespace(+ fs=SimpleNamespace(read_visible=AsyncMock(return_value="line one\nline two"))+ )++ uri = "viking://user/memories/notes/demo.md"+ assert await client.read(uri, offset=1, limit=1) == "line one\nline two"+ client._service.fs.read_visible.assert_awaited_once_with(+ uri,+ ctx=client._ctx,+ offset=1,+ limit=1,+ )+++async def test_local_client_read_does_not_strip_resource_comments():+ client = object.__new__(LocalClient)+ client._ctx = object()+ raw = 'visible\n\n<!-- MEMORY_FIELDS\n{"looks_like":"user content"}\n-->'+ client._service = SimpleNamespace(fs=SimpleNamespace(read_visible=AsyncMock(return_value=raw)))++ assert await client.read("viking://resources/demo.md") == raw+++async def test_local_client_read_raw_preserves_service_slicing():+ client = object.__new__(LocalClient)+ client._ctx = object()+ client._service = SimpleNamespace(fs=SimpleNamespace(read=AsyncMock(return_value="raw")))++ uri = "viking://user/memories/notes/demo.md"+ assert await client.read_raw(uri, offset=4, limit=1) == "raw"+ client._service.fs.read.assert_awaited_once_with(+ uri,+ ctx=client._ctx,+ offset=4,+ limit=1,+ )+++def test_sync_client_forwards_missing_async_surface():+ client = object.__new__(SyncOpenViking)+ client._async_client = SimpleNamespace(+ build_index=Mock(return_value="index"),+ summarize=Mock(return_value="summary"),+ read_raw=Mock(return_value="raw"),+ )+ with patch("openviking.sync_client.run_async", side_effect=lambda result: result):+ assert client.build_index(["uri"], wait=True) == "index"+ assert client.summarize("uri", mode="fast") == "summary"+ assert client.read_raw("uri", offset=1, limit=2) == "raw"++ client._async_client.build_index.assert_called_once_with(["uri"], wait=True)+ client._async_client.summarize.assert_called_once_with("uri", mode="fast")+ client._async_client.read_raw.assert_called_once_with("uri", offset=1, limit=2)+++def test_sync_import_ovpack_accepts_parent_and_legacy_target():+ client = object.__new__(SyncOpenViking)+ import_ovpack = Mock(return_value="imported")+ client._async_client = SimpleNamespace(import_ovpack=import_ovpack)+ with patch("openviking.sync_client.run_async", side_effect=lambda result: result):+ assert client.import_ovpack("pack", parent="parent") == "imported"+ assert client.import_ovpack("pack", target="legacy") == "imported"+ with pytest.raises(ValueError):+ client.import_ovpack("pack", parent="parent", target="legacy")++ assert import_ovpack.call_args_list[0].args == ("pack", "parent")+ assert import_ovpack.call_args_list[1].args == ("pack", "legacy")++async def test_local_client_batch_add_messages_forwards_to_session():class FakeSession:def __init__(self):diff --git a/tests/server/test_api_content.py b/tests/server/test_api_content.pyindex 1c00b1198..87b69e0e5 100644--- a/tests/server/test_api_content.py+++ b/tests/server/test_api_content.py@@ -3,9 +3,12 @@"""Tests for content endpoints: read, abstract, overview, reindex."""+from types import SimpleNamespace+import pytestfrom openviking.server.identity import RequestContext, Role+from openviking.server.routers import content as content_routerfrom openviking.server.routers.content import ReindexRequest, reindexfrom openviking_cli.session.user_id import UserIdentifier@@ -32,6 +35,35 @@ async def test_read_content(client_with_resource):assert body["result"] is not None+async def test_read_memory_uses_visible_projection_and_raw_bypasses_it(monkeypatch):+ ctx = RequestContext(user=UserIdentifier.the_default_user("test_user"), role=Role.USER)+ uri = "viking://user/test_user/memories/notes/private.md"+ raw = 'line one\nline two\n\n<!-- MEMORY_FIELDS\n{"secret": "do-not-leak"}\n-->'+ calls = []++ async def fake_read_visible(_uri, *, ctx, offset, limit):+ calls.append((offset, limit))+ return "visible slice"++ async def fake_read(_uri, *, ctx, offset, limit):+ lines = raw.splitlines(keepends=True)+ return "".join(lines[offset:] if limit == -1 else lines[offset : offset + limit])++ monkeypatch.setattr(+ content_router,+ "get_service",+ lambda: SimpleNamespace(fs=SimpleNamespace(read=fake_read, read_visible=fake_read_visible)),+ )++ response = await content_router.read(uri=uri, offset=4, limit=1, raw=False, _ctx=ctx)+ assert response.result == "visible slice"+ assert calls[-1] == (4, 1)++ raw_response = await content_router.read(uri=uri, offset=4, limit=1, raw=True, _ctx=ctx)+ assert raw_response.result == '{"secret": "do-not-leak"}\n'+ assert calls[-1] == (4, 1)++async def test_read_directory_uri_returns_invalid_argument(client_with_resource):client, uri = client_with_resourcediff --git a/tests/server/test_mcp_endpoint.py b/tests/server/test_mcp_endpoint.pyindex 27ab9d81f..eb74c28a5 100644--- a/tests/server/test_mcp_endpoint.py+++ b/tests/server/test_mcp_endpoint.py@@ -409,6 +409,19 @@ async def test_read_batch(service):assert "nothing found" in result.lower()+async def test_read_uses_public_content_projection(monkeypatch):+ read_visible = AsyncMock(return_value="visible memory")+ monkeypatch.setattr(+ mcp_endpoint,+ "get_service",+ lambda: SimpleNamespace(fs=SimpleNamespace(read_visible=read_visible)),+ )+ uri = "viking://user/default/memories/private.md"++ assert await read(uri) == "visible memory"+ read_visible.assert_awaited_once_with(uri, ctx=DEFAULT_CTX)++# ---------------------------------------------------------------------------# list tool# ---------------------------------------------------------------------------diff --git a/tests/service/test_fs_service.py b/tests/service/test_fs_service.pyindex 94e3dc988..ff6b315f4 100644--- a/tests/service/test_fs_service.py+++ b/tests/service/test_fs_service.py@@ -54,9 +54,7 @@ class _FakeWatchManager:move_resource,rollback_resource=None,):- self.sync_calls.append(- {"from_uri": from_uri, "to_uri": to_uri, "account_id": account_id}- )+ self.sync_calls.append({"from_uri": from_uri, "to_uri": to_uri, "account_id": account_id})if self.plan_error:raise self.plan_errorawait move_resource()@@ -140,6 +138,80 @@ def request_context():)+async def test_read_visible_strips_memory_metadata_before_slicing(request_context):+ raw = 'line one\nline two\n\n<!-- MEMORY_FIELDS\n{"secret":"hidden"}\n-->'+ viking_fs = SimpleNamespace(read_file=AsyncMock(return_value=raw))+ service = FSService(viking_fs=viking_fs)+ uri = "viking://user/ryoma/memories/notes/private.md"++ assert await service.read_visible(uri, ctx=request_context, offset=3, limit=1) == ""+ viking_fs.read_file.assert_awaited_once_with(uri, ctx=request_context)+++ "raw",+ [+ 'visible\n<!-- MEMORY_FIELDS\n{"secret":"hidden"}\n-->',+ 'visible\n\n<!-- MEMORY_FIELDS {"secret":"hidden"} -->',+ 'visible <!-- MEMORY_FIELDS {"secret":"hidden"} -->',+ ],+)+async def test_read_visible_strips_supported_memory_metadata_trailers(+ request_context,+ raw,+):+ uri = "viking://user/ryoma/memories/notes/private.md"+ service = FSService(+ viking_fs=SimpleNamespace(read_file=AsyncMock(return_value=raw)),+ )++ assert await service.read_visible(uri, ctx=request_context) == "visible"+++async def test_read_visible_preserves_non_memory_content(request_context):+ raw = 'visible\n<!-- MEMORY_FIELDS {"example":true} -->'+ viking_fs = SimpleNamespace(read_file=AsyncMock(return_value=raw))+ service = FSService(viking_fs=viking_fs)++ assert (+ await service.read_visible(+ "viking://resources/example.md",+ ctx=request_context,+ offset=1,+ limit=1,+ )+ == '<!-- MEMORY_FIELDS {"example":true} -->'+ )+++async def test_grep_projects_memory_content_but_keeps_resource_fast_path(request_context):+ viking_fs = SimpleNamespace(grep=AsyncMock(return_value={"matches": []}))+ service = FSService(viking_fs=viking_fs)++ await service.grep(+ "viking://user/ryoma/memories",+ "secret",+ ctx=request_context,+ )+ memory_kwargs = viking_fs.grep.await_args.kwargs+ transform = memory_kwargs["content_transform"]+ assert (+ transform(+ 'visible\n<!-- MEMORY_FIELDS {"secret":"hidden"} -->',+ "viking://user/ryoma/memories/private.md",+ )+ == "visible"+ )++ viking_fs.grep.reset_mock()+ await service.grep("viking://resources", "secret", ctx=request_context)+ assert "content_transform" not in viking_fs.grep.await_args.kwargs++@pytest.mark.asyncioasync def test_resource_rm_enqueues_parent_delete_refresh_and_waits(request_context):viking_fs = _FakeVikingFS()diff --git a/tests/storage/test_embedding_tracker.py b/tests/storage/test_embedding_tracker.pyindex 8b9df4ab0..ea6ad0b72 100644--- a/tests/storage/test_embedding_tracker.py+++ b/tests/storage/test_embedding_tracker.py@@ -185,3 +185,29 @@ async def test_tracker_clears_zero_task_entry_without_callback():await tracker.register("semantic-msg", 0, on_complete=None)assert await tracker.decrement("semantic-msg") is None+++async def test_tracker_abort_ignores_late_decrements_without_completing():+ tracker = EmbeddingTaskTracker.get_instance()+ callback_calls = []++ await tracker.register(+ "semantic-msg:attempt-1",+ 2,+ on_complete=lambda: callback_calls.append("attempt-1"),+ )++ assert await tracker.abort("semantic-msg:attempt-1") is True+ assert await tracker.abort("semantic-msg:attempt-1") is False++ await tracker.register(+ "semantic-msg:attempt-2",+ 1,+ on_complete=lambda: callback_calls.append("attempt-2"),+ )++ assert await tracker.decrement("semantic-msg:attempt-1") is None+ assert callback_calls == []+ assert await tracker.decrement("semantic-msg:attempt-2") == 0+ assert callback_calls == ["attempt-2"]diff --git a/tests/storage/test_memory_semantic_stall.py b/tests/storage/test_memory_semantic_stall.pyindex 628851c62..099fa35f5 100644--- a/tests/storage/test_memory_semantic_stall.py+++ b/tests/storage/test_memory_semantic_stall.py@@ -198,6 +198,10 @@ async def test_memory_write_error_reports_error():fake_fs.ls = AsyncMock(return_value=[{"name": "file1.md", "isDir": False}])fake_fs.read_file = AsyncMock(return_value="some content")fake_fs.write_file = AsyncMock(side_effect=PermissionError("Permission denied"))+ fake_fs._async_agfs.pathlock_acquire_exact_batch = AsyncMock(+ return_value={"lease_ref": "test"}+ )+ fake_fs._async_agfs.pathlock_release = AsyncMock()fake_fs._uri_to_path = MagicMock(side_effect=lambda uri, ctx=None: f"/local/acc1/{uri.removeprefix('viking://')}")diff --git a/tests/storage/test_semantic_dag_incremental.py b/tests/storage/test_semantic_dag_incremental.pyindex 38527ed54..b478c5fd0 100644--- a/tests/storage/test_semantic_dag_incremental.py+++ b/tests/storage/test_semantic_dag_incremental.py@@ -11,11 +11,20 @@ from openviking.storage.queuefs.semantic_dag import SemanticDagExecutorfrom openviking_cli.session.user_id import UserIdentifier+class _FakeAgfs:+ async def pathlock_acquire_exact_batch(self, _paths):+ return {"lease_ref": "test"}++ async def pathlock_release(self, _lease):+ return None++class _FakeVikingFS:def __init__(self, tree, file_contents):self._tree = {self._norm(k): v for k, v in tree.items()}self._file_contents = {self._norm(k): v for k, v in file_contents.items()}self.writes = []+ self._async_agfs = _FakeAgfs()def _norm(self, path):if "://" not in path:@@ -34,7 +43,7 @@ class _FakeVikingFS:async def read_file(self, path, ctx=None):return self._file_contents.get(self._norm(path), "")- async def write_file(self, path, content, ctx=None):+ async def write_file(self, path, content, ctx=None, lease_ref=None):norm_path = self._norm(path)self._file_contents[norm_path] = contentself.writes.append((norm_path, content))diff --git a/tests/storage/test_semantic_dag_skip_files.py b/tests/storage/test_semantic_dag_skip_files.pyindex a195a4210..947a206ba 100644--- a/tests/storage/test_semantic_dag_skip_files.py+++ b/tests/storage/test_semantic_dag_skip_files.py@@ -8,15 +8,24 @@ from openviking.storage.queuefs.semantic_dag import SemanticDagExecutorfrom openviking_cli.session.user_id import UserIdentifier+class _FakeAgfs:+ async def pathlock_acquire_exact_batch(self, _paths):+ return {"lease_ref": "test"}++ async def pathlock_release(self, _lease):+ return None++class _FakeVikingFS:def __init__(self, tree):self._tree = treeself.writes = []+ self._async_agfs = _FakeAgfs()async def ls(self, uri, node_limit=None, ctx=None):return self._tree.get(uri, [])- async def write_file(self, path, content, ctx=None):+ async def write_file(self, path, content, ctx=None, lease_ref=None):self.writes.append((path, content))def _uri_to_path(self, uri, ctx=None):@@ -39,7 +48,14 @@ class _FakeProcessor:return overview, "abstract"async def _vectorize_directory(- self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None+ self,+ uri,+ context_type,+ abstract,+ overview,+ ctx=None,+ semantic_msg_id=None,+ ingest_options=None,):pass@@ -55,6 +71,7 @@ class _FakeProcessor:ctx=None,semantic_msg_id=None,use_summary=False,+ ingest_options=None,):self.vectorized_files.append(file_path)diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.pyindex 6ba49edcb..b3619f762 100644--- a/tests/storage/test_semantic_dag_stats.py+++ b/tests/storage/test_semantic_dag_stats.py@@ -17,16 +17,33 @@ from openviking.storage.queuefs.semantic_dag import (from openviking_cli.session.user_id import UserIdentifier+class _FakeAgfs:+ async def pathlock_acquire_exact_batch(self, _paths):+ return {"lease_ref": "test"}++ async def pathlock_release(self, _lease):+ return None++class _FakeVikingFS:def __init__(self, tree):self._tree = treeself.writes = []+ self._async_agfs = _FakeAgfs()async def ls(self, uri, node_limit=None, ctx=None):del node_limitreturn self._tree.get(uri, [])- async def write_file(self, path, content, ctx=None, lock_handle=None):+ async def write_file(+ self,+ path,+ content,+ ctx=None,+ lease_ref=None,+ lock_handle=None,+ ):+ del ctx, lease_ref, lock_handleself.writes.append((path, content))def _uri_to_path(self, uri, ctx=None):@@ -48,7 +65,14 @@ class _FakeProcessor:return overview, "abstract"async def _vectorize_directory(- self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None+ self,+ uri,+ context_type,+ abstract,+ overview,+ ctx=None,+ semantic_msg_id=None,+ ingest_options=None,):self.vectorized_dirs.append(uri)@@ -61,6 +85,7 @@ class _FakeProcessor:ctx=None,semantic_msg_id=None,use_summary=False,+ ingest_options=None,):self.vectorized_files.append(file_path)@@ -93,6 +118,159 @@ class _DummyTracker:return None+class _CompletingTracker:+ def __init__(self):+ self.remaining = 0+ self.total = 0+ self.on_complete = None+ self.completed = False+ self.active_id = None+ self.aborted_ids = []++ async def register(self, *, semantic_msg_id, total_count, on_complete, **_kwargs):+ self.active_id = semantic_msg_id+ self.remaining = total_count+ self.total = total_count+ self.on_complete = on_complete++ async def decrement(self, semantic_msg_id):+ if semantic_msg_id != self.active_id:+ return None+ self.remaining -= 1+ if self.remaining == 0:+ self.completed = True+ self.active_id = None+ await self.on_complete()+ return self.remaining++ async def abort(self, semantic_msg_id):+ if semantic_msg_id != self.active_id:+ return False+ self.aborted_ids.append(semantic_msg_id)+ self.active_id = None+ return True+++class _FailingVectorizeProcessor(_FakeProcessor):+ def __init__(self, tracker):+ super().__init__()+ self._tracker = tracker++ async def _vectorize_single_file(+ self,+ parent_uri,+ context_type,+ file_path,+ summary_dict,+ ctx=None,+ semantic_msg_id=None,+ use_summary=False,+ ingest_options=None,+ ):+ self.vectorized_files.append(file_path)+ try:+ raise RuntimeError("embedding enqueue unavailable")+ finally:+ await self._tracker.decrement(semantic_msg_id)++ async def _vectorize_directory(+ self,+ uri,+ context_type,+ abstract,+ overview,+ ctx=None,+ semantic_msg_id=None,+ ingest_options=None,+ ):+ self.vectorized_dirs.append(uri)+ await self._tracker.decrement(semantic_msg_id)+ await self._tracker.decrement(semantic_msg_id)+++class _StalledEmbeddingProcessor(_FailingVectorizeProcessor):+ async def _vectorize_directory(+ self,+ uri,+ context_type,+ abstract,+ overview,+ ctx=None,+ semantic_msg_id=None,+ ingest_options=None,+ ):+ self.vectorized_dirs.append(uri)+ # Both directory messages were accepted, but the embedding worker is+ # unavailable and therefore never decrements their tracker slots.+++async def test_semantic_dag_propagates_vectorization_failure_after_tracker_drains(monkeypatch):+ root_uri = "viking://resources/root"+ tree = {root_uri: [{"name": "a.txt", "isDir": False}]}+ fake_fs = _FakeVikingFS(tree)+ tracker = _CompletingTracker()+ monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)+ monkeypatch.setattr(+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",+ lambda: tracker,+ )++ processor = _FailingVectorizeProcessor(tracker)+ ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)+ executor = SemanticDagExecutor(+ processor=processor,+ context_type="resource",+ max_concurrent_llm=2,+ ctx=ctx,+ semantic_msg_id="semantic-root",+ )++ with pytest.raises(RuntimeError, match="embedding enqueue unavailable"):+ await executor.run(root_uri)++ assert processor.vectorized_files == [f"{root_uri}/a.txt"]+ assert processor.vectorized_dirs == [root_uri]+ assert tracker.total == 3+ assert tracker.remaining == 0+ assert tracker.completed is True+++async def test_semantic_dag_aborts_stalled_embedding_attempt_after_dispatch_failure(monkeypatch):+ root_uri = "viking://resources/root"+ tree = {root_uri: [{"name": "a.txt", "isDir": False}]}+ fake_fs = _FakeVikingFS(tree)+ tracker = _CompletingTracker()+ monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)+ monkeypatch.setattr(+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",+ lambda: tracker,+ )++ processor = _StalledEmbeddingProcessor(tracker)+ ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)+ executor = SemanticDagExecutor(+ processor=processor,+ context_type="resource",+ max_concurrent_llm=2,+ ctx=ctx,+ semantic_msg_id="semantic-root",+ )++ with pytest.raises(RuntimeError, match="embedding enqueue unavailable"):+ await asyncio.wait_for(executor.run(root_uri), timeout=0.5)++ assert processor.vectorized_files == [f"{root_uri}/a.txt"]+ assert processor.vectorized_dirs == [root_uri]+ assert tracker.remaining == 2+ assert tracker.completed is False+ assert len(tracker.aborted_ids) == 1+ aborted_id = tracker.aborted_ids[0]+ assert aborted_id.startswith("semantic-root:")+ assert await tracker.decrement(aborted_id) is None++class _ScheduledExecutor:def __init__(self, run) -> None:self.closed = Falsediff --git a/tests/storage/test_viking_fs_grep.py b/tests/storage/test_viking_fs_grep.pyindex 47aa71ad5..d707d5b9e 100644--- a/tests/storage/test_viking_fs_grep.py+++ b/tests/storage/test_viking_fs_grep.py@@ -273,6 +273,45 @@ async def test_grep_preserves_dfs_order_and_node_limit(monkeypatch):]+async def test_grep_applies_content_transform_before_matching(monkeypatch):+ fs = VikingFS(agfs=_DummyAgfs())++ async def fake_stat(uri, ctx=None, skip_count=False):+ return {"isDir": True}++ async def fake_ls(uri, ctx=None, **kwargs):+ if uri == "viking://resources":+ return [{"name": "memory.md", "isDir": False}]+ return []++ def fake_agfs_read(path, offset=0, size=-1):+ return b'visible\n<!-- MEMORY_FIELDS {"secret":"hidden"} -->'++ monkeypatch.setattr(fs, "stat", fake_stat)+ monkeypatch.setattr(fs, "ls", fake_ls)+ monkeypatch.setattr(+ fs,+ "_uri_to_path",+ lambda uri, ctx=None: uri.replace("viking://", "/"),+ )+ monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False)++ result = await fs.grep(+ "viking://resources",+ pattern="visible|secret",+ content_transform=lambda content, _uri: content.split("<!--", 1)[0].rstrip(),+ )++ assert result["matches"] == [+ {+ "line": 1,+ "uri": "viking://resources/memory.md",+ "content": "visible",+ }+ ]++@pytest.mark.asyncioasync def test_grep_parallel_reads_respect_concurrency_limit(monkeypatch):fs = VikingFS(agfs=_DummyAgfs())diff --git a/tests/telemetry/test_http_observability_middleware.py b/tests/telemetry/test_http_observability_middleware.pyindex 05e15c0cf..c3a9568e5 100644--- a/tests/telemetry/test_http_observability_middleware.py+++ b/tests/telemetry/test_http_observability_middleware.py@@ -68,9 +68,11 @@ def test_http_observability_middleware_updates_route_template_after_routing(monkreturn {"ok": True}with TestClient(app) as client:- resp = client.get("/hello")+ resp = client.get("/hello?token=top-secret")assert resp.status_code == 200root_attrs = captured["root_attrs"]assert root_attrs.http_route == "/hello"+ assert root_attrs.url_query is None+ assert "url.query" not in root_attrs.to_otel_attributes()assert "GET /hello" in dummy_span.updated_namesdiff --git a/tests/unit/session/test_session_phase2_replay_tolerance.py b/tests/unit/session/test_session_phase2_replay_tolerance.pynew file mode 100644index 000000000..db2229ea1--- /dev/null+++ b/tests/unit/session/test_session_phase2_replay_tolerance.py@@ -0,0 +1,117 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Regression tests for Phase-2 archive replay tolerance (follow-up to #3417).++#3417 made ``Session._read_archive_messages`` strict: it now raises on a+missing/corrupt ``messages.jsonl`` instead of returning ``[]``. The read path+(``_get_uncovered_archive_messages``) gained not-found tolerance, but the+Phase-2 commit replay path (``_prepare_phase2_archive_messages``) did not.++A terminally-failed earlier archive whose ``messages.jsonl`` is missing would+therefore make every subsequent commit's Phase-2 replay raise, terminal-fail+the current archive, and permanently poison the session's memory extraction.+"""++from unittest.mock import AsyncMock++import pytest++from openviking.session.session import ArchiveState, Session+++class _MemoryVikingFS:+ def __init__(self, files):+ self.files = files++ def _uri_to_path(self, uri, ctx=None):+ return "/local/session-1"++ async def read_file(self, uri, ctx=None):+ if uri not in self.files:+ raise FileNotFoundError(uri)+ value = self.files[uri]+ if isinstance(value, Exception):+ raise value+ return value++ async def write_file(self, uri, content, ctx=None):+ self.files[uri] = content+++def _session(files, states, monkeypatch):+ session_uri = "viking://user/sessions/session-1"+ session = Session(+ viking_fs=_MemoryVikingFS(files),+ session_id="session-1",+ session_uri=session_uri,+ )+ monkeypatch.setattr(session, "_scan_archive_states", AsyncMock(return_value=states))+ return session, session_uri+++def _failed_then_pending(session_uri):+ """archive_001 = terminally failed (missing messages), archive_002 = current pending."""+ a1 = f"{session_uri}/history/archive_001"+ a2 = f"{session_uri}/history/archive_002"+ states = [+ ArchiveState(+ archive_id="archive_001",+ archive_uri=a1,+ index=1,+ state="failed",+ failed={"stage": "archive_read", "error": "no messages"},+ ),+ ArchiveState(+ archive_id="archive_002",+ archive_uri=a2,+ index=2,+ state="pending",+ ),+ ]+ return a1, a2, states+++async def test_phase2_replay_skips_failed_archive_with_missing_messages(monkeypatch):+ session_uri = "viking://user/sessions/session-1"+ a1, a2, states = _failed_then_pending(session_uri)+ # archive_001/messages.jsonl is absent -> not-found; archive_002 has no+ # replay reads of its own (current messages are passed in directly).+ session, _ = _session({}, states, monkeypatch)++ from openviking.message import Message, TextPart++ current = [Message(id="cur", role="user", parts=[TextPart("hi")])]++ combined, _start, _end, covered_failed, _steps = await session._prepare_phase2_archive_messages(+ a2, current+ )++ # The poisoned archive is skipped; the current commit still gets its own+ # messages and can complete Phase 2 instead of being terminal-failed.+ assert [m.id for m in combined] == ["cur"]+ # The skipped archive is still reported as covered_failed so the current+ # archive's .done marks it covered — clearing the poison permanently+ # instead of re-reading (and skipping) it on every future commit.+ assert covered_failed == ["archive_001"]++ # Asymmetry check: the read path already tolerates the exact same scenario.+ uncovered = await session._get_uncovered_archive_messages(states)+ assert uncovered == []+++async def test_phase2_replay_reraises_real_storage_failure(monkeypatch):+ session_uri = "viking://user/sessions/session-1"+ a1, a2, states = _failed_then_pending(session_uri)+ # A real storage failure (not a not-found) reading archive_001 must still+ # propagate so it is not silently swallowed.+ boom = RuntimeError("storage backend unavailable")+ session, _ = _session({f"{a1}/messages.jsonl": boom}, states, monkeypatch)++ from openviking.message import Message, TextPart++ current = [Message(id="cur", role="user", parts=[TextPart("hi")])]++ with pytest.raises(RuntimeError, match="storage backend unavailable"):+ await session._prepare_phase2_archive_messages(a2, current)diff --git a/tests/unit/test_body_dump_middleware.py b/tests/unit/test_body_dump_middleware.pynew file mode 100644index 000000000..b5141c710--- /dev/null+++ b/tests/unit/test_body_dump_middleware.py@@ -0,0 +1,9 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++from openviking.server.body_dump_middleware import _should_skip+++def test_zip_responses_skip_body_capture():+ assert _should_skip("application/zip")+ assert _should_skip("application/zip; charset=binary")diff --git a/tests/unit/test_content_visibility.py b/tests/unit/test_content_visibility.pynew file mode 100644index 000000000..984ad9efc--- /dev/null+++ b/tests/unit/test_content_visibility.py@@ -0,0 +1,52 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++import pytest++from openviking.session.memory.utils.content_visibility import visible_content++MEMORY_URI = "viking://user/default/memories/notes/private.md"+++ "raw_content",+ [+ 'visible\n\n<!-- MEMORY_FIELDS\n{"secret":"hidden"}\n-->',+ 'visible\n<!-- MEMORY_FIELDS\n{"secret":"hidden"}\n-->',+ 'visible\n\n<!-- MEMORY_FIELDS {"secret":"hidden"} -->',+ 'visible <!-- MEMORY_FIELDS {"secret":"hidden"} -->',+ '<!-- MEMORY_FIELDS {"secret":"hidden"} -->',+ ],+)+def test_visible_content_strips_supported_memory_fields_trailers(raw_content):+ expected = "" if raw_content.startswith("<!--") else "visible"++ assert visible_content(raw_content, uri=MEMORY_URI) == expected+++ "raw_content",+ [+ "visible <!-- ordinary comment -->",+ "visible <!-- MEMORY_FIELDS_EXAMPLE {} -->",+ 'visible <!-- MEMORY_FIELDS {"example":true} -->\nmore',+ ],+)+def test_visible_content_preserves_non_reserved_or_non_trailing_comments(raw_content):+ assert visible_content(raw_content, uri=MEMORY_URI) == raw_content+++def test_visible_content_preserves_reserved_comment_outside_memory_namespace():+ raw_content = 'visible <!-- MEMORY_FIELDS {"example":true} -->'++ assert visible_content(raw_content, uri="viking://resources/example.md") == raw_content+++def test_visible_content_strips_repeated_memory_fields_trailers():+ raw_content = """visible+<!-- MEMORY_FIELDS {"secret":"old"} -->+<!-- MEMORY_FIELDS+{"secret":"new"}+-->"""++ assert visible_content(raw_content, uri=MEMORY_URI) == "visible"diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.pyindex d1717a88f..35df6ac22 100644--- a/tests/unit/test_vectorize_file_strategy.py+++ b/tests/unit/test_vectorize_file_strategy.py@@ -1,10 +1,11 @@import types+from unittest.mock import AsyncMockimport pytest-from openviking.utils.ingest_options import IngestOptionsfrom openviking.core.context import Context, ResourceContentTypefrom openviking.utils import embedding_utils+from openviking.utils.ingest_options import IngestOptionsclass DummyQueue:@@ -200,13 +201,14 @@ async def test_vectorize_file_marks_registered_wait_root_failed_when_enqueue_rai),)- await embedding_utils.vectorize_file(- file_path="viking://user/default/resources/test.md",- summary_dict={"name": "test.md", "summary": ""},- parent_uri="viking://user/default/resources",- ctx=DummyReq(),- register_request_wait=True,- )+ with pytest.raises(RuntimeError, match="queue unavailable"):+ await embedding_utils.vectorize_file(+ file_path="viking://user/default/resources/test.md",+ summary_dict={"name": "test.md", "summary": ""},+ parent_uri="viking://user/default/resources",+ ctx=DummyReq(),+ register_request_wait=True,+ )assert len(queue.items) == 1assert registered == [(queue.items[0].telemetry_id, queue.items[0].id)]@@ -214,6 +216,52 @@ async def test_vectorize_file_marks_registered_wait_root_failed_when_enqueue_raiassert failed[0][0:2] == registered[0]+async def test_vectorize_file_propagates_enqueue_failure(monkeypatch):+ monkeypatch.setattr(+ embedding_utils, "get_queue_manager", lambda: DummyQueueManager(FailingQueue())+ )+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: DummyFS("content"))+ monkeypatch.setattr(+ embedding_utils,+ "get_openviking_config",+ lambda: types.SimpleNamespace(+ embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000)+ ),+ )+ monkeypatch.setattr(+ embedding_utils.EmbeddingMsgConverter, "from_context", lambda context: context+ )++ with pytest.raises(RuntimeError, match="queue unavailable"):+ await embedding_utils.vectorize_file(+ "viking://user/default/resources/test.md",+ {"name": "test.md", "summary": "summary"},+ "viking://user/default/resources",+ ctx=DummyReq(),+ )+++async def test_vectorize_directory_propagates_enqueue_failures_and_drains_tracker(monkeypatch):+ decrement = AsyncMock()+ monkeypatch.setattr(+ embedding_utils, "get_queue_manager", lambda: DummyQueueManager(FailingQueue())+ )+ monkeypatch.setattr(embedding_utils, "_decrement_embedding_tracker", decrement)++ with pytest.raises(RuntimeError, match="queue unavailable"):+ await embedding_utils.vectorize_directory_meta(+ "viking://user/default/resources",+ abstract="abstract",+ overview="overview",+ ctx=DummyReq(),+ semantic_msg_id="semantic-root",+ )++ decrement.assert_awaited_once_with("semantic-root", 2)++@pytest.mark.asyncioasync def test_vectorize_unknown_text_file_embeds_summary_but_indexes_raw_content(monkeypatch):queue = DummyQueue()