Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(core): harden privacy and background failure handling (#3548)

Hao Zheaf914ca27b36
diff --git a/openviking/async_client.py b/openviking/async_client.py
index 1d98c40af..7b09a6a6f 100644
--- a/openviking/async_client.py
+++ b/openviking/async_client.py
@@ -729,7 +729,7 @@ class AsyncOpenViking:
simple = kwargs.get("simple", False)
output = kwargs.get("output", "original")
abs_limit = kwargs.get("abs_limit", 256)
- show_all_hidden = kwargs.get("show_all_hidden", True)
+ show_all_hidden = kwargs.get("show_all_hidden", False)
node_limit = kwargs.get("node_limit", 1000)
sort_by = kwargs.get("sort_by")
sort_order = kwargs.get("sort_order", "asc")
@@ -791,7 +791,7 @@ class AsyncOpenViking:
await self._ensure_initialized()
output = kwargs.get("output", "original")
abs_limit = kwargs.get("abs_limit", 128)
- show_all_hidden = kwargs.get("show_all_hidden", True)
+ show_all_hidden = kwargs.get("show_all_hidden", False)
node_limit = kwargs.get("node_limit", 1000)
return await self._client.tree(
uri,
diff --git a/openviking/client/local.py b/openviking/client/local.py
index a9baba1f3..8a50c8ff6 100644
--- a/openviking/client/local.py
+++ b/openviking/client/local.py
@@ -633,7 +633,12 @@ class LocalClient(BaseClient):
offset: Starting line number (0-indexed). Default 0.
limit: Number of lines to read. -1 means read to end. Default -1.
"""
- return await self._service.fs.read(uri, ctx=self._ctx, offset=offset, limit=limit)
+ return await self._service.fs.read_visible(
+ uri,
+ ctx=self._ctx,
+ offset=offset,
+ limit=limit,
+ )
async def read_raw(self, uri: str, offset: int = 0, limit: int = -1) -> str:
"""Read raw file content, including hidden MEMORY_FIELDS metadata."""
diff --git a/openviking/observability/http_observability_middleware.py b/openviking/observability/http_observability_middleware.py
index 2ab88ba8b..48536cbc6 100644
--- a/openviking/observability/http_observability_middleware.py
+++ b/openviking/observability/http_observability_middleware.py
@@ -824,7 +824,6 @@ def create_http_observability_middleware() -> Callable[[Request, Callable], Resp
# Extract request information
raw_path = str(request.url.path)
- raw_query = request.url.query or None
route_template = _get_route_template(request)
request_id = request.state.request_id
@@ -834,7 +833,6 @@ def create_http_observability_middleware() -> Callable[[Request, Callable], Resp
http_route=route_template,
request_id=request_id,
url_path=raw_path,
- url_query=raw_query,
url_scheme=request.url.scheme,
http_host=request.url.netloc,
source_type=request.headers.get("x-source-type"),
diff --git a/openviking/server/body_dump_middleware.py b/openviking/server/body_dump_middleware.py
index 28f308023..f27f78ea0 100644
--- a/openviking/server/body_dump_middleware.py
+++ b/openviking/server/body_dump_middleware.py
@@ -25,6 +25,7 @@ except ImportError: # pragma: no cover - OTel optional
_SKIP_CONTENT_TYPE_PREFIXES = (
"multipart/form-data",
"application/octet-stream",
+ "application/zip",
"text/event-stream",
"audio/",
"video/",
diff --git a/openviking/server/mcp_endpoint.py b/openviking/server/mcp_endpoint.py
index 0c7073aa4..712a4ff9f 100644
--- a/openviking/server/mcp_endpoint.py
+++ b/openviking/server/mcp_endpoint.py
@@ -363,7 +363,7 @@ async def read(uris: str | list[str]) -> str:
async def _read_one(uri: str) -> str:
async with semaphore:
try:
- body = await service.fs.read(uri, ctx=ctx)
+ body = await service.fs.read_visible(uri, ctx=ctx)
if isinstance(body, str) and body.strip():
return body
except Exception:
diff --git a/openviking/server/routers/content.py b/openviking/server/routers/content.py
index ceb44d260..fc6a7f97e 100644
--- a/openviking/server/routers/content.py
+++ b/openviking/server/routers/content.py
@@ -150,7 +150,10 @@ async def read(
service = get_service()
uri = resolve_path_variables(uri)
try:
- result = await service.fs.read(uri, ctx=_ctx, offset=offset, limit=limit)
+ if raw:
+ result = await service.fs.read(uri, ctx=_ctx, offset=offset, limit=limit)
+ else:
+ result = await service.fs.read_visible(uri, ctx=_ctx, offset=offset, limit=limit)
except AGFSNotFoundError:
raise NotFoundError(uri, "file")
except AGFSClientError as e:
@@ -159,21 +162,6 @@ async def read(
raise mapped from e
raise
- if not raw:
- # 清理MEMORY_FIELDS隐藏注释(v2记忆加工过程中的临时内部数据,不暴露给外部用户)
- if isinstance(result, bytes):
- text = result.decode("utf-8")
- elif isinstance(result, str):
- text = result
- else:
- text = None
-
- if text:
- from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils
-
- mf = MemoryFileUtils.read(text)
- result = mf.content
-
return Response(status="ok", result=result)
diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.py
index 17d628848..2814e39b3 100644
--- a/openviking/service/fs_service.py
+++ b/openviking/service/fs_service.py
@@ -8,7 +8,7 @@ Provides file system operations: ls, mkdir, rm, mv, tree, stat, read, abstract,
from typing import TYPE_CHECKING, Any, Dict, List, Optional
-from openviking.core.namespace import context_type_for_uri
+from openviking.core.namespace import classify_uri, context_type_for_uri
from openviking.core.uri_validation import validate_optional_viking_uri, validate_viking_uri
from openviking.privacy import (
UserPrivacyConfigService,
@@ -18,6 +18,7 @@ from openviking.privacy import (
from openviking.resource.watch_storage import is_watch_task_control_uri
from openviking.server.identity import RequestContext
from openviking.session.memory.memory_updater import MemoryUpdater
+from openviking.session.memory.utils.content_visibility import visible_content
from openviking.storage.content_write import ContentWriteCoordinator
from openviking.storage.queuefs import SemanticMsg, get_queue_manager
from openviking.storage.queuefs.semantic_msg import build_semantic_coalesce_key
@@ -31,6 +32,21 @@ from openviking_cli.utils import VikingURI, get_logger
logger = get_logger(__name__)
+
+def _may_include_memory_content(uri: str) -> bool:
+ """Return whether a public subtree read can contain memory files."""
+ classification = classify_uri(uri)
+ if classification.is_memory:
+ return True
+ if classification.content_index is not None:
+ return False
+ return not classification.parts or classification.scope in {"user", "agent"}
+
+
+def _visible_grep_content(content: str, uri: str) -> str:
+ return visible_content(content, uri=uri)
+
+
if TYPE_CHECKING:
from openviking.resource.watch_manager import WatchManager
from openviking.resource.watch_scheduler import WatchScheduler
@@ -413,9 +429,7 @@ class FSService:
rollback_resource=lambda: viking_fs.mv(to_uri, from_uri, ctx=ctx),
)
- async def _sync_watch_after_rm(
- self, uri: str, *, account_id: str, context_type: str
- ) -> None:
+ async def _sync_watch_after_rm(self, uri: str, *, account_id: str, context_type: str) -> None:
if context_type != "resource":
return
if is_watch_task_control_uri(uri):
@@ -493,6 +507,20 @@ class FSService:
sliced = lines[offset:] if limit == -1 else lines[offset : offset + limit]
return "".join(sliced)
+ async def read_visible(
+ self,
+ uri: str,
+ ctx: RequestContext,
+ offset: int = 0,
+ limit: int = -1,
+ ) -> str:
+ """Read public content, hiding reserved metadata from memory files."""
+ uri = validate_viking_uri(uri)
+ if not classify_uri(uri).is_memory:
+ return await self.read(uri, ctx=ctx, offset=offset, limit=limit)
+ content = await self.read(uri, ctx=ctx)
+ return visible_content(content, uri=uri, offset=offset, limit=limit)
+
async def abstract(self, uri: str, ctx: RequestContext) -> str:
"""Read L0 abstract (.abstract.md)."""
viking_fs = self._ensure_initialized()
@@ -519,15 +547,16 @@ class FSService:
viking_fs = self._ensure_initialized()
uri = validate_viking_uri(uri)
exclude_uri = validate_optional_viking_uri(exclude_uri, field_name="exclude_uri") or None
- return await viking_fs.grep(
- uri,
- pattern,
- exclude_uri=exclude_uri,
- case_insensitive=case_insensitive,
- node_limit=node_limit,
- level_limit=level_limit,
- ctx=ctx,
- )
+ kwargs = {
+ "exclude_uri": exclude_uri,
+ "case_insensitive": case_insensitive,
+ "node_limit": node_limit,
+ "level_limit": level_limit,
+ "ctx": ctx,
+ }
+ if _may_include_memory_content(uri):
+ kwargs["content_transform"] = _visible_grep_content
+ return await viking_fs.grep(uri, pattern, **kwargs)
async def glob(
self,
@@ -622,9 +651,7 @@ class FSService:
) -> Dict[str, Any]:
"""Forward to VikingFS.commit. See viking_fs.commit for semantics."""
viking_fs = self._ensure_initialized()
- validated = (
- [validate_viking_uri(p) for p in paths] if paths is not None else None
- )
+ validated = [validate_viking_uri(p) for p in paths] if paths is not None else None
return await viking_fs.commit(
message=message,
paths=validated,
@@ -721,9 +748,7 @@ class FSService:
viking_fs = self._ensure_initialized()
return await viking_fs.get_gitignore(ctx=ctx)
- async def set_gitignore(
- self, *, content: str, ctx: RequestContext
- ) -> None:
+ async def set_gitignore(self, *, content: str, ctx: RequestContext) -> None:
"""Forward to VikingFS.set_gitignore. Writes the account .ovgitignore
control file (validates the size limit)."""
viking_fs = self._ensure_initialized()
diff --git a/openviking/session/memory/utils/content_visibility.py b/openviking/session/memory/utils/content_visibility.py
new file mode 100644
index 000000000..a73b4aff3
--- /dev/null
+++ b/openviking/session/memory/utils/content_visibility.py
@@ -0,0 +1,30 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Public-content projection for persisted memory files."""
+
+from openviking.core.namespace import classify_uri
+from openviking.session.memory.utils.memory_fields import strip_memory_fields_trailer
+
+
+def visible_content(
+ raw_content: str,
+ *,
+ uri: str,
+ offset: int = 0,
+ limit: int = -1,
+) -> str:
+ """Hide a memory file's reserved metadata trailer, then apply line slicing.
+
+ MEMORY_FIELDS is a reserved trailer only in memory namespaces. Resource and
+ skill files may legitimately contain similarly shaped HTML comments, so
+ their content is returned verbatim apart from the requested line slice.
+ """
+ content = raw_content
+ if classify_uri(uri).is_memory:
+ content = strip_memory_fields_trailer(content)
+
+ if offset == 0 and limit == -1:
+ return content
+ lines = content.splitlines(keepends=True)
+ sliced = lines[offset:] if limit == -1 else lines[offset : offset + limit]
+ return "".join(sliced)
diff --git a/openviking/session/memory/utils/memory_fields.py b/openviking/session/memory/utils/memory_fields.py
new file mode 100644
index 000000000..1f336b4d7
--- /dev/null
+++ b/openviking/session/memory/utils/memory_fields.py
@@ -0,0 +1,27 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Shared grammar for reserved ``MEMORY_FIELDS`` comments."""
+
+from __future__ import annotations
+
+import re
+from typing import Match, Optional
+
+MEMORY_FIELDS_COMMENT_RE = re.compile(r"<!--\s*MEMORY_FIELDS\b(?P<fields>[\s\S]*?)\s*-->")
+
+
+def find_memory_fields_trailer(content: str) -> Optional[Match[str]]:
+ """Return the reserved comment when it is the final content element."""
+ trailer = None
+ for match in MEMORY_FIELDS_COMMENT_RE.finditer(content):
+ if not content[match.end() :].strip():
+ trailer = match
+ return trailer
+
+
+def strip_memory_fields_trailer(content: str) -> str:
+ """Remove all trailing reserved comments and their whitespace separators."""
+ visible = content
+ while match := find_memory_fields_trailer(visible):
+ visible = visible[: match.start()].rstrip()
+ return visible
diff --git a/openviking/session/memory/utils/messages.py b/openviking/session/memory/utils/messages.py
index 657e85b37..b3ccb1a26 100644
--- a/openviking/session/memory/utils/messages.py
+++ b/openviking/session/memory/utils/messages.py
@@ -5,11 +5,11 @@ Message formatting and memory file parsing utilities.
"""
import json
-import re
from typing import Any, Dict, List
import json_repair
+from openviking.session.memory.utils.memory_fields import MEMORY_FIELDS_COMMENT_RE
from openviking.telemetry import tracer
from openviking_cli.utils import get_logger
@@ -92,16 +92,12 @@ def parse_memory_file_with_fields(content: str) -> Dict[str, Any]:
if not content:
return {"content": ""}
- # Pattern to match: <!-- MEMORY_FIELDS ... -->
- # Matches multi-line JSON inside the comment
- pattern = r"<!--\s*MEMORY_FIELDS\s*([\s\S]*?)\s*-->"
-
- match = re.search(pattern, content)
+ match = MEMORY_FIELDS_COMMENT_RE.search(content)
result = {}
if match:
- fields_json_str = match.group(1).strip()
+ fields_json_str = match.group("fields").strip()
if fields_json_str:
try:
fields = json_repair.loads(fields_json_str)
@@ -114,7 +110,7 @@ def parse_memory_file_with_fields(content: str) -> Dict[str, Any]:
tracer.warning(f"Failed to parse MEMORY_FIELDS JSON: {e}")
# Remove the comment from content
- content_without_comment = re.sub(pattern, "", content).strip()
+ content_without_comment = MEMORY_FIELDS_COMMENT_RE.sub("", content).strip()
result["content"] = content_without_comment
return result
diff --git a/openviking/session/session.py b/openviking/session/session.py
index dd8fc71ed..c7a3383de 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -3798,7 +3798,29 @@ class Session:
combined: List[Message] = []
completed_memory_steps: Dict[str, set[str]] = {}
for state in replay_states:
- combined.extend(await self._read_archive_messages(state.archive_uri))
+ # A terminally-failed earlier archive can have a missing or corrupt
+ # messages.jsonl (legacy "no messages" data, or #3417's own
+ # archive_read terminal path). Tolerate that exactly like
+ # _get_uncovered_archive_messages does, otherwise every subsequent
+ # commit's Phase 2 re-raises here and stays permanently poisoned.
+ # Real storage failures still propagate.
+ try:
+ replayed = await self._read_archive_messages(state.archive_uri)
+ except _ArchiveMessagesCorruptError:
+ logger.warning(
+ "Skipping failed archive %s in Phase-2 replay: messages.jsonl is corrupt",
+ state.archive_uri,
+ )
+ continue
+ except Exception as exc:
+ if not _is_storage_not_found(exc):
+ raise
+ logger.warning(
+ "Skipping failed archive %s in Phase-2 replay: messages.jsonl is missing",
+ state.archive_uri,
+ )
+ continue
+ combined.extend(replayed)
marker = state.failed
self._merge_completed_memory_steps(
completed_memory_steps,
diff --git a/openviking/storage/queuefs/embedding_tracker.py b/openviking/storage/queuefs/embedding_tracker.py
index 2da33ecec..a993184c2 100644
--- a/openviking/storage/queuefs/embedding_tracker.py
+++ b/openviking/storage/queuefs/embedding_tracker.py
@@ -200,3 +200,27 @@ class EmbeddingTaskTracker:
if record_to_finalize is not None:
await self._run_on_complete(semantic_msg_id, record_to_finalize)
return remaining
+
+ async def abort(self, semantic_msg_id: str) -> bool:
+ """Retire one tracking attempt without running its completion callback.
+
+ Late decrements for an aborted attempt are ignored because its record no
+ longer exists. Callers that may retry a logical semantic message must
+ register each attempt under a unique tracking ID.
+
+ Returns:
+ True when an active record was removed, otherwise False.
+ """
+ with self._lock:
+ record = self._tasks.pop(semantic_msg_id, None)
+
+ if record is None:
+ return False
+
+ logger.warning(
+ "Aborted embedding tracker for SemanticMsg %s with %s/%s tasks remaining",
+ semantic_msg_id,
+ record.remaining,
+ record.total,
+ )
+ return True
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py
index d2f8617aa..d2590e87d 100644
--- a/openviking/storage/queuefs/semantic_dag.py
+++ b/openviking/storage/queuefs/semantic_dag.py
@@ -6,6 +6,7 @@ import asyncio
import threading
from dataclasses import dataclass, field
from typing import Any, ClassVar, Dict, List, Optional, Set
+from uuid import uuid4
from weakref import WeakKeyDictionary
from openviking.server.identity import RequestContext
@@ -217,6 +218,15 @@ class SemanticDagExecutor:
self._pending_vectorize_tasks: List[VectorizeTask] = []
self._pending_vectorize_work = 0
self._vectorize_done: Optional[asyncio.Event] = None
+ self._vectorize_failure: Optional[Exception] = None
+ self._vectorize_dispatch_complete = False
+ self._embedding_tracker_done: Optional[asyncio.Event] = None
+ self._embedding_tracker = None
+ # Retries preserve the logical SemanticMsg ID, so each execution needs
+ # a distinct tracker key to quarantine late decrements from old attempts.
+ self._embedding_tracker_id = f"{self._semantic_msg_id or 'semantic'}:{uuid4()}"
+ self._vectorize_finalized = False
+ self._vectorize_finalize_lock = asyncio.Lock()
self._vectorize_lock = asyncio.Lock()
self._file_change_status: Dict[str, bool] = {}
self._dir_change_status: Dict[str, bool] = {}
@@ -237,12 +247,6 @@ class SemanticDagExecutor:
raise self._failure
# Mark semantic done after downstream vectorization finishes.
- async def wrapped_on_complete() -> None:
- if self._telemetry_id and self._semantic_msg_id:
- get_request_wait_tracker().mark_semantic_done(
- self._telemetry_id, self._semantic_msg_id
- )
-
async with self._vectorize_lock:
task_count = self._vectorize_task_count
tasks = list(self._pending_vectorize_tasks)
@@ -250,24 +254,47 @@ class SemanticDagExecutor:
if task_count > 0:
from .embedding_tracker import EmbeddingTaskTracker
+ embedding_tracker_done = asyncio.Event()
+ self._embedding_tracker_done = embedding_tracker_done
tracker = EmbeddingTaskTracker.get_instance()
+ self._embedding_tracker = tracker
await tracker.register(
- semantic_msg_id=self._semantic_msg_id,
+ semantic_msg_id=self._embedding_tracker_id,
total_count=task_count,
- on_complete=wrapped_on_complete,
- metadata={"uri": root_uri},
+ on_complete=self._on_embedding_tasks_complete,
+ metadata={
+ "uri": root_uri,
+ "semantic_msg_id": self._semantic_msg_id,
+ },
)
await self._dispatch_vectorize_tasks(tasks)
+ self._vectorize_dispatch_complete = True
+
+ if embedding_tracker_done.is_set():
+ await self._finalize_vectorization()
+ if self._vectorize_failure is not None:
+ raise self._vectorize_failure
else:
- # No vectorize tasks — release lock immediately (via wrapped callback)
+ # No vectorize tasks — mark semantic work done immediately.
try:
- await wrapped_on_complete()
+ await self._finalize_vectorization()
except Exception as e:
logger.error(f"Error in on_complete callback: {e}", exc_info=True)
except BaseException:
self._closed = True
await self._active_work_idle.wait()
+ if self._embedding_tracker is not None:
+ try:
+ # Pending embedding messages may still finish, but their
+ # attempt-specific decrements become harmless after abort.
+ await self._embedding_tracker.abort(self._embedding_tracker_id)
+ except Exception:
+ logger.warning(
+ "Failed to abort embedding tracker %s",
+ self._embedding_tracker_id,
+ exc_info=True,
+ )
raise
finally:
self._closed = True
@@ -416,11 +443,29 @@ class SemanticDagExecutor:
await self._run_vectorize_task(task)
except Exception as exc:
logger.error("Vectorization dispatch task failed: %s", exc, exc_info=True)
+ if self._vectorize_failure is None:
+ self._vectorize_failure = exc
finally:
self._pending_vectorize_work = max(0, self._pending_vectorize_work - 1)
if self._pending_vectorize_work == 0 and self._vectorize_done:
self._vectorize_done.set()
+ async def _on_embedding_tasks_complete(self) -> None:
+ if self._embedding_tracker_done is not None:
+ self._embedding_tracker_done.set()
+ if self._vectorize_dispatch_complete:
+ await self._finalize_vectorization()
+
+ async def _finalize_vectorization(self) -> None:
+ async with self._vectorize_finalize_lock:
+ if self._vectorize_finalized:
+ return
+ if self._vectorize_failure is None and self._telemetry_id and self._semantic_msg_id:
+ get_request_wait_tracker().mark_semantic_done(
+ self._telemetry_id, self._semantic_msg_id
+ )
+ self._vectorize_finalized = True
+
async def _run_vectorize_task(self, task: VectorizeTask) -> None:
if task.task_type == "file":
await self._processor._vectorize_single_file(
@@ -715,7 +760,7 @@ class SemanticDagExecutor:
uri=file_path,
context_type=self._context_type,
ctx=self._ctx,
- semantic_msg_id=self._semantic_msg_id,
+ semantic_msg_id=self._embedding_tracker_id,
file_path=file_path,
summary_dict=summary_dict,
parent_uri=parent_uri,
@@ -862,7 +907,7 @@ class SemanticDagExecutor:
uri=dir_uri,
context_type=self._context_type,
ctx=self._ctx,
- semantic_msg_id=self._semantic_msg_id,
+ semantic_msg_id=self._embedding_tracker_id,
abstract=abstract,
overview=overview,
ingest_options=self._ingest_options,
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 3d9ae81a2..edae4b4ec 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -22,7 +22,7 @@ import time
from contextlib import contextmanager
from dataclasses import dataclass, field
from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, TypeVar, Union
from openviking.core.context import ContextLevel
from openviking.core.namespace import (
@@ -1031,6 +1031,7 @@ class VikingFS:
node_limit: Optional[int] = None,
level_limit: int = 10,
ctx: Optional[RequestContext] = None,
+ content_transform: Optional[Callable[[str, str], str]] = None,
) -> Dict:
"""Content search by pattern or keywords.
@@ -1049,6 +1050,7 @@ class VikingFS:
node_limit: Maximum number of results to return
level_limit: Maximum depth level to traverse (default: 10)
ctx: Request context
+ content_transform: Optional projection applied before regex matching.
Internal bm25 recall limit is auto-adapted from node_limit as
min(node_limit * 5, 100000); when node_limit is unset, use 100000.
@@ -1070,8 +1072,12 @@ class VikingFS:
self.grep_config.switch_to_remote_threshold if self.grep_config else 10000
)
- resolved_engine = await self._resolve_grep_engine(
- engine, uri, ctx, switch_to_remote_threshold
+ # A projection must run before matching. The remote BM25 index contains
+ # persisted raw content, so it cannot safely recall projected results.
+ resolved_engine = (
+ "fs"
+ if content_transform is not None
+ else await self._resolve_grep_engine(engine, uri, ctx, switch_to_remote_threshold)
)
if resolved_engine == "fs":
@@ -1083,6 +1089,7 @@ class VikingFS:
node_limit=node_limit,
level_limit=level_limit,
ctx=ctx,
+ content_transform=content_transform,
)
else: # "vikingdb_then_fs"
return await self._grep_vikingdb_then_fs(
@@ -1188,21 +1195,30 @@ class VikingFS:
return count
async def _grep_fs(
- self, uri, pattern, exclude_uri, case_insensitive, node_limit, level_limit, ctx
+ self,
+ uri,
+ pattern,
+ exclude_uri,
+ case_insensitive,
+ node_limit,
+ level_limit,
+ ctx,
+ content_transform=None,
):
"""Filesystem grep path: prefer native agfs grep and fall back if unavailable."""
- try:
- return await self._grep_with_agfs(
- uri=uri,
- pattern=pattern,
- exclude_uri=exclude_uri,
- case_insensitive=case_insensitive,
- node_limit=node_limit,
- level_limit=level_limit,
- ctx=ctx,
- )
- except (AttributeError, AGFSNotSupportedError, NotImplementedError) as e:
- logger.debug(f"agfs grep unavailable, falling back to VikingFS implementation: {e}")
+ if content_transform is None:
+ try:
+ return await self._grep_with_agfs(
+ uri=uri,
+ pattern=pattern,
+ exclude_uri=exclude_uri,
+ case_insensitive=case_insensitive,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ )
+ except (AttributeError, AGFSNotSupportedError, NotImplementedError) as e:
+ logger.debug(f"agfs grep unavailable, falling back to VikingFS implementation: {e}")
return await self._grep_encrypted(
uri=uri,
@@ -1212,6 +1228,7 @@ class VikingFS:
node_limit=node_limit,
level_limit=level_limit,
ctx=ctx,
+ content_transform=content_transform,
)
async def _grep_vikingdb_then_fs(
@@ -1449,6 +1466,7 @@ class VikingFS:
node_limit: Optional[int] = None,
level_limit: int = 10,
ctx: Optional[RequestContext] = None,
+ content_transform: Optional[Callable[[str, str], str]] = None,
) -> Dict:
"""Grep implementation for encrypted files.
@@ -1484,6 +1502,7 @@ class VikingFS:
compiled_pattern=compiled_pattern,
node_limit=node_limit,
ctx=ctx,
+ content_transform=content_transform,
)
return {
@@ -1555,13 +1574,20 @@ class VikingFS:
compiled_pattern: re.Pattern,
node_limit: Optional[int],
ctx: Optional[RequestContext] = None,
+ content_transform: Optional[Callable[[str, str], str]] = None,
) -> tuple[List[Dict[str, Any]], int]:
results: List[Dict[str, Any]] = []
files_scanned = 0
for start in range(0, len(file_uris), _DEFAULT_GREP_FILE_CONCURRENCY):
batch_uris = file_uris[start : start + _DEFAULT_GREP_FILE_CONCURRENCY]
batch_jobs = [
- self._grep_single_file(entry_uri, compiled_pattern, ctx) for entry_uri in batch_uris
+ self._grep_single_file(
+ entry_uri,
+ compiled_pattern,
+ ctx,
+ content_transform=content_transform,
+ )
+ for entry_uri in batch_uris
]
batch_results = await asyncio.gather(*batch_jobs)
for matches, scanned_count in batch_results:
@@ -1578,11 +1604,14 @@ class VikingFS:
entry_uri: str,
compiled_pattern: re.Pattern,
ctx: Optional[RequestContext] = None,
+ content_transform: Optional[Callable[[str, str], str]] = None,
) -> tuple[List[Dict[str, Any]], int]:
try:
content = await self.read(entry_uri, ctx=ctx)
if isinstance(content, bytes):
content = content.decode("utf-8", errors="replace")
+ if content_transform is not None:
+ content = content_transform(content, entry_uri)
matches: List[Dict[str, Any]] = []
lines = content.split("\n")
diff --git a/openviking/sync_client.py b/openviking/sync_client.py
index 657b2ebb3..a2a060443 100644
--- a/openviking/sync_client.py
+++ b/openviking/sync_client.py
@@ -509,6 +509,10 @@ class SyncOpenViking:
"""Read file"""
return run_async(self._async_client.read(uri, offset=offset, limit=limit))
+ def read_raw(self, uri: str, offset: int = 0, limit: int = -1) -> str:
+ """Read raw file content, including hidden MEMORY_FIELDS metadata."""
+ return run_async(self._async_client.read_raw(uri, offset=offset, limit=limit))
+
def write(
self,
uri: str,
@@ -579,15 +583,22 @@ class SyncOpenViking:
def import_ovpack(
self,
file_path: str,
- target: str,
+ parent: Optional[str] = None,
on_conflict: Optional[str] = None,
vector_mode: Optional[str] = None,
+ *,
+ target: Optional[str] = None,
) -> str:
"""Import .ovpack file (triggers vectorization by default)"""
+ if parent is not None and target is not None:
+ raise ValueError("parent cannot be used with legacy target")
+ parent = parent if parent is not None else target
+ if parent is None:
+ raise TypeError("parent or legacy target is required")
return run_async(
self._async_client.import_ovpack(
file_path,
- target,
+ parent,
on_conflict=on_conflict,
vector_mode=vector_mode,
)
@@ -634,6 +645,14 @@ class SyncOpenViking:
"""Wait for all async operations to complete"""
return run_async(self._async_client.wait_processed(timeout))
+ def build_index(self, resource_uris: Union[str, List[str]], **kwargs) -> Dict[str, Any]:
+ """Manually trigger index building for resources."""
+ return run_async(self._async_client.build_index(resource_uris, **kwargs))
+
+ def summarize(self, resource_uris: Union[str, List[str]], **kwargs) -> Dict[str, Any]:
+ """Manually trigger summarization for resources."""
+ return run_async(self._async_client.summarize(resource_uris, **kwargs))
+
def grep(
self,
uri: str,
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index 4d3beb1c4..978cd75f3 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -342,6 +342,7 @@ async def vectorize_directory_meta(
"""
enqueued = 0
expected = 2 if include_overview else 1
+ first_enqueue_error: Optional[Exception] = None
try:
if not ctx:
logger.warning("No context provided for vectorization")
@@ -395,6 +396,7 @@ async def vectorize_directory_meta(
f"Failed to enqueue directory L0 (abstract) for vectorization: {uri}: {e}",
exc_info=True,
)
+ first_enqueue_error = e
if include_overview:
# Vectorize L1: .overview.md (overview)
@@ -432,6 +434,10 @@ async def vectorize_directory_meta(
f"Failed to enqueue directory L1 (overview) for vectorization: {uri}: {e}",
exc_info=True,
)
+ if first_enqueue_error is None:
+ first_enqueue_error = e
+ if first_enqueue_error is not None:
+ raise first_enqueue_error
except Exception as e:
logger.error(
f"Failed to vectorize directory metadata for {uri}: {e}",
@@ -607,6 +613,7 @@ async def vectorize_file(
registered_wait_root[1],
f"Failed to enqueue file vector for {file_path}: {e}",
)
+ raise
finally:
if not enqueued:
await _decrement_embedding_tracker(semantic_msg_id, 1)
@@ -688,3 +695,4 @@ async def index_resource(
except Exception as e:
logger.error(f"Failed to scan directory {uri} for indexing: {e}")
+ raise
diff --git a/tests/client/test_ls_forwarding.py b/tests/client/test_ls_forwarding.py
index 101eea1b0..b920d8e93 100644
--- a/tests/client/test_ls_forwarding.py
+++ b/tests/client/test_ls_forwarding.py
@@ -27,8 +27,24 @@ async def test_async_openviking_ls_forwards_ordering_and_limit_options():
simple=False,
output="original",
abs_limit=256,
- show_all_hidden=True,
+ show_all_hidden=False,
node_limit=200,
sort_by="mtime",
sort_order="desc",
)
+
+
+async def test_async_openviking_tree_hides_internal_files_by_default():
+ client = object.__new__(AsyncOpenViking)
+ client._ensure_initialized = AsyncMock()
+ client._client = SimpleNamespace(tree=AsyncMock(return_value={}))
+
+ await client.tree("viking://session")
+
+ client._client.tree.assert_awaited_once_with(
+ "viking://session",
+ output="original",
+ abs_limit=128,
+ show_all_hidden=False,
+ node_limit=1000,
+ )
diff --git a/tests/client/test_rebuild_clients.py b/tests/client/test_rebuild_clients.py
index ecd32761d..da80aaf68 100644
--- a/tests/client/test_rebuild_clients.py
+++ b/tests/client/test_rebuild_clients.py
@@ -204,6 +204,78 @@ async def test_local_client_reindex_forwards_to_service():
)
+async def test_local_client_read_uses_public_content_projection():
+ client = object.__new__(LocalClient)
+ client._ctx = object()
+ client._service = SimpleNamespace(
+ fs=SimpleNamespace(read_visible=AsyncMock(return_value="line one\nline two"))
+ )
+
+ uri = "viking://user/memories/notes/demo.md"
+ assert await client.read(uri, offset=1, limit=1) == "line one\nline two"
+ client._service.fs.read_visible.assert_awaited_once_with(
+ uri,
+ ctx=client._ctx,
+ offset=1,
+ limit=1,
+ )
+
+
+async def test_local_client_read_does_not_strip_resource_comments():
+ client = object.__new__(LocalClient)
+ client._ctx = object()
+ raw = 'visible\n\n<!-- MEMORY_FIELDS\n{"looks_like":"user content"}\n-->'
+ client._service = SimpleNamespace(fs=SimpleNamespace(read_visible=AsyncMock(return_value=raw)))
+
+ assert await client.read("viking://resources/demo.md") == raw
+
+
+async def test_local_client_read_raw_preserves_service_slicing():
+ client = object.__new__(LocalClient)
+ client._ctx = object()
+ client._service = SimpleNamespace(fs=SimpleNamespace(read=AsyncMock(return_value="raw")))
+
+ uri = "viking://user/memories/notes/demo.md"
+ assert await client.read_raw(uri, offset=4, limit=1) == "raw"
+ client._service.fs.read.assert_awaited_once_with(
+ uri,
+ ctx=client._ctx,
+ offset=4,
+ limit=1,
+ )
+
+
+def test_sync_client_forwards_missing_async_surface():
+ client = object.__new__(SyncOpenViking)
+ client._async_client = SimpleNamespace(
+ build_index=Mock(return_value="index"),
+ summarize=Mock(return_value="summary"),
+ read_raw=Mock(return_value="raw"),
+ )
+ with patch("openviking.sync_client.run_async", side_effect=lambda result: result):
+ assert client.build_index(["uri"], wait=True) == "index"
+ assert client.summarize("uri", mode="fast") == "summary"
+ assert client.read_raw("uri", offset=1, limit=2) == "raw"
+
+ client._async_client.build_index.assert_called_once_with(["uri"], wait=True)
+ client._async_client.summarize.assert_called_once_with("uri", mode="fast")
+ client._async_client.read_raw.assert_called_once_with("uri", offset=1, limit=2)
+
+
+def test_sync_import_ovpack_accepts_parent_and_legacy_target():
+ client = object.__new__(SyncOpenViking)
+ import_ovpack = Mock(return_value="imported")
+ client._async_client = SimpleNamespace(import_ovpack=import_ovpack)
+ with patch("openviking.sync_client.run_async", side_effect=lambda result: result):
+ assert client.import_ovpack("pack", parent="parent") == "imported"
+ assert client.import_ovpack("pack", target="legacy") == "imported"
+ with pytest.raises(ValueError):
+ client.import_ovpack("pack", parent="parent", target="legacy")
+
+ assert import_ovpack.call_args_list[0].args == ("pack", "parent")
+ assert import_ovpack.call_args_list[1].args == ("pack", "legacy")
+
+
async def test_local_client_batch_add_messages_forwards_to_session():
class FakeSession:
def __init__(self):
diff --git a/tests/server/test_api_content.py b/tests/server/test_api_content.py
index 1c00b1198..87b69e0e5 100644
--- a/tests/server/test_api_content.py
+++ b/tests/server/test_api_content.py
@@ -3,9 +3,12 @@
"""Tests for content endpoints: read, abstract, overview, reindex."""
+from types import SimpleNamespace
+
import pytest
from openviking.server.identity import RequestContext, Role
+from openviking.server.routers import content as content_router
from openviking.server.routers.content import ReindexRequest, reindex
from openviking_cli.session.user_id import UserIdentifier
@@ -32,6 +35,35 @@ async def test_read_content(client_with_resource):
assert body["result"] is not None
+async def test_read_memory_uses_visible_projection_and_raw_bypasses_it(monkeypatch):
+ ctx = RequestContext(user=UserIdentifier.the_default_user("test_user"), role=Role.USER)
+ uri = "viking://user/test_user/memories/notes/private.md"
+ raw = 'line one\nline two\n\n<!-- MEMORY_FIELDS\n{"secret": "do-not-leak"}\n-->'
+ calls = []
+
+ async def fake_read_visible(_uri, *, ctx, offset, limit):
+ calls.append((offset, limit))
+ return "visible slice"
+
+ async def fake_read(_uri, *, ctx, offset, limit):
+ lines = raw.splitlines(keepends=True)
+ return "".join(lines[offset:] if limit == -1 else lines[offset : offset + limit])
+
+ monkeypatch.setattr(
+ content_router,
+ "get_service",
+ lambda: SimpleNamespace(fs=SimpleNamespace(read=fake_read, read_visible=fake_read_visible)),
+ )
+
+ response = await content_router.read(uri=uri, offset=4, limit=1, raw=False, _ctx=ctx)
+ assert response.result == "visible slice"
+ assert calls[-1] == (4, 1)
+
+ raw_response = await content_router.read(uri=uri, offset=4, limit=1, raw=True, _ctx=ctx)
+ assert raw_response.result == '{"secret": "do-not-leak"}\n'
+ assert calls[-1] == (4, 1)
+
+
async def test_read_directory_uri_returns_invalid_argument(client_with_resource):
client, uri = client_with_resource
diff --git a/tests/server/test_mcp_endpoint.py b/tests/server/test_mcp_endpoint.py
index 27ab9d81f..eb74c28a5 100644
--- a/tests/server/test_mcp_endpoint.py
+++ b/tests/server/test_mcp_endpoint.py
@@ -409,6 +409,19 @@ async def test_read_batch(service):
assert "nothing found" in result.lower()
+async def test_read_uses_public_content_projection(monkeypatch):
+ read_visible = AsyncMock(return_value="visible memory")
+ monkeypatch.setattr(
+ mcp_endpoint,
+ "get_service",
+ lambda: SimpleNamespace(fs=SimpleNamespace(read_visible=read_visible)),
+ )
+ uri = "viking://user/default/memories/private.md"
+
+ assert await read(uri) == "visible memory"
+ read_visible.assert_awaited_once_with(uri, ctx=DEFAULT_CTX)
+
+
# ---------------------------------------------------------------------------
# list tool
# ---------------------------------------------------------------------------
diff --git a/tests/service/test_fs_service.py b/tests/service/test_fs_service.py
index 94e3dc988..ff6b315f4 100644
--- a/tests/service/test_fs_service.py
+++ b/tests/service/test_fs_service.py
@@ -54,9 +54,7 @@ class _FakeWatchManager:
move_resource,
rollback_resource=None,
):
- self.sync_calls.append(
- {"from_uri": from_uri, "to_uri": to_uri, "account_id": account_id}
- )
+ self.sync_calls.append({"from_uri": from_uri, "to_uri": to_uri, "account_id": account_id})
if self.plan_error:
raise self.plan_error
await move_resource()
@@ -140,6 +138,80 @@ def request_context():
)
+async def test_read_visible_strips_memory_metadata_before_slicing(request_context):
+ raw = 'line one\nline two\n\n<!-- MEMORY_FIELDS\n{"secret":"hidden"}\n-->'
+ viking_fs = SimpleNamespace(read_file=AsyncMock(return_value=raw))
+ service = FSService(viking_fs=viking_fs)
+ uri = "viking://user/ryoma/memories/notes/private.md"
+
+ assert await service.read_visible(uri, ctx=request_context, offset=3, limit=1) == ""
+ viking_fs.read_file.assert_awaited_once_with(uri, ctx=request_context)
+
+
+ "raw",
+ [
+ 'visible\n<!-- MEMORY_FIELDS\n{"secret":"hidden"}\n-->',
+ 'visible\n\n<!-- MEMORY_FIELDS {"secret":"hidden"} -->',
+ 'visible <!-- MEMORY_FIELDS {"secret":"hidden"} -->',
+ ],
+)
+async def test_read_visible_strips_supported_memory_metadata_trailers(
+ request_context,
+ raw,
+):
+ uri = "viking://user/ryoma/memories/notes/private.md"
+ service = FSService(
+ viking_fs=SimpleNamespace(read_file=AsyncMock(return_value=raw)),
+ )
+
+ assert await service.read_visible(uri, ctx=request_context) == "visible"
+
+
+async def test_read_visible_preserves_non_memory_content(request_context):
+ raw = 'visible\n<!-- MEMORY_FIELDS {"example":true} -->'
+ viking_fs = SimpleNamespace(read_file=AsyncMock(return_value=raw))
+ service = FSService(viking_fs=viking_fs)
+
+ assert (
+ await service.read_visible(
+ "viking://resources/example.md",
+ ctx=request_context,
+ offset=1,
+ limit=1,
+ )
+ == '<!-- MEMORY_FIELDS {"example":true} -->'
+ )
+
+
+async def test_grep_projects_memory_content_but_keeps_resource_fast_path(request_context):
+ viking_fs = SimpleNamespace(grep=AsyncMock(return_value={"matches": []}))
+ service = FSService(viking_fs=viking_fs)
+
+ await service.grep(
+ "viking://user/ryoma/memories",
+ "secret",
+ ctx=request_context,
+ )
+ memory_kwargs = viking_fs.grep.await_args.kwargs
+ transform = memory_kwargs["content_transform"]
+ assert (
+ transform(
+ 'visible\n<!-- MEMORY_FIELDS {"secret":"hidden"} -->',
+ "viking://user/ryoma/memories/private.md",
+ )
+ == "visible"
+ )
+
+ viking_fs.grep.reset_mock()
+ await service.grep("viking://resources", "secret", ctx=request_context)
+ assert "content_transform" not in viking_fs.grep.await_args.kwargs
+
+
@pytest.mark.asyncio
async def test_resource_rm_enqueues_parent_delete_refresh_and_waits(request_context):
viking_fs = _FakeVikingFS()
diff --git a/tests/storage/test_embedding_tracker.py b/tests/storage/test_embedding_tracker.py
index 8b9df4ab0..ea6ad0b72 100644
--- a/tests/storage/test_embedding_tracker.py
+++ b/tests/storage/test_embedding_tracker.py
@@ -185,3 +185,29 @@ async def test_tracker_clears_zero_task_entry_without_callback():
await tracker.register("semantic-msg", 0, on_complete=None)
assert await tracker.decrement("semantic-msg") is None
+
+
+async def test_tracker_abort_ignores_late_decrements_without_completing():
+ tracker = EmbeddingTaskTracker.get_instance()
+ callback_calls = []
+
+ await tracker.register(
+ "semantic-msg:attempt-1",
+ 2,
+ on_complete=lambda: callback_calls.append("attempt-1"),
+ )
+
+ assert await tracker.abort("semantic-msg:attempt-1") is True
+ assert await tracker.abort("semantic-msg:attempt-1") is False
+
+ await tracker.register(
+ "semantic-msg:attempt-2",
+ 1,
+ on_complete=lambda: callback_calls.append("attempt-2"),
+ )
+
+ assert await tracker.decrement("semantic-msg:attempt-1") is None
+ assert callback_calls == []
+ assert await tracker.decrement("semantic-msg:attempt-2") == 0
+ assert callback_calls == ["attempt-2"]
diff --git a/tests/storage/test_memory_semantic_stall.py b/tests/storage/test_memory_semantic_stall.py
index 628851c62..099fa35f5 100644
--- a/tests/storage/test_memory_semantic_stall.py
+++ b/tests/storage/test_memory_semantic_stall.py
@@ -198,6 +198,10 @@ async def test_memory_write_error_reports_error():
fake_fs.ls = AsyncMock(return_value=[{"name": "file1.md", "isDir": False}])
fake_fs.read_file = AsyncMock(return_value="some content")
fake_fs.write_file = AsyncMock(side_effect=PermissionError("Permission denied"))
+ fake_fs._async_agfs.pathlock_acquire_exact_batch = AsyncMock(
+ return_value={"lease_ref": "test"}
+ )
+ fake_fs._async_agfs.pathlock_release = AsyncMock()
fake_fs._uri_to_path = MagicMock(
side_effect=lambda uri, ctx=None: f"/local/acc1/{uri.removeprefix('viking://')}"
)
diff --git a/tests/storage/test_semantic_dag_incremental.py b/tests/storage/test_semantic_dag_incremental.py
index 38527ed54..b478c5fd0 100644
--- a/tests/storage/test_semantic_dag_incremental.py
+++ b/tests/storage/test_semantic_dag_incremental.py
@@ -11,11 +11,20 @@ from openviking.storage.queuefs.semantic_dag import SemanticDagExecutor
from openviking_cli.session.user_id import UserIdentifier
+class _FakeAgfs:
+ async def pathlock_acquire_exact_batch(self, _paths):
+ return {"lease_ref": "test"}
+
+ async def pathlock_release(self, _lease):
+ return None
+
+
class _FakeVikingFS:
def __init__(self, tree, file_contents):
self._tree = {self._norm(k): v for k, v in tree.items()}
self._file_contents = {self._norm(k): v for k, v in file_contents.items()}
self.writes = []
+ self._async_agfs = _FakeAgfs()
def _norm(self, path):
if "://" not in path:
@@ -34,7 +43,7 @@ class _FakeVikingFS:
async def read_file(self, path, ctx=None):
return self._file_contents.get(self._norm(path), "")
- async def write_file(self, path, content, ctx=None):
+ async def write_file(self, path, content, ctx=None, lease_ref=None):
norm_path = self._norm(path)
self._file_contents[norm_path] = content
self.writes.append((norm_path, content))
diff --git a/tests/storage/test_semantic_dag_skip_files.py b/tests/storage/test_semantic_dag_skip_files.py
index a195a4210..947a206ba 100644
--- a/tests/storage/test_semantic_dag_skip_files.py
+++ b/tests/storage/test_semantic_dag_skip_files.py
@@ -8,15 +8,24 @@ from openviking.storage.queuefs.semantic_dag import SemanticDagExecutor
from openviking_cli.session.user_id import UserIdentifier
+class _FakeAgfs:
+ async def pathlock_acquire_exact_batch(self, _paths):
+ return {"lease_ref": "test"}
+
+ async def pathlock_release(self, _lease):
+ return None
+
+
class _FakeVikingFS:
def __init__(self, tree):
self._tree = tree
self.writes = []
+ self._async_agfs = _FakeAgfs()
async def ls(self, uri, node_limit=None, ctx=None):
return self._tree.get(uri, [])
- async def write_file(self, path, content, ctx=None):
+ async def write_file(self, path, content, ctx=None, lease_ref=None):
self.writes.append((path, content))
def _uri_to_path(self, uri, ctx=None):
@@ -39,7 +48,14 @@ class _FakeProcessor:
return overview, "abstract"
async def _vectorize_directory(
- self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None
+ self,
+ uri,
+ context_type,
+ abstract,
+ overview,
+ ctx=None,
+ semantic_msg_id=None,
+ ingest_options=None,
):
pass
@@ -55,6 +71,7 @@ class _FakeProcessor:
ctx=None,
semantic_msg_id=None,
use_summary=False,
+ ingest_options=None,
):
self.vectorized_files.append(file_path)
diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.py
index 6ba49edcb..b3619f762 100644
--- a/tests/storage/test_semantic_dag_stats.py
+++ b/tests/storage/test_semantic_dag_stats.py
@@ -17,16 +17,33 @@ from openviking.storage.queuefs.semantic_dag import (
from openviking_cli.session.user_id import UserIdentifier
+class _FakeAgfs:
+ async def pathlock_acquire_exact_batch(self, _paths):
+ return {"lease_ref": "test"}
+
+ async def pathlock_release(self, _lease):
+ return None
+
+
class _FakeVikingFS:
def __init__(self, tree):
self._tree = tree
self.writes = []
+ self._async_agfs = _FakeAgfs()
async def ls(self, uri, node_limit=None, ctx=None):
del node_limit
return self._tree.get(uri, [])
- async def write_file(self, path, content, ctx=None, lock_handle=None):
+ async def write_file(
+ self,
+ path,
+ content,
+ ctx=None,
+ lease_ref=None,
+ lock_handle=None,
+ ):
+ del ctx, lease_ref, lock_handle
self.writes.append((path, content))
def _uri_to_path(self, uri, ctx=None):
@@ -48,7 +65,14 @@ class _FakeProcessor:
return overview, "abstract"
async def _vectorize_directory(
- self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None
+ self,
+ uri,
+ context_type,
+ abstract,
+ overview,
+ ctx=None,
+ semantic_msg_id=None,
+ ingest_options=None,
):
self.vectorized_dirs.append(uri)
@@ -61,6 +85,7 @@ class _FakeProcessor:
ctx=None,
semantic_msg_id=None,
use_summary=False,
+ ingest_options=None,
):
self.vectorized_files.append(file_path)
@@ -93,6 +118,159 @@ class _DummyTracker:
return None
+class _CompletingTracker:
+ def __init__(self):
+ self.remaining = 0
+ self.total = 0
+ self.on_complete = None
+ self.completed = False
+ self.active_id = None
+ self.aborted_ids = []
+
+ async def register(self, *, semantic_msg_id, total_count, on_complete, **_kwargs):
+ self.active_id = semantic_msg_id
+ self.remaining = total_count
+ self.total = total_count
+ self.on_complete = on_complete
+
+ async def decrement(self, semantic_msg_id):
+ if semantic_msg_id != self.active_id:
+ return None
+ self.remaining -= 1
+ if self.remaining == 0:
+ self.completed = True
+ self.active_id = None
+ await self.on_complete()
+ return self.remaining
+
+ async def abort(self, semantic_msg_id):
+ if semantic_msg_id != self.active_id:
+ return False
+ self.aborted_ids.append(semantic_msg_id)
+ self.active_id = None
+ return True
+
+
+class _FailingVectorizeProcessor(_FakeProcessor):
+ def __init__(self, tracker):
+ super().__init__()
+ self._tracker = tracker
+
+ async def _vectorize_single_file(
+ self,
+ parent_uri,
+ context_type,
+ file_path,
+ summary_dict,
+ ctx=None,
+ semantic_msg_id=None,
+ use_summary=False,
+ ingest_options=None,
+ ):
+ self.vectorized_files.append(file_path)
+ try:
+ raise RuntimeError("embedding enqueue unavailable")
+ finally:
+ await self._tracker.decrement(semantic_msg_id)
+
+ async def _vectorize_directory(
+ self,
+ uri,
+ context_type,
+ abstract,
+ overview,
+ ctx=None,
+ semantic_msg_id=None,
+ ingest_options=None,
+ ):
+ self.vectorized_dirs.append(uri)
+ await self._tracker.decrement(semantic_msg_id)
+ await self._tracker.decrement(semantic_msg_id)
+
+
+class _StalledEmbeddingProcessor(_FailingVectorizeProcessor):
+ async def _vectorize_directory(
+ self,
+ uri,
+ context_type,
+ abstract,
+ overview,
+ ctx=None,
+ semantic_msg_id=None,
+ ingest_options=None,
+ ):
+ self.vectorized_dirs.append(uri)
+ # Both directory messages were accepted, but the embedding worker is
+ # unavailable and therefore never decrements their tracker slots.
+
+
+async def test_semantic_dag_propagates_vectorization_failure_after_tracker_drains(monkeypatch):
+ root_uri = "viking://resources/root"
+ tree = {root_uri: [{"name": "a.txt", "isDir": False}]}
+ fake_fs = _FakeVikingFS(tree)
+ tracker = _CompletingTracker()
+ monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
+ lambda: tracker,
+ )
+
+ processor = _FailingVectorizeProcessor(tracker)
+ ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
+ executor = SemanticDagExecutor(
+ processor=processor,
+ context_type="resource",
+ max_concurrent_llm=2,
+ ctx=ctx,
+ semantic_msg_id="semantic-root",
+ )
+
+ with pytest.raises(RuntimeError, match="embedding enqueue unavailable"):
+ await executor.run(root_uri)
+
+ assert processor.vectorized_files == [f"{root_uri}/a.txt"]
+ assert processor.vectorized_dirs == [root_uri]
+ assert tracker.total == 3
+ assert tracker.remaining == 0
+ assert tracker.completed is True
+
+
+async def test_semantic_dag_aborts_stalled_embedding_attempt_after_dispatch_failure(monkeypatch):
+ root_uri = "viking://resources/root"
+ tree = {root_uri: [{"name": "a.txt", "isDir": False}]}
+ fake_fs = _FakeVikingFS(tree)
+ tracker = _CompletingTracker()
+ monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
+ lambda: tracker,
+ )
+
+ processor = _StalledEmbeddingProcessor(tracker)
+ ctx = RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
+ executor = SemanticDagExecutor(
+ processor=processor,
+ context_type="resource",
+ max_concurrent_llm=2,
+ ctx=ctx,
+ semantic_msg_id="semantic-root",
+ )
+
+ with pytest.raises(RuntimeError, match="embedding enqueue unavailable"):
+ await asyncio.wait_for(executor.run(root_uri), timeout=0.5)
+
+ assert processor.vectorized_files == [f"{root_uri}/a.txt"]
+ assert processor.vectorized_dirs == [root_uri]
+ assert tracker.remaining == 2
+ assert tracker.completed is False
+ assert len(tracker.aborted_ids) == 1
+ aborted_id = tracker.aborted_ids[0]
+ assert aborted_id.startswith("semantic-root:")
+ assert await tracker.decrement(aborted_id) is None
+
+
class _ScheduledExecutor:
def __init__(self, run) -> None:
self.closed = False
diff --git a/tests/storage/test_viking_fs_grep.py b/tests/storage/test_viking_fs_grep.py
index 47aa71ad5..d707d5b9e 100644
--- a/tests/storage/test_viking_fs_grep.py
+++ b/tests/storage/test_viking_fs_grep.py
@@ -273,6 +273,45 @@ async def test_grep_preserves_dfs_order_and_node_limit(monkeypatch):
]
+async def test_grep_applies_content_transform_before_matching(monkeypatch):
+ fs = VikingFS(agfs=_DummyAgfs())
+
+ async def fake_stat(uri, ctx=None, skip_count=False):
+ return {"isDir": True}
+
+ async def fake_ls(uri, ctx=None, **kwargs):
+ if uri == "viking://resources":
+ return [{"name": "memory.md", "isDir": False}]
+ return []
+
+ def fake_agfs_read(path, offset=0, size=-1):
+ return b'visible\n<!-- MEMORY_FIELDS {"secret":"hidden"} -->'
+
+ monkeypatch.setattr(fs, "stat", fake_stat)
+ monkeypatch.setattr(fs, "ls", fake_ls)
+ monkeypatch.setattr(
+ fs,
+ "_uri_to_path",
+ lambda uri, ctx=None: uri.replace("viking://", "/"),
+ )
+ monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False)
+
+ result = await fs.grep(
+ "viking://resources",
+ pattern="visible|secret",
+ content_transform=lambda content, _uri: content.split("<!--", 1)[0].rstrip(),
+ )
+
+ assert result["matches"] == [
+ {
+ "line": 1,
+ "uri": "viking://resources/memory.md",
+ "content": "visible",
+ }
+ ]
+
+
@pytest.mark.asyncio
async def test_grep_parallel_reads_respect_concurrency_limit(monkeypatch):
fs = VikingFS(agfs=_DummyAgfs())
diff --git a/tests/telemetry/test_http_observability_middleware.py b/tests/telemetry/test_http_observability_middleware.py
index 05e15c0cf..c3a9568e5 100644
--- a/tests/telemetry/test_http_observability_middleware.py
+++ b/tests/telemetry/test_http_observability_middleware.py
@@ -68,9 +68,11 @@ def test_http_observability_middleware_updates_route_template_after_routing(monk
return {"ok": True}
with TestClient(app) as client:
- resp = client.get("/hello")
+ resp = client.get("/hello?token=top-secret")
assert resp.status_code == 200
root_attrs = captured["root_attrs"]
assert root_attrs.http_route == "/hello"
+ assert root_attrs.url_query is None
+ assert "url.query" not in root_attrs.to_otel_attributes()
assert "GET /hello" in dummy_span.updated_names
diff --git a/tests/unit/session/test_session_phase2_replay_tolerance.py b/tests/unit/session/test_session_phase2_replay_tolerance.py
new file mode 100644
index 000000000..db2229ea1
--- /dev/null
+++ b/tests/unit/session/test_session_phase2_replay_tolerance.py
@@ -0,0 +1,117 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Regression tests for Phase-2 archive replay tolerance (follow-up to #3417).
+
+#3417 made ``Session._read_archive_messages`` strict: it now raises on a
+missing/corrupt ``messages.jsonl`` instead of returning ``[]``. The read path
+(``_get_uncovered_archive_messages``) gained not-found tolerance, but the
+Phase-2 commit replay path (``_prepare_phase2_archive_messages``) did not.
+
+A terminally-failed earlier archive whose ``messages.jsonl`` is missing would
+therefore make every subsequent commit's Phase-2 replay raise, terminal-fail
+the current archive, and permanently poison the session's memory extraction.
+"""
+
+from unittest.mock import AsyncMock
+
+import pytest
+
+from openviking.session.session import ArchiveState, Session
+
+
+class _MemoryVikingFS:
+ def __init__(self, files):
+ self.files = files
+
+ def _uri_to_path(self, uri, ctx=None):
+ return "/local/session-1"
+
+ async def read_file(self, uri, ctx=None):
+ if uri not in self.files:
+ raise FileNotFoundError(uri)
+ value = self.files[uri]
+ if isinstance(value, Exception):
+ raise value
+ return value
+
+ async def write_file(self, uri, content, ctx=None):
+ self.files[uri] = content
+
+
+def _session(files, states, monkeypatch):
+ session_uri = "viking://user/sessions/session-1"
+ session = Session(
+ viking_fs=_MemoryVikingFS(files),
+ session_id="session-1",
+ session_uri=session_uri,
+ )
+ monkeypatch.setattr(session, "_scan_archive_states", AsyncMock(return_value=states))
+ return session, session_uri
+
+
+def _failed_then_pending(session_uri):
+ """archive_001 = terminally failed (missing messages), archive_002 = current pending."""
+ a1 = f"{session_uri}/history/archive_001"
+ a2 = f"{session_uri}/history/archive_002"
+ states = [
+ ArchiveState(
+ archive_id="archive_001",
+ archive_uri=a1,
+ index=1,
+ state="failed",
+ failed={"stage": "archive_read", "error": "no messages"},
+ ),
+ ArchiveState(
+ archive_id="archive_002",
+ archive_uri=a2,
+ index=2,
+ state="pending",
+ ),
+ ]
+ return a1, a2, states
+
+
+async def test_phase2_replay_skips_failed_archive_with_missing_messages(monkeypatch):
+ session_uri = "viking://user/sessions/session-1"
+ a1, a2, states = _failed_then_pending(session_uri)
+ # archive_001/messages.jsonl is absent -> not-found; archive_002 has no
+ # replay reads of its own (current messages are passed in directly).
+ session, _ = _session({}, states, monkeypatch)
+
+ from openviking.message import Message, TextPart
+
+ current = [Message(id="cur", role="user", parts=[TextPart("hi")])]
+
+ combined, _start, _end, covered_failed, _steps = await session._prepare_phase2_archive_messages(
+ a2, current
+ )
+
+ # The poisoned archive is skipped; the current commit still gets its own
+ # messages and can complete Phase 2 instead of being terminal-failed.
+ assert [m.id for m in combined] == ["cur"]
+ # The skipped archive is still reported as covered_failed so the current
+ # archive's .done marks it covered — clearing the poison permanently
+ # instead of re-reading (and skipping) it on every future commit.
+ assert covered_failed == ["archive_001"]
+
+ # Asymmetry check: the read path already tolerates the exact same scenario.
+ uncovered = await session._get_uncovered_archive_messages(states)
+ assert uncovered == []
+
+
+async def test_phase2_replay_reraises_real_storage_failure(monkeypatch):
+ session_uri = "viking://user/sessions/session-1"
+ a1, a2, states = _failed_then_pending(session_uri)
+ # A real storage failure (not a not-found) reading archive_001 must still
+ # propagate so it is not silently swallowed.
+ boom = RuntimeError("storage backend unavailable")
+ session, _ = _session({f"{a1}/messages.jsonl": boom}, states, monkeypatch)
+
+ from openviking.message import Message, TextPart
+
+ current = [Message(id="cur", role="user", parts=[TextPart("hi")])]
+
+ with pytest.raises(RuntimeError, match="storage backend unavailable"):
+ await session._prepare_phase2_archive_messages(a2, current)
diff --git a/tests/unit/test_body_dump_middleware.py b/tests/unit/test_body_dump_middleware.py
new file mode 100644
index 000000000..b5141c710
--- /dev/null
+++ b/tests/unit/test_body_dump_middleware.py
@@ -0,0 +1,9 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+from openviking.server.body_dump_middleware import _should_skip
+
+
+def test_zip_responses_skip_body_capture():
+ assert _should_skip("application/zip")
+ assert _should_skip("application/zip; charset=binary")
diff --git a/tests/unit/test_content_visibility.py b/tests/unit/test_content_visibility.py
new file mode 100644
index 000000000..984ad9efc
--- /dev/null
+++ b/tests/unit/test_content_visibility.py
@@ -0,0 +1,52 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import pytest
+
+from openviking.session.memory.utils.content_visibility import visible_content
+
+MEMORY_URI = "viking://user/default/memories/notes/private.md"
+
+
+ "raw_content",
+ [
+ 'visible\n\n<!-- MEMORY_FIELDS\n{"secret":"hidden"}\n-->',
+ 'visible\n<!-- MEMORY_FIELDS\n{"secret":"hidden"}\n-->',
+ 'visible\n\n<!-- MEMORY_FIELDS {"secret":"hidden"} -->',
+ 'visible <!-- MEMORY_FIELDS {"secret":"hidden"} -->',
+ '<!-- MEMORY_FIELDS {"secret":"hidden"} -->',
+ ],
+)
+def test_visible_content_strips_supported_memory_fields_trailers(raw_content):
+ expected = "" if raw_content.startswith("<!--") else "visible"
+
+ assert visible_content(raw_content, uri=MEMORY_URI) == expected
+
+
+ "raw_content",
+ [
+ "visible <!-- ordinary comment -->",
+ "visible <!-- MEMORY_FIELDS_EXAMPLE {} -->",
+ 'visible <!-- MEMORY_FIELDS {"example":true} -->\nmore',
+ ],
+)
+def test_visible_content_preserves_non_reserved_or_non_trailing_comments(raw_content):
+ assert visible_content(raw_content, uri=MEMORY_URI) == raw_content
+
+
+def test_visible_content_preserves_reserved_comment_outside_memory_namespace():
+ raw_content = 'visible <!-- MEMORY_FIELDS {"example":true} -->'
+
+ assert visible_content(raw_content, uri="viking://resources/example.md") == raw_content
+
+
+def test_visible_content_strips_repeated_memory_fields_trailers():
+ raw_content = """visible
+<!-- MEMORY_FIELDS {"secret":"old"} -->
+<!-- MEMORY_FIELDS
+{"secret":"new"}
+-->"""
+
+ assert visible_content(raw_content, uri=MEMORY_URI) == "visible"
diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.py
index d1717a88f..35df6ac22 100644
--- a/tests/unit/test_vectorize_file_strategy.py
+++ b/tests/unit/test_vectorize_file_strategy.py
@@ -1,10 +1,11 @@
import types
+from unittest.mock import AsyncMock
import pytest
-from openviking.utils.ingest_options import IngestOptions
from openviking.core.context import Context, ResourceContentType
from openviking.utils import embedding_utils
+from openviking.utils.ingest_options import IngestOptions
class DummyQueue:
@@ -200,13 +201,14 @@ async def test_vectorize_file_marks_registered_wait_root_failed_when_enqueue_rai
),
)
- await embedding_utils.vectorize_file(
- file_path="viking://user/default/resources/test.md",
- summary_dict={"name": "test.md", "summary": ""},
- parent_uri="viking://user/default/resources",
- ctx=DummyReq(),
- register_request_wait=True,
- )
+ with pytest.raises(RuntimeError, match="queue unavailable"):
+ await embedding_utils.vectorize_file(
+ file_path="viking://user/default/resources/test.md",
+ summary_dict={"name": "test.md", "summary": ""},
+ parent_uri="viking://user/default/resources",
+ ctx=DummyReq(),
+ register_request_wait=True,
+ )
assert len(queue.items) == 1
assert registered == [(queue.items[0].telemetry_id, queue.items[0].id)]
@@ -214,6 +216,52 @@ async def test_vectorize_file_marks_registered_wait_root_failed_when_enqueue_rai
assert failed[0][0:2] == registered[0]
+async def test_vectorize_file_propagates_enqueue_failure(monkeypatch):
+ monkeypatch.setattr(
+ embedding_utils, "get_queue_manager", lambda: DummyQueueManager(FailingQueue())
+ )
+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: DummyFS("content"))
+ monkeypatch.setattr(
+ embedding_utils,
+ "get_openviking_config",
+ lambda: types.SimpleNamespace(
+ embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000)
+ ),
+ )
+ monkeypatch.setattr(
+ embedding_utils.EmbeddingMsgConverter, "from_context", lambda context: context
+ )
+
+ with pytest.raises(RuntimeError, match="queue unavailable"):
+ await embedding_utils.vectorize_file(
+ "viking://user/default/resources/test.md",
+ {"name": "test.md", "summary": "summary"},
+ "viking://user/default/resources",
+ ctx=DummyReq(),
+ )
+
+
+async def test_vectorize_directory_propagates_enqueue_failures_and_drains_tracker(monkeypatch):
+ decrement = AsyncMock()
+ monkeypatch.setattr(
+ embedding_utils, "get_queue_manager", lambda: DummyQueueManager(FailingQueue())
+ )
+ monkeypatch.setattr(embedding_utils, "_decrement_embedding_tracker", decrement)
+
+ with pytest.raises(RuntimeError, match="queue unavailable"):
+ await embedding_utils.vectorize_directory_meta(
+ "viking://user/default/resources",
+ abstract="abstract",
+ overview="overview",
+ ctx=DummyReq(),
+ semantic_msg_id="semantic-root",
+ )
+
+ decrement.assert_awaited_once_with("semantic-root", 2)
+
+
@pytest.mark.asyncio
async def test_vectorize_unknown_text_file_embeds_summary_but_indexes_raw_content(monkeypatch):
queue = DummyQueue()