Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

refactor: vikingfs.py (#3947)

MaojiaShengb84395d6dd6d
diff --git a/openviking/server/auth/__init__.py b/openviking/server/auth/__init__.py
index 85d1af631..9d0e74518 100644
--- a/openviking/server/auth/__init__.py
+++ b/openviking/server/auth/__init__.py
@@ -303,9 +303,8 @@ def require_auth_role(*allowed_roles: Role):
"Admin API authentication failed: unable to resolve request context."
)
- plugin = _get_plugin(request)
manager = getattr(request.app.state, "api_key_manager", None)
- if manager is None and plugin.requires_api_key_manager():
+ if manager is None:
raise PermissionDeniedError(_DEV_MODE_ADMIN_API_MESSAGE)
if ctx.role not in allowed_roles:
@@ -345,11 +344,9 @@ def get_api_key_manager_or_raise(request: Request):
"""Get APIKeyManager from app state or raise appropriate error.
Raises:
- PermissionDeniedError: When the current auth plugin requires an
- APIKeyManager but none is available.
+ PermissionDeniedError: When no APIKeyManager is available.
"""
manager = getattr(request.app.state, "api_key_manager", None)
- plugin = _get_plugin(request)
- if manager is None and plugin.requires_api_key_manager():
+ if manager is None:
raise PermissionDeniedError(_DEV_MODE_ADMIN_API_MESSAGE)
return manager
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index c73d6158c..dfc04ceb7 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -6,7 +6,6 @@ import asyncio
import re
import threading
from contextlib import nullcontext
-from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Set, Tuple
from openviking.observability.context import (
@@ -40,7 +39,7 @@ from openviking.storage.queuefs.semantic_lock import SemanticLockScope
from openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_coalesce_key
from openviking.storage.queuefs.semantic_queue import is_semantic_msg_stale
from openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecars
-from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
+from openviking.storage.viking_fs import LS_ALL_NODES, SyncDiff, get_viking_fs
from openviking.telemetry import bind_telemetry, bind_telemetry_stage, resolve_telemetry
from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.telemetry.span_models import create_root_span_attributes
@@ -59,24 +58,6 @@ from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
-@dataclass
-class DiffResult:
- """Directory diff result for sync operations."""
-
- added_files: List[str] = field(default_factory=list)
- deleted_files: List[str] = field(default_factory=list)
- updated_files: List[str] = field(default_factory=list)
- added_dirs: List[str] = field(default_factory=list)
- deleted_dirs: List[str] = field(default_factory=list)
-
- def to_changes(self) -> Dict[str, List[str]]:
- return {
- "added": self.added_files + self.added_dirs,
- "modified": self.updated_files,
- "deleted": self.deleted_files + self.deleted_dirs,
- }
-
-
class RequestQueueStats:
processed: int = 0
requeue_count: int = 0
@@ -209,24 +190,6 @@ class SemanticProcessor(DequeueHandlerBase):
# Default to other
return FILE_TYPE_OTHER
- async def _check_file_content_changed(
- self, file_path: str, target_file: str, ctx: Optional[RequestContext] = None
- ) -> bool:
- """Check if file content has changed compared to target file."""
- viking_fs = get_viking_fs()
- try:
- current_stat = await viking_fs.stat(file_path, ctx=ctx)
- target_stat = await viking_fs.stat(target_file, ctx=ctx)
- current_size = current_stat.get("size") if isinstance(current_stat, dict) else None
- target_size = target_stat.get("size") if isinstance(target_stat, dict) else None
- if current_size is not None and target_size is not None and current_size != target_size:
- return True
- current_content = await viking_fs.read_file(file_path, ctx=ctx)
- target_content = await viking_fs.read_file(target_file, ctx=ctx)
- return current_content != target_content
- except Exception:
- return True
-
async def _reenqueue_semantic_msg(self, msg: SemanticMsg) -> None:
"""Re-enqueue a semantic message for later processing.
@@ -744,203 +707,39 @@ class SemanticProcessor(DequeueHandlerBase):
ctx: Optional[RequestContext] = None,
file_change_status: Optional[Dict[str, bool]] = None,
lock: Optional[Dict[str, Any]] = None,
- ) -> DiffResult:
+ ) -> SyncDiff:
+ """Merge a temp/staging source tree into the target via VikingFS.sync_tree.
+
+ The pure filesystem diff-and-move is delegated to
+ :py:meth:`VikingFS.sync_tree`; this wrapper carries over parser
+ sidecars (``.image_mappings.json``) and rewrites markdown image
+ references once the visible files have been moved into place.
+ Hidden entries are skipped by ``sync_tree``, which is why the
+ sidecar handling lives here rather than in the FS layer.
+ """
viking_fs = get_viking_fs()
if not await viking_fs.exists(root_uri, ctx=ctx):
raise FileNotFoundError(
f"Semantic source no longer exists; refusing to sync into {target_uri}: {root_uri}"
)
- diff = DiffResult()
-
- async def list_children(dir_uri: str) -> Tuple[Dict[str, str], Dict[str, str]]:
- files: Dict[str, str] = {}
- dirs: Dict[str, str] = {}
- entries = await viking_fs.ls(
- dir_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx
- )
-
- for entry in entries:
- name = entry.get("name", "")
- if not name or name in [".", ".."]:
- continue
- if name.startswith("."):
- continue
- item_uri = VikingURI(dir_uri).join(name).uri
- if entry.get("isDir", False):
- dirs[name] = item_uri
- else:
- files[name] = item_uri
- return files, dirs
-
- async def sync_dir(root_dir: str, target_dir: str) -> None:
- root_files, root_dirs = await list_children(root_dir)
- target_files, target_dirs = await list_children(target_dir)
-
- file_names = set(root_files.keys()) | set(target_files.keys())
- for name in sorted(file_names):
- root_file = root_files.get(name)
- target_file = target_files.get(name)
-
- if root_file and name in target_dirs:
- target_conflict_dir = target_dirs[name]
- try:
- await viking_fs.rm(
- target_conflict_dir,
- recursive=True,
- ctx=ctx,
- lease_ref=lock,
- )
- diff.deleted_dirs.append(target_conflict_dir)
- target_dirs.pop(name, None)
- except Exception as e:
- logger.error(
- f"[SyncDiff] Failed to delete directory for file conflict: {target_conflict_dir}, error={e}"
- )
- target_file = None
-
- if target_file and name in root_dirs and not root_file:
- try:
- await viking_fs.rm(target_file, ctx=ctx, lease_ref=lock)
- diff.deleted_files.append(target_file)
- target_files.pop(name, None)
- except Exception as e:
- logger.error(
- f"[SyncDiff] Failed to delete file for dir conflict: {target_file}, error={e}"
- )
- continue
-
- if target_file and not root_file:
- try:
- await viking_fs.rm(target_file, ctx=ctx, lease_ref=lock)
- diff.deleted_files.append(target_file)
- except Exception as e:
- logger.error(f"[SyncDiff] Failed to delete file: {target_file}, error={e}")
- continue
-
- if root_file and target_file:
- changed = False
- if file_change_status and root_file in file_change_status:
- changed = file_change_status[root_file]
- else:
- try:
- changed = await self._check_file_content_changed(
- root_file, target_file, ctx=ctx
- )
- except Exception as e:
- logger.error(
- f"[SyncDiff] Failed to compare file content for {root_file}: {e}, treating as unchanged"
- )
- changed = False
- if changed:
- diff.updated_files.append(target_file)
- try:
- await viking_fs.rm(target_file, ctx=ctx, lease_ref=lock)
- except Exception as e:
- logger.error(
- f"[SyncDiff] Failed to remove old file before update: {target_file}, error={e}"
- )
- try:
- await viking_fs.mv(
- root_file,
- target_file,
- ctx=ctx,
- lease_ref=lock,
- )
- except Exception as e:
- logger.error(
- f"[SyncDiff] Failed to move updated file: {root_file} -> {target_file}, error={e}"
- )
- continue
-
- if root_file and not target_file:
- target_file_uri = VikingURI(target_dir).join(name).uri
- diff.added_files.append(target_file_uri)
- try:
- await viking_fs.mv(
- root_file,
- target_file_uri,
- ctx=ctx,
- lease_ref=lock,
- )
- except Exception as e:
- logger.error(
- f"[SyncDiff] Failed to move added file: {root_file} -> {target_file_uri}, error={e}"
- )
-
- dir_names = set(root_dirs.keys()) | set(target_dirs.keys())
- for name in sorted(dir_names):
- root_subdir = root_dirs.get(name)
- target_subdir = target_dirs.get(name)
-
- if root_subdir and name in target_files:
- target_conflict_file = target_files[name]
- try:
- await viking_fs.rm(
- target_conflict_file,
- ctx=ctx,
- lease_ref=lock,
- )
- diff.deleted_files.append(target_conflict_file)
- target_files.pop(name, None)
- except Exception as e:
- logger.error(
- f"[SyncDiff] Failed to delete file for dir conflict: {target_conflict_file}, error={e}"
- )
- target_subdir = None
-
- if target_subdir and not root_subdir:
- try:
- await viking_fs.rm(
- target_subdir,
- recursive=True,
- ctx=ctx,
- lease_ref=lock,
- )
- diff.deleted_dirs.append(target_subdir)
- except Exception as e:
- logger.error(
- f"[SyncDiff] Failed to delete directory: {target_subdir}, error={e}"
- )
- continue
-
- if root_subdir and not target_subdir:
- target_subdir_uri = VikingURI(target_dir).join(name).uri
- diff.added_dirs.append(target_subdir_uri)
- try:
- await viking_fs.mv(
- root_subdir,
- target_subdir_uri,
- ctx=ctx,
- lease_ref=lock,
- )
- except Exception as e:
- logger.error(
- f"[SyncDiff] Failed to move added directory: {root_subdir} -> {target_subdir_uri}, error={e}"
- )
- continue
-
- if root_subdir and target_subdir:
- await sync_dir(root_subdir, target_subdir)
target_exists = await viking_fs.exists(target_uri, ctx=ctx)
+ diff = await viking_fs.sync_tree(
+ root_uri,
+ target_uri,
+ ctx=ctx,
+ file_change_status=file_change_status,
+ lease_ref=lock,
+ delete_temp_after=False,
+ )
+
if not target_exists:
- parent_uri = VikingURI(target_uri).parent
- if parent_uri:
- await viking_fs.mkdir(
- parent_uri.uri,
- exist_ok=True,
- ctx=ctx,
- lease_ref=lock,
- )
- diff.added_dirs.append(target_uri)
- await viking_fs.mv(root_uri, target_uri, ctx=ctx, lease_ref=lock)
# The whole temp tree (including the hidden .image_mappings.json
# sidecar) was moved into the target; rewrite local image paths now.
await self._rewrite_target_image_uris(root_uri, target_uri, ctx=ctx, lock=lock)
return diff
- await sync_dir(root_uri, target_uri)
- # sync_dir skips hidden files, so the .image_mappings.json sidecar is
+ # sync_tree skips hidden files, so the .image_mappings.json sidecar is
# still at the temp root. Carry it over and rewrite the synced markdown
# before the temp tree is deleted below.
await self._rewrite_target_image_uris(root_uri, target_uri, ctx=ctx, lock=lock)
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
deleted file mode 100644
index 59c6347dc..000000000
--- a/openviking/storage/viking_fs.py
+++ /dev/null
@@ -1,4513 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""
-VikingFS: OpenViking file system abstraction layer
-
-Encapsulates the AGFS binding client, providing file operation interface based on Viking URI.
-Responsibilities:
-- URI conversion (viking:// <-> /local/)
-- L0/L1 reading (.abstract.md, .overview.md)
-- Relation management (.relations.json)
-- Semantic search (vector retrieval + rerank)
-- Vector sync (sync vector store on rm/mv)
-"""
-
-import asyncio
-import contextvars
-import hashlib
-import json
-import os
-import re
-import time
-from contextlib import contextmanager
-from dataclasses import dataclass, field
-from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, TypeVar, Union
-
-from openviking.core.context import ContextLevel
-from openviking.core.namespace import (
- canonicalize_uri,
- is_hidden_by_actor_peer_view,
- may_include_hidden_actor_peers,
-)
-from openviking.core.namespace import (
- is_accessible as namespace_is_accessible,
-)
-from openviking.core.retrieval_targets import resolve_retrieval_targets
-from openviking.pyagfs import AsyncAGFSClient
-from openviking.pyagfs.exceptions import (
- AGFSClientError,
- AGFSDirectoryNotEmptyError,
- AGFSHTTPError,
- AGFSInvalidOperationError,
- AGFSNotFoundError,
- AGFSNotSupportedError,
- AGFSPathNotFoundError,
- AGFSResourceExhaustedError,
-)
-from openviking.resource.watch_storage import is_watch_task_control_uri
-from openviking.server.error_mapping import is_not_found_error, map_exception
-from openviking.server.identity import RequestContext, Role
-from openviking.storage.expr import And, PathScope, RawDSL
-from openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMES
-from openviking.telemetry import get_current_telemetry
-from openviking.utils.image_search import build_multimodal_embedding_input
-from openviking.utils.time_utils import format_iso8601, get_current_timestamp, parse_iso_datetime
-from openviking_cli.exceptions import (
- FailedPreconditionError,
- InvalidArgumentError,
- NotFoundError,
- PermissionDeniedError,
- ResourceExhaustedError,
-)
-from openviking_cli.session.user_id import UserIdentifier
-from openviking_cli.utils.config.grep_config import GrepEngine
-from openviking_cli.utils.logger import get_logger
-from openviking_cli.utils.uri import VikingURI
-
-if TYPE_CHECKING:
- from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
- from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig
-
-logger = get_logger(__name__)
-
-# Sentinel node_limit for internal callers that MUST enumerate an entire
-# directory. ``ls()`` defaults to ``node_limit=1000`` to protect agent-facing
-# context from being flooded, but internal system operations (parse merge,
-# temp->final sync, summary DAG, vectorization) must see every child or they
-# silently drop entries beyond the cap — e.g. a >1000-doc directory ingest only
-# materializes its first 1000 subdirectories. Pass this explicitly at those
-# call sites.
-LS_ALL_NODES = 2**31 - 1
-SNAPSHOT_DIFF_MAX_FILE_BYTES = 10 * 1024 * 1024
-SNAPSHOT_DIFF_MAX_OUTPUT_BYTES = 20 * 1024 * 1024
-SNAPSHOT_DIFF_MAX_LINES = 100_000
-SNAPSHOT_DIFF_TIMEOUT_MS = 500
-_T = TypeVar("_T")
-
-
-def _snapshot_line_count(text: str) -> int:
- if not text:
- return 0
- line_breaks = (
- "\n",
- "\r",
- "\v",
- "\f",
- "\x1c",
- "\x1d",
- "\x1e",
- "\x85",
- "\u2028",
- "\u2029",
- )
- count = sum(text.count(separator) for separator in line_breaks)
- count -= text.count("\r\n")
- return count if text.endswith(line_breaks) else count + 1
-
-
-def _prepare_snapshot_diff(
- *,
- path: str,
- before_bytes: Optional[bytes],
- after_bytes: Optional[bytes],
- max_lines: int,
-) -> tuple[str, str, str]:
- try:
- before = before_bytes.decode("utf-8") if before_bytes is not None else ""
- after = after_bytes.decode("utf-8") if after_bytes is not None else ""
- except UnicodeDecodeError as exc:
- raise InvalidArgumentError("snapshot diff only supports UTF-8 text files") from exc
-
- for text in (before, after):
- line_count = _snapshot_line_count(text)
- if line_count > max_lines:
- raise ResourceExhaustedError(
- f"snapshot diff line count limit exceeded ({max_lines} lines per file)",
- details={"limit_lines": max_lines, "path": path},
- )
-
- if before_bytes is None:
- change_type = "added"
- elif after_bytes is None:
- change_type = "deleted"
- elif before_bytes == after_bytes:
- change_type = "unchanged"
- else:
- change_type = "modified"
-
- return change_type, before, after
-
-
-def _ensure_non_empty_search_query(query: str, image_url: Optional[str] = None) -> None:
- if not query.strip() and not image_url:
- raise InvalidArgumentError("Search query or image_url must not be empty.")
-
-
-def _is_directory_not_empty_error(message: str) -> bool:
- """Check if an error message indicates a directory not empty error.
-
- Handles multiple possible error message formats from different backends.
- """
- msg = message.lower()
- return any(
- pattern in msg
- for pattern in [
- "directory not empty",
- "dir not empty",
- "directory is not empty",
- ]
- )
-
-
-def _get_cpu_count() -> int:
- """Return the number of CPUs available to this process.
-
- Tries process_cpu_count (Python 3.13+, cgroup-aware),
- falls back to sched_getaffinity (Linux),
- then os.cpu_count (may report host CPUs in containers).
- """
- if hasattr(os, "process_cpu_count"):
- return os.process_cpu_count() or 1
- try:
- return len(os.sched_getaffinity(0))
- except (AttributeError, NotImplementedError):
- return os.cpu_count() or 1
-
-
-def _get_abstract_worker_count() -> int:
- default = max(4, min(12, min(32, _get_cpu_count() + 4) // 2))
- env_val = os.getenv("OPENVIKING_FILE_OPS_CONCURRENCY")
- if env_val is not None:
- try:
- return max(1, int(env_val))
- except ValueError:
- pass
- return max(1, default)
-
-
-_ABSTRACT_WORKER_COUNT = _get_abstract_worker_count()
-_DEFAULT_GREP_FILE_CONCURRENCY = 32
-
-
-# ========== Dataclass ==========
-
-
-@dataclass
-class RelationEntry:
- """Relation table entry."""
-
- id: str
- uris: List[str]
- reason: str = ""
- created_at: str = field(default_factory=get_current_timestamp)
-
- def to_dict(self) -> Dict[str, Any]:
- return {
- "id": self.id,
- "uris": self.uris,
- "reason": self.reason,
- "created_at": self.created_at,
- }
-
- @staticmethod
- def from_dict(data: Dict[str, Any]) -> "RelationEntry":
- return RelationEntry(**data)
-
-
-# ========== Singleton Pattern ==========
-
-_instance: Optional["VikingFS"] = None
-
-
-def init_viking_fs(
- agfs: Any,
- query_embedder: Optional[Any] = None,
- rerank_config: Optional["RerankConfig"] = None,
- vector_store: Optional["VikingVectorIndexBackend"] = None,
- retrieval_config: Optional["RetrievalConfig"] = None,
- grep_config: Optional["GrepConfig"] = None,
- timeout: int = 10,
- enable_recorder: bool = False,
- encryptor: Optional[Any] = None,
-) -> "VikingFS":
- """Initialize VikingFS singleton.
-
- Args:
- agfs: Pre-initialized AGFS client (HTTP or Binding)
- query_embedder: Embedder instance
- rerank_config: Rerank configuration
- retrieval_config: Retrieval ranking configuration
- grep_config: Grep engine configuration
- vector_store: Vector store instance
- enable_recorder: Whether to enable IO recording
- encryptor: FileEncryptor instance for encryption/decryption
- """
- global _instance
-
- _instance = VikingFS(
- agfs=agfs,
- query_embedder=query_embedder,
- rerank_config=rerank_config,
- vector_store=vector_store,
- retrieval_config=retrieval_config,
- grep_config=grep_config,
- encryptor=encryptor,
- )
-
- if enable_recorder:
- _enable_viking_fs_recorder(_instance)
-
- return _instance
-
-
-def _enable_viking_fs_recorder(viking_fs: "VikingFS") -> None:
- """
- Enable recorder for a VikingFS instance.
-
- This wraps the VikingFS instance with recording capabilities.
- Called automatically when enable_recorder=True in init_viking_fs.
-
- Args:
- viking_fs: VikingFS instance to enable recording for
- """
- from openviking.eval.recorder import RecordingVikingFS, get_recorder
-
- recorder = get_recorder()
- if not recorder.enabled:
- from openviking.eval.recorder import init_recorder
-
- init_recorder(enabled=True)
-
- global _instance
- _instance = RecordingVikingFS(viking_fs)
- logger.info("[VikingFS] IO Recorder enabled")
-
-
-def enable_viking_fs_recorder() -> None:
- """
- Enable recorder for the global VikingFS singleton.
-
- This function wraps the existing VikingFS's AGFS client with recording.
- Must be called after init_viking_fs().
- """
- global _instance
- if _instance is None:
- raise RuntimeError("VikingFS not initialized. Call init_viking_fs() first.")
- _enable_viking_fs_recorder(_instance)
-
-
-def get_viking_fs() -> "VikingFS":
- """Get VikingFS singleton."""
- if _instance is None:
- raise RuntimeError("VikingFS not initialized. Call init_viking_fs() first.")
- return _instance
-
-
-# ========== VikingFS Main Class ==========
-
-
-class VikingFS:
- """RAGFS-based OpenViking file system.
-
- APIs are divided into two categories:
- - RAGFS basic commands (direct forwarding): read, ls, write, mkdir, rm, mv, grep, stat
- - VikingFS specific capabilities: abstract, overview, find, search, relations, link, unlink
-
- Uses Rust binding mode: Use RAGFSBindingClient to directly use RAGFS implementation
- """
-
- def set_user_deletion_guard(self, guard: Optional[Callable[[str, str], bool]]) -> None:
- self._user_deletion_guard = guard
-
- def __init__(
- self,
- agfs: Any,
- query_embedder: Optional[Any] = None,
- rerank_config: Optional["RerankConfig"] = None,
- vector_store: Optional["VikingVectorIndexBackend"] = None,
- retrieval_config: Optional["RetrievalConfig"] = None,
- grep_config: Optional["GrepConfig"] = None,
- timeout: int = 10,
- encryptor: Optional[Any] = None,
- ):
- self.agfs = agfs
- self._async_agfs = AsyncAGFSClient(agfs)
- self.query_embedder = query_embedder
- self.rerank_config = rerank_config
- self.vector_store = vector_store
- self.retrieval_config = retrieval_config
- self.grep_config = grep_config
- self._encryptor = encryptor
- self._count_cache: Dict[str, tuple] = {} # cache_key → (count, timestamp)
- self._count_cache_max_size = 1024
- self._fulltext_available: Optional[bool] = None # cached result of _collection_has_fulltext
- self._bound_ctx: contextvars.ContextVar[Optional[RequestContext]] = contextvars.ContextVar(
- "vikingfs_bound_ctx", default=None
- )
- self._background_tasks: set = set()
- self._user_deletion_guard: Optional[Callable[[str, str], bool]] = None
-
- @staticmethod
- def _default_ctx() -> RequestContext:
- return RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
-
- def _ctx_or_default(self, ctx: Optional[RequestContext]) -> RequestContext:
- if ctx is not None:
- return ctx
- bound = self._bound_ctx.get()
- return bound or self._default_ctx()
-
- @contextmanager
- def bind_request_context(self, ctx: RequestContext):
- """Temporarily bind ctx for legacy internal call paths without explicit ctx param."""
- token = self._bound_ctx.set(ctx)
- try:
- yield
- finally:
- self._bound_ctx.reset(token)
-
- @staticmethod
- def _normalize_uri(uri: str) -> str:
- """Normalize short-format URIs to the canonical viking:// form."""
- if uri.startswith("viking://"):
- return uri
- return VikingURI.normalize(uri)
-
- @classmethod
- def _normalized_uri_parts(cls, uri: str) -> tuple[str, List[str]]:
- """Normalize a URI and reject ambiguous or platform-specific path traversal forms."""
- normalized = cls._normalize_uri(uri)
- parts = [p for p in normalized[len("viking://") :].strip("/").split("/") if p]
-
- for part in parts:
- if part in {".", ".."}:
- raise PermissionDeniedError(
- f"Unsafe URI traversal segment '{part}' in {normalized}",
- resource=normalized,
- )
- if "\\" in part:
- raise PermissionDeniedError(
- f"Unsafe URI path separator '\\\\' in component '{part}' of {normalized}",
- resource=normalized,
- )
- if len(part) >= 2 and part[1] == ":" and part[0].isalpha():
- raise PermissionDeniedError(
- f"Unsafe URI drive-prefixed component '{part}' in {normalized}",
- resource=normalized,
- )
-
- return normalized, parts
-
- # TODO: Once pathlock moves down into ragfs, stop reconstructing the
- # encrypted mount-relative path in Python and derive the lock target from
- # the same backend-side source of truth.
- def _encrypted_mount_relative_path(self, path: str) -> tuple[str, str]:
- """Return the mount prefix and mount-relative path used by Rust encrypted writes."""
- normalized = path if path.startswith("/") else f"/{path}"
- parts = [part for part in normalized.split("/") if part]
- if len(parts) < 2:
- return "", normalized
- return f"/{parts[0]}", f"/{'/'.join(parts[1:])}"
-
- def _encrypted_temp_path(self, path: str) -> str:
- """Build the deterministic internal `.encrypt` temp-file path for a final file path."""
- mount_prefix, relative_path = self._encrypted_mount_relative_path(path)
- relative_parts = [part for part in relative_path.split("/") if part]
- if len(relative_parts) >= 2 and relative_parts[0] == "local":
- temp_root = f"/local/{relative_parts[1]}/temp/.encrypt_stage"
- elif len(relative_parts) >= 2:
- temp_root = f"/{relative_parts[0]}/temp/.encrypt_stage"
- else:
- temp_root = "/temp/.encrypt_stage"
- digest = hashlib.sha256(relative_path.encode("utf-8")).hexdigest()
- return f"{mount_prefix}{temp_root}/{digest}.encrypt"
-
- def _ensure_access(self, uri: str, ctx: Optional[RequestContext]) -> None:
- real_ctx = self._ctx_or_default(ctx)
- normalized_uri, _ = self._normalized_uri_parts(uri)
- if not self._is_accessible(normalized_uri, real_ctx):
- raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)
-
- def _ensure_mutable_access(self, uri: str, ctx: Optional[RequestContext]) -> None:
- self._ensure_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- self._ensure_user_not_deleting(real_ctx)
- normalized_uri, _ = self._normalized_uri_parts(uri)
- if is_hidden_by_actor_peer_view(normalized_uri, real_ctx) or may_include_hidden_actor_peers(
- normalized_uri, real_ctx
- ):
- raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)
- self._ensure_supported_write_namespace(normalized_uri)
- if real_ctx.role != Role.ROOT and normalized_uri.rstrip("/") == "viking://temp":
- raise PermissionDeniedError(
- "Temp root is read-only for non-root users",
- resource=normalized_uri,
- )
-
- def _ensure_delete_access(self, uri: str, ctx: Optional[RequestContext]) -> None:
- self._ensure_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- self._ensure_user_not_deleting(real_ctx)
- normalized_uri, _ = self._normalized_uri_parts(uri)
- if is_hidden_by_actor_peer_view(normalized_uri, real_ctx) or may_include_hidden_actor_peers(
- normalized_uri, real_ctx
- ):
- raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)
- self._ensure_supported_delete_namespace(normalized_uri)
- if real_ctx.role != Role.ROOT and normalized_uri.rstrip("/") == "viking://temp":
- raise PermissionDeniedError(
- "Temp root is read-only for non-root users",
- resource=normalized_uri,
- )
-
- def _ensure_user_not_deleting(self, ctx: RequestContext) -> None:
- if (
- ctx.role != Role.ROOT
- and self._user_deletion_guard is not None
- and self._user_deletion_guard(ctx.account_id, ctx.user.user_id)
- ):
- raise FailedPreconditionError("User deletion is in progress")
-
- def _ensure_supported_delete_namespace(self, normalized_uri: str) -> None:
- parts = [p for p in normalized_uri[len("viking://") :].strip("/").split("/") if p]
- if not parts:
- raise PermissionDeniedError(
- "Deleting viking:// is not supported; use a concrete scope instead.",
- resource=normalized_uri,
- )
- if parts == ["user"]:
- raise PermissionDeniedError(
- "Deleting viking://user is not supported; use an explicit user namespace "
- "or current-user content path instead.",
- resource=normalized_uri,
- )
- if parts == ["agent"]:
- # Parity with _ensure_supported_write_namespace, which forbids the
- # account-shared viking://agent root: deleting it would recursively
- # wipe every account's agent skills/endpoints/tools/payments. Concrete
- # sub-paths (viking://agent/skills/...) remain deletable.
- raise PermissionDeniedError(
- "Deleting viking://agent root is not supported; use a concrete "
- "agent sub-path (e.g. viking://agent/skills/...) instead.",
- resource=normalized_uri,
- )
-
- def _ensure_supported_write_namespace(self, normalized_uri: str) -> None:
- parts = [p for p in normalized_uri[len("viking://") :].strip("/").split("/") if p]
- if parts == ["user"]:
- raise PermissionDeniedError(
- "Writing viking://user is not supported; use an explicit user namespace "
- "or current-user content path instead.",
- resource=normalized_uri,
- )
- if parts and parts[0] == "session":
- raise PermissionDeniedError(
- f"Writing {normalized_uri} is not supported; use user-owned namespaces instead.",
- resource=normalized_uri,
- )
- if parts and parts[0] == "agent":
- if len(parts) >= 2 and parts[1] not in {"skills", "endpoints", "tools", "payments"}:
- raise PermissionDeniedError(
- "viking://agent/{agent_id} is deprecated. Use viking://user/.../peers/{agent_id} instead.",
- resource=normalized_uri,
- )
- if len(parts) < 2:
- raise PermissionDeniedError(
- "Writing to viking://agent root is not supported.",
- resource=normalized_uri,
- )
-
- # ========== AGFS Basic Commands ==========
-
- async def read(
- self,
- uri: str,
- offset: int = 0,
- size: int = -1,
- ctx: Optional[RequestContext] = None,
- ) -> bytes:
- """Read file"""
- self._ensure_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- primary_path = self._uri_to_path(uri, ctx=ctx)
-
- # Decryption + offset/size slicing now happen inside the ragfs encryption layer
- # (when configured); the plaintext stack reads bytes directly. Either way, pass the
- # offset/size through and let the Rust layer return the requested slice.
- last_not_found: Optional[Exception] = None
- for path in self._read_paths(uri, ctx=ctx):
- if not await self._read_path_visible(uri, path, primary_path, real_ctx):
- continue
- try:
- result = await self._async_agfs.read(path, offset, size)
- break
- except Exception as exc:
- if is_not_found_error(exc):
- last_not_found = exc
- continue
- raise
- else:
- raise NotFoundError(uri, "file") from last_not_found
- if isinstance(result, bytes):
- raw = result
- elif result is not None and hasattr(result, "content"):
- raw = result.content
- else:
- raw = b""
-
- return raw
-
- async def write(
- self,
- uri: str,
- data: Union[bytes, str],
- ctx: Optional[RequestContext] = None,
- ) -> str:
- """Write file"""
- self._ensure_mutable_access(uri, ctx)
- path = self._uri_to_path(uri, ctx=ctx)
- if isinstance(data, str):
- data = data.encode("utf-8")
-
- # Encryption (when configured) happens inside the ragfs layer keyed by account_id.
- return await self._async_agfs.write(path, data)
-
- def _pathlock_fs_ctx(
- self,
- ctx: Optional[RequestContext],
- lease_ref: Optional[Dict[str, Any] | str],
- ) -> Dict[str, str] | None:
- """Build an AGFS fs_ctx carrying account_id and an opaque pathlock lease_ref."""
- if lease_ref is None:
- return None
- if isinstance(lease_ref, str):
- ref = lease_ref
- elif isinstance(lease_ref, dict):
- ref = lease_ref.get("lease_ref")
- else:
- raise ValueError("lease_ref must be a non-empty string or lease dictionary")
- if not isinstance(ref, str) or not ref:
- raise ValueError("lease_ref must contain a non-empty lease_ref")
- return {"account_id": self._ctx_or_default(ctx).account_id, "lease_ref": ref}
-
- async def mkdir(
- self,
- uri: str,
- mode: str = "755",
- exist_ok: bool = False,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> None:
- """Create directory."""
- self._ensure_mutable_access(uri, ctx)
- path = self._uri_to_path(uri, ctx=ctx)
- # Always ensure parent directories exist before creating this directory
- await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)
- try:
- await self._async_agfs.mkdir(path, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))
- except Exception as exc:
- message = str(exc).lower()
- already_exists = "exist" in message or "already" in message
- if exist_ok and already_exists:
- return
- raise
-
- async def rm(
- self,
- uri: str,
- recursive: bool = False,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> Dict[str, Any]:
- """Delete file/directory + recursively update vector index.
-
- This method is idempotent: deleting a non-existent file succeeds
- after cleaning up any orphan index records.
-
- Acquires a path lock, deletes VectorDB records, then FS files.
- Raises ResourceBusyError when the target is locked by an ongoing
- operation (e.g. semantic processing).
-
- Returns:
- Dict with 'estimated_deleted_count' indicating the estimated number
- of nodes deleted from vector index.
- """
- from openviking.storage.errors import LockAcquisitionError, ResourceBusyError
-
- self._ensure_delete_access(uri, ctx)
- path = self._uri_to_path(uri, ctx=ctx)
- target_uri = self._path_to_uri(path, ctx=ctx)
-
- async def _estimate_deleted_count(target_path: str, real_ctx: RequestContext) -> int:
- """Estimate number of nodes to be deleted using vector index."""
- vector_store = self._get_vector_store()
- if not vector_store:
- return 0
- try:
- target_canonical_uri = canonicalize_uri(
- self._path_to_uri(target_path, ctx=real_ctx), real_ctx
- )
- filter_expr = PathScope("uri", target_canonical_uri, depth=-1)
- return await vector_store.count(filter=filter_expr, ctx=real_ctx)
- except Exception as e:
- logger.warning(f"[VikingFS] Failed to count nodes before delete: {e}")
- return 0
-
- # Check existence and determine lock strategy
- try:
- stat = await self._async_agfs.stat(path)
- is_dir = stat.get("isDir", False) if isinstance(stat, dict) else False
- except Exception as exc:
- if not is_not_found_error(exc):
- mapped = map_exception(exc, resource=uri)
- if mapped is not None:
- raise mapped from exc
- raise
- # Path does not exist: clean up any orphan index records and return
- uris_to_delete = await self._collect_uris(path, recursive, ctx=ctx)
- uris_to_delete.append(target_uri)
- real_ctx = self._ctx_or_default(ctx)
- estimated_count = await _estimate_deleted_count(path, real_ctx)
- await self._delete_from_vector_store(uris_to_delete, ctx=ctx)
- logger.info(f"[VikingFS] rm target not found, cleaned orphan index: {uri}")
- return {"estimated_deleted_count": estimated_count}
-
- if is_dir:
- if not recursive:
- raise FailedPreconditionError(
- f"Cannot remove directory without --recursive: {uri}",
- details={"resource": uri, "expected_flag": "recursive"},
- )
- lock_method = self._async_agfs.pathlock_acquire_tree
- else:
- lock_method = self._async_agfs.pathlock_acquire_exact
-
- lease = lease_ref
- if lease is None:
- try:
- lease = await lock_method(path)
- except LockAcquisitionError:
- raise ResourceBusyError(f"Resource is being processed: {uri}", uri=uri)
-
- try:
- uris_to_delete = await self._collect_uris(path, recursive, ctx=ctx) if is_dir else []
- uris_to_delete.append(target_uri)
- real_ctx = self._ctx_or_default(ctx)
- estimated_count = await _estimate_deleted_count(path, real_ctx)
- await self._delete_from_vector_store(uris_to_delete, ctx=ctx)
- try:
- result = await self._async_agfs.rm(
- path,
- recursive=recursive,
- fs_ctx=self._pathlock_fs_ctx(ctx, lease),
- )
- except AGFSDirectoryNotEmptyError:
- raise FailedPreconditionError(
- f"Directory not empty: {uri}. Use recursive=True to delete non-empty directories."
- )
- except RuntimeError as e:
- # Fallback for older versions without typed exceptions
- if _is_directory_not_empty_error(str(e)):
- raise FailedPreconditionError(
- f"Directory not empty: {uri}. Use recursive=True to delete non-empty directories."
- )
- raise
- # Add estimated_deleted_count to the result
- if isinstance(result, dict):
- result["estimated_deleted_count"] = estimated_count
- else:
- result = {"estimated_deleted_count": estimated_count}
- return result
- finally:
- if lease_ref is None:
- await self._async_agfs.pathlock_release(lease)
-
- async def mv(
- self,
- old_uri: str,
- new_uri: str,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> Dict[str, Any]:
- """Move file/directory while extending an optional outer pathlock lease.
-
- Implemented as cp + rm to avoid lock files being carried by FS mv.
- On VectorDB update failure the copy is cleaned up so the source stays intact.
- """
-
- self._ensure_mutable_access(old_uri, ctx)
- # mv is implemented as copy + recursive rm of the source (see the
- # ``rm(old_path, recursive=is_dir)`` below), so the source must also clear
- # the delete guard. Without this, a protected account root such as
- # ``viking://`` — which rm() rejects up front (#2873) — could still be
- # destroyed via mv, since the write guard alone permits the bare root.
- self._ensure_delete_access(old_uri, ctx)
- self._ensure_mutable_access(new_uri, ctx)
- old_path = self._uri_to_path(old_uri, ctx=ctx)
- new_path = self._uri_to_path(new_uri, ctx=ctx)
- target_uri = self._path_to_uri(old_path, ctx=ctx)
-
- # Verify source exists and determine type before locking.
- try:
- stat = await self._async_agfs.stat(old_path)
- is_dir = stat.get("isDir", False) if isinstance(stat, dict) else False
- except Exception as exc:
- if not is_not_found_error(exc):
- mapped = map_exception(exc, resource=old_uri)
- if mapped is not None:
- raise mapped from exc
- raise
- raise FileNotFoundError(f"mv source not found: {old_uri}") from exc
-
- if not is_dir:
- if new_uri.rstrip("/") != new_uri:
- raise InvalidArgumentError(
- f"mv destination for a file must include the target file name: {new_uri}",
- details={"from_uri": old_uri, "to_uri": new_uri},
- )
- try:
- destination_stat = await self._async_agfs.stat(new_path)
- except Exception as exc:
- if not is_not_found_error(exc):
- mapped = map_exception(exc, resource=new_uri)
- if mapped is not None:
- raise mapped from exc
- raise
- else:
- if isinstance(destination_stat, dict) and destination_stat.get("isDir", False):
- raise InvalidArgumentError(
- f"mv destination for a file must include the target file name: {new_uri}",
- details={"from_uri": old_uri, "to_uri": new_uri},
- )
-
- real_ctx = self._ctx_or_default(ctx)
- old_uri = canonicalize_uri(old_uri, real_ctx)
- new_uri = canonicalize_uri(new_uri, real_ctx)
-
- if is_dir:
- lease = await self._async_agfs.pathlock_acquire_batch(
- [
- {"path": old_path, "kind": "tree"},
- {"path": new_path, "kind": "exact"},
- ],
- owner_lease_ref=lease_ref,
- )
- else:
- lease = await self._async_agfs.pathlock_acquire_batch(
- [
- {"path": old_path, "kind": "exact"},
- {"path": new_path, "kind": "exact"},
- ],
- owner_lease_ref=lease_ref,
- )
-
- try:
- uris_to_move = (
- await self._collect_uris(old_path, recursive=True, ctx=ctx) if is_dir else []
- )
- uris_to_move.append(target_uri)
-
- # Check if it's temp directory (files already encrypted)
- is_temp = old_uri.startswith("viking://temp/")
-
- # Copy source to destination. Source must stay intact until vector updates succeed.
- try:
- await self._copy_for_mv(
- old_uri=old_uri,
- new_uri=new_uri,
- old_path=old_path,
- new_path=new_path,
- is_dir=is_dir,
- is_temp=is_temp,
- ctx=ctx,
- lease_ref=lease,
- )
- except Exception as e:
- if "not found" in str(e).lower():
- try:
- await self._delete_from_vector_store(uris_to_move, ctx=ctx)
- except Exception:
- # Orphan cleanup is best effort here; preserve the copy error.
- pass
- else:
- logger.info(
- f"[VikingFS] mv source not found, cleaned orphan index: {old_uri}"
- )
- raise
-
- # Update VectorDB URIs (on failure, clean up the copy)
- try:
- await self._update_vector_store_uris(uris_to_move, old_uri, new_uri, ctx=ctx)
- except Exception:
- try:
- if is_dir:
- cleanup_lease = await self._async_agfs.pathlock_acquire_tree(
- new_path,
- owner_lease_ref=lease,
- )
- try:
- await self._async_agfs.rm(
- new_path,
- recursive=True,
- fs_ctx=self._pathlock_fs_ctx(ctx, cleanup_lease),
- )
- finally:
- await self._async_agfs.pathlock_release(cleanup_lease)
- else:
- await self._async_agfs.rm(
- new_path,
- fs_ctx=self._pathlock_fs_ctx(ctx, lease),
- )
- except Exception:
- pass
- raise
-
- # Delete source
- await self._async_agfs.rm(
- old_path,
- recursive=is_dir,
- fs_ctx=self._pathlock_fs_ctx(ctx, lease),
- )
- return {}
- finally:
- await self._async_agfs.pathlock_release(lease)
-
- async def system_sync_status(
- self, uri: str, ctx: Optional[RequestContext] = None
- ) -> Dict[str, Any]:
- """Return multi-write sync status for one Viking URI subtree."""
- self._ensure_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- path = self._uri_to_path(uri, ctx=ctx)
- return await self._async_agfs.system_sync_status(
- path,
- fs_ctx={"account_id": real_ctx.account_id},
- )
-
- async def system_sync_retry(
- self, uri: str, ctx: Optional[RequestContext] = None
- ) -> Dict[str, Any]:
- """Retry multi-write sync for one Viking URI subtree."""
- self._ensure_mutable_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- path = self._uri_to_path(uri, ctx=ctx)
- return await self._async_agfs.system_sync_retry(
- path,
- fs_ctx={"account_id": real_ctx.account_id},
- )
-
- async def _copy_for_mv(
- self,
- old_uri: str,
- new_uri: str,
- old_path: str,
- new_path: str,
- is_dir: bool,
- is_temp: bool,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> None:
- """Copy source to destination for mv without deleting source."""
- if is_temp:
- if is_dir:
- await self._copy_temp_dir_with_exact_locks(
- old_path,
- new_path,
- ctx=ctx,
- lease_ref=lease_ref,
- )
- else:
- await self._async_agfs.cp(
- old_path,
- new_path,
- recursive=False,
- fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),
- )
- return
-
- if is_dir:
- await self._copy_dir_through_vikingfs(old_uri, new_uri, ctx=ctx, lease_ref=lease_ref)
- else:
- await self._copy_file_through_vikingfs(old_uri, new_uri, ctx=ctx, lease_ref=lease_ref)
-
- async def _copy_temp_dir_with_exact_locks(
- self,
- old_path: str,
- new_path: str,
- ctx: Optional[RequestContext],
- lease_ref: Dict[str, Any] | None,
- ) -> None:
- """Copy an encrypted temp directory while locking every destination entry.
-
- Args:
- old_path: Source backend directory path.
- new_path: Destination backend directory path.
- ctx: Request context used for filesystem operations.
- lease_ref: Exact lease covering the current destination directory.
-
- Returns:
- None.
- """
- if lease_ref is None:
- raise ValueError("temp directory copy requires a pathlock lease")
- fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)
- await self._async_agfs.mkdir(new_path, fs_ctx=fs_ctx)
- entries = await self._async_agfs.ls(old_path, fs_ctx=fs_ctx)
- for entry in entries:
- name = entry.get("name", "")
- if not name or name in (".", ".."):
- continue
- old_child = f"{old_path.rstrip('/')}/{name}"
- new_child = f"{new_path.rstrip('/')}/{name}"
- child_lease = await self._async_agfs.pathlock_acquire_exact(
- new_child,
- owner_lease_ref=lease_ref,
- )
- try:
- if entry.get("isDir", False):
- await self._copy_temp_dir_with_exact_locks(
- old_child,
- new_child,
- ctx=ctx,
- lease_ref=child_lease,
- )
- else:
- await self._async_agfs.cp(
- old_child,
- new_child,
- recursive=False,
- fs_ctx=self._pathlock_fs_ctx(ctx, child_lease),
- )
- finally:
- await self._async_agfs.pathlock_release(child_lease)
-
- async def _copy_dir_through_vikingfs(
- self,
- old_uri: str,
- new_uri: str,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> None:
- """Recursively copy a directory through VikingFS read/write hooks."""
- await self.mkdir(new_uri, exist_ok=True, ctx=ctx, lease_ref=lease_ref)
-
- entries = await self.ls(old_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx)
- for entry in entries:
- name = entry.get("name", "")
- if not name or name in (".", ".."):
- continue
- old_child_uri = f"{old_uri.rstrip('/')}/{name}"
- new_child_uri = f"{new_uri.rstrip('/')}/{name}"
- if entry.get("isDir"):
- await self._copy_dir_through_vikingfs(
- old_child_uri,
- new_child_uri,
- ctx=ctx,
- lease_ref=lease_ref,
- )
- else:
- await self._copy_file_through_vikingfs(
- old_child_uri,
- new_child_uri,
- ctx=ctx,
- lease_ref=lease_ref,
- )
-
- async def _copy_file_through_vikingfs(
- self,
- from_uri: str,
- to_uri: str,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> None:
- """Copy one file through VikingFS read/write hooks without deleting source."""
- content_bytes = await self.read_file_bytes(from_uri, ctx=ctx)
- if lease_ref is None:
- await self.write_file_bytes(to_uri, content_bytes, ctx=ctx)
- return
-
- child_path = self._uri_to_path(to_uri, ctx=ctx)
- child_lease = await self._async_agfs.pathlock_acquire_exact(
- child_path,
- owner_lease_ref=lease_ref,
- )
- try:
- await self.write_file_bytes(to_uri, content_bytes, ctx=ctx, lease_ref=child_lease)
- finally:
- await self._async_agfs.pathlock_release(child_lease)
-
- async def grep(
- self,
- uri: str,
- pattern: str,
- exclude_uri: Optional[str] = None,
- case_insensitive: bool = False,
- node_limit: Optional[int] = None,
- level_limit: int = 10,
- ctx: Optional[RequestContext] = None,
- content_transform: Optional[Callable[[str, str], str]] = None,
- ) -> Dict:
- """Content search by pattern or keywords.
-
- Optimized implementation that uses agfs native grep when possible.
- The ragfs layer greps transparently over encrypted and plaintext files
- (it decrypts via account_id when an encryption layer is configured).
- Falls back to VikingFS layer implementation if native grep is unavailable.
- When engine="auto" and vikingdb is available with sufficient data,
- uses vikingdb bm25 recall + local fs precise matching.
-
- Args:
- uri: Viking URI
- pattern: Regular expression pattern to search for
- exclude_uri: Optional URI prefix to exclude from search
- case_insensitive: Whether to perform case-insensitive matching
- node_limit: Maximum number of results to return
- level_limit: Maximum depth level to traverse (default: 10)
- ctx: Request context
- content_transform: Optional projection applied before regex matching.
- Internal bm25 recall limit is auto-adapted from node_limit as
- min(node_limit * 5, 100000); when node_limit is unset, use 100000.
-
- Returns:
- Dict with matches, count, match_count, files_scanned
- """
- real_ctx = self._ctx_or_default(ctx)
- uri = canonicalize_uri(uri, real_ctx)
- if exclude_uri:
- exclude_uri = canonicalize_uri(exclude_uri, real_ctx)
- self._ensure_access(uri, ctx)
- # Skip vector_store.count() — the count field is not needed for grep,
- # and avoiding it saves one VikingDB API call.
- await self.stat(uri, ctx=ctx, skip_count=True)
-
- # Read engine and threshold from grep_config (ov.conf)
- engine = self.grep_config.engine if self.grep_config else "auto"
- switch_to_remote_threshold = (
- self.grep_config.switch_to_remote_threshold if self.grep_config else 10000
- )
-
- # A projection must run before matching. The remote BM25 index contains
- # persisted raw content, so it cannot safely recall projected results.
- resolved_engine = (
- "fs"
- if content_transform is not None
- else await self._resolve_grep_engine(engine, uri, ctx, switch_to_remote_threshold)
- )
-
- if resolved_engine == "fs":
- return await self._grep_fs(
- uri=uri,
- pattern=pattern,
- exclude_uri=exclude_uri,
- case_insensitive=case_insensitive,
- node_limit=node_limit,
- level_limit=level_limit,
- ctx=ctx,
- content_transform=content_transform,
- )
- else: # "vikingdb_then_fs"
- return await self._grep_vikingdb_then_fs(
- uri=uri,
- pattern=pattern,
- exclude_uri=exclude_uri,
- case_insensitive=case_insensitive,
- node_limit=node_limit,
- level_limit=level_limit,
- ctx=ctx,
- )
-
- async def _resolve_grep_engine(
- self, engine: GrepEngine, uri: str, ctx, switch_to_remote_threshold: int = 10000
- ) -> str:
- """Resolve the actual grep engine to use."""
- if engine == "fs":
- return "fs"
-
- # auto mode: check vikingdb availability
- vector_store = self._get_vector_store()
- if not vector_store:
- return "fs"
-
- backend_type = getattr(vector_store, "_backend_type", "unknown")
- # Keep this set consistent with ``CollectionAdapter.USE_CONTENT_FIELD``:
- # only these backends store the ``content`` field required for full-text grep.
- if backend_type not in ("volcengine", "vikingdb"):
- return "fs"
-
- # Check collection has content field and FullText config
- if not await self._collection_has_fulltext(vector_store, ctx):
- return "fs"
-
- # switch_to_remote_threshold=0 means always use vikingdb
- if switch_to_remote_threshold == 0:
- return "vikingdb_then_fs"
-
- # Check data volume threshold
- try:
- count = await self._get_cached_count(uri, ctx)
- if count < switch_to_remote_threshold:
- return "fs"
- except Exception:
- logger.debug(
- "grep engine=auto: count() check failed, falling back to fs", exc_info=True
- )
- return "fs"
-
- return "vikingdb_then_fs"
-
- async def _collection_has_fulltext(self, vector_store, ctx) -> bool:
- """Check if collection has content field and FullText config.
-
- Result is cached on the VikingFS instance since collection schema
- does not change at runtime.
- """
- if self._fulltext_available is not None:
- return self._fulltext_available
- try:
- meta = None
- if hasattr(vector_store, "get_collection_meta"):
- meta = await vector_store.get_collection_meta(ctx=ctx)
- if not meta:
- self._fulltext_available = False
- return False
- fields = meta.get("Fields", [])
- has_content = any(
- f.get("FieldName") == "content" and f.get("FieldType") == "text" for f in fields
- )
- fulltext = meta.get("FullText") or []
- has_content_fulltext = any(ft.get("Field") == "content" for ft in fulltext)
- result = has_content and has_content_fulltext
- self._fulltext_available = result
- return result
- except Exception:
- logger.debug(
- "Failed to check collection fulltext config, assuming no fulltext", exc_info=True
- )
- return False
-
- async def _get_cached_count(self, uri: str, ctx) -> int:
- """Get cached count of records for a URI (TTL=1h)."""
- _COUNT_CACHE_TTL = 3600
- vector_store = self._get_vector_store()
-
- # Include account_id in cache key for multi-tenant safety
- account_id = getattr(ctx, "account_id", None) if ctx else None
- cache_key = f"{account_id}:{uri}" if account_id else uri
-
- now = time.time()
- cached = self._count_cache.get(cache_key)
- if cached and (now - cached[1]) < _COUNT_CACHE_TTL:
- return cached[0]
-
- count = await vector_store.count(filter=PathScope("uri", uri, depth=-1), ctx=ctx)
- # Evict oldest entries if cache exceeds max size
- if len(self._count_cache) >= self._count_cache_max_size:
- oldest_keys = sorted(self._count_cache, key=lambda k: self._count_cache[k][1])
- for k in oldest_keys[: len(oldest_keys) // 2]:
- del self._count_cache[k]
- self._count_cache[cache_key] = (count, now)
- return count
-
- async def _grep_fs(
- self,
- uri,
- pattern,
- exclude_uri,
- case_insensitive,
- node_limit,
- level_limit,
- ctx,
- content_transform=None,
- ):
- """Filesystem grep path: prefer native agfs grep and fall back if unavailable."""
- if content_transform is None:
- try:
- return await self._grep_with_agfs(
- uri=uri,
- pattern=pattern,
- exclude_uri=exclude_uri,
- case_insensitive=case_insensitive,
- node_limit=node_limit,
- level_limit=level_limit,
- ctx=ctx,
- )
- except (AttributeError, AGFSNotSupportedError, NotImplementedError) as e:
- logger.debug(f"agfs grep unavailable, falling back to VikingFS implementation: {e}")
-
- return await self._grep_encrypted(
- uri=uri,
- pattern=pattern,
- exclude_uri=exclude_uri,
- case_insensitive=case_insensitive,
- node_limit=node_limit,
- level_limit=level_limit,
- ctx=ctx,
- content_transform=content_transform,
- )
-
- async def _grep_vikingdb_then_fs(
- self,
- uri,
- pattern,
- exclude_uri,
- case_insensitive,
- node_limit,
- level_limit,
- ctx,
- ):
- """VikingDB bm25 recall + local fs precise matching."""
- vector_store = self._get_vector_store()
-
- # Split regex alternation (e.g. "error|warning|fail") and join as a
- # single query string for bm25 search. VikingDB's standard tokenizer
- # will handle the tokenization of the query string.
- query = " ".join(kw.strip() for kw in pattern.split("|") if kw.strip())
- filter_expr = PathScope("uri", uri, depth=level_limit)
- excluded_prefix = None
- if exclude_uri:
- excluded_prefix = exclude_uri.rstrip("/")
- self._ensure_access(excluded_prefix, ctx)
- filter_expr = And([
- filter_expr,
- RawDSL({
- "op": "must_not",
- "field": "uri",
- "conds": [excluded_prefix],
- "para": "-d=-1",
- }),
- ])
-
- # Auto-adapt bm25 recall limit: recall up to 5x requested matches
- # while capping at VikingDB's max limit. If node_limit is unset,
- # use the maximum limit to avoid truncation.
- remote_return_limit = min(node_limit * 5, 100000) if node_limit else 100000
-
- # Step 1: vikingdb recall candidate files
- try:
- logger.debug(
- "grep vikingdb search_by_keywords request: query=%r limit=%s filter=%r "
- "output_fields=%s",
- query,
- remote_return_limit,
- filter_expr,
- ["uri"],
- )
- result = await vector_store.search_by_keywords(
- query=query,
- limit=remote_return_limit,
- filter=filter_expr,
- output_fields=["uri"],
- ctx=ctx,
- )
- except Exception as e:
- logger.warning(f"grep vikingdb step failed, falling back to fs: {e}")
- return await self._grep_fs(
- uri=uri,
- pattern=pattern,
- exclude_uri=exclude_uri,
- case_insensitive=case_insensitive,
- node_limit=node_limit,
- level_limit=level_limit,
- ctx=ctx,
- )
-
- candidate_uris = [r["uri"] for r in result if r.get("uri")]
- if excluded_prefix:
- candidate_uris = [
- u
- for u in candidate_uris
- if u != excluded_prefix and not u.startswith(excluded_prefix + "/")
- ]
- if not candidate_uris:
- # BM25 returned no candidates — the index confirms no matching content
- return {"matches": [], "count": 0, "match_count": 0, "files_scanned": 0}
-
- # Step 2: local fs precise matching on candidate files
- return await self._grep_in_files(
- candidate_uris,
- pattern,
- case_insensitive,
- node_limit,
- ctx,
- )
-
- async def _grep_in_files(
- self,
- file_uris: List[str],
- pattern: str,
- case_insensitive: bool,
- node_limit: Optional[int],
- ctx: Optional[RequestContext],
- ) -> Dict:
- """Execute regex matching in specified file list (vikingdb_then_fs Step 2)."""
- flags = re.IGNORECASE if case_insensitive else 0
- compiled = re.compile(pattern, flags)
-
- results = []
- files_scanned = 0
-
- for file_uri in file_uris:
- files_scanned += 1
- try:
- content_bytes = await self.read(file_uri, ctx=ctx)
- content = content_bytes.decode("utf-8", errors="replace")
- except Exception:
- continue
-
- for line_no, line in enumerate(content.splitlines(), 1):
- if compiled.search(line):
- results.append({"uri": file_uri, "line": line_no, "content": line})
- if node_limit and len(results) >= node_limit:
- return {
- "matches": results,
- "count": len(results),
- "match_count": len(results),
- "files_scanned": files_scanned,
- }
-
- return {
- "matches": results,
- "count": len(results),
- "match_count": len(results),
- "files_scanned": files_scanned,
- }
-
- async def _grep_with_agfs(
- self,
- uri: str,
- pattern: str,
- exclude_uri: Optional[str] = None,
- case_insensitive: bool = False,
- node_limit: Optional[int] = None,
- level_limit: int = 10,
- ctx: Optional[RequestContext] = None,
- ) -> Dict:
- """Grep using agfs native implementation.
-
- This is the optimized path for non-encrypted files.
- Uses agfs.grep() which performs matching on the server side.
-
- Prefer pushing filters down to agfs backend:
- - exclude_uri -> exclude_path
- - level_limit -> level_limit
-
- Args:
- uri: Viking URI
- pattern: Regular expression pattern to search for
- exclude_uri: Optional URI prefix to exclude from search
- case_insensitive: Whether to perform case-insensitive matching
- node_limit: Maximum number of results to return
- level_limit: Maximum depth level to traverse
- ctx: Request context
-
- Returns:
- Dict with matches, count, match_count, files_scanned
- """
- path = self._uri_to_path(uri, ctx=ctx)
-
- excluded_path = None
- if exclude_uri:
- normalized_excluded_uri = exclude_uri.rstrip("/")
- self._ensure_access(normalized_excluded_uri, ctx)
- excluded_path = self._uri_to_path(normalized_excluded_uri, ctx=ctx)
-
- try:
- result = await self._async_agfs.grep(
- path=path,
- pattern=pattern,
- recursive=True,
- case_insensitive=case_insensitive,
- stream=False,
- node_limit=node_limit,
- exclude_path=excluded_path,
- level_limit=level_limit,
- )
- except (AttributeError, AGFSNotSupportedError, NotImplementedError):
- # Capability missing: let the outer caller fall back to the VikingFS implementation.
- logger.warning("agfs grep unavailable, falling back to VikingFS implementation")
- raise
-
- matches = result.get("matches", [])
- results = []
- files_scanned_set = set()
- real_ctx = self._ctx_or_default(ctx)
-
- for match in matches:
- match_file = match.get("file", "")
- if not match_file:
- continue
-
- agfs_file_path = self._resolve_grep_match_agfs_path(path, match_file)
-
- file_uri = self._path_to_uri(agfs_file_path, ctx=ctx)
- if not self._is_accessible(file_uri, real_ctx):
- continue
-
- files_scanned_set.add(file_uri)
-
- results.append({
- "line": match.get("line", match.get("line_number", 0)),
- "uri": file_uri,
- "content": match.get("content", ""),
- })
-
- if node_limit and len(results) >= node_limit:
- break
-
- # Prefer backend-provided scanned file count if available; otherwise fall back to
- # counting files that produced at least one match (best-effort).
- backend_files_scanned = result.get("files_scanned")
- if isinstance(backend_files_scanned, int) and backend_files_scanned >= 0:
- files_scanned = (
- len(files_scanned_set) if real_ctx.actor_peer_id else backend_files_scanned
- )
- else:
- files_scanned = len(files_scanned_set)
-
- return {
- "matches": results,
- "count": len(results),
- "match_count": len(results),
- "files_scanned": files_scanned,
- }
-
- async def _grep_encrypted(
- self,
- uri: str,
- pattern: str,
- exclude_uri: Optional[str] = None,
- case_insensitive: bool = False,
- node_limit: Optional[int] = None,
- level_limit: int = 10,
- ctx: Optional[RequestContext] = None,
- content_transform: Optional[Callable[[str, str], str]] = None,
- ) -> Dict:
- """Grep implementation for encrypted files.
-
- This implementation decrypts files at VikingFS layer before matching.
- Used when encryption is enabled or when agfs.grep is not available.
-
- Args:
- uri: Viking URI
- pattern: Regular expression pattern to search for
- exclude_uri: Optional URI prefix to exclude from search
- case_insensitive: Whether to perform case-insensitive matching
- node_limit: Maximum number of results to return
- level_limit: Maximum depth level to traverse (default: 10)
- ctx: Request context
-
- Returns:
- Dict with matches, count, match_count, files_scanned
- """
- flags = re.IGNORECASE if case_insensitive else 0
- compiled_pattern = re.compile(pattern, flags)
- excluded_prefix = None
- if exclude_uri:
- excluded_prefix = exclude_uri.rstrip("/")
- self._ensure_access(excluded_prefix, ctx)
- file_uris = await self._collect_grep_files(
- uri,
- excluded_prefix=excluded_prefix,
- level_limit=level_limit,
- ctx=ctx,
- )
- results, files_scanned = await self._grep_files_parallel(
- file_uris,
- compiled_pattern=compiled_pattern,
- node_limit=node_limit,
- ctx=ctx,
- content_transform=content_transform,
- )
-
- return {
- "matches": results,
- "count": len(results),
- "match_count": len(results),
- "files_scanned": files_scanned,
- }
-
- async def _collect_grep_files(
- self,
- uri: str,
- excluded_prefix: Optional[str],
- level_limit: int,
- ctx: Optional[RequestContext] = None,
- ) -> List[str]:
- file_uris: List[str] = []
-
- async def search_recursive(current_uri: str, current_depth: int) -> None:
- if current_depth > level_limit:
- return
-
- normalized_current_uri = current_uri
- if excluded_prefix and (
- normalized_current_uri == excluded_prefix
- or normalized_current_uri.startswith(excluded_prefix + "/")
- ):
- logger.debug(f"Skipping excluded uri during grep: {normalized_current_uri}")
- return
-
- try:
- entries = await self.ls(normalized_current_uri, ctx=ctx)
- except Exception:
- return
-
- for entry in entries:
- entry_uri = f"{normalized_current_uri.rstrip('/')}/{entry['name']}"
- if excluded_prefix and (
- entry_uri == excluded_prefix or entry_uri.startswith(excluded_prefix + "/")
- ):
- logger.debug(f"Skipping excluded uri during grep: {entry_uri}")
- continue
-
- if entry.get("isDir"):
- await search_recursive(entry_uri, current_depth + 1)
- else:
- file_uris.append(entry_uri)
-
- normalized_uri = uri
- if excluded_prefix and (
- normalized_uri == excluded_prefix or normalized_uri.startswith(excluded_prefix + "/")
- ):
- logger.debug(f"Skipping excluded uri during grep: {normalized_uri}")
- return file_uris
- try:
- root_stat = await self.stat(normalized_uri, ctx=ctx)
- except Exception:
- return file_uris
- if not root_stat.get("isDir", False):
- file_uris.append(normalized_uri)
- return file_uris
-
- await search_recursive(uri, 0)
- return file_uris
-
- async def _grep_files_parallel(
- self,
- file_uris: List[str],
- compiled_pattern: re.Pattern,
- node_limit: Optional[int],
- ctx: Optional[RequestContext] = None,
- content_transform: Optional[Callable[[str, str], str]] = None,
- ) -> tuple[List[Dict[str, Any]], int]:
- results: List[Dict[str, Any]] = []
- files_scanned = 0
- for start in range(0, len(file_uris), _DEFAULT_GREP_FILE_CONCURRENCY):
- batch_uris = file_uris[start : start + _DEFAULT_GREP_FILE_CONCURRENCY]
- batch_jobs = [
- self._grep_single_file(
- entry_uri,
- compiled_pattern,
- ctx,
- content_transform=content_transform,
- )
- for entry_uri in batch_uris
- ]
- batch_results = await asyncio.gather(*batch_jobs)
- for matches, scanned_count in batch_results:
- files_scanned += scanned_count
- for match in matches:
- results.append(match)
- if node_limit and len(results) >= node_limit:
- return results, files_scanned
-
- return results, files_scanned
-
- async def _grep_single_file(
- self,
- entry_uri: str,
- compiled_pattern: re.Pattern,
- ctx: Optional[RequestContext] = None,
- content_transform: Optional[Callable[[str, str], str]] = None,
- ) -> tuple[List[Dict[str, Any]], int]:
- try:
- content = await self.read(entry_uri, ctx=ctx)
- if isinstance(content, bytes):
- content = content.decode("utf-8", errors="replace")
- if content_transform is not None:
- content = content_transform(content, entry_uri)
-
- matches: List[Dict[str, Any]] = []
- lines = content.split("\n")
- for line_num, line in enumerate(lines, 1):
- if compiled_pattern.search(line):
- matches.append({
- "line": line_num,
- "uri": entry_uri,
- "content": line,
- })
- return matches, 1
- except Exception as e:
- logger.debug(f"Failed to grep {entry_uri}: {e}")
- return [], 1
-
- def _resolve_grep_match_agfs_path(self, base_path: str, match_file: str) -> str:
- """Resolve a grep match path (relative to query root) into a full AGFS path."""
- if match_file == ".":
- return base_path
- return f"{base_path.rstrip('/')}/{match_file.lstrip('/')}"
-
- async def stat(
- self, uri: str, ctx: Optional[RequestContext] = None, skip_count: bool = False
- ) -> Dict[str, Any]:
- """
- File/directory information.
-
- example: {'name': 'resources', 'size': 128, 'mode': 2147484141, 'modTime': '2026-02-10T21:26:02.934376379+08:00', 'isDir': True, 'isLocked': False, 'count': 42, 'meta': {'Name': 'localfs', 'Type': 'local', 'Content': {'local_path': '...'}}}
-
- Extra field:
- isLocked (bool): Whether the path is currently held by a path lock
- (either the path itself or any ancestor directory). Returns
- False when the pathlock system is not enabled or the lookup
- fails.
- count (int): For directories, the number of nodes in the vector index
- under this directory (including subdirectories). For files, this
- field is not included.
-
- Args:
- uri: Viking URI
- ctx: Request context
- skip_count: If True, skip the vector_store.count() call for directories.
- Use this when the count field is not needed (e.g. in grep) to avoid
- an extra VikingDB API call.
- """
- self._ensure_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- primary_path = self._uri_to_path(uri, ctx=ctx)
- path = primary_path
- last_not_found: Optional[Exception] = None
- for candidate_path in self._read_paths(uri, ctx=ctx):
- if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):
- continue
- try:
- result = await self._async_agfs.stat(candidate_path)
- path = candidate_path
- break
- except Exception as exc:
- if is_not_found_error(exc):
- last_not_found = exc
- continue
- raise
- else:
- if self._is_legacy_session_root_uri(uri):
- now = datetime.now(timezone.utc).isoformat()
- return {
- "name": "session",
- "size": 0,
- "mode": 0o755,
- "modTime": now,
- "isDir": True,
- "isLocked": False,
- }
- raise NotFoundError(uri, "file") from last_not_found
- if isinstance(result, dict):
- result["isLocked"] = await self._is_path_locked_async(path)
- # Add count for directories if vector store available
- if not skip_count and result.get("isDir", False):
- try:
- vector_store = self._get_vector_store()
- if vector_store:
- target_canonical_uri = canonicalize_uri(
- self._path_to_uri(path, ctx=real_ctx), real_ctx
- )
- if not may_include_hidden_actor_peers(target_canonical_uri, real_ctx):
- filter_expr = PathScope("uri", target_canonical_uri, depth=-1)
- result["count"] = await vector_store.count(
- filter=filter_expr,
- ctx=real_ctx,
- )
- except Exception as e:
- logger.warning(f"[VikingFS] Failed to count nodes for directory stat: {e}")
- return result
-
- async def _is_path_locked_async(self, path: str) -> bool:
- """Best-effort async path-lock lookup; returns False on any error."""
- try:
- return await self._async_agfs.pathlock_is_locked(path)
- except Exception:
- return False
-
- async def exists(self, uri: str, ctx: Optional[RequestContext] = None) -> bool:
- """Check if a URI exists.
-
- Args:
- uri: Viking URI
- ctx: Request context
-
- Returns:
- bool: True if the URI exists, False otherwise
- """
- try:
- await self.stat(uri, ctx=ctx)
- return True
- except Exception:
- return False
-
- async def glob(
- self,
- pattern: str,
- uri: str = "viking://",
- node_limit: Optional[int] = None,
- ctx: Optional[RequestContext] = None,
- ) -> Dict:
- """File pattern matching, supports **/*.md recursive."""
- _ensure_non_empty_search_query(pattern)
- self._ensure_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- primary_path = self._uri_to_path(uri, ctx=ctx)
- path: Optional[str] = None
- for candidate_path in self._read_paths(uri, ctx=ctx):
- if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):
- continue
- if await self._agfs_path_exists(candidate_path):
- path = candidate_path
- break
- if path is None:
- if self._is_legacy_session_root_uri(uri):
- return {"matches": [], "count": 0}
- raise NotFoundError(uri, "directory")
-
- page_size = self._glob_page_size(node_limit)
- continuation_token: Optional[str] = None
- matches = []
- while True:
- page = await self._async_agfs.glob_directory(
- path,
- pattern,
- show_hidden=False,
- page_size=page_size,
- level_limit=None,
- continuation_token=continuation_token,
- )
-
- for entry in page.get("entries", []):
- if node_limit is not None and node_limit > 0 and len(matches) >= node_limit:
- return {"matches": matches, "count": len(matches)}
- if not self._is_path_entry_visible(
- entry["path"],
- entry.get("name") or entry["path"].rsplit("/", 1)[-1],
- path,
- real_ctx,
- ):
- continue
- if not await self._read_path_visible(uri, entry["path"], primary_path, real_ctx):
- continue
- entry_uri = self._alias_uri_for_path(
- request_uri=uri,
- base_path=path,
- entry_path=entry["path"],
- ctx=ctx,
- )
- matches.append(entry_uri)
-
- if node_limit is not None and node_limit > 0 and len(matches) >= node_limit:
- return {"matches": matches, "count": len(matches)}
- continuation_token = page.get("next_token")
- if not continuation_token:
- break
- return {"matches": matches, "count": len(matches)}
-
- async def _batch_fetch_abstracts(
- self,
- entries: List[Dict[str, Any]],
- abs_limit: int,
- ctx: Optional[RequestContext] = None,
- ) -> None:
- """Batch fetch abstracts for entries using a fixed-size worker pool.
-
- Non-directory entries receive an empty abstract immediately.
- Directory entries are processed concurrently via a worker pool,
- using _read_abstract_for_known_dir to skip redundant stat() calls.
-
- Args:
- entries: List of entries to fetch abstracts for
- abs_limit: Maximum length for abstract truncation
- """
- dir_jobs = []
- for index, entry in enumerate(entries):
- if not entry.get("isDir", False):
- entry["abstract"] = ""
- continue
- dir_jobs.append((index, entry))
-
- if not dir_jobs:
- return
-
- worker_count = min(_ABSTRACT_WORKER_COUNT, len(dir_jobs))
-
- cursor = 0
- cursor_lock = asyncio.Lock()
- results: Dict[int, str] = {}
-
- async def worker() -> None:
- nonlocal cursor
- while True:
- async with cursor_lock:
- if cursor >= len(dir_jobs):
- return
- index, entry = dir_jobs[cursor]
- cursor += 1
-
- try:
- abstract = await self._read_abstract_for_known_dir(entry["uri"], ctx=ctx)
- except Exception:
- abstract = "[.abstract.md is not ready]"
-
- results[index] = abstract
-
- await asyncio.gather(*(worker() for _ in range(worker_count)))
-
- for index, abstract in results.items():
- if len(abstract) > abs_limit:
- abstract = abstract[: abs_limit - 3] + "..."
- entries[index]["abstract"] = abstract
-
- async def tree(
- self,
- uri: str = "viking://",
- output: str = "original",
- abs_limit: int = 256,
- show_all_hidden: bool = False,
- node_limit: Optional[int] = 1000,
- level_limit: Optional[int] = 3,
- ctx: Optional[RequestContext] = None,
- ) -> List[Dict[str, Any]]:
- """
- Recursively list all contents (includes rel_path).
-
- Args:
- uri: Viking URI
- output: str = "original" or "agent"
- abs_limit: int = 256 (for agent output abstract truncation)
- show_all_hidden: bool = False (list all hidden files, like -a)
- node_limit: int | None = 1000 (maximum number of nodes to list, None means unlimited)
- level_limit: int | None = 3 (maximum depth level to traverse, None means unlimited)
-
- output="original"
- [{'name': '.abstract.md', 'size': 100, 'mode': 420, 'modTime': '2026-02-11T16:52:16.256334192+08:00', 'isDir': False, 'rel_path': '.abstract.md', 'uri': 'viking://resources...'}]
-
- output="agent"
- [{'uri': 'viking://resources...', 'size': 100, 'isDir': False, 'modTime': '2026-02-11T08:52:16.256Z', 'rel_path': '.abstract.md', 'abstract': "..."}]
- """
- self._ensure_access(uri, ctx)
- if output == "original":
- return await self._tree_original(uri, show_all_hidden, node_limit, level_limit, ctx=ctx)
- elif output == "agent":
- return await self._tree_agent(
- uri, abs_limit, show_all_hidden, node_limit, level_limit, ctx=ctx
- )
- else:
- raise ValueError(f"Invalid output format: {output}")
-
- async def _tree_original(
- self,
- uri: str,
- show_all_hidden: bool = False,
- node_limit: Optional[int] = 1000,
- level_limit: Optional[int] = 3,
- ctx: Optional[RequestContext] = None,
- ) -> List[Dict[str, Any]]:
- """Recursively list all contents (original format)."""
- result = []
- async for entry, entry_uri in self._iter_visible_tree_entries(
- uri,
- show_all_hidden=show_all_hidden,
- node_limit=node_limit,
- level_limit=level_limit,
- ctx=ctx,
- ):
- info = entry["info"]
- new_entry = dict(entry.get("extra", {}))
- new_entry.update({
- "name": info["name"],
- "size": info["size"],
- "mode": info["mode"],
- "modTime": info["modTime"],
- "isDir": info["isDir"],
- "rel_path": entry["rel_path"],
- "uri": entry_uri,
- })
- result.append(new_entry)
- return result
-
- async def _tree_agent(
- self,
- uri: str,
- abs_limit: int,
- show_all_hidden: bool = False,
- node_limit: Optional[int] = 1000,
- level_limit: Optional[int] = 3,
- ctx: Optional[RequestContext] = None,
- ) -> List[Dict[str, Any]]:
- """Recursively list all contents (agent format with abstracts)."""
- result = []
-
- async for entry, entry_uri in self._iter_visible_tree_entries(
- uri,
- show_all_hidden=show_all_hidden,
- node_limit=node_limit,
- level_limit=level_limit,
- ctx=ctx,
- ):
- info = entry["info"]
- is_dir = info["isDir"]
- result.append({
- "uri": entry_uri,
- "size": 0 if is_dir else info["size"],
- "isDir": is_dir,
- "modTime": format_iso8601(parse_iso_datetime(info["modTime"])),
- "rel_path": entry["rel_path"],
- })
-
- await self._batch_fetch_abstracts(result, abs_limit, ctx=ctx)
-
- return result
-
- # ========== VikingFS Specific Capabilities ==========
-
- async def _read_abstract_file(
- self,
- path: str,
- uri: str,
- ctx: Optional[RequestContext] = None,
- ) -> str:
- """Read and decrypt/decode .abstract.md from a known directory path.
-
- Does NOT perform stat or isDir check -- caller is responsible for
- ensuring the path points to a directory.
- """
- file_path = f"{path}/.abstract.md"
- try:
- content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))
- except Exception as exc:
- if not is_not_found_error(exc):
- mapped = map_exception(exc, resource=uri)
- if mapped is not None:
- raise mapped from exc
- raise
- return f"# {uri} [Directory abstract is not ready]"
-
- return self._decode_bytes(content_bytes)
-
- async def _read_abstract_for_known_dir(
- self,
- uri: str,
- ctx: Optional[RequestContext] = None,
- ) -> str:
- """Read .abstract.md for a directory that is already known to be a directory.
-
- Bypasses stat() and isDir check. Caller (i.e. _batch_fetch_abstracts)
- must guarantee that the URI points to a directory.
- """
- self._ensure_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- primary_path = self._uri_to_path(uri, ctx=ctx)
- for path in self._read_paths(uri, ctx=ctx):
- if not await self._read_path_visible(uri, path, primary_path, real_ctx):
- continue
- try:
- if not await self._agfs_path_exists(path):
- continue
- return await self._read_abstract_file(path, uri, ctx=ctx)
- except Exception as exc:
- if is_not_found_error(exc):
- continue
- raise
- return f"# {uri} [Directory abstract is not ready]"
-
- async def abstract(
- self,
- uri: str,
- ctx: Optional[RequestContext] = None,
- ) -> str:
- """Read directory's L0 summary (.abstract.md).
-
- If the caller points to a file, its parent directory is used instead so
- the endpoint remains usable for both file and directory URIs.
- """
- self._ensure_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- primary_path = self._uri_to_path(uri, ctx=ctx)
- path = primary_path
- last_exc: Optional[Exception] = None
- for candidate_path in self._read_paths(uri, ctx=ctx):
- if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):
- continue
- try:
- info = await self._async_agfs.stat(candidate_path)
- path = candidate_path
- break
- except Exception as exc:
- if is_not_found_error(exc):
- last_exc = exc
- continue
- mapped = map_exception(exc, resource=uri)
- if mapped is not None:
- raise mapped from exc
- raise
- else:
- if last_exc is not None:
- mapped = map_exception(last_exc, resource=uri)
- if mapped is not None:
- raise mapped from last_exc
- raise NotFoundError(uri, "directory") from last_exc
- if not info.get("isDir", info.get("is_dir")):
- parent_path = path.rsplit("/", 1)[0] or "/"
- parent_uri = self._path_to_uri(parent_path, ctx=ctx)
- logger.info(
- "content/abstract: %s is a file, falling back to parent directory %s",
- uri,
- parent_uri,
- )
- return await self.abstract(parent_uri, ctx=ctx)
- return await self._read_abstract_file(path, uri, ctx=ctx)
-
- async def overview(
- self,
- uri: str,
- ctx: Optional[RequestContext] = None,
- ) -> str:
- """Read directory's L1 overview (.overview.md).
-
- If the caller points to a file, its parent directory is used instead so
- the endpoint remains usable for both file and directory URIs.
- """
- self._ensure_access(uri, ctx=ctx)
- real_ctx = self._ctx_or_default(ctx)
- primary_path = self._uri_to_path(uri, ctx=ctx)
- path = primary_path
- last_exc: Optional[Exception] = None
- for candidate_path in self._read_paths(uri, ctx=ctx):
- if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):
- continue
- try:
- info = await self._async_agfs.stat(candidate_path)
- path = candidate_path
- break
- except Exception as exc:
- if is_not_found_error(exc):
- last_exc = exc
- continue
- mapped = map_exception(exc, resource=uri)
- if mapped is not None:
- raise mapped from exc
- raise
- else:
- if last_exc is not None:
- mapped = map_exception(last_exc, resource=uri)
- if mapped is not None:
- raise mapped from last_exc
- raise NotFoundError(uri, "directory") from last_exc
- if not info.get("isDir", info.get("is_dir")):
- parent_path = path.rsplit("/", 1)[0] or "/"
- parent_uri = self._path_to_uri(parent_path, ctx=ctx)
- logger.info(
- "content/overview: %s is a file, falling back to parent directory %s",
- uri,
- parent_uri,
- )
- return await self.overview(parent_uri, ctx=ctx)
- file_path = f"{path}/.overview.md"
- try:
- content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))
- except Exception as exc:
- if not is_not_found_error(exc):
- mapped = map_exception(exc, resource=uri)
- if mapped is not None:
- raise mapped from exc
- raise
- # Fallback to default if .overview.md doesn't exist
- return f"# {uri}\n\n[Directory overview is not ready]"
-
- return self._decode_bytes(content_bytes)
-
- async def relations(
- self,
- uri: str,
- ctx: Optional[RequestContext] = None,
- ) -> List[Dict[str, Any]]:
- """Get relation list.
-
- Returns: [{"uri": "...", "reason": "..."}, ...]
- """
- self._ensure_access(uri, ctx)
- entries = await self.get_relation_table(uri, ctx=ctx)
- result = []
- for entry in entries:
- for u in entry.uris:
- if self._is_accessible(u, self._ctx_or_default(ctx)):
- result.append({"uri": u, "reason": entry.reason})
- return result
-
- async def find(
- self,
- query: str,
- target_uri: Union[str, List[str]] = "",
- limit: int = 10,
- score_threshold: Optional[float] = None,
- filter: Optional[Dict] = None,
- ctx: Optional[RequestContext] = None,
- level: Optional[List[int]] = None,
- image_url: Optional[str] = None,
- ):
- """Semantic search.
-
- Args:
- query: Search query
- target_uri: Target directory URI(s), supports str or List[str]
- limit: Return count
- score_threshold: Score threshold
- filter: Metadata filter
-
- Returns:
- FindResult
- """
- _ensure_non_empty_search_query(query, image_url)
- telemetry = get_current_telemetry()
- from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever
- from openviking_cli.retrieve import (
- ContextType,
- FindResult,
- TypedQuery,
- )
-
- real_ctx = self._ctx_or_default(ctx)
- retrieval_targets = resolve_retrieval_targets(target_uri, real_ctx)
-
- for target_dir in retrieval_targets.target_directories:
- self._ensure_access(target_dir, ctx)
-
- storage = self._get_vector_store()
- if not storage:
- raise RuntimeError("Vector store not initialized. Call OpenViking.initialize() first.")
-
- embedder = self._get_embedder()
- if not embedder:
- raise RuntimeError("Embedder not configured.")
-
- retriever = HierarchicalRetriever(
- storage=storage,
- embedder=embedder,
- rerank_config=self.rerank_config,
- retrieval_config=self.retrieval_config,
- )
-
- typed_query = TypedQuery(
- query=query,
- context_type=None,
- intent="",
- target_directories=retrieval_targets.target_directories,
- embedding_input=(
- build_multimodal_embedding_input(query, image_url) if image_url else None
- ),
- image_query=bool(image_url),
- )
-
- logger.debug(
- "[VikingFS.find] Calling retriever.retrieve with "
- f"ctx.account_id={real_ctx.account_id}, ctx.user={real_ctx.user}"
- )
-
- result = await retriever.retrieve(
- typed_query,
- ctx=real_ctx,
- limit=limit,
- score_threshold=score_threshold,
- scope_dsl=filter,
- level=level,
- )
-
- # Convert QueryResult to FindResult
- memories, resources, skills = [], [], []
- for ctx in result.matched_contexts:
- if ctx.context_type == ContextType.MEMORY:
- memories.append(ctx)
- elif ctx.context_type == ContextType.RESOURCE:
- resources.append(ctx)
- elif ctx.context_type == ContextType.SKILL:
- skills.append(ctx)
-
- find_result = FindResult(
- memories=memories,
- resources=resources,
- skills=skills,
- )
- telemetry.set("vector.returned", find_result.total)
- return find_result
-
- async def search(
- self,
- query: str,
- target_uri: Union[str, List[str]] = "",
- session_info: Optional[Dict] = None,
- limit: int = 10,
- score_threshold: Optional[float] = None,
- filter: Optional[Dict] = None,
- ctx: Optional[RequestContext] = None,
- level: Optional[List[int]] = None,
- image_url: Optional[str] = None,
- ):
- """Complex search with session context.
-
- Args:
- query: Search query
- target_uri: Target directory URI(s), supports str or List[str]
- session_info: Session information
- limit: Return count
- filter: Metadata filter
-
- Returns:
- FindResult
- """
- _ensure_non_empty_search_query(query, image_url)
- telemetry = get_current_telemetry()
- from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever
- from openviking.retrieve.intent_analyzer import IntentAnalyzer
- from openviking_cli.retrieve import (
- ContextType,
- FindResult,
- QueryPlan,
- TypedQuery,
- )
-
- real_ctx = self._ctx_or_default(ctx)
- retrieval_targets = resolve_retrieval_targets(target_uri, real_ctx)
- primary_target_uri = retrieval_targets.first_explicit_directory
-
- session_summary = (
- str(session_info.get("latest_archive_overview") or "") if session_info else ""
- )
- current_messages = session_info.get("current_messages") if session_info else None
-
- query_plan: Optional[QueryPlan] = None
- for target_dir in retrieval_targets.target_directories:
- self._ensure_access(target_dir, ctx)
-
- # When target_uri exists, read its abstract as optional query-planning context.
- target_abstract = ""
- if primary_target_uri:
- try:
- with telemetry.measure("search.target_abstract"):
- target_abstract = await self.abstract(primary_target_uri, ctx=ctx)
- except Exception:
- target_abstract = ""
-
- intent_enabled = (
- bool(self.retrieval_config.enable_intent)
- if self.retrieval_config is not None
- else True
- )
-
- # With session context: optional intent analysis
- if image_url:
- typed_queries = [
- TypedQuery(
- query=query,
- context_type=None,
- intent="",
- priority=1,
- target_directories=retrieval_targets.target_directories,
- embedding_input=build_multimodal_embedding_input(query, image_url),
- image_query=True,
- )
- ]
- elif intent_enabled and (session_summary or current_messages):
- analyzer = IntentAnalyzer(max_recent_messages=5)
- with telemetry.measure("search.intent_analysis"):
- query_plan = await analyzer.analyze(
- compression_summary=session_summary or "",
- messages=current_messages or [],
- current_message=query,
- target_abstract=target_abstract,
- )
- typed_queries = query_plan.queries
- for tq in typed_queries:
- tq.target_directories = retrieval_targets.target_directories
- else:
- # No session context, or intent disabled: search with the raw query.
- typed_queries = [
- TypedQuery(
- query=query,
- context_type=None,
- intent="",
- priority=1,
- target_directories=retrieval_targets.target_directories,
- )
- ]
- telemetry.set("search.typed_queries_count", len(typed_queries))
-
- # Concurrent execution
- storage = self._get_vector_store()
- embedder = self._get_embedder()
- retriever = HierarchicalRetriever(
- storage=storage,
- embedder=embedder,
- rerank_config=self.rerank_config,
- retrieval_config=self.retrieval_config,
- )
-
- async def _execute(tq: TypedQuery):
- real_ctx = self._ctx_or_default(ctx)
- logger.debug(
- "[VikingFS.search._execute] Calling retriever.retrieve with "
- f"ctx.account_id={real_ctx.account_id}, ctx.user={real_ctx.user}"
- )
- return await retriever.retrieve(
- tq,
- ctx=real_ctx,
- limit=limit,
- score_threshold=score_threshold,
- scope_dsl=filter,
- level=level,
- )
-
- query_results = await asyncio.gather(*[_execute(tq) for tq in typed_queries])
-
- # Aggregate results to FindResult
- memories, resources, skills = [], [], []
- for result in query_results:
- for ctx in result.matched_contexts:
- if ctx.context_type == ContextType.MEMORY:
- memories.append(ctx)
- elif ctx.context_type == ContextType.RESOURCE:
- resources.append(ctx)
- elif ctx.context_type == ContextType.SKILL:
- skills.append(ctx)
-
- find_result = FindResult(
- memories=memories,
- resources=resources,
- skills=skills,
- query_plan=query_plan,
- query_results=query_results,
- )
- telemetry.set("vector.returned", find_result.total)
- return find_result
-
- # ========== Relation Management ==========
-
- async def link(
- self,
- from_uri: str,
- uris: Union[str, List[str]],
- reason: str = "",
- ctx: Optional[RequestContext] = None,
- ) -> None:
- """Create relation (maintained in .relations.json)."""
- if isinstance(uris, str):
- uris = [uris]
- self._ensure_mutable_access(from_uri, ctx)
- for uri in uris:
- self._ensure_access(uri, ctx)
-
- from_path = self._uri_to_path(from_uri, ctx=ctx)
-
- entries = await self._read_relation_table(from_path, ctx=ctx)
- existing_ids = {e.id for e in entries}
-
- link_id = next(f"link_{i}" for i in range(1, 10000) if f"link_{i}" not in existing_ids)
-
- entries.append(RelationEntry(id=link_id, uris=uris, reason=reason))
-
- await self._write_relation_table(from_path, entries, ctx=ctx)
- logger.debug(f"[VikingFS] Created link: {from_uri} -> {uris}")
-
- async def unlink(
- self,
- from_uri: str,
- uri: str,
- ctx: Optional[RequestContext] = None,
- ) -> None:
- """Delete relation."""
- self._ensure_mutable_access(from_uri, ctx)
- self._ensure_access(uri, ctx)
- from_path = self._uri_to_path(from_uri, ctx=ctx)
-
- try:
- entries = await self._read_relation_table(from_path, ctx=ctx)
-
- entry_to_modify = None
- for entry in entries:
- if uri in entry.uris:
- entry_to_modify = entry
- break
-
- if not entry_to_modify:
- logger.debug(f"[VikingFS] URI not found in relations: {uri}")
- return
-
- entry_to_modify.uris.remove(uri)
-
- if not entry_to_modify.uris:
- entries.remove(entry_to_modify)
- logger.debug(f"[VikingFS] Removed empty entry: {entry_to_modify.id}")
-
- await self._write_relation_table(from_path, entries, ctx=ctx)
- logger.debug(f"[VikingFS] Removed link: {from_uri} -> {uri}")
-
- except Exception as e:
- logger.error(f"[VikingFS] Failed to unlink {from_uri} -> {uri}: {e}")
- raise IOError(f"Failed to unlink: {e}")
-
- async def get_relation_table(
- self, uri: str, ctx: Optional[RequestContext] = None
- ) -> List[RelationEntry]:
- """Get relation table."""
- self._ensure_access(uri, ctx)
- path = self._uri_to_path(uri, ctx=ctx)
- return await self._read_relation_table(path, ctx=ctx)
-
- # ========== Tree Traversal (Refactored) ==========
-
- def _is_name_visible_at_path(self, name: str, parent_path: str) -> bool:
- """Check if name would appear in _ls_entries(parent_path).
-
- At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.
- At other levels, uses the shared storage internal-name blacklist.
- """
- parts = [p for p in parent_path.strip("/").split("/") if p]
- if len(parts) == 2 and parts[0] == "local":
- return name in VikingURI.LISTABLE_SCOPES
- return name not in STORAGE_INTERNAL_ENTRY_NAMES
-
- def _ancestor_is_filtered(self, entry_path: str, base_path: str) -> bool:
- """Check if any ancestor directory of entry_path would be filtered by _ls_entries.
-
- Walks from base_path (exclusive) to entry's parent directory (exclusive),
- checking each component against _is_name_visible_at_path.
- """
- base_parts = [p for p in base_path.strip("/").split("/") if p]
- entry_parts = [p for p in entry_path.strip("/").split("/") if p]
-
- for i in range(len(base_parts), len(entry_parts) - 1):
- name = entry_parts[i]
- parent_parts = entry_parts[:i]
- parent_path = "/" + "/".join(parent_parts) if parent_parts else "/"
- if not self._is_name_visible_at_path(name, parent_path):
- return True
- return False
-
- def _is_path_entry_visible(
- self, entry_path: str, name: str, base_path: str, ctx: RequestContext
- ) -> bool:
- """Check visibility for one flattened path entry returned by Rust."""
- if self._ancestor_is_filtered(entry_path, base_path):
- return False
-
- entry_parts = [p for p in entry_path.strip("/").split("/") if p]
- if entry_parts:
- parent_parts = entry_parts[:-1]
- parent_path = "/" + "/".join(parent_parts) if parent_parts else "/"
- if not self._is_name_visible_at_path(name, parent_path):
- return False
-
- uri = self._path_to_uri(entry_path, ctx=ctx)
- if not self._is_accessible(uri, ctx):
- return False
-
- return True
-
- def _is_tree_entry_visible(
- self, entry: Dict[str, Any], base_path: str, ctx: RequestContext
- ) -> bool:
- """Check visibility for a single TreeEntry returned by Rust tree_directory."""
- entry_path = entry["path"]
- entry_info = entry.get("info", {})
- name = entry_info.get("name") or entry_path.rstrip("/").rsplit("/", 1)[-1]
- return self._is_path_entry_visible(entry_path, name, base_path, ctx)
-
- # Over-fetch multiplier for bounded tree traversal. When a node_limit is
- # set, we push down node_limit * this factor as the raw-node limit to Rust,
- # leaving headroom for ACL/internal-name filtering before re-fetching.
- _TREE_OVERFETCH_FACTOR = 4
- _GLOB_PAGE_SIZE_DEFAULT = 1024
-
- def _glob_page_size(self, node_limit: Optional[int]) -> int:
- """Return the backend page size used by glob_directory."""
- if node_limit is None or node_limit <= 0:
- return self._GLOB_PAGE_SIZE_DEFAULT
- return node_limit
-
- async def _iter_visible_tree_entries(
- self,
- uri: str,
- show_all_hidden: bool = False,
- node_limit: Optional[int] = None,
- level_limit: Optional[int] = None,
- ctx: Optional[RequestContext] = None,
- ):
- """Shared generator: fetch raw TreeEntry list from Rust, yield (entry, uri) tuples.
-
- node_limit counts ACL-visible entries (see design §6.5), so the user's
- node_limit cannot be pushed directly to Rust — doing so would truncate
- before filtering and drop entries that should be visible.
-
- To keep memory bounded without changing that semantic, we push down an
- *amplified* raw-node limit (node_limit * _TREE_OVERFETCH_FACTOR). If ACL
- filtering leaves fewer than node_limit visible entries while Rust still
- returned a full page (i.e. more raw nodes may exist), we double the raw
- limit and re-fetch. Because Rust truncates a deterministic sorted prefix,
- this yields exactly the same result as an unbounded fetch, while avoiding
- materializing the entire prefix in the common case.
-
- When node_limit is None (full-tree callers), no limit is pushed down.
- level_limit IS always passed to Rust.
- """
- real_ctx = self._ctx_or_default(ctx)
- primary_path = self._uri_to_path(uri, ctx=ctx)
- path: Optional[str] = None
- for candidate_path in self._read_paths(uri, ctx=ctx):
- if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):
- continue
- if await self._agfs_path_exists(candidate_path):
- path = candidate_path
- break
- if path is None:
- if self._is_legacy_session_root_uri(uri):
- return
- raise NotFoundError(uri, "directory")
-
- if node_limit is None:
- raw_limit: Optional[int] = None
- else:
- raw_limit = max(node_limit * self._TREE_OVERFETCH_FACTOR, node_limit)
-
- while True:
- raw_entries = await self._async_agfs.tree_directory(
- path,
- show_hidden=show_all_hidden,
- node_limit=raw_limit,
- level_limit=level_limit,
- )
-
- visible: List[tuple] = []
- for entry in raw_entries:
- if node_limit is not None and len(visible) >= node_limit:
- break
- if not self._is_tree_entry_visible(entry, path, real_ctx):
- continue
- if not await self._read_path_visible(uri, entry["path"], primary_path, real_ctx):
- continue
- entry_uri = self._alias_uri_for_path(
- request_uri=uri,
- base_path=path,
- entry_path=entry["path"],
- ctx=ctx,
- )
- visible.append((entry, entry_uri))
-
- # If we still lack enough visible entries but Rust returned a full
- # page (raw_limit reached), more raw nodes may exist — re-fetch with
- # a doubled limit. Otherwise Rust is exhausted and we yield as-is.
- need_more = (
- node_limit is not None
- and len(visible) < node_limit
- and raw_limit is not None
- and len(raw_entries) >= raw_limit
- )
- if need_more:
- raw_limit *= 2
- continue
-
- for item in visible:
- yield item
- return
-
- # ========== URI Conversion ==========
-
- # Maximum bytes for a single filename component (filesystem limit is typically 255)
- _MAX_FILENAME_BYTES = 255
-
- @staticmethod
- def _shorten_component(component: str, max_bytes: int = 255) -> str:
- """Shorten a path component if its UTF-8 encoding exceeds max_bytes."""
- if len(component.encode("utf-8")) <= max_bytes:
- return component
- hash_suffix = hashlib.sha256(component.encode("utf-8")).hexdigest()[:8]
- # Trim to fit within max_bytes after adding hash suffix
- prefix = component
- target = max_bytes - len(f"_{hash_suffix}".encode("utf-8"))
- while len(prefix.encode("utf-8")) > target and prefix:
- prefix = prefix[:-1]
- return f"{prefix}_{hash_suffix}"
-
- def _uri_to_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:
- """Map virtual URI to account-isolated AGFS path.
-
- Pure prefix replacement: viking://{remainder} -> /local/{account_id}/{remainder}.
- No implicit space injection — URIs must include space segments explicitly.
- """
- real_ctx = self._ctx_or_default(ctx)
- account_id = real_ctx.account_id
- normalized_uri, legacy_parts = self._normalized_uri_parts(uri)
- if legacy_parts and legacy_parts[0] == "agent" and self._is_legacy_agent_id_uri(uri):
- # Old format: viking://agent/{agent_id}/... — direct mapping for read-only compat
- safe_parts = [
- self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in legacy_parts
- ]
- return f"/local/{account_id}/{'/'.join(safe_parts)}"
- canonical_uri = canonicalize_uri(uri, real_ctx)
- _, parts = self._normalized_uri_parts(canonical_uri)
- if not parts:
- return f"/local/{account_id}"
-
- safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]
- return f"/local/{account_id}/{'/'.join(safe_parts)}"
-
- def _legacy_session_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:
- """Map a legacy viking://session URI to its pre-user-namespace path."""
- real_ctx = self._ctx_or_default(ctx)
- _, parts = self._normalized_uri_parts(uri)
- safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]
- return f"/local/{real_ctx.account_id}/{'/'.join(safe_parts)}"
-
- def _legacy_current_user_session_path(
- self, uri: str, ctx: Optional[RequestContext] = None
- ) -> Optional[str]:
- """Return the legacy nested /session/{user_id}/{session_id} candidate."""
- real_ctx = self._ctx_or_default(ctx)
- _, parts = self._normalized_uri_parts(uri)
- if len(parts) <= 1 or parts[0] != "session":
- return None
- nested_parts = ["session", real_ctx.user.user_id, *parts[1:]]
- safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in nested_parts]
- return f"/local/{real_ctx.account_id}/{'/'.join(safe_parts)}"
-
- def _is_legacy_session_uri(self, uri: str) -> bool:
- _, parts = self._normalized_uri_parts(uri)
- return bool(parts and parts[0] == "session")
-
- def _is_legacy_session_root_uri(self, uri: str) -> bool:
- _, parts = self._normalized_uri_parts(uri)
- return parts == ["session"]
-
- def _is_legacy_agent_id_uri(self, uri: str) -> bool:
- _, parts = self._normalized_uri_parts(uri)
- return bool(
- parts
- and parts[0] == "agent"
- and len(parts) >= 2
- and parts[1] not in {"skills", "endpoints", "tools", "payments"}
- )
-
- def _read_paths(self, uri: str, ctx: Optional[RequestContext] = None) -> List[str]:
- """Return read candidates for a URI, including legacy alias fallbacks."""
- paths = [self._uri_to_path(uri, ctx=ctx)]
-
- if self._is_legacy_session_uri(uri):
- for candidate in (
- self._legacy_session_path(uri, ctx=ctx),
- self._legacy_current_user_session_path(uri, ctx=ctx),
- ):
- if candidate and candidate not in paths:
- paths.append(candidate)
- return paths
-
- async def _read_path_visible(
- self,
- request_uri: str,
- path: str,
- primary_path: str,
- ctx: RequestContext,
- ) -> bool:
- if path == primary_path:
- return True
- if self._is_legacy_session_uri(request_uri):
- return await self._legacy_session_path_visible(path, ctx)
- return True
-
- def _alias_uri_for_path(
- self,
- *,
- request_uri: str,
- base_path: str,
- entry_path: str,
- ctx: Optional[RequestContext],
- ) -> str:
- base = base_path.rstrip("/")
- normalized_request, request_parts = self._normalized_uri_parts(request_uri)
- request_root = (
- normalized_request
- if normalized_request == "viking://"
- else normalized_request.rstrip("/")
- )
- preserve_request_alias = normalized_request in {"viking://", "viking://user"} or bool(
- request_parts
- and (
- request_parts[0] == "session"
- or (request_parts[0] == "agent" and self._is_legacy_agent_id_uri(request_uri))
- )
- )
- rel_path = entry_path[len(base) :].strip("/") if entry_path.startswith(base) else ""
- if entry_path.startswith(base):
- separator = "" if request_root.endswith("://") else "/"
- candidate_uri = request_root if not rel_path else f"{request_root}{separator}{rel_path}"
- if preserve_request_alias:
- return candidate_uri
- try:
- if self._uri_to_path(candidate_uri, ctx=ctx) == entry_path:
- return candidate_uri
- except Exception:
- pass
- return self._path_to_uri(entry_path, ctx=ctx)
-
- async def _agfs_path_exists(self, path: str) -> bool:
- try:
- await self._async_agfs.stat(path)
- return True
- except Exception as exc:
- if is_not_found_error(exc):
- return False
- raise
-
- async def _looks_like_legacy_session_dir(self, path: str) -> bool:
- for leaf in (".meta.json", "messages.jsonl", "history", "tool-results", "tools"):
- if await self._agfs_path_exists(f"{path}/{leaf}"):
- return True
- return False
-
- async def _legacy_session_owner(self, session_root_path: str) -> str:
- try:
- raw = self._handle_agfs_read(
- await self._async_agfs.read(f"{session_root_path}/.meta.json")
- )
- except Exception as exc:
- if is_not_found_error(exc):
- return ""
- raise
- try:
- data = json.loads(self._decode_bytes(raw))
- except json.JSONDecodeError:
- return ""
- if not isinstance(data, dict):
- return ""
- for key in ("created_by_user_id", "user_id", "owner_user_id", "created_by"):
- value = data.get(key)
- if isinstance(value, str) and value.strip():
- return value.strip()
- return ""
-
- async def _legacy_session_visible(
- self,
- session_root_path: str,
- ctx: RequestContext,
- *,
- owner_hint: Optional[str] = None,
- ) -> bool:
- if ctx.role == Role.ROOT:
- return True
- owner = await self._legacy_session_owner(session_root_path)
- if owner:
- return owner == ctx.user.user_id
- if owner_hint:
- return owner_hint == ctx.user.user_id
- return True
-
- async def _legacy_session_path_visible(self, path: str, ctx: RequestContext) -> bool:
- parts = [p for p in path.strip("/").split("/") if p]
- try:
- session_index = parts.index("session")
- except ValueError:
- return True
- suffix = parts[session_index + 1 :]
- if not suffix:
- return True
-
- root_prefix = "/" + "/".join(parts[: session_index + 1])
- direct_root = f"{root_prefix}/{suffix[0]}"
- if await self._looks_like_legacy_session_dir(direct_root):
- return await self._legacy_session_visible(direct_root, ctx)
-
- if len(suffix) >= 2:
- nested_root = f"{root_prefix}/{suffix[0]}/{suffix[1]}"
- if await self._looks_like_legacy_session_dir(nested_root):
- return await self._legacy_session_visible(
- nested_root,
- ctx,
- owner_hint=suffix[0],
- )
- return True
-
- async def _legacy_session_root_items(
- self,
- path: str,
- ctx: RequestContext,
- ) -> List[tuple[Dict[str, Any], str]]:
- try:
- entries = await self._ls_entries(path)
- except Exception as exc:
- if is_not_found_error(exc):
- return []
- raise
-
- items: List[tuple[Dict[str, Any], str]] = []
- for entry in entries:
- name = entry.get("name", "")
- if not name or name in {".", ".."} or not entry.get("isDir"):
- continue
- child_path = f"{path.rstrip('/')}/{name}"
- if await self._looks_like_legacy_session_dir(child_path):
- if await self._legacy_session_visible(child_path, ctx):
- items.append((entry, f"viking://session/{name}"))
- continue
-
- if ctx.role != Role.ROOT and name != ctx.user.user_id:
- continue
- try:
- nested_entries = await self._ls_entries(child_path)
- except Exception as exc:
- if is_not_found_error(exc):
- continue
- raise
- for nested in nested_entries:
- nested_name = nested.get("name", "")
- if not nested_name or nested_name in {".", ".."} or not nested.get("isDir"):
- continue
- nested_path = f"{child_path.rstrip('/')}/{nested_name}"
- if not await self._looks_like_legacy_session_dir(nested_path):
- continue
- if await self._legacy_session_visible(nested_path, ctx, owner_hint=name):
- items.append((nested, f"viking://session/{nested_name}"))
- return items
-
- async def _session_root_items(
- self,
- uri: str,
- ctx: RequestContext,
- ) -> List[tuple[Dict[str, Any], str]]:
- primary_path = self._uri_to_path(uri, ctx=ctx)
- by_name: Dict[str, tuple[Dict[str, Any], str]] = {}
- try:
- for entry in await self._ls_entries(primary_path, ctx=ctx):
- name = entry.get("name", "")
- if not name or name in {".", ".."}:
- continue
- by_name[name] = (entry, f"viking://session/{name}")
- except Exception as exc:
- if not is_not_found_error(exc):
- raise
-
- legacy_path = self._legacy_session_path(uri, ctx=ctx)
- for entry, entry_uri in await self._legacy_session_root_items(legacy_path, ctx):
- name = entry.get("name", "")
- if name and name not in by_name:
- by_name[name] = (entry, entry_uri)
- return list(by_name.values())
-
- async def _list_read_path_items(
- self,
- uri: str,
- ctx: Optional[RequestContext] = None,
- ) -> List[tuple[Dict[str, Any], str]]:
- real_ctx = self._ctx_or_default(ctx)
- if self._is_legacy_session_root_uri(uri):
- return await self._session_root_items(uri, real_ctx)
-
- primary_path = self._uri_to_path(uri, ctx=ctx)
- merge_paths = self._is_legacy_session_uri(uri)
- found_path = False
- last_not_found: Optional[Exception] = None
- by_uri: Dict[str, tuple[Dict[str, Any], str]] = {}
-
- for path in self._read_paths(uri, ctx=ctx):
- if not await self._read_path_visible(uri, path, primary_path, real_ctx):
- continue
- try:
- entries = await self._ls_entries(path, ctx=ctx)
- except Exception as exc:
- if is_not_found_error(exc):
- last_not_found = exc
- continue
- raise
-
- found_path = True
- for entry in entries:
- entry_uri = self._alias_uri_for_path(
- request_uri=uri,
- base_path=path,
- entry_path=f"{path.rstrip('/')}/{entry.get('name', '')}",
- ctx=ctx,
- )
- by_uri.setdefault(entry_uri, (entry, entry_uri))
- if not merge_paths:
- break
-
- if found_path:
- return list(by_uri.values())
- raise NotFoundError(uri, "directory") from last_not_found
-
- _ROOT_PATH = "/local"
-
- async def _ls_entries(
- self, path: str, ctx: Optional[RequestContext] = None
- ) -> List[Dict[str, Any]]:
- """List directory entries, filtering out internal directories.
-
- At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.
- At other levels, uses the shared storage internal-name blacklist.
- """
- entries = await self._async_agfs.ls(path)
- parts = [p for p in path.strip("/").split("/") if p]
- if len(parts) == 2 and parts[0] == "local":
- return [e for e in entries if e.get("name") in VikingURI.LISTABLE_SCOPES]
- return [e for e in entries if e.get("name") not in STORAGE_INTERNAL_ENTRY_NAMES]
-
- def _path_to_uri(self, path: str, ctx: Optional[RequestContext] = None) -> str:
- """/local/{account}/... -> viking://...
-
- Pure prefix replacement: strips /local/{account_id}/ and prepends viking://.
- No implicit space stripping.
- """
- if path.startswith("viking://"):
- return path
- elif path.startswith("/local/"):
- inner = path[7:].strip("/")
- if not inner:
- return "viking://"
- real_ctx = self._ctx_or_default(ctx)
- parts = [p for p in inner.split("/") if p]
- if parts and parts[0] == real_ctx.account_id:
- parts = parts[1:]
- if not parts:
- return "viking://"
- return f"viking://{'/'.join(parts)}"
- elif path.startswith("/"):
- return f"viking:/{path}"
- else:
- return f"viking://{path}"
-
- def _looks_like_legacy_temp_leaf(self, value: str) -> bool:
- return bool(re.match(r"^\d{8}_[0-9a-f]{6}$", value or ""))
-
- def _is_legacy_temp_uri_parts(self, parts: List[str]) -> bool:
- if len(parts) < 2 or parts[0] != "temp" or not self._looks_like_legacy_temp_leaf(parts[1]):
- return False
- if len(parts) == 2:
- return True
- return not self._looks_like_legacy_temp_leaf(parts[2])
-
- def _is_accessible(self, uri: str, ctx: RequestContext) -> bool:
- """Check whether a URI is visible/accessible under current request context."""
- normalized_uri, parts = self._normalized_uri_parts(uri)
- if ctx.role == Role.ROOT:
- return True
- if is_hidden_by_actor_peer_view(normalized_uri, ctx):
- return False
- if not parts:
- return True
- if is_watch_task_control_uri(normalized_uri):
- return False
-
- scope = parts[0]
- if scope == "resources":
- return True
- if scope == "temp":
- if len(parts) == 1:
- return True
- if parts[1] == ctx.user.user_space_name():
- return True
- return self._is_legacy_temp_uri_parts(parts)
- if scope == "upload":
- return ctx.role == Role.ROOT
- if scope == "_system":
- return False
- if scope == "agent":
- # New format: agent/skills/..., agent/endpoints/... — globally readable (account scope)
- if len(parts) >= 2 and parts[1] in {"skills", "endpoints", "tools", "payments"}:
- return True
- # Old format: agent/{agent_id}/... — actor_peer_id match for read-only access
- if not ctx.actor_peer_id or len(parts) < 2:
- return True
- return parts[1] == ctx.actor_peer_id
- return namespace_is_accessible(normalized_uri, ctx)
-
- def _handle_agfs_read(self, result: Union[bytes, Any, None]) -> bytes:
- """Handle AGFSClient read return types consistently."""
- if isinstance(result, bytes):
- return result
- elif result is None:
- return b""
- elif hasattr(result, "content") and result.content is not None:
- return result.content
- else:
- # Try to convert to bytes
- try:
- return str(result).encode("utf-8")
- except Exception:
- return b""
-
- def _decode_bytes(self, data: bytes) -> str:
- """Robustly decode bytes to string."""
- if not data:
- return ""
- try:
- return data.decode("utf-8")
- except UnicodeDecodeError:
- try:
- # Try common encoding for Windows/legacy files in China
- return data.decode("gbk")
- except UnicodeDecodeError:
- try:
- return data.decode("latin-1")
- except UnicodeDecodeError:
- return data.decode("utf-8", errors="replace")
-
- # ========== Vector Sync Helper Methods ==========
-
- async def _collect_uris(
- self, path: str, recursive: bool, ctx: Optional[RequestContext] = None
- ) -> List[str]:
- """Recursively collect all URIs (for rm/mv), including directories."""
- uris = []
-
- async def _collect(p: str):
- try:
- entries = await self._ls_entries(p, ctx=ctx)
- except Exception as exc:
- if is_not_found_error(exc):
- return
- raise
-
- for entry in entries:
- name = entry.get("name", "")
- if name in [".", ".."]:
- continue
- full_path = f"{p}/{name}".replace("//", "/")
- if entry.get("isDir"):
- uris.append(self._path_to_uri(full_path, ctx=ctx))
- if recursive:
- await _collect(full_path)
- else:
- uris.append(self._path_to_uri(full_path, ctx=ctx))
-
- await _collect(path)
- return uris
-
- async def _delete_from_vector_store(
- self, uris: List[str], ctx: Optional[RequestContext] = None
- ) -> None:
- """Delete records with specified URIs from vector store.
-
- Uses tenant-safe URI deletion semantics from vector store.
- """
- vector_store = self._get_vector_store()
- if not vector_store:
- return
- real_ctx = self._ctx_or_default(ctx)
-
- try:
- await vector_store.delete_uris(real_ctx, uris)
- for uri in uris:
- logger.debug(f"[VikingFS] Deleted from vector store: {uri}")
- except Exception as e:
- logger.warning(f"[VikingFS] Failed to delete from vector store: {e}")
- raise
-
- async def _update_vector_store_uris(
- self,
- uris: List[str],
- old_base: str,
- new_base: str,
- ctx: Optional[RequestContext] = None,
- ) -> None:
- """Update URIs in vector store (when moving files).
-
- Preserves vector data and updates URI-derived identifiers without regenerating embeddings.
- """
- vector_store = self._get_vector_store()
- if not vector_store:
- return
-
- real_ctx = self._ctx_or_default(ctx)
-
- for uri in uris:
- try:
- new_uri = new_base + uri[len(old_base) :]
- if await vector_store.update_uri_mapping(
- ctx=real_ctx,
- uri=uri,
- new_uri=new_uri,
- ):
- logger.debug(f"[VikingFS] Updated URI: {uri} -> {new_uri}")
- except Exception as e:
- logger.warning(f"[VikingFS] Failed to update {uri} in vector store: {e}")
-
- def _get_vector_store(self) -> Optional["VikingVectorIndexBackend"]:
- """Get vector store instance."""
- return self.vector_store
-
- def _get_embedder(self) -> Any:
- """Get embedder instance."""
- return self.query_embedder
-
- # ========== Parent Directory Creation ==========
-
- async def _ensure_parent_dirs(
- self,
- path: str,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> None:
- """Recursively create all parent directories."""
- try:
- await self._async_agfs.ensure_parent_dirs(
- path,
- fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),
- )
- except Exception as e:
- logger.debug(f"Failed to ensure parent directories for {path}: {e}")
- parent = path.rstrip("/").rsplit("/", 1)[0]
- await self._mkdir_path_with_parents(parent, ctx=ctx, lease_ref=lease_ref)
-
- async def _mkdir_path_with_parents(
- self,
- dir_path: str,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> None:
- """Create a directory path segment-by-segment using the same fs context."""
- parts = [part for part in dir_path.strip("/").split("/") if part]
- current = ""
- for part in parts:
- current = f"{current}/{part}"
- try:
- await self._async_agfs.mkdir(
- current,
- fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),
- )
- except Exception as e:
- message = str(e).lower()
- if "exist" in message or "already" in message:
- continue
- logger.debug(f"Failed to create parent directory {current}: {e}")
-
- # ========== Relation Table Internal Methods ==========
-
- async def _read_relation_table(
- self, dir_path: str, ctx: Optional[RequestContext] = None
- ) -> List[RelationEntry]:
- """Read .relations.json."""
- table_path = f"{dir_path}/.relations.json"
- try:
- content = self._handle_agfs_read(await self._async_agfs.read(table_path))
- data = json.loads(content.decode("utf-8"))
- except FileNotFoundError:
- return []
- except Exception:
- # logger.warning(f"[VikingFS] Failed to read relation table {table_path}: {e}")
- return []
-
- entries = []
- # Compatible with old format (nested) and new format (flat)
- if isinstance(data, list):
- # New format: flat list
- for entry_data in data:
- entries.append(RelationEntry.from_dict(entry_data))
- elif isinstance(data, dict):
- # Old format: nested {namespace: {user: [entries]}}
- for _namespace, user_dict in data.items():
- for _user, entry_list in user_dict.items():
- for entry_data in entry_list:
- entries.append(RelationEntry.from_dict(entry_data))
- return entries
-
- async def _write_relation_table(
- self, dir_path: str, entries: List[RelationEntry], ctx: Optional[RequestContext] = None
- ) -> None:
- """Write .relations.json."""
- # Use flat list format
- data = [entry.to_dict() for entry in entries]
-
- content = json.dumps(data, ensure_ascii=False, indent=2)
- table_path = f"{dir_path}/.relations.json"
- if isinstance(content, str):
- content = content.encode("utf-8")
-
- await self._async_agfs.write(table_path, content)
-
- # ========== Batch Read (backward compatible) ==========
-
- async def read_batch(
- self, uris: List[str], level: str = "l0", ctx: Optional[RequestContext] = None
- ) -> Dict[str, str]:
- """Batch read content from multiple URIs."""
- results = {}
- for uri in uris:
- try:
- content = ""
- if level == "l0":
- content = await self.abstract(uri, ctx=ctx)
- elif level == "l1":
- content = await self.overview(uri, ctx=ctx)
- results[uri] = content
- except Exception:
- pass
- return results
-
- # ========== Other Preserved Methods ==========
-
- async def write_file(
- self,
- uri: str,
- content: Union[str, bytes],
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> None:
- """Write file directly. Encryption lock handled internally by EncryptionWrappedFS."""
- self._ensure_mutable_access(uri, ctx)
- path = self._uri_to_path(uri, ctx=ctx)
- await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)
-
- if isinstance(content, str):
- content = content.encode("utf-8")
-
- await self._async_agfs.write(path, content, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))
-
- async def read_file(
- self,
- uri: str,
- offset: int = 0,
- limit: int = -1,
- ctx: Optional[RequestContext] = None,
- ) -> str:
- """Read single file, optionally sliced by line range.
-
- Args:
- uri: Viking URI
- offset: Starting line number (0-indexed). Default 0.
- limit: Number of lines to read. -1 means read to end. Default -1.
-
- Raises:
- FileNotFoundError: If the file does not exist.
- """
- self._ensure_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- primary_path = self._uri_to_path(uri, ctx=ctx)
- # Verify the file exists before reading, because AGFS read returns
- # empty bytes for non-existent files instead of raising an error.
- last_not_found: Optional[Exception] = None
- for path in self._read_paths(uri, ctx=ctx):
- if not await self._read_path_visible(uri, path, primary_path, real_ctx):
- continue
- try:
- stat = await self._async_agfs.stat(path)
- break
- except Exception as exc:
- if is_not_found_error(exc):
- last_not_found = exc
- continue
- raise
- else:
- raise NotFoundError(uri, "file") from last_not_found
- if isinstance(stat, dict) and stat.get("isDir", False):
- raise InvalidArgumentError(
- f"Cannot read directory as file: {uri}",
- details={"resource": uri, "expected": "file", "actual": "directory"},
- )
- try:
- content = await self._async_agfs.read(path)
- if isinstance(content, bytes):
- raw = content
- elif content is not None and hasattr(content, "content"):
- raw = content.content
- else:
- raw = b""
-
- text = self._decode_bytes(raw)
- except Exception:
- raise NotFoundError(uri, "file")
-
- if offset == 0 and limit == -1:
- return text
- lines = text.splitlines(keepends=True)
- sliced = lines[offset:] if limit == -1 else lines[offset : offset + limit]
- return "".join(sliced)
-
- async def read_file_bytes(
- self,
- uri: str,
- ctx: Optional[RequestContext] = None,
- ) -> bytes:
- """Read single binary file."""
- self._ensure_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- primary_path = self._uri_to_path(uri, ctx=ctx)
- last_not_found: Optional[Exception] = None
- for path in self._read_paths(uri, ctx=ctx):
- if not await self._read_path_visible(uri, path, primary_path, real_ctx):
- continue
- try:
- stat = await self._async_agfs.stat(path)
- break
- except Exception as exc:
- if is_not_found_error(exc):
- last_not_found = exc
- continue
- raise
- else:
- raise NotFoundError(uri, "file") from last_not_found
- if isinstance(stat, dict) and stat.get("isDir", False):
- raise InvalidArgumentError(
- f"Cannot read directory as file: {uri}",
- details={"resource": uri, "expected": "file", "actual": "directory"},
- )
- try:
- raw = self._handle_agfs_read(await self._async_agfs.read(path))
- return raw
- except Exception:
- raise NotFoundError(uri, "file")
-
- async def write_file_bytes(
- self,
- uri: str,
- content: bytes,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> None:
- """Write single binary file. Encryption lock handled internally by EncryptionWrappedFS."""
- self._ensure_mutable_access(uri, ctx)
- path = self._uri_to_path(uri, ctx=ctx)
- await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)
-
- await self._async_agfs.write(path, content, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))
-
- async def append_file(
- self,
- uri: str,
- content: str,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> None:
- """Append content to file while holding one exact pathlock lease."""
- self._ensure_mutable_access(uri, ctx)
- path = self._uri_to_path(uri, ctx=ctx)
-
- owned_lease = None
- try:
- await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)
- lease = lease_ref
- if lease is None:
- lease = await self._async_agfs.pathlock_acquire_exact(path)
- owned_lease = lease
- fs_ctx = self._pathlock_fs_ctx(ctx, lease)
-
- # Read old content and rewrite the whole file to avoid lost updates.
- existing = ""
- try:
- existing_bytes = self._handle_agfs_read(
- await self._async_agfs.read(path, fs_ctx=fs_ctx)
- )
- existing = self._decode_bytes(existing_bytes)
- except FileNotFoundError:
- pass
- except AGFSHTTPError as e:
- if e.status_code != 404:
- raise
- except AGFSClientError:
- raise
-
- final_content = (existing + content).encode("utf-8")
- await self._async_agfs.write(
- path,
- final_content,
- fs_ctx=fs_ctx,
- )
-
- except Exception as e:
- logger.error(f"[VikingFS] Failed to append to file {uri}: {e}")
- raise IOError(f"Failed to append to file {uri}: {e}")
- finally:
- if owned_lease is not None:
- await self._async_agfs.pathlock_release(owned_lease)
-
- async def ls(
- self,
- uri: str,
- output: str = "original",
- abs_limit: int = 256,
- show_all_hidden: bool = False,
- node_limit: int = 1000,
- sort_by: Optional[str] = None,
- sort_order: str = "asc",
- ctx: Optional[RequestContext] = None,
- ) -> List[Dict[str, Any]]:
- """
- List directory contents (URI version).
-
- Args:
- uri: Viking URI
- output: str = "original"
- abs_limit: int = 256
- show_all_hidden: bool = False (list all hidden files, like -a)
- node_limit: int = 1000 (maximum number of nodes to list)
- sort_by: Optional sort field, "name" or "mtime"
- sort_order: Sort direction, "asc" or "desc"
-
- output="original"
- [{'name': '.abstract.md', 'size': 100, 'mode': 420, 'modTime': '2026-02-11T16:52:16.256334192+08:00', 'isDir': False, 'meta': {'Name': 'localfs', 'Type': 'local', 'Content': None}, 'uri': 'viking://resources/.abstract.md'}]
-
- output="agent"
- [{'name': '.abstract.md', 'size': 100, 'modTime': '2026-02-11T08:52:16.256Z', 'isDir': False, 'uri': 'viking://resources/.abstract.md', 'abstract': "..."}]
- """
- self._ensure_access(uri, ctx)
- if sort_by not in {None, "name", "mtime"}:
- raise ValueError("sort_by must be 'name' or 'mtime'")
- if sort_order not in {"asc", "desc"}:
- raise ValueError("sort_order must be 'asc' or 'desc'")
- if output == "original":
- return await self._ls_original(
- uri,
- show_all_hidden,
- node_limit,
- sort_by=sort_by,
- sort_order=sort_order,
- ctx=ctx,
- )
- elif output == "agent":
- return await self._ls_agent(
- uri,
- abs_limit,
- show_all_hidden,
- node_limit,
- sort_by=sort_by,
- sort_order=sort_order,
- ctx=ctx,
- )
- else:
- raise ValueError(f"Invalid output format: {output}")
-
- @staticmethod
- def _ls_entry_mtime(entry: Dict[str, Any]) -> Optional[float]:
- raw_time = entry.get("modTime")
- if isinstance(raw_time, (int, float)):
- return float(raw_time)
- if isinstance(raw_time, str) and raw_time:
- try:
- return parse_iso_datetime(raw_time).timestamp()
- except (TypeError, ValueError, OverflowError):
- return None
-
- legacy_time = entry.get("mtime")
- if isinstance(legacy_time, (int, float)):
- return float(legacy_time)
- return None
-
- @classmethod
- def _sort_ls_entry_items(
- cls,
- entry_items: List[tuple[Dict[str, Any], str]],
- sort_by: Optional[str],
- sort_order: str,
- ) -> List[tuple[Dict[str, Any], str]]:
- if sort_by is None:
- return entry_items
-
- descending = sort_order == "desc"
- directories = [item for item in entry_items if item[0].get("isDir", False)]
- files = [item for item in entry_items if not item[0].get("isDir", False)]
-
- if sort_by == "name":
-
- def name_key(item: tuple[Dict[str, Any], str]) -> tuple[str, str]:
- name = str(item[0].get("name", ""))
- return name.lower(), name
-
- directories.sort(key=name_key, reverse=descending)
- files.sort(key=name_key, reverse=descending)
- return directories + files
-
- def sort_by_mtime(
- items: List[tuple[Dict[str, Any], str]],
- ) -> List[tuple[Dict[str, Any], str]]:
- timestamped = []
- missing = []
- for item in items:
- timestamp = cls._ls_entry_mtime(item[0])
- if timestamp is None:
- missing.append(item)
- else:
- timestamped.append((timestamp, item))
- timestamped.sort(
- key=lambda pair: pair[0],
- reverse=descending,
- )
- return [item for _, item in timestamped] + missing
-
- return sort_by_mtime(directories) + sort_by_mtime(files)
-
- async def _ls_agent(
- self,
- uri: str,
- abs_limit: int,
- show_all_hidden: bool,
- node_limit: int = 1000,
- sort_by: Optional[str] = None,
- sort_order: str = "asc",
- ctx: Optional[RequestContext] = None,
- ) -> List[Dict[str, Any]]:
- """List directory contents (URI version)."""
- real_ctx = self._ctx_or_default(ctx)
- entry_items = await self._list_read_path_items(uri, ctx=ctx)
- entry_items = self._sort_ls_entry_items(entry_items, sort_by, sort_order)
- # basic info
- fallback_time = datetime.now(timezone.utc)
- all_entries = []
- for entry, entry_uri in entry_items:
- if len(all_entries) >= node_limit:
- break
- name = entry.get("name", "")
- raw_time = entry.get("modTime", "")
- parsed_time = fallback_time
- if isinstance(raw_time, (int, float)):
- parsed_time = datetime.fromtimestamp(raw_time, tz=timezone.utc)
- elif raw_time:
- if len(raw_time) > 26 and "+" in raw_time:
- parts = raw_time.split("+")
- raw_time = parts[0][:26] + "+" + parts[1]
- parsed_time = parse_iso_datetime(raw_time)
- elif isinstance(entry.get("mtime"), (int, float)):
- parsed_time = datetime.fromtimestamp(entry["mtime"], tz=timezone.utc)
- is_dir = entry.get("isDir", False)
- new_entry = {
- "uri": entry_uri,
- "size": 0 if is_dir else entry.get("size", 0),
- "isDir": is_dir,
- "modTime": format_iso8601(parsed_time),
- }
- if not self._is_accessible(new_entry["uri"], real_ctx):
- continue
- if is_dir:
- all_entries.append(new_entry)
- elif not name.startswith("."):
- all_entries.append(new_entry)
- elif show_all_hidden:
- all_entries.append(new_entry)
- await self._batch_fetch_abstracts(all_entries, abs_limit, ctx=ctx)
- return all_entries
-
- async def _ls_original(
- self,
- uri: str,
- show_all_hidden: bool = False,
- node_limit: int = 1000,
- sort_by: Optional[str] = None,
- sort_order: str = "asc",
- ctx: Optional[RequestContext] = None,
- ) -> List[Dict[str, Any]]:
- """List directory contents (URI version)."""
- real_ctx = self._ctx_or_default(ctx)
- try:
- entry_items = await self._list_read_path_items(uri, ctx=ctx)
- entry_items = self._sort_ls_entry_items(entry_items, sort_by, sort_order)
- # AGFS returns read-only structure, need to create new dict
- all_entries = []
- for entry, entry_uri in entry_items:
- if len(all_entries) >= node_limit:
- break
- name = entry.get("name", "")
- new_entry = dict(entry) # Copy original data
- new_entry["uri"] = entry_uri
- if not self._is_accessible(new_entry["uri"], real_ctx):
- continue
- if entry.get("isDir"):
- all_entries.append(new_entry)
- elif not name.startswith("."):
- all_entries.append(new_entry)
- elif show_all_hidden:
- all_entries.append(new_entry)
- return all_entries
- except Exception:
- raise NotFoundError(uri, "directory")
-
- async def move_file(
- self,
- from_uri: str,
- to_uri: str,
- ctx: Optional[RequestContext] = None,
- ) -> None:
- """Move file."""
- self._ensure_mutable_access(from_uri, ctx)
- self._ensure_mutable_access(to_uri, ctx)
- from_path = self._uri_to_path(from_uri, ctx=ctx)
-
- await self._copy_file_through_vikingfs(from_uri, to_uri, ctx=ctx)
- await self._async_agfs.rm(from_path)
-
- # ========== Temp File Operations (backward compatible) ==========
-
- def create_temp_uri(self, ctx: Optional[RequestContext] = None) -> str:
- """Create a temp directory URI.
-
- - explicit ctx or bound request context -> user-scoped temp URI
- - no explicit/bound context -> legacy temp URI shape for backward compatibility
- """
- real_ctx = ctx if ctx is not None else self._bound_ctx.get()
- if real_ctx is None:
- return VikingURI.create_temp_uri()
- return VikingURI.create_temp_uri(space=real_ctx.user.user_space_name())
-
- async def persist_temp_tree(
- self,
- temp_uri: str,
- target_uri: str,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> None:
- """Persist an already-encrypted temp tree without rewriting file bytes."""
- self._ensure_access(temp_uri, ctx)
- self._ensure_mutable_access(target_uri, ctx)
- src_path = self._uri_to_path(temp_uri, ctx=ctx)
- dst_path = self._uri_to_path(target_uri, ctx=ctx)
- fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)
- await self._ensure_parent_dirs(dst_path, ctx=ctx, lease_ref=lease_ref)
- await self._async_agfs.cp(
- src_path,
- dst_path,
- recursive=True,
- fs_ctx=fs_ctx or {"account_id": self._ctx_or_default(ctx).account_id},
- )
-
- async def delete_temp(
- self,
- temp_uri: str,
- ctx: Optional[RequestContext] = None,
- lease_ref: Dict[str, Any] | None = None,
- ) -> None:
- """Delete temp directory and its contents."""
- self._ensure_mutable_access(temp_uri, ctx)
- path = self._uri_to_path(temp_uri, ctx=ctx)
- fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)
- try:
- await self._async_agfs.rm(path, recursive=True, fs_ctx=fs_ctx)
- except Exception as e:
- logger.warning(f"[VikingFS] Failed to delete temp {temp_uri}: {e}")
-
- async def get_relations(self, uri: str, ctx: Optional[RequestContext] = None) -> List[str]:
- """Get all related URIs (backward compatible)."""
- entries = await self.get_relation_table(uri, ctx=ctx)
- real_ctx = self._ctx_or_default(ctx)
- all_uris = []
- for entry in entries:
- for related in entry.uris:
- if self._is_accessible(related, real_ctx):
- all_uris.append(related)
- return all_uris
-
- async def get_relations_with_content(
- self,
- uri: str,
- include_l0: bool = True,
- include_l1: bool = False,
- ctx: Optional[RequestContext] = None,
- ) -> List[Dict[str, Any]]:
- """Get related URIs and their content (backward compatible)."""
- relation_uris = await self.get_relations(uri, ctx=ctx)
- if not relation_uris:
- return []
-
- results = []
- abstracts = {}
- overviews = {}
- if include_l0:
- abstracts = await self.read_batch(relation_uris, level="l0", ctx=ctx)
- if include_l1:
- overviews = await self.read_batch(relation_uris, level="l1", ctx=ctx)
-
- for rel_uri in relation_uris:
- info = {"uri": rel_uri}
- if include_l0:
- info["abstract"] = abstracts.get(rel_uri, "")
- if include_l1:
- info["overview"] = overviews.get(rel_uri, "")
- results.append(info)
-
- return results
-
- async def write_context(
- self,
- uri: str,
- content: Union[str, bytes] = "",
- abstract: str = "",
- overview: str = "",
- content_filename: str = "content.md",
- is_leaf: bool = False,
- ctx: Optional[RequestContext] = None,
- ) -> None:
- """Write context to AGFS (L0/L1/L2)."""
-
- self._ensure_mutable_access(uri, ctx)
- path = self._uri_to_path(uri, ctx=ctx)
-
- try:
- await self._ensure_parent_dirs(path, ctx=ctx)
- try:
- await self._async_agfs.mkdir(path)
- except Exception as e:
- if "exist" not in str(e).lower():
- raise
-
- if content:
- content_uri = f"{uri}/{content_filename}"
- await self.write_file(content_uri, content, ctx=ctx)
-
- if abstract:
- abstract_uri = f"{uri}/.abstract.md"
- await self.write_file(abstract_uri, abstract, ctx=ctx)
-
- if overview:
- overview_uri = f"{uri}/.overview.md"
- await self.write_file(overview_uri, overview, ctx=ctx)
-
- except Exception as e:
- logger.error(f"[VikingFS] Failed to write {uri}: {e}")
- raise IOError(f"Failed to write {uri}: {e}")
-
- # ------------------------------------------------------------------
- # Git version control (commit / restore / show / log)
- # ------------------------------------------------------------------
-
- # First path segments that the Rust git enumerate.rs prunes from snapshots,
- # plus the runtime lock name. Mirrors INTERNAL_FIRST_SEGMENTS in
- # crates/ragfs/src/git/enumerate.rs and VikingFS._INTERNAL_NAMES so that
- # callers fail fast in Python with a clear error rather than passing a
- # path that the Rust side will silently drop.
- _GIT_INTERNAL_FIRST_SEGMENTS = frozenset({
- "_system",
- "tasks",
- "temp",
- "queue",
- "upload",
- ".path.ovlock",
- })
-
- _DEFAULT_GIT_AUTHOR_NAME = "viking-bot"
- _DEFAULT_GIT_AUTHOR_EMAIL = "[email protected]"
-
- _OVGITIGNORE_TREE_PATH = ".ovgitignore"
-
- # Must stay in sync with `OVGITIGNORE_MAX_BYTES` in the Rust layer
- # (`crates/ragfs/src/git/ignore.rs`); enforced here at write time so a bad
- # file can never be persisted only to poison every later commit.
- _OVGITIGNORE_MAX_BYTES = 64 * 1024
-
- def _gitignore_agfs_path(self, ctx: Optional[RequestContext] = None) -> str:
- real_ctx = self._ctx_or_default(ctx)
- return f"/local/{real_ctx.account_id}/{self._OVGITIGNORE_TREE_PATH}"
-
- def _uri_to_tree_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:
- """Convert a viking:// URI to an account-relative git tree path.
-
- ``viking://resources/proj_a/docs/a.md`` -> ``resources/proj_a/docs/a.md``.
-
- Pure prefix stripping: removes the ``viking://`` scheme and any
- ``/local/{account}/`` segment. Internal scopes that the Rust git layer
- would prune (`_system`, `tasks`, `temp`, `queue`, `upload`) and the
- runtime lock name (`.path.ovlock`) are rejected with ``ValueError``
- — passing them through silently would result in a no-op commit and
- confuse callers.
- """
- real_ctx = self._ctx_or_default(ctx)
- canonical = canonicalize_uri(uri, real_ctx)
- _, parts = self._normalized_uri_parts(canonical)
- if not parts:
- raise ValueError(f"git tree path cannot be the account root: {uri!r}")
- first = parts[0]
- if first in self._GIT_INTERNAL_FIRST_SEGMENTS:
- raise ValueError(f"git tree path rejects internal scope/segment {first!r}: {uri!r}")
- return "/".join(parts)
-
- def _tree_path_to_uri(self, tree_path: str) -> str:
- """Convert an account-relative git tree path to a viking:// URI.
-
- Inverse of :py:meth:`_uri_to_tree_path` (without context canonicalization).
- """
- cleaned = tree_path.strip("/")
- if not cleaned:
- raise ValueError("tree path must not be empty")
- return f"viking://{cleaned}"
-
- _DIR_MARKER_LEVELS = {
- ".abstract.md": ContextLevel.ABSTRACT,
- ".overview.md": ContextLevel.OVERVIEW,
- }
- _NO_VECTOR_DERIVED = frozenset({".relations.json", ".ovgitignore"})
-
- def _classify_restore_path(self, tree_path: str, *, deleted: bool) -> Optional[tuple]:
- """Classify a restore-affected tree path into a vector maintenance task.
-
- Returns a ``(op, uri, level)`` triple, or ``None`` when the path has no
- vector side-effect:
-
- - ``dir/.abstract.md`` / ``dir/.overview.md`` → recompute (write) or
- delete (removal) ONLY that directory's L0/L1 vector:
- ``("reindex_marker"|"delete", dir_uri, ABSTRACT|OVERVIEW)``.
- - ``.relations.json`` → ``None`` (not a vector text source).
- - anything else (a source file) → reindex (write) or delete (removal)
- its DETAIL vector:
- ``("reindex_file", file_uri, DETAIL)`` / ``("delete", file_uri, DETAIL)``.
-
- ``None`` is also returned for a directory marker at the account root
- (no parent directory to scope an L0/L1 vector to).
- """
- parent, _, name = tree_path.rpartition("/")
- if name in self._NO_VECTOR_DERIVED:
- return None
- level = self._DIR_MARKER_LEVELS.get(name)
- if level is not None:
- if not parent:
- return None
- dir_uri = self._tree_path_to_uri(parent)
- op = "delete" if deleted else "reindex_marker"
- return (op, dir_uri, level)
- # Source file.
- file_uri = self._tree_path_to_uri(tree_path)
- op = "delete" if deleted else "reindex_file"
- return (op, file_uri, ContextLevel.DETAIL)
-
- async def get_gitignore(self, ctx: Optional[RequestContext] = None) -> str:
- """Return the account-level .ovgitignore content, or an empty string if absent.
-
- Raises ``AGFSInvalidOperationError`` if the stored content is not valid
- UTF-8, mirroring the Rust layer's reject-non-UTF-8 behavior at commit
- time rather than leaking a raw ``UnicodeDecodeError``.
- """
- path = self._gitignore_agfs_path(ctx)
- try:
- raw = await self._async_agfs.read(path, 0, -1)
- except Exception as exc:
- if is_not_found_error(exc):
- return ""
- mapped = map_exception(exc, resource=path)
- if mapped is not None:
- raise mapped from exc
- raise
- if isinstance(raw, bytes):
- data = raw
- elif raw is not None and hasattr(raw, "content"):
- data = raw.content
- else:
- data = b""
- try:
- return data.decode("utf-8")
- except UnicodeDecodeError as exc:
- raise AGFSInvalidOperationError(
- f"invalid ignore file {path}: must be UTF-8: {exc}"
- ) from exc
-
- async def set_gitignore(
- self,
- content: str,
- ctx: Optional[RequestContext] = None,
- ) -> None:
- """Write the account-level .ovgitignore control file without semantic indexing.
-
- Validates the size limit up front so a too-large file can never be
- persisted only to make every later ``commit`` fail. Syntax (negation,
- escaping) is still validated at commit time by the Rust layer.
- """
- if not isinstance(content, str):
- raise TypeError("content must be a string")
- path = self._gitignore_agfs_path(ctx)
- data = content.encode("utf-8")
- if len(data) > self._OVGITIGNORE_MAX_BYTES:
- raise AGFSInvalidOperationError(
- f"ignore file too large: {path} is {len(data)} bytes, "
- f"limit {self._OVGITIGNORE_MAX_BYTES} bytes"
- )
- await self._ensure_parent_dirs(path, ctx=ctx)
- await self._async_agfs.write(path, data)
-
- async def delete_gitignore(self, ctx: Optional[RequestContext] = None) -> None:
- """Delete the account-level .ovgitignore control file. Missing is success."""
- path = self._gitignore_agfs_path(ctx)
- try:
- await self._async_agfs.rm(path, recursive=False)
- except Exception as exc:
- if is_not_found_error(exc):
- return
- mapped = map_exception(exc, resource=path)
- if mapped is not None:
- raise mapped from exc
- raise
-
- async def commit(
- self,
- *,
- message: str,
- paths: Optional[List[str]] = None,
- branch: str = "main",
- author_name: Optional[str] = None,
- author_email: Optional[str] = None,
- ctx: Optional[RequestContext] = None,
- ) -> Dict[str, Any]:
- """Create a git snapshot of the account's tree.
-
- Args:
- message: Commit message.
- paths: Optional list of ``viking://`` URIs to scope the commit to;
- entries may be files or directories. Directories are expanded
- recursively with the snapshot pruning rules applied. ``None``
- (default) enumerates the whole account tree. An empty list is
- forwarded as an explicit empty path list (no-op commit). A
- path that exists in neither the VFS nor the previous snapshot
- logs a warning and is treated as a no-op deletion.
- branch: Branch to advance. Defaults to ``"main"``.
- author_name / author_email: Override the default bot author.
- ctx: Request context (provides ``account_id``).
-
- Returns:
- Dict with ``result`` (``"created"`` / ``"noop"``) and ``commit_oid``.
- When ``result == "created"`` it also includes ``changed`` (paths
- added/updated/removed). Both variants include ``ignored``: the
- number of candidate paths skipped by the account ``.ovgitignore``
- rules (system pruning is not counted). Noop omits ``changed``.
- """
- real_ctx = self._ctx_or_default(ctx)
- account = real_ctx.account_id
- if paths is None:
- tree_paths: Optional[List[str]] = None
- else:
- tree_paths = [self._uri_to_tree_path(p, ctx=real_ctx) for p in paths]
- return await self._async_agfs.run(
- "git_commit",
- account=account,
- branch=branch,
- message=message,
- paths=tree_paths,
- author_name=author_name or self._DEFAULT_GIT_AUTHOR_NAME,
- author_email=author_email or self._DEFAULT_GIT_AUTHOR_EMAIL,
- )
-
- async def restore(
- self,
- *,
- project_dir: Optional[str] = None,
- source_commit: str,
- branch: str = "main",
- dry_run: bool = False,
- message: Optional[str] = None,
- author_name: Optional[str] = None,
- author_email: Optional[str] = None,
- ctx: Optional[RequestContext] = None,
- ) -> Dict[str, Any]:
- """Restore a project subtree to the state at ``source_commit``.
-
- Generates a new commit whose parent is the current HEAD (not the
- source commit) and writes the diff through the VFS. Paths outside
- ``project_dir`` are left untouched.
-
- Args:
- project_dir: ``viking://`` URI of the subtree, e.g.
- ``"viking://resources/proj_a"``. May also be passed as a
- short form like ``"resources/proj_a"``. Trailing slashes are
- stripped. Internal scopes are rejected with ``ValueError``.
- If None, restore the entire tree.
- source_commit: 40-hex OID, branch name, or full ref path.
- branch: Branch to advance. Defaults to ``"main"``.
- dry_run: If True, returns the planned diff without writing.
- message: Optional commit message; defaults to a generated string.
- author_name / author_email: Override the default bot author.
- ctx: Request context (provides ``account_id``).
-
- Returns:
- Dict containing ``result`` (``"applied"`` / ``"noop"`` / ``"dry_run"``)
- and corresponding oid / diff fields. When an ``Applied`` result has
- vector side-effects, a ``task_id`` is included for polling the
- background reindex via ``GET /api/v1/tasks/{task_id}``.
-
- After an ``Applied`` result, this method schedules background vector
- maintenance for the affected paths via :class:`ReindexExecutor`:
- directory markers (``.abstract.md`` / ``.overview.md``) recompute or
- delete only that directory's L0/L1 vector; source files reindex (write)
- or delete (removal) their DETAIL vector. ``.relations.json`` has no
- vector side-effect. The rebuild is tracked as a single task
- (``snapshot_restore_reindex``); per-path failures are logged and do not
- block the return value. The task reaches ``completed`` only after each
- affected vector has been written to (or deleted from) the index, so
- polling ``task_id`` to ``completed`` guarantees subsequent ``find``
- reads see the restored state.
- """
- real_ctx = self._ctx_or_default(ctx)
- account = real_ctx.account_id
- tree_dir: Optional[str]
- if project_dir is None:
- tree_dir = None
- else:
- tree_dir = self._uri_to_tree_path(project_dir, ctx=real_ctx).rstrip("/")
- if not tree_dir:
- raise ValueError(f"project_dir must not be empty: {project_dir!r}")
- # Build kwargs dynamically, only include project_dir if it's not None
- kwargs = {
- "account": account,
- "branch": branch,
- "source_commit": source_commit,
- "dry_run": dry_run,
- "message": message,
- "author_name": author_name or self._DEFAULT_GIT_AUTHOR_NAME,
- "author_email": author_email or self._DEFAULT_GIT_AUTHOR_EMAIL,
- }
- if tree_dir is not None:
- kwargs["project_dir"] = tree_dir
- # dry_run only computes the diff; it never writes the VFS, so it needs
- # no lock.
- if dry_run:
- return await self._async_agfs.run("git_restore", **kwargs)
-
- from openviking.pyagfs.exceptions import GitRestoreWritebackPartialError
- from openviking.storage.errors import LockAcquisitionError, ResourceBusyError
-
- # Serialize the writeback against concurrent VFS mutations on the same
- # subtree. A scoped restore tree-locks project_dir; a full restore
- # (project_dir is None) locks the account root so only one restore runs
- # per account at a time. The lock covers only the writeback — the
- # background reindex is scheduled after release, otherwise its per-path
- # child locks would conflict with this tree lock.
- lock_path = (
- self._uri_to_path(project_dir, ctx=real_ctx)
- if project_dir is not None
- else f"/local/{account}"
- )
- partial_exc: Optional[GitRestoreWritebackPartialError] = None
- try:
- lease = await self._async_agfs.pathlock_acquire_tree(lock_path)
- except LockAcquisitionError:
- raise ResourceBusyError(
- f"Resource is being processed: {project_dir or '*'}",
- uri=project_dir or "*",
- )
-
- try:
- try:
- result = await self._async_agfs.run("git_restore", **kwargs)
- except GitRestoreWritebackPartialError as exc:
- # The ref already advanced — capture the exception and
- # finish the lock scope cleanly. Reindex scheduling and
- # re-raise happen below, outside the tree lock.
- partial_exc = exc
- result = None
- finally:
- await self._async_agfs.pathlock_release(lease)
-
- if partial_exc is not None:
- # HEAD has moved forward but some VFS writes/deletes failed.
- # Still schedule reindex for the paths that *did* reach the VFS so
- # the vector index doesn't stay stale, then re-raise so the caller
- # learns about the partial failure (and can inspect failed_writes
- # / failed_deletes / task_id on the exception).
- try:
- partial_exc.task_id = await self._schedule_restore_reindex_for_paths(
- written_paths=partial_exc.written_paths,
- deleted_paths=partial_exc.deleted_paths,
- project_dir=project_dir,
- real_ctx=real_ctx,
- )
- except Exception:
- logger.exception(
- "[VikingFS] git restore partial: reindex scheduling failed; "
- "HEAD advanced but reindex was not queued"
- )
- raise partial_exc
-
- if result.get("result") != "applied":
- return result
-
- written = list(result.get("written_paths") or [])
- deleted = list(result.get("deleted_paths") or [])
- if written or deleted:
- try:
- task_id = await self._schedule_restore_reindex_for_paths(
- written_paths=written,
- deleted_paths=deleted,
- project_dir=project_dir,
- real_ctx=real_ctx,
- )
- if task_id is not None:
- result["task_id"] = task_id
- except Exception:
- logger.exception(
- "[VikingFS] git restore reindex task creation failed; "
- "falling back to fire-and-forget rebuild"
- )
- self._schedule_vector_rebuild(written=written, deleted=deleted, ctx=real_ctx)
- return result
-
- async def _schedule_restore_reindex_for_paths(
- self,
- *,
- written_paths: List[str],
- deleted_paths: List[str],
- project_dir: Optional[str],
- real_ctx: RequestContext,
- ) -> Optional[str]:
- """Classify ``written``/``deleted`` paths into vector tasks and queue
- them as a single tracked background rebuild. Returns the task id, or
- ``None`` if there is nothing to do.
-
- Shared by the applied path and the partial-writeback recovery path —
- both schedule reindex for paths that actually reached the VFS.
- """
- if not written_paths and not deleted_paths:
- return None
- tasks = self._collect_restore_vector_tasks(written_paths, deleted_paths)
- if not tasks:
- return None
-
- from openviking.service.task_tracker import get_task_tracker
-
- tracker = get_task_tracker()
- task = await tracker.create(
- "snapshot_restore_reindex",
- resource_id=project_dir or "*",
- account_id=real_ctx.account_id,
- user_id=real_ctx.user.user_id,
- )
- await tracker.update_stage(
- task.task_id,
- "queued",
- account_id=real_ctx.account_id,
- user_id=real_ctx.user.user_id,
- )
- background = asyncio.create_task(
- self._run_restore_rebuild_tracked(task.task_id, tasks, real_ctx),
- name=f"vikingfs-git-restore-reindex:{task.task_id}",
- )
- self._background_tasks.add(background)
- background.add_done_callback(self._background_tasks.discard)
- return task.task_id
-
- async def show(
- self,
- target_ref: str,
- *,
- path: Optional[str] = None,
- max_blob_bytes: Optional[int] = None,
- ctx: Optional[RequestContext] = None,
- ) -> Union[Dict[str, Any], bytes]:
- """Read a commit's metadata or a single blob.
-
- ``path=None`` returns the commit metadata dict (oid, tree, parents,
- author, committer, message). ``path=str`` returns the blob bytes
- directly, stripping the oid/size envelope returned by the binding.
- """
- real_ctx = self._ctx_or_default(ctx)
- account = real_ctx.account_id
- tree_path = self._uri_to_tree_path(path, ctx=real_ctx) if path else None
- kwargs: Dict[str, Any] = {
- "account": account,
- "target_ref": target_ref,
- "path": tree_path,
- }
- if max_blob_bytes is not None:
- kwargs["max_blob_bytes"] = max_blob_bytes
- resp = await self._async_agfs.run("git_show", **kwargs)
- if path is not None and isinstance(resp, dict) and "bytes" in resp:
- return resp["bytes"]
- return resp
-
- async def show_blob_raw(
- self,
- target_ref: str,
- *,
- path: str,
- ctx: Optional[RequestContext] = None,
- ) -> Dict[str, Any]:
- """Like ``show(target_ref, path=...)`` but returns the full envelope.
-
- Returns ``{"oid": str, "size": int, "bytes": bytes}`` without
- stripping. Used by the HTTP snapshot router to populate
- ``X-Snapshot-Oid`` / ``X-Snapshot-Size`` response headers.
- """
- real_ctx = self._ctx_or_default(ctx)
- account = real_ctx.account_id
- tree_path = self._uri_to_tree_path(path, ctx=real_ctx)
- resp = await self._async_agfs.run(
- "git_show",
- account=account,
- target_ref=target_ref,
- path=tree_path,
- )
- if not isinstance(resp, dict) or "bytes" not in resp:
- raise TypeError(
- f"git_show returned unexpected shape for blob path: {type(resp).__name__}"
- )
- return resp
-
- async def diff(
- self,
- *,
- path: str,
- from_ref: Optional[str],
- to_ref: str,
- raw: bool = True,
- ctx: Optional[RequestContext] = None,
- ) -> Dict[str, Any]:
- """Return a unified text diff for one path between two snapshots."""
- real_ctx = self._ctx_or_default(ctx)
- path = canonicalize_uri(path, ctx=real_ctx)
- self._ensure_access(path, real_ctx)
-
- from_meta: Optional[Dict[str, Any]] = None
- if from_ref:
- try:
- from_meta = await self.show(from_ref, ctx=real_ctx)
- except AGFSNotFoundError as exc:
- raise NotFoundError(from_ref, "git_ref") from exc
- try:
- to_meta = await self.show(to_ref, ctx=real_ctx)
- except AGFSNotFoundError as exc:
- raise NotFoundError(to_ref, "git_ref") from exc
-
- from_oid = str(from_meta.get("oid", from_ref)) if isinstance(from_meta, dict) else ""
- to_oid = str(to_meta.get("oid", to_ref)) if isinstance(to_meta, dict) else to_ref
-
- async def read_optional(ref: str) -> Optional[bytes]:
- try:
- value = await self.show(
- ref,
- path=path,
- max_blob_bytes=SNAPSHOT_DIFF_MAX_FILE_BYTES,
- ctx=real_ctx,
- )
- except AGFSPathNotFoundError:
- return None
- except AGFSResourceExhaustedError as exc:
- raise ResourceExhaustedError(
- f"snapshot diff file size limit exceeded "
- f"({SNAPSHOT_DIFF_MAX_FILE_BYTES} bytes)",
- details={"limit_bytes": SNAPSHOT_DIFF_MAX_FILE_BYTES, "path": path},
- ) from exc
- if not isinstance(value, bytes):
- raise TypeError(f"git_show returned unexpected blob type: {type(value).__name__}")
- return value
-
- before_bytes = await read_optional(from_oid) if from_ref else None
- after_bytes = await read_optional(to_oid)
- if before_bytes is None and after_bytes is None:
- raise NotFoundError(path, "git_blob")
-
- for value in (before_bytes, after_bytes):
- if value is not None and len(value) > SNAPSHOT_DIFF_MAX_FILE_BYTES:
- raise ResourceExhaustedError(
- f"snapshot diff file size limit exceeded ({SNAPSHOT_DIFF_MAX_FILE_BYTES} bytes)",
- details={"limit_bytes": SNAPSHOT_DIFF_MAX_FILE_BYTES, "path": path},
- )
-
- if not raw:
- from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils
-
- def visible_content(value: Optional[bytes]) -> Optional[bytes]:
- if value is None:
- return None
- try:
- text = value.decode("utf-8")
- except UnicodeDecodeError as exc:
- raise InvalidArgumentError("raw=false requires UTF-8 snapshot content") from exc
- return MemoryFileUtils.read(text, uri=path).content.encode("utf-8")
-
- before_bytes = visible_content(before_bytes)
- after_bytes = visible_content(after_bytes)
-
- change_type, before, after = await asyncio.to_thread(
- _prepare_snapshot_diff,
- path=path,
- before_bytes=before_bytes,
- after_bytes=after_bytes,
- max_lines=SNAPSHOT_DIFF_MAX_LINES,
- )
- try:
- diff_text = await self._async_agfs.run(
- "git_diff_text",
- before=before,
- after=after,
- fromfile=f"{path}@{from_oid}",
- tofile=f"{path}@{to_oid}",
- timeout_ms=SNAPSHOT_DIFF_TIMEOUT_MS,
- max_output_bytes=SNAPSHOT_DIFF_MAX_OUTPUT_BYTES,
- )
- except AGFSResourceExhaustedError as exc:
- raise ResourceExhaustedError(
- f"snapshot diff output size limit exceeded "
- f"({SNAPSHOT_DIFF_MAX_OUTPUT_BYTES} bytes)",
- details={"limit_bytes": SNAPSHOT_DIFF_MAX_OUTPUT_BYTES, "path": path},
- ) from exc
- return {
- "path": path,
- "from_commit": from_oid,
- "to_commit": to_oid,
- "change_type": change_type,
- "diff_text": diff_text,
- }
-
- async def log(
- self,
- *,
- branch: str = "main",
- limit: int = 20,
- paths: Optional[List[str]] = None,
- ctx: Optional[RequestContext] = None,
- ) -> List[Dict[str, Any]]:
- """Walk back from ``branch``'s HEAD along ``parents[0]`` up to ``limit`` commits.
-
- Returns a list of commit metadata dicts (same shape as ``show(target_ref)``).
- """
- if limit <= 0:
- return []
- real_ctx = self._ctx_or_default(ctx)
- account = real_ctx.account_id
- tree_paths = (
- None if not paths else [self._uri_to_tree_path(path, ctx=real_ctx) for path in paths]
- )
- return await self._async_agfs.run(
- "git_log",
- account=account,
- branch=branch,
- limit=limit,
- paths=tree_paths,
- )
-
- def _collect_restore_vector_tasks(
- self,
- written: List[str],
- deleted: List[str],
- ) -> set[tuple]:
- """Classify restore-affected paths into deduplicated ``(op, uri, level)`` tasks.
-
- Tasks are deduplicated on the exact ``(op, uri, level)`` key (no ancestor
- subsumption — each change is handled independently because no operation
- recurses into descendants).
- """
- tasks: set[tuple] = set()
- for tree_path in written:
- try:
- task = self._classify_restore_path(tree_path, deleted=False)
- except ValueError:
- continue
- if task is not None:
- tasks.add(task)
- for tree_path in deleted:
- try:
- task = self._classify_restore_path(tree_path, deleted=True)
- except ValueError:
- continue
- if task is not None:
- tasks.add(task)
- return tasks
-
- def _schedule_vector_rebuild(
- self,
- *,
- written: List[str],
- deleted: List[str],
- ctx: RequestContext,
- ) -> None:
- """Fire-and-forget precise vector maintenance for a git restore.
-
- Each affected path is classified by :py:meth:`_classify_restore_path`
- into a ``(op, uri, level)`` task and scheduled independently:
-
- - ``reindex_marker`` — recompute only a directory's L0/L1 vector.
- - ``reindex_file`` — recompute only a source file's DETAIL vector
- (non-recursive ``execute(mode="vectors_only")``).
- - ``delete`` — remove only the ``(uri, level)`` vector (directory
- marker removal, or deleted source file).
-
- Failures are logged and never propagate.
- """
- try:
- from openviking.service.reindex_executor import get_reindex_executor
- except Exception:
- logger.exception("[VikingFS] ReindexExecutor import failed; skipping rebuild")
- return
-
- try:
- loop = asyncio.get_running_loop()
- except RuntimeError:
- logger.warning("[VikingFS] git restore vector rebuild skipped: no running event loop")
- return
-
- tasks = self._collect_restore_vector_tasks(written, deleted)
- if not tasks:
- return
-
- executor = get_reindex_executor()
- for op, uri, level in tasks:
- loop.create_task(
- self._run_vector_rebuild(executor, op, uri, level, ctx),
- name=f"vikingfs-git-{op}:{uri}:{int(level)}",
- )
-
- async def _run_restore_rebuild_tracked(
- self,
- task_id: str,
- tasks: set[tuple],
- ctx: RequestContext,
- ) -> None:
- """Background worker driving a tracked restore vector rebuild.
-
- Runs all classified ``(op, uri, level)`` rebuild tasks concurrently and
- drives the task through start → complete/fail. Per-task failures are
- swallowed (and logged) inside :py:meth:`_run_vector_rebuild`, preserving
- the "failures do not block" semantics.
- """
- from openviking.service.task_tracker import get_task_tracker
- from openviking.service.task_work_index import bind_task_context
-
- tracker = get_task_tracker()
- tracker.register_running_task(task_id)
- try:
- await tracker.start(
- task_id,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- stage="reindexing",
- )
- from openviking.service.reindex_executor import get_reindex_executor
-
- executor = get_reindex_executor()
- with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):
- await asyncio.gather(
- *[
- self._run_vector_rebuild(executor, op, uri, level, ctx)
- for (op, uri, level) in tasks
- ]
- )
- await tracker.complete(
- task_id,
- {"status": "completed", "task_count": len(tasks)},
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- except asyncio.CancelledError:
- # TaskWorkIndex finalizes after this active task and its queue work settle.
- return
- except Exception as exc:
- await tracker.fail(
- task_id,
- str(exc),
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- finally:
- await tracker.unregister_running_task(task_id)
-
- async def _run_vector_rebuild(
- self,
- executor: Any,
- op: str,
- uri: str,
- level: ContextLevel,
- ctx: RequestContext,
- ) -> None:
- """Wrapper coroutine: dispatch one vector task and swallow errors."""
- try:
- if op == "reindex_marker":
- await executor.reindex_directory_marker(dir_uri=uri, level=level, ctx=ctx)
- elif op == "reindex_file":
- await executor.execute(uri=uri, mode="vectors_only", wait=True, ctx=ctx)
- elif op == "delete":
- await executor.delete_uri_level(uri=uri, level=level, ctx=ctx)
- else: # pragma: no cover - defensive
- logger.warning("[VikingFS] unknown vector rebuild op %r for %s", op, uri)
- except Exception:
- logger.exception("[VikingFS] git restore vector task %s failed for %s", op, uri)
diff --git a/openviking/storage/viking_fs/__init__.py b/openviking/storage/viking_fs/__init__.py
new file mode 100644
index 000000000..d543ac689
--- /dev/null
+++ b/openviking/storage/viking_fs/__init__.py
@@ -0,0 +1,205 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""
+VikingFS: OpenViking file system abstraction layer
+
+Encapsulates the AGFS binding client, providing file operation interface based on Viking URI.
+Responsibilities:
+- URI conversion (viking:// <-> /local/)
+- L0/L1 reading (.abstract.md, .overview.md)
+- Relation management (.relations.json)
+- Semantic search (vector retrieval + rerank)
+- Vector sync (sync vector store on rm/mv)
+
+This package is assembled from mixin submodules. The ``VikingFS`` class
+combines all mixins; shared constants, helpers, and singleton management
+live in :mod:`openviking.storage.viking_fs._base`.
+"""
+
+import asyncio
+import contextvars
+import hashlib
+import json
+import os
+import re
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, TypeVar, Union
+
+from openviking.core.context import ContextLevel
+from openviking.core.namespace import (
+ canonicalize_uri,
+ is_hidden_by_actor_peer_view,
+ may_include_hidden_actor_peers,
+)
+from openviking.core.namespace import (
+ is_accessible as namespace_is_accessible,
+)
+from openviking.core.retrieval_targets import resolve_retrieval_targets
+from openviking.pyagfs import AsyncAGFSClient
+from openviking.pyagfs.exceptions import (
+ AGFSClientError,
+ AGFSDirectoryNotEmptyError,
+ AGFSHTTPError,
+ AGFSInvalidOperationError,
+ AGFSNotFoundError,
+ AGFSNotSupportedError,
+ AGFSPathNotFoundError,
+ AGFSResourceExhaustedError,
+)
+from openviking.resource.watch_storage import is_watch_task_control_uri
+from openviking.server.error_mapping import is_not_found_error, map_exception
+from openviking.server.identity import RequestContext, Role
+from openviking.storage.expr import And, PathScope, RawDSL
+from openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMES
+from openviking.telemetry import get_current_telemetry
+from openviking.utils.image_search import build_multimodal_embedding_input
+from openviking.utils.time_utils import format_iso8601, get_current_timestamp, parse_iso_datetime
+from openviking_cli.exceptions import (
+ FailedPreconditionError,
+ InvalidArgumentError,
+ NotFoundError,
+ PermissionDeniedError,
+ ResourceExhaustedError,
+)
+from openviking_cli.session.user_id import UserIdentifier
+from openviking_cli.utils.config.grep_config import GrepEngine
+from openviking_cli.utils.logger import get_logger
+from openviking_cli.utils.uri import VikingURI
+
+# Import mixins
+from openviking.storage.viking_fs import _base as _base_mod
+from openviking.storage.viking_fs._access import _AccessMixin
+from openviking.storage.viking_fs._base import (
+ LS_ALL_NODES,
+ SNAPSHOT_DIFF_MAX_FILE_BYTES,
+ SNAPSHOT_DIFF_MAX_LINES,
+ SNAPSHOT_DIFF_MAX_OUTPUT_BYTES,
+ SNAPSHOT_DIFF_TIMEOUT_MS,
+ RelationEntry,
+ _ABSTRACT_WORKER_COUNT,
+ _DEFAULT_GREP_FILE_CONCURRENCY,
+ _T,
+ _ensure_non_empty_search_query,
+ _get_abstract_worker_count,
+ _get_cpu_count,
+ _is_directory_not_empty_error,
+ _prepare_snapshot_diff,
+ _snapshot_line_count,
+ enable_viking_fs_recorder,
+ get_viking_fs,
+ init_viking_fs,
+ logger,
+)
+from openviking.storage.viking_fs._grep import _GrepMixin
+from openviking.storage.viking_fs._ops import _OpsMixin
+from openviking.storage.viking_fs._semantic import _SemanticMixin
+from openviking.storage.viking_fs._snapshot import _SnapshotMixin
+from openviking.storage.viking_fs._sync import SyncDiff, _SyncMixin
+from openviking.storage.viking_fs._vector import _VectorMixin
+
+if TYPE_CHECKING:
+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig
+
+
+# ========== VikingFS Main Class ==========
+
+
+class VikingFS(
+ _AccessMixin,
+ _OpsMixin,
+ _SyncMixin,
+ _GrepMixin,
+ _SemanticMixin,
+ _SnapshotMixin,
+ _VectorMixin,
+):
+ """RAGFS-based OpenViking file system.
+
+ APIs are divided into two categories:
+ - RAGFS basic commands (direct forwarding): read, ls, write, mkdir, rm, mv, grep, stat
+ - VikingFS specific capabilities: abstract, overview, find, search, relations, link, unlink
+
+ Uses Rust binding mode: Use RAGFSBindingClient to directly use RAGFS implementation
+ """
+
+ def __init__(
+ self,
+ agfs: Any,
+ query_embedder: Optional[Any] = None,
+ rerank_config: Optional["RerankConfig"] = None,
+ vector_store: Optional["VikingVectorIndexBackend"] = None,
+ retrieval_config: Optional["RetrievalConfig"] = None,
+ grep_config: Optional["GrepConfig"] = None,
+ timeout: int = 10,
+ encryptor: Optional[Any] = None,
+ ):
+ self.agfs = agfs
+ self._async_agfs = AsyncAGFSClient(agfs)
+ self.query_embedder = query_embedder
+ self.rerank_config = rerank_config
+ self.vector_store = vector_store
+ self.retrieval_config = retrieval_config
+ self.grep_config = grep_config
+ self._encryptor = encryptor
+ self._count_cache: Dict[str, tuple] = {} # cache_key → (count, timestamp)
+ self._count_cache_max_size = 1024
+ self._fulltext_available: Optional[bool] = None # cached result of _collection_has_fulltext
+ self._bound_ctx: contextvars.ContextVar[Optional[RequestContext]] = contextvars.ContextVar(
+ "vikingfs_bound_ctx", default=None
+ )
+ self._background_tasks: set = set()
+ self._user_deletion_guard: Optional[Callable[[str, str], bool]] = None
+
+
+VikingFS.__module__ = __name__
+
+# Provide _instance on the package module that transparently proxies to
+# _base._instance so that test fixtures (conftest.py) can save/restore the
+# singleton across init_viking_fs boundaries. PEP 562 only gives us
+# __getattr__; we also need to intercept writes so that assigning to
+# ``openviking.storage.viking_fs._instance`` updates the real singleton in
+# ``_base`` where ``get_viking_fs``/``init_viking_fs`` read/write it.
+import sys as _sys
+
+
+class _ModuleProxy: # pragma: no cover - trivial proxy
+ def __init__(self, wrapped):
+ object.__setattr__(self, "_wrapped", wrapped)
+
+ def __getattr__(self, name):
+ if name == "_instance":
+ return _base_mod._instance
+ return getattr(object.__getattribute__(self, "_wrapped"), name)
+
+ def __setattr__(self, name, value):
+ if name == "_instance":
+ _base_mod._instance = value
+ return
+ setattr(object.__getattribute__(self, "_wrapped"), name, value)
+
+ def __delattr__(self, name):
+ delattr(object.__getattribute__(self, "_wrapped"), name)
+
+ def __dir__(self):
+ return dir(object.__getattribute__(self, "_wrapped"))
+
+
+_sys.modules[__name__] = _ModuleProxy(_sys.modules[__name__]) # type: ignore[assignment]
+
+__all__ = [
+ "LS_ALL_NODES",
+ "SNAPSHOT_DIFF_MAX_FILE_BYTES",
+ "SNAPSHOT_DIFF_MAX_LINES",
+ "SNAPSHOT_DIFF_MAX_OUTPUT_BYTES",
+ "SNAPSHOT_DIFF_TIMEOUT_MS",
+ "RelationEntry",
+ "SyncDiff",
+ "VikingFS",
+ "enable_viking_fs_recorder",
+ "get_viking_fs",
+ "init_viking_fs",
+]
diff --git a/openviking/storage/viking_fs/_access.py b/openviking/storage/viking_fs/_access.py
new file mode 100644
index 000000000..dc9cd3e6f
--- /dev/null
+++ b/openviking/storage/viking_fs/_access.py
@@ -0,0 +1,917 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Access control, URI normalization, path conversion, and visibility mixin for VikingFS."""
+
+import contextvars
+import hashlib
+import json
+import re
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
+
+from openviking.core.context import ContextLevel
+from openviking.core.namespace import (
+ canonicalize_uri,
+ is_hidden_by_actor_peer_view,
+ may_include_hidden_actor_peers,
+)
+from openviking.core.namespace import (
+ is_accessible as namespace_is_accessible,
+)
+from openviking.resource.watch_storage import is_watch_task_control_uri
+from openviking.server.error_mapping import is_not_found_error, map_exception
+from openviking.server.identity import RequestContext, Role
+from openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMES
+from openviking_cli.exceptions import (
+ FailedPreconditionError,
+ NotFoundError,
+ PermissionDeniedError,
+)
+from openviking_cli.session.user_id import UserIdentifier
+from openviking_cli.utils.uri import VikingURI
+
+if TYPE_CHECKING:
+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig
+
+
+class _AccessMixin:
+ """URI normalization, access control, path conversion, and visibility helpers."""
+
+ # Over-fetch multiplier for bounded tree traversal. When a node_limit is
+ # set, we push down node_limit * this factor as the raw-node limit to Rust,
+ # leaving headroom for ACL/internal-name filtering before re-fetching.
+ _TREE_OVERFETCH_FACTOR = 4
+ _GLOB_PAGE_SIZE_DEFAULT = 1024
+
+ # Maximum bytes for a single filename component (filesystem limit is typically 255)
+ _MAX_FILENAME_BYTES = 255
+
+ _ROOT_PATH = "/local"
+
+ # First path segments that the Rust git enumerate.rs prunes from snapshots,
+ # plus the runtime lock name. Mirrors INTERNAL_FIRST_SEGMENTS in
+ # crates/ragfs/src/git/enumerate.rs and VikingFS._INTERNAL_NAMES so that
+ # callers fail fast in Python with a clear error rather than passing a
+ # path that the Rust side will silently drop.
+ _GIT_INTERNAL_FIRST_SEGMENTS = frozenset({
+ "_system",
+ "tasks",
+ "temp",
+ "queue",
+ "upload",
+ ".path.ovlock",
+ })
+
+ _DEFAULT_GIT_AUTHOR_NAME = "viking-bot"
+ _DEFAULT_GIT_AUTHOR_EMAIL = "[email protected]"
+
+ _OVGITIGNORE_TREE_PATH = ".ovgitignore"
+
+ # Must stay in sync with `OVGITIGNORE_MAX_BYTES` in the Rust layer
+ # (`crates/ragfs/src/git/ignore.rs`); enforced here at write time so a bad
+ # file can never be persisted only to poison every later commit.
+ _OVGITIGNORE_MAX_BYTES = 64 * 1024
+
+ _DIR_MARKER_LEVELS = {
+ ".abstract.md": ContextLevel.ABSTRACT,
+ ".overview.md": ContextLevel.OVERVIEW,
+ }
+ _NO_VECTOR_DERIVED = frozenset({".relations.json", ".ovgitignore"})
+
+ def set_user_deletion_guard(self, guard: Optional[Callable[[str, str], bool]]) -> None:
+ self._user_deletion_guard = guard
+
+ @staticmethod
+ def _default_ctx() -> RequestContext:
+ return RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
+
+ def _ctx_or_default(self, ctx: Optional[RequestContext]) -> RequestContext:
+ if ctx is not None:
+ return ctx
+ bound = self._bound_ctx.get()
+ return bound or self._default_ctx()
+
+ @contextmanager
+ def bind_request_context(self, ctx: RequestContext):
+ """Temporarily bind ctx for legacy internal call paths without explicit ctx param."""
+ token = self._bound_ctx.set(ctx)
+ try:
+ yield
+ finally:
+ self._bound_ctx.reset(token)
+
+ @staticmethod
+ def _normalize_uri(uri: str) -> str:
+ """Normalize short-format URIs to the canonical viking:// form."""
+ if uri.startswith("viking://"):
+ return uri
+ return VikingURI.normalize(uri)
+
+ @classmethod
+ def _normalized_uri_parts(cls, uri: str) -> tuple[str, List[str]]:
+ """Normalize a URI and reject ambiguous or platform-specific path traversal forms."""
+ normalized = cls._normalize_uri(uri)
+ parts = [p for p in normalized[len("viking://") :].strip("/").split("/") if p]
+
+ for part in parts:
+ if part in {".", ".."}:
+ raise PermissionDeniedError(
+ f"Unsafe URI traversal segment '{part}' in {normalized}",
+ resource=normalized,
+ )
+ if "\\" in part:
+ raise PermissionDeniedError(
+ f"Unsafe URI path separator '\\\\' in component '{part}' of {normalized}",
+ resource=normalized,
+ )
+ if len(part) >= 2 and part[1] == ":" and part[0].isalpha():
+ raise PermissionDeniedError(
+ f"Unsafe URI drive-prefixed component '{part}' in {normalized}",
+ resource=normalized,
+ )
+
+ return normalized, parts
+
+ # TODO: Once pathlock moves down into ragfs, stop reconstructing the
+ # encrypted mount-relative path in Python and derive the lock target from
+ # the same backend-side source of truth.
+ def _encrypted_mount_relative_path(self, path: str) -> tuple[str, str]:
+ """Return the mount prefix and mount-relative path used by Rust encrypted writes."""
+ normalized = path if path.startswith("/") else f"/{path}"
+ parts = [part for part in normalized.split("/") if part]
+ if len(parts) < 2:
+ return "", normalized
+ return f"/{parts[0]}", f"/{'/'.join(parts[1:])}"
+
+ def _encrypted_temp_path(self, path: str) -> str:
+ """Build the deterministic internal `.encrypt` temp-file path for a final file path."""
+ mount_prefix, relative_path = self._encrypted_mount_relative_path(path)
+ relative_parts = [part for part in relative_path.split("/") if part]
+ if len(relative_parts) >= 2 and relative_parts[0] == "local":
+ temp_root = f"/local/{relative_parts[1]}/temp/.encrypt_stage"
+ elif len(relative_parts) >= 2:
+ temp_root = f"/{relative_parts[0]}/temp/.encrypt_stage"
+ else:
+ temp_root = "/temp/.encrypt_stage"
+ digest = hashlib.sha256(relative_path.encode("utf-8")).hexdigest()
+ return f"{mount_prefix}{temp_root}/{digest}.encrypt"
+
+ def _ensure_access(self, uri: str, ctx: Optional[RequestContext]) -> None:
+ real_ctx = self._ctx_or_default(ctx)
+ normalized_uri, _ = self._normalized_uri_parts(uri)
+ if not self._is_accessible(normalized_uri, real_ctx):
+ raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)
+
+ def _ensure_mutable_access(self, uri: str, ctx: Optional[RequestContext]) -> None:
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ self._ensure_user_not_deleting(real_ctx)
+ normalized_uri, _ = self._normalized_uri_parts(uri)
+ if is_hidden_by_actor_peer_view(normalized_uri, real_ctx) or may_include_hidden_actor_peers(
+ normalized_uri, real_ctx
+ ):
+ raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)
+ self._ensure_supported_write_namespace(normalized_uri)
+ if real_ctx.role != Role.ROOT and normalized_uri.rstrip("/") == "viking://temp":
+ raise PermissionDeniedError(
+ "Temp root is read-only for non-root users",
+ resource=normalized_uri,
+ )
+
+ def _ensure_delete_access(self, uri: str, ctx: Optional[RequestContext]) -> None:
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ self._ensure_user_not_deleting(real_ctx)
+ normalized_uri, _ = self._normalized_uri_parts(uri)
+ if is_hidden_by_actor_peer_view(normalized_uri, real_ctx) or may_include_hidden_actor_peers(
+ normalized_uri, real_ctx
+ ):
+ raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)
+ self._ensure_supported_delete_namespace(normalized_uri)
+ if real_ctx.role != Role.ROOT and normalized_uri.rstrip("/") == "viking://temp":
+ raise PermissionDeniedError(
+ "Temp root is read-only for non-root users",
+ resource=normalized_uri,
+ )
+
+ def _ensure_user_not_deleting(self, ctx: RequestContext) -> None:
+ guard = getattr(self, "_user_deletion_guard", None)
+ if (
+ ctx.role != Role.ROOT
+ and guard is not None
+ and guard(ctx.account_id, ctx.user.user_id)
+ ):
+ raise FailedPreconditionError("User deletion is in progress")
+
+ def _ensure_supported_delete_namespace(self, normalized_uri: str) -> None:
+ parts = [p for p in normalized_uri[len("viking://") :].strip("/").split("/") if p]
+ if not parts:
+ raise PermissionDeniedError(
+ "Deleting viking:// is not supported; use a concrete scope instead.",
+ resource=normalized_uri,
+ )
+ if parts == ["user"]:
+ raise PermissionDeniedError(
+ "Deleting viking://user is not supported; use an explicit user namespace "
+ "or current-user content path instead.",
+ resource=normalized_uri,
+ )
+ if parts == ["agent"]:
+ # Parity with _ensure_supported_write_namespace, which forbids the
+ # account-shared viking://agent root: deleting it would recursively
+ # wipe every account's agent skills/endpoints/tools/payments. Concrete
+ # sub-paths (viking://agent/skills/...) remain deletable.
+ raise PermissionDeniedError(
+ "Deleting viking://agent root is not supported; use a concrete "
+ "agent sub-path (e.g. viking://agent/skills/...) instead.",
+ resource=normalized_uri,
+ )
+
+ def _ensure_supported_write_namespace(self, normalized_uri: str) -> None:
+ parts = [p for p in normalized_uri[len("viking://") :].strip("/").split("/") if p]
+ if parts == ["user"]:
+ raise PermissionDeniedError(
+ "Writing viking://user is not supported; use an explicit user namespace "
+ "or current-user content path instead.",
+ resource=normalized_uri,
+ )
+ if parts and parts[0] == "session":
+ raise PermissionDeniedError(
+ f"Writing {normalized_uri} is not supported; use user-owned namespaces instead.",
+ resource=normalized_uri,
+ )
+ if parts and parts[0] == "agent":
+ if len(parts) >= 2 and parts[1] not in {"skills", "endpoints", "tools", "payments"}:
+ raise PermissionDeniedError(
+ "viking://agent/{agent_id} is deprecated. Use viking://user/.../peers/{agent_id} instead.",
+ resource=normalized_uri,
+ )
+ if len(parts) < 2:
+ raise PermissionDeniedError(
+ "Writing to viking://agent root is not supported.",
+ resource=normalized_uri,
+ )
+
+ def _pathlock_fs_ctx(
+ self,
+ ctx: Optional[RequestContext],
+ lease_ref: Optional[Dict[str, Any] | str],
+ ) -> Dict[str, str] | None:
+ """Build an AGFS fs_ctx carrying account_id and an opaque pathlock lease_ref."""
+ if lease_ref is None:
+ return None
+ if isinstance(lease_ref, str):
+ ref = lease_ref
+ elif isinstance(lease_ref, dict):
+ ref = lease_ref.get("lease_ref")
+ else:
+ raise ValueError("lease_ref must be a non-empty string or lease dictionary")
+ if not isinstance(ref, str) or not ref:
+ raise ValueError("lease_ref must contain a non-empty lease_ref")
+ return {"account_id": self._ctx_or_default(ctx).account_id, "lease_ref": ref}
+
+ # ========== Tree Traversal (Refactored) ==========
+
+ def _is_name_visible_at_path(self, name: str, parent_path: str) -> bool:
+ """Check if name would appear in _ls_entries(parent_path).
+
+ At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.
+ At other levels, uses the shared storage internal-name blacklist.
+ """
+ parts = [p for p in parent_path.strip("/").split("/") if p]
+ if len(parts) == 2 and parts[0] == "local":
+ return name in VikingURI.LISTABLE_SCOPES
+ return name not in STORAGE_INTERNAL_ENTRY_NAMES
+
+ def _ancestor_is_filtered(self, entry_path: str, base_path: str) -> bool:
+ """Check if any ancestor directory of entry_path would be filtered by _ls_entries.
+
+ Walks from base_path (exclusive) to entry's parent directory (exclusive),
+ checking each component against _is_name_visible_at_path.
+ """
+ base_parts = [p for p in base_path.strip("/").split("/") if p]
+ entry_parts = [p for p in entry_path.strip("/").split("/") if p]
+
+ for i in range(len(base_parts), len(entry_parts) - 1):
+ name = entry_parts[i]
+ parent_parts = entry_parts[:i]
+ parent_path = "/" + "/".join(parent_parts) if parent_parts else "/"
+ if not self._is_name_visible_at_path(name, parent_path):
+ return True
+ return False
+
+ def _is_path_entry_visible(
+ self, entry_path: str, name: str, base_path: str, ctx: RequestContext
+ ) -> bool:
+ """Check visibility for one flattened path entry returned by Rust."""
+ if self._ancestor_is_filtered(entry_path, base_path):
+ return False
+
+ entry_parts = [p for p in entry_path.strip("/").split("/") if p]
+ if entry_parts:
+ parent_parts = entry_parts[:-1]
+ parent_path = "/" + "/".join(parent_parts) if parent_parts else "/"
+ if not self._is_name_visible_at_path(name, parent_path):
+ return False
+
+ uri = self._path_to_uri(entry_path, ctx=ctx)
+ if not self._is_accessible(uri, ctx):
+ return False
+
+ return True
+
+ def _is_tree_entry_visible(
+ self, entry: Dict[str, Any], base_path: str, ctx: RequestContext
+ ) -> bool:
+ """Check visibility for a single TreeEntry returned by Rust tree_directory."""
+ entry_path = entry["path"]
+ entry_info = entry.get("info", {})
+ name = entry_info.get("name") or entry_path.rstrip("/").rsplit("/", 1)[-1]
+ return self._is_path_entry_visible(entry_path, name, base_path, ctx)
+
+ def _glob_page_size(self, node_limit: Optional[int]) -> int:
+ """Return the backend page size used by glob_directory."""
+ if node_limit is None or node_limit <= 0:
+ return self._GLOB_PAGE_SIZE_DEFAULT
+ return node_limit
+
+ async def _iter_visible_tree_entries(
+ self,
+ uri: str,
+ show_all_hidden: bool = False,
+ node_limit: Optional[int] = None,
+ level_limit: Optional[int] = None,
+ ctx: Optional[RequestContext] = None,
+ ):
+ """Shared generator: fetch raw TreeEntry list from Rust, yield (entry, uri) tuples.
+
+ node_limit counts ACL-visible entries (see design §6.5), so the user's
+ node_limit cannot be pushed directly to Rust — doing so would truncate
+ before filtering and drop entries that should be visible.
+
+ To keep memory bounded without changing that semantic, we push down an
+ *amplified* raw-node limit (node_limit * _TREE_OVERFETCH_FACTOR). If ACL
+ filtering leaves fewer than node_limit visible entries while Rust still
+ returned a full page (i.e. more raw nodes may exist), we double the raw
+ limit and re-fetch. Because Rust truncates a deterministic sorted prefix,
+ this yields exactly the same result as an unbounded fetch, while avoiding
+ materializing the entire prefix in the common case.
+
+ When node_limit is None (full-tree callers), no limit is pushed down.
+ level_limit IS always passed to Rust.
+ """
+ real_ctx = self._ctx_or_default(ctx)
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+ path: Optional[str] = None
+ for candidate_path in self._read_paths(uri, ctx=ctx):
+ if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):
+ continue
+ if await self._agfs_path_exists(candidate_path):
+ path = candidate_path
+ break
+ if path is None:
+ if self._is_legacy_session_root_uri(uri):
+ return
+ raise NotFoundError(uri, "directory")
+
+ if node_limit is None:
+ raw_limit: Optional[int] = None
+ else:
+ raw_limit = max(node_limit * self._TREE_OVERFETCH_FACTOR, node_limit)
+
+ while True:
+ raw_entries = await self._async_agfs.tree_directory(
+ path,
+ show_hidden=show_all_hidden,
+ node_limit=raw_limit,
+ level_limit=level_limit,
+ )
+
+ visible: List[tuple] = []
+ for entry in raw_entries:
+ if node_limit is not None and len(visible) >= node_limit:
+ break
+ if not self._is_tree_entry_visible(entry, path, real_ctx):
+ continue
+ if not await self._read_path_visible(uri, entry["path"], primary_path, real_ctx):
+ continue
+ entry_uri = self._alias_uri_for_path(
+ request_uri=uri,
+ base_path=path,
+ entry_path=entry["path"],
+ ctx=ctx,
+ )
+ visible.append((entry, entry_uri))
+
+ # If we still lack enough visible entries but Rust returned a full
+ # page (raw_limit reached), more raw nodes may exist — re-fetch with
+ # a doubled limit. Otherwise Rust is exhausted and we yield as-is.
+ need_more = (
+ node_limit is not None
+ and len(visible) < node_limit
+ and raw_limit is not None
+ and len(raw_entries) >= raw_limit
+ )
+ if need_more:
+ raw_limit *= 2
+ continue
+
+ for item in visible:
+ yield item
+ return
+
+ # ========== URI Conversion ==========
+
+ @staticmethod
+ def _shorten_component(component: str, max_bytes: int = 255) -> str:
+ """Shorten a path component if its UTF-8 encoding exceeds max_bytes."""
+ if len(component.encode("utf-8")) <= max_bytes:
+ return component
+ hash_suffix = hashlib.sha256(component.encode("utf-8")).hexdigest()[:8]
+ # Trim to fit within max_bytes after adding hash suffix
+ prefix = component
+ target = max_bytes - len(f"_{hash_suffix}".encode("utf-8"))
+ while len(prefix.encode("utf-8")) > target and prefix:
+ prefix = prefix[:-1]
+ return f"{prefix}_{hash_suffix}"
+
+ def _uri_to_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:
+ """Map virtual URI to account-isolated AGFS path.
+
+ Pure prefix replacement: viking://{remainder} -> /local/{account_id}/{remainder}.
+ No implicit space injection — URIs must include space segments explicitly.
+ """
+ real_ctx = self._ctx_or_default(ctx)
+ account_id = real_ctx.account_id
+ normalized_uri, legacy_parts = self._normalized_uri_parts(uri)
+ if legacy_parts and legacy_parts[0] == "agent" and self._is_legacy_agent_id_uri(uri):
+ # Old format: viking://agent/{agent_id}/... — direct mapping for read-only compat
+ safe_parts = [
+ self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in legacy_parts
+ ]
+ return f"/local/{account_id}/{'/'.join(safe_parts)}"
+ canonical_uri = canonicalize_uri(uri, real_ctx)
+ _, parts = self._normalized_uri_parts(canonical_uri)
+ if not parts:
+ return f"/local/{account_id}"
+
+ safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]
+ return f"/local/{account_id}/{'/'.join(safe_parts)}"
+
+ def _legacy_session_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:
+ """Map a legacy viking://session URI to its pre-user-namespace path."""
+ real_ctx = self._ctx_or_default(ctx)
+ _, parts = self._normalized_uri_parts(uri)
+ safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]
+ return f"/local/{real_ctx.account_id}/{'/'.join(safe_parts)}"
+
+ def _legacy_current_user_session_path(
+ self, uri: str, ctx: Optional[RequestContext] = None
+ ) -> Optional[str]:
+ """Return the legacy nested /session/{user_id}/{session_id} candidate."""
+ real_ctx = self._ctx_or_default(ctx)
+ _, parts = self._normalized_uri_parts(uri)
+ if len(parts) <= 1 or parts[0] != "session":
+ return None
+ nested_parts = ["session", real_ctx.user.user_id, *parts[1:]]
+ safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in nested_parts]
+ return f"/local/{real_ctx.account_id}/{'/'.join(safe_parts)}"
+
+ def _is_legacy_session_uri(self, uri: str) -> bool:
+ _, parts = self._normalized_uri_parts(uri)
+ return bool(parts and parts[0] == "session")
+
+ def _is_legacy_session_root_uri(self, uri: str) -> bool:
+ _, parts = self._normalized_uri_parts(uri)
+ return parts == ["session"]
+
+ def _is_legacy_agent_id_uri(self, uri: str) -> bool:
+ _, parts = self._normalized_uri_parts(uri)
+ return bool(
+ parts
+ and parts[0] == "agent"
+ and len(parts) >= 2
+ and parts[1] not in {"skills", "endpoints", "tools", "payments"}
+ )
+
+ def _read_paths(self, uri: str, ctx: Optional[RequestContext] = None) -> List[str]:
+ """Return read candidates for a URI, including legacy alias fallbacks."""
+ paths = [self._uri_to_path(uri, ctx=ctx)]
+
+ if self._is_legacy_session_uri(uri):
+ for candidate in (
+ self._legacy_session_path(uri, ctx=ctx),
+ self._legacy_current_user_session_path(uri, ctx=ctx),
+ ):
+ if candidate and candidate not in paths:
+ paths.append(candidate)
+ return paths
+
+ async def _read_path_visible(
+ self,
+ request_uri: str,
+ path: str,
+ primary_path: str,
+ ctx: RequestContext,
+ ) -> bool:
+ if path == primary_path:
+ return True
+ if self._is_legacy_session_uri(request_uri):
+ return await self._legacy_session_path_visible(path, ctx)
+ return True
+
+ def _alias_uri_for_path(
+ self,
+ *,
+ request_uri: str,
+ base_path: str,
+ entry_path: str,
+ ctx: Optional[RequestContext],
+ ) -> str:
+ base = base_path.rstrip("/")
+ normalized_request, request_parts = self._normalized_uri_parts(request_uri)
+ request_root = (
+ normalized_request
+ if normalized_request == "viking://"
+ else normalized_request.rstrip("/")
+ )
+ preserve_request_alias = normalized_request in {"viking://", "viking://user"} or bool(
+ request_parts
+ and (
+ request_parts[0] == "session"
+ or (request_parts[0] == "agent" and self._is_legacy_agent_id_uri(request_uri))
+ )
+ )
+ rel_path = entry_path[len(base) :].strip("/") if entry_path.startswith(base) else ""
+ if entry_path.startswith(base):
+ separator = "" if request_root.endswith("://") else "/"
+ candidate_uri = request_root if not rel_path else f"{request_root}{separator}{rel_path}"
+ if preserve_request_alias:
+ return candidate_uri
+ try:
+ if self._uri_to_path(candidate_uri, ctx=ctx) == entry_path:
+ return candidate_uri
+ except Exception:
+ pass
+ return self._path_to_uri(entry_path, ctx=ctx)
+
+ async def _agfs_path_exists(self, path: str) -> bool:
+ try:
+ await self._async_agfs.stat(path)
+ return True
+ except Exception as exc:
+ if is_not_found_error(exc):
+ return False
+ raise
+
+ async def _looks_like_legacy_session_dir(self, path: str) -> bool:
+ for leaf in (".meta.json", "messages.jsonl", "history", "tool-results", "tools"):
+ if await self._agfs_path_exists(f"{path}/{leaf}"):
+ return True
+ return False
+
+ async def _legacy_session_owner(self, session_root_path: str) -> str:
+ try:
+ raw = self._handle_agfs_read(
+ await self._async_agfs.read(f"{session_root_path}/.meta.json")
+ )
+ except Exception as exc:
+ if is_not_found_error(exc):
+ return ""
+ raise
+ try:
+ data = json.loads(self._decode_bytes(raw))
+ except json.JSONDecodeError:
+ return ""
+ if not isinstance(data, dict):
+ return ""
+ for key in ("created_by_user_id", "user_id", "owner_user_id", "created_by"):
+ value = data.get(key)
+ if isinstance(value, str) and value.strip():
+ return value.strip()
+ return ""
+
+ async def _legacy_session_visible(
+ self,
+ session_root_path: str,
+ ctx: RequestContext,
+ *,
+ owner_hint: Optional[str] = None,
+ ) -> bool:
+ if ctx.role == Role.ROOT:
+ return True
+ owner = await self._legacy_session_owner(session_root_path)
+ if owner:
+ return owner == ctx.user.user_id
+ if owner_hint:
+ return owner_hint == ctx.user.user_id
+ return True
+
+ async def _legacy_session_path_visible(self, path: str, ctx: RequestContext) -> bool:
+ parts = [p for p in path.strip("/").split("/") if p]
+ try:
+ session_index = parts.index("session")
+ except ValueError:
+ return True
+ suffix = parts[session_index + 1 :]
+ if not suffix:
+ return True
+
+ root_prefix = "/" + "/".join(parts[: session_index + 1])
+ direct_root = f"{root_prefix}/{suffix[0]}"
+ if await self._looks_like_legacy_session_dir(direct_root):
+ return await self._legacy_session_visible(direct_root, ctx)
+
+ if len(suffix) >= 2:
+ nested_root = f"{root_prefix}/{suffix[0]}/{suffix[1]}"
+ if await self._looks_like_legacy_session_dir(nested_root):
+ return await self._legacy_session_visible(
+ nested_root,
+ ctx,
+ owner_hint=suffix[0],
+ )
+ return True
+
+ async def _legacy_session_root_items(
+ self,
+ path: str,
+ ctx: RequestContext,
+ ) -> List[tuple[Dict[str, Any], str]]:
+ try:
+ entries = await self._ls_entries(path)
+ except Exception as exc:
+ if is_not_found_error(exc):
+ return []
+ raise
+
+ items: List[tuple[Dict[str, Any], str]] = []
+ for entry in entries:
+ name = entry.get("name", "")
+ if not name or name in {".", ".."} or not entry.get("isDir"):
+ continue
+ child_path = f"{path.rstrip('/')}/{name}"
+ if await self._looks_like_legacy_session_dir(child_path):
+ if await self._legacy_session_visible(child_path, ctx):
+ items.append((entry, f"viking://session/{name}"))
+ continue
+
+ if ctx.role != Role.ROOT and name != ctx.user.user_id:
+ continue
+ try:
+ nested_entries = await self._ls_entries(child_path)
+ except Exception as exc:
+ if is_not_found_error(exc):
+ continue
+ raise
+ for nested in nested_entries:
+ nested_name = nested.get("name", "")
+ if not nested_name or nested_name in {".", ".."} or not nested.get("isDir"):
+ continue
+ nested_path = f"{child_path.rstrip('/')}/{nested_name}"
+ if not await self._looks_like_legacy_session_dir(nested_path):
+ continue
+ if await self._legacy_session_visible(nested_path, ctx, owner_hint=name):
+ items.append((nested, f"viking://session/{nested_name}"))
+ return items
+
+ async def _session_root_items(
+ self,
+ uri: str,
+ ctx: RequestContext,
+ ) -> List[tuple[Dict[str, Any], str]]:
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+ by_name: Dict[str, tuple[Dict[str, Any], str]] = {}
+ try:
+ for entry in await self._ls_entries(primary_path, ctx=ctx):
+ name = entry.get("name", "")
+ if not name or name in {".", ".."}:
+ continue
+ by_name[name] = (entry, f"viking://session/{name}")
+ except Exception as exc:
+ if not is_not_found_error(exc):
+ raise
+
+ legacy_path = self._legacy_session_path(uri, ctx=ctx)
+ for entry, entry_uri in await self._legacy_session_root_items(legacy_path, ctx):
+ name = entry.get("name", "")
+ if name and name not in by_name:
+ by_name[name] = (entry, entry_uri)
+ return list(by_name.values())
+
+ async def _list_read_path_items(
+ self,
+ uri: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> List[tuple[Dict[str, Any], str]]:
+ real_ctx = self._ctx_or_default(ctx)
+ if self._is_legacy_session_root_uri(uri):
+ return await self._session_root_items(uri, real_ctx)
+
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+ merge_paths = self._is_legacy_session_uri(uri)
+ found_path = False
+ last_not_found: Optional[Exception] = None
+ by_uri: Dict[str, tuple[Dict[str, Any], str]] = {}
+
+ for path in self._read_paths(uri, ctx=ctx):
+ if not await self._read_path_visible(uri, path, primary_path, real_ctx):
+ continue
+ try:
+ entries = await self._ls_entries(path, ctx=ctx)
+ except Exception as exc:
+ if is_not_found_error(exc):
+ last_not_found = exc
+ continue
+ raise
+
+ found_path = True
+ for entry in entries:
+ entry_uri = self._alias_uri_for_path(
+ request_uri=uri,
+ base_path=path,
+ entry_path=f"{path.rstrip('/')}/{entry.get('name', '')}",
+ ctx=ctx,
+ )
+ by_uri.setdefault(entry_uri, (entry, entry_uri))
+ if not merge_paths:
+ break
+
+ if found_path:
+ return list(by_uri.values())
+ raise NotFoundError(uri, "directory") from last_not_found
+
+ def _path_to_uri(self, path: str, ctx: Optional[RequestContext] = None) -> str:
+ """/local/{account}/... -> viking://...
+
+ Pure prefix replacement: strips /local/{account_id}/ and prepends viking://.
+ No implicit space stripping.
+ """
+ if path.startswith("viking://"):
+ return path
+ elif path.startswith("/local/"):
+ inner = path[7:].strip("/")
+ if not inner:
+ return "viking://"
+ real_ctx = self._ctx_or_default(ctx)
+ parts = [p for p in inner.split("/") if p]
+ if parts and parts[0] == real_ctx.account_id:
+ parts = parts[1:]
+ if not parts:
+ return "viking://"
+ return f"viking://{'/'.join(parts)}"
+ elif path.startswith("/"):
+ return f"viking:/{path}"
+ else:
+ return f"viking://{path}"
+
+ def _looks_like_legacy_temp_leaf(self, value: str) -> bool:
+ return bool(re.match(r"^\d{8}_[0-9a-f]{6}$", value or ""))
+
+ def _is_legacy_temp_uri_parts(self, parts: List[str]) -> bool:
+ if len(parts) < 2 or parts[0] != "temp" or not self._looks_like_legacy_temp_leaf(parts[1]):
+ return False
+ if len(parts) == 2:
+ return True
+ return not self._looks_like_legacy_temp_leaf(parts[2])
+
+ def _is_accessible(self, uri: str, ctx: RequestContext) -> bool:
+ """Check whether a URI is visible/accessible under current request context."""
+ normalized_uri, parts = self._normalized_uri_parts(uri)
+ if ctx.role == Role.ROOT:
+ return True
+ if is_hidden_by_actor_peer_view(normalized_uri, ctx):
+ return False
+ if not parts:
+ return True
+ if is_watch_task_control_uri(normalized_uri):
+ return False
+
+ scope = parts[0]
+ if scope == "resources":
+ return True
+ if scope == "temp":
+ if len(parts) == 1:
+ return True
+ if parts[1] == ctx.user.user_space_name():
+ return True
+ return self._is_legacy_temp_uri_parts(parts)
+ if scope == "upload":
+ return ctx.role == Role.ROOT
+ if scope == "_system":
+ return False
+ if scope == "agent":
+ # New format: agent/skills/..., agent/endpoints/... — globally readable (account scope)
+ if len(parts) >= 2 and parts[1] in {"skills", "endpoints", "tools", "payments"}:
+ return True
+ # Old format: agent/{agent_id}/... — actor_peer_id match for read-only access
+ if not ctx.actor_peer_id or len(parts) < 2:
+ return True
+ return parts[1] == ctx.actor_peer_id
+ return namespace_is_accessible(normalized_uri, ctx)
+
+ def _handle_agfs_read(self, result: Union[bytes, Any, None]) -> bytes:
+ """Handle AGFSClient read return types consistently."""
+ if isinstance(result, bytes):
+ return result
+ elif result is None:
+ return b""
+ elif hasattr(result, "content") and result.content is not None:
+ return result.content
+ else:
+ # Try to convert to bytes
+ try:
+ return str(result).encode("utf-8")
+ except Exception:
+ return b""
+
+ def _decode_bytes(self, data: bytes) -> str:
+ """Robustly decode bytes to string."""
+ if not data:
+ return ""
+ try:
+ return data.decode("utf-8")
+ except UnicodeDecodeError:
+ try:
+ # Try common encoding for Windows/legacy files in China
+ return data.decode("gbk")
+ except UnicodeDecodeError:
+ try:
+ return data.decode("latin-1")
+ except UnicodeDecodeError:
+ return data.decode("utf-8", errors="replace")
+
+ async def _is_path_locked_async(self, path: str) -> bool:
+ """Best-effort async path-lock lookup; returns False on any error."""
+ try:
+ return await self._async_agfs.pathlock_is_locked(path)
+ except Exception:
+ return False
+
+ def _gitignore_agfs_path(self, ctx: Optional[RequestContext] = None) -> str:
+ real_ctx = self._ctx_or_default(ctx)
+ return f"/local/{real_ctx.account_id}/{self._OVGITIGNORE_TREE_PATH}"
+
+ def _uri_to_tree_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:
+ """Convert a viking:// URI to an account-relative git tree path.
+
+ ``viking://resources/proj_a/docs/a.md`` -> ``resources/proj_a/docs/a.md``.
+
+ Pure prefix stripping: removes the ``viking://`` scheme and any
+ ``/local/{account}/`` segment. Internal scopes that the Rust git layer
+ would prune (`_system`, `tasks`, `temp`, `queue`, `upload`) and the
+ runtime lock name (`.path.ovlock`) are rejected with ``ValueError``
+ — passing them through silently would result in a no-op commit and
+ confuse callers.
+ """
+ real_ctx = self._ctx_or_default(ctx)
+ canonical = canonicalize_uri(uri, real_ctx)
+ _, parts = self._normalized_uri_parts(canonical)
+ if not parts:
+ raise ValueError(f"git tree path cannot be the account root: {uri!r}")
+ first = parts[0]
+ if first in self._GIT_INTERNAL_FIRST_SEGMENTS:
+ raise ValueError(f"git tree path rejects internal scope/segment {first!r}: {uri!r}")
+ return "/".join(parts)
+
+ def _tree_path_to_uri(self, tree_path: str) -> str:
+ """Convert an account-relative git tree path to a viking:// URI.
+
+ Inverse of :py:meth:`_uri_to_tree_path` (without context canonicalization).
+ """
+ cleaned = tree_path.strip("/")
+ if not cleaned:
+ raise ValueError("tree path must not be empty")
+ return f"viking://{cleaned}"
+
+ def _classify_restore_path(self, tree_path: str, *, deleted: bool) -> Optional[tuple]:
+ """Classify a restore-affected tree path into a vector maintenance task.
+
+ Returns a ``(op, uri, level)`` triple, or ``None`` when the path has no
+ vector side-effect:
+
+ - ``dir/.abstract.md`` / ``dir/.overview.md`` → recompute (write) or
+ delete (removal) ONLY that directory's L0/L1 vector:
+ ``("reindex_marker"|"delete", dir_uri, ABSTRACT|OVERVIEW)``.
+ - ``.relations.json`` → ``None`` (not a vector text source).
+ - anything else (a source file) → reindex (write) or delete (removal)
+ its DETAIL vector:
+ ``("reindex_file", file_uri, DETAIL)`` / ``("delete", file_uri, DETAIL)``.
+
+ ``None`` is also returned for a directory marker at the account root
+ (no parent directory to scope an L0/L1 vector to).
+ """
+ parent, _, name = tree_path.rpartition("/")
+ if name in self._NO_VECTOR_DERIVED:
+ return None
+ level = self._DIR_MARKER_LEVELS.get(name)
+ if level is not None:
+ if not parent:
+ return None
+ dir_uri = self._tree_path_to_uri(parent)
+ op = "delete" if deleted else "reindex_marker"
+ return (op, dir_uri, level)
+ # Source file.
+ file_uri = self._tree_path_to_uri(tree_path)
+ op = "delete" if deleted else "reindex_file"
+ return (op, file_uri, ContextLevel.DETAIL)
diff --git a/openviking/storage/viking_fs/_base.py b/openviking/storage/viking_fs/_base.py
new file mode 100644
index 000000000..bde4fdf8e
--- /dev/null
+++ b/openviking/storage/viking_fs/_base.py
@@ -0,0 +1,254 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Shared constants, helpers, dataclasses, and singleton management for VikingFS."""
+
+import os
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypeVar
+
+from openviking.utils.time_utils import get_current_timestamp
+from openviking_cli.exceptions import (
+ InvalidArgumentError,
+ ResourceExhaustedError,
+)
+from openviking_cli.utils.logger import get_logger
+
+if TYPE_CHECKING:
+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig
+
+logger = get_logger(__name__)
+
+# Sentinel node_limit for internal callers that MUST enumerate an entire
+# directory. ``ls()`` defaults to ``node_limit=1000`` to protect agent-facing
+# context from being flooded, but internal system operations (parse merge,
+# temp->final sync, summary DAG, vectorization) must see every child or they
+# silently drop entries beyond the cap — e.g. a >1000-doc directory ingest only
+# materializes its first 1000 subdirectories. Pass this explicitly at those
+# call sites.
+LS_ALL_NODES = 2**31 - 1
+SNAPSHOT_DIFF_MAX_FILE_BYTES = 10 * 1024 * 1024
+SNAPSHOT_DIFF_MAX_OUTPUT_BYTES = 20 * 1024 * 1024
+SNAPSHOT_DIFF_MAX_LINES = 100_000
+SNAPSHOT_DIFF_TIMEOUT_MS = 500
+_T = TypeVar("_T")
+
+
+def _snapshot_line_count(text: str) -> int:
+ if not text:
+ return 0
+ line_breaks = (
+ "\n",
+ "\r",
+ "\v",
+ "\f",
+ "\x1c",
+ "\x1d",
+ "\x1e",
+ "\x85",
+ "\u2028",
+ "\u2029",
+ )
+ count = sum(text.count(separator) for separator in line_breaks)
+ count -= text.count("\r\n")
+ return count if text.endswith(line_breaks) else count + 1
+
+
+def _prepare_snapshot_diff(
+ *,
+ path: str,
+ before_bytes: Optional[bytes],
+ after_bytes: Optional[bytes],
+ max_lines: int,
+) -> tuple[str, str, str]:
+ try:
+ before = before_bytes.decode("utf-8") if before_bytes is not None else ""
+ after = after_bytes.decode("utf-8") if after_bytes is not None else ""
+ except UnicodeDecodeError as exc:
+ raise InvalidArgumentError("snapshot diff only supports UTF-8 text files") from exc
+
+ for text in (before, after):
+ line_count = _snapshot_line_count(text)
+ if line_count > max_lines:
+ raise ResourceExhaustedError(
+ f"snapshot diff line count limit exceeded ({max_lines} lines per file)",
+ details={"limit_lines": max_lines, "path": path},
+ )
+
+ if before_bytes is None:
+ change_type = "added"
+ elif after_bytes is None:
+ change_type = "deleted"
+ elif before_bytes == after_bytes:
+ change_type = "unchanged"
+ else:
+ change_type = "modified"
+
+ return change_type, before, after
+
+
+def _ensure_non_empty_search_query(query: str, image_url: Optional[str] = None) -> None:
+ if not query.strip() and not image_url:
+ raise InvalidArgumentError("Search query or image_url must not be empty.")
+
+
+def _is_directory_not_empty_error(message: str) -> bool:
+ """Check if an error message indicates a directory not empty error.
+
+ Handles multiple possible error message formats from different backends.
+ """
+ msg = message.lower()
+ return any(
+ pattern in msg
+ for pattern in [
+ "directory not empty",
+ "dir not empty",
+ "directory is not empty",
+ ]
+ )
+
+
+def _get_cpu_count() -> int:
+ """Return the number of CPUs available to this process.
+
+ Tries process_cpu_count (Python 3.13+, cgroup-aware),
+ falls back to sched_getaffinity (Linux),
+ then os.cpu_count (may report host CPUs in containers).
+ """
+ if hasattr(os, "process_cpu_count"):
+ return os.process_cpu_count() or 1
+ try:
+ return len(os.sched_getaffinity(0))
+ except (AttributeError, NotImplementedError):
+ return os.cpu_count() or 1
+
+
+def _get_abstract_worker_count() -> int:
+ default = max(4, min(12, min(32, _get_cpu_count() + 4) // 2))
+ env_val = os.getenv("OPENVIKING_FILE_OPS_CONCURRENCY")
+ if env_val is not None:
+ try:
+ return max(1, int(env_val))
+ except ValueError:
+ pass
+ return max(1, default)
+
+
+_ABSTRACT_WORKER_COUNT = _get_abstract_worker_count()
+_DEFAULT_GREP_FILE_CONCURRENCY = 32
+
+
+# ========== Dataclass ==========
+
+
+@dataclass
+class RelationEntry:
+ """Relation table entry."""
+
+ id: str
+ uris: List[str]
+ reason: str = ""
+ created_at: str = field(default_factory=get_current_timestamp)
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "id": self.id,
+ "uris": self.uris,
+ "reason": self.reason,
+ "created_at": self.created_at,
+ }
+
+ @staticmethod
+ def from_dict(data: Dict[str, Any]) -> "RelationEntry":
+ return RelationEntry(**data)
+
+
+# ========== Singleton Pattern ==========
+
+_instance: Optional["Any"] = None
+
+
+def init_viking_fs(
+ agfs: Any,
+ query_embedder: Optional[Any] = None,
+ rerank_config: Optional["RerankConfig"] = None,
+ vector_store: Optional["VikingVectorIndexBackend"] = None,
+ retrieval_config: Optional["RetrievalConfig"] = None,
+ grep_config: Optional["GrepConfig"] = None,
+ timeout: int = 10,
+ enable_recorder: bool = False,
+ encryptor: Optional[Any] = None,
+):
+ """Initialize VikingFS singleton.
+
+ Args:
+ agfs: Pre-initialized AGFS client (HTTP or Binding)
+ query_embedder: Embedder instance
+ rerank_config: Rerank configuration
+ retrieval_config: Retrieval ranking configuration
+ grep_config: Grep engine configuration
+ vector_store: Vector store instance
+ enable_recorder: Whether to enable IO recording
+ encryptor: FileEncryptor instance for encryption/decryption
+ """
+ from openviking.storage.viking_fs import VikingFS
+
+ global _instance
+
+ _instance = VikingFS(
+ agfs=agfs,
+ query_embedder=query_embedder,
+ rerank_config=rerank_config,
+ vector_store=vector_store,
+ retrieval_config=retrieval_config,
+ grep_config=grep_config,
+ encryptor=encryptor,
+ )
+
+ if enable_recorder:
+ _enable_viking_fs_recorder(_instance)
+
+ return _instance
+
+
+def _enable_viking_fs_recorder(viking_fs) -> None:
+ """
+ Enable recorder for a VikingFS instance.
+
+ This wraps the VikingFS instance with recording capabilities.
+ Called automatically when enable_recorder=True in init_viking_fs.
+
+ Args:
+ viking_fs: VikingFS instance to enable recording for
+ """
+ from openviking.eval.recorder import RecordingVikingFS, get_recorder
+
+ recorder = get_recorder()
+ if not recorder.enabled:
+ from openviking.eval.recorder import init_recorder
+
+ init_recorder(enabled=True)
+
+ global _instance
+ _instance = RecordingVikingFS(viking_fs)
+ logger.info("[VikingFS] IO Recorder enabled")
+
+
+def enable_viking_fs_recorder() -> None:
+ """
+ Enable recorder for the global VikingFS singleton.
+
+ This function wraps the existing VikingFS's AGFS client with recording.
+ Must be called after init_viking_fs().
+ """
+ global _instance
+ if _instance is None:
+ raise RuntimeError("VikingFS not initialized. Call init_viking_fs() first.")
+ _enable_viking_fs_recorder(_instance)
+
+
+def get_viking_fs():
+ """Get VikingFS singleton."""
+ if _instance is None:
+ raise RuntimeError("VikingFS not initialized. Call init_viking_fs() first.")
+ return _instance
diff --git a/openviking/storage/viking_fs/_grep.py b/openviking/storage/viking_fs/_grep.py
new file mode 100644
index 000000000..aa9b8ca44
--- /dev/null
+++ b/openviking/storage/viking_fs/_grep.py
@@ -0,0 +1,641 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Grep and search-backend mixin for VikingFS."""
+
+import asyncio
+import re
+import sys
+import time
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
+
+from openviking.core.namespace import canonicalize_uri
+from openviking.pyagfs.exceptions import AGFSNotSupportedError
+from openviking.server.identity import RequestContext
+from openviking.storage.expr import And, PathScope, RawDSL
+from openviking.storage.viking_fs._base import logger
+from openviking_cli.utils.config.grep_config import GrepEngine
+
+
+def _pkg():
+ return sys.modules[__package__]
+
+
+if TYPE_CHECKING:
+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig
+
+
+class _GrepMixin:
+ """Grep and search-backend methods."""
+
+ async def grep(
+ self,
+ uri: str,
+ pattern: str,
+ exclude_uri: Optional[str] = None,
+ case_insensitive: bool = False,
+ node_limit: Optional[int] = None,
+ level_limit: int = 10,
+ ctx: Optional[RequestContext] = None,
+ content_transform: Optional[Callable[[str, str], str]] = None,
+ ) -> Dict:
+ """Content search by pattern or keywords.
+
+ Optimized implementation that uses agfs native grep when possible.
+ The ragfs layer greps transparently over encrypted and plaintext files
+ (it decrypts via account_id when an encryption layer is configured).
+ Falls back to VikingFS layer implementation if native grep is unavailable.
+ When engine="auto" and vikingdb is available with sufficient data,
+ uses vikingdb bm25 recall + local fs precise matching.
+
+ Args:
+ uri: Viking URI
+ pattern: Regular expression pattern to search for
+ exclude_uri: Optional URI prefix to exclude from search
+ case_insensitive: Whether to perform case-insensitive matching
+ node_limit: Maximum number of results to return
+ level_limit: Maximum depth level to traverse (default: 10)
+ ctx: Request context
+ content_transform: Optional projection applied before regex matching.
+ Internal bm25 recall limit is auto-adapted from node_limit as
+ min(node_limit * 5, 100000); when node_limit is unset, use 100000.
+
+ Returns:
+ Dict with matches, count, match_count, files_scanned
+ """
+ real_ctx = self._ctx_or_default(ctx)
+ uri = canonicalize_uri(uri, real_ctx)
+ if exclude_uri:
+ exclude_uri = canonicalize_uri(exclude_uri, real_ctx)
+ self._ensure_access(uri, ctx)
+ # Skip vector_store.count() — the count field is not needed for grep,
+ # and avoiding it saves one VikingDB API call.
+ await self.stat(uri, ctx=ctx, skip_count=True)
+
+ # Read engine and threshold from grep_config (ov.conf)
+ engine = self.grep_config.engine if self.grep_config else "auto"
+ switch_to_remote_threshold = (
+ self.grep_config.switch_to_remote_threshold if self.grep_config else 10000
+ )
+
+ # A projection must run before matching. The remote BM25 index contains
+ # persisted raw content, so it cannot safely recall projected results.
+ resolved_engine = (
+ "fs"
+ if content_transform is not None
+ else await self._resolve_grep_engine(engine, uri, ctx, switch_to_remote_threshold)
+ )
+
+ if resolved_engine == "fs":
+ return await self._grep_fs(
+ uri=uri,
+ pattern=pattern,
+ exclude_uri=exclude_uri,
+ case_insensitive=case_insensitive,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ content_transform=content_transform,
+ )
+ else: # "vikingdb_then_fs"
+ return await self._grep_vikingdb_then_fs(
+ uri=uri,
+ pattern=pattern,
+ exclude_uri=exclude_uri,
+ case_insensitive=case_insensitive,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ )
+
+ async def _resolve_grep_engine(
+ self, engine: GrepEngine, uri: str, ctx, switch_to_remote_threshold: int = 10000
+ ) -> str:
+ """Resolve the actual grep engine to use."""
+ if engine == "fs":
+ return "fs"
+
+ # auto mode: check vikingdb availability
+ vector_store = self._get_vector_store()
+ if not vector_store:
+ return "fs"
+
+ backend_type = getattr(vector_store, "_backend_type", "unknown")
+ # Keep this set consistent with ``CollectionAdapter.USE_CONTENT_FIELD``:
+ # only these backends store the ``content`` field required for full-text grep.
+ if backend_type not in ("volcengine", "vikingdb"):
+ return "fs"
+
+ # Check collection has content field and FullText config
+ if not await self._collection_has_fulltext(vector_store, ctx):
+ return "fs"
+
+ # switch_to_remote_threshold=0 means always use vikingdb
+ if switch_to_remote_threshold == 0:
+ return "vikingdb_then_fs"
+
+ # Check data volume threshold
+ try:
+ count = await self._get_cached_count(uri, ctx)
+ if count < switch_to_remote_threshold:
+ return "fs"
+ except Exception:
+ logger.debug(
+ "grep engine=auto: count() check failed, falling back to fs", exc_info=True
+ )
+ return "fs"
+
+ return "vikingdb_then_fs"
+
+ async def _collection_has_fulltext(self, vector_store, ctx) -> bool:
+ """Check if collection has content field and FullText config.
+
+ Result is cached on the VikingFS instance since collection schema
+ does not change at runtime.
+ """
+ if self._fulltext_available is not None:
+ return self._fulltext_available
+ try:
+ meta = None
+ if hasattr(vector_store, "get_collection_meta"):
+ meta = await vector_store.get_collection_meta(ctx=ctx)
+ if not meta:
+ self._fulltext_available = False
+ return False
+ fields = meta.get("Fields", [])
+ has_content = any(
+ f.get("FieldName") == "content" and f.get("FieldType") == "text" for f in fields
+ )
+ fulltext = meta.get("FullText") or []
+ has_content_fulltext = any(ft.get("Field") == "content" for ft in fulltext)
+ result = has_content and has_content_fulltext
+ self._fulltext_available = result
+ return result
+ except Exception:
+ logger.debug(
+ "Failed to check collection fulltext config, assuming no fulltext", exc_info=True
+ )
+ return False
+
+ async def _get_cached_count(self, uri: str, ctx) -> int:
+ """Get cached count of records for a URI (TTL=1h)."""
+ _COUNT_CACHE_TTL = 3600
+ vector_store = self._get_vector_store()
+
+ # Include account_id in cache key for multi-tenant safety
+ account_id = getattr(ctx, "account_id", None) if ctx else None
+ cache_key = f"{account_id}:{uri}" if account_id else uri
+
+ now = time.time()
+ cached = self._count_cache.get(cache_key)
+ if cached and (now - cached[1]) < _COUNT_CACHE_TTL:
+ return cached[0]
+
+ count = await vector_store.count(filter=PathScope("uri", uri, depth=-1), ctx=ctx)
+ # Evict oldest entries if cache exceeds max size
+ if len(self._count_cache) >= self._count_cache_max_size:
+ oldest_keys = sorted(self._count_cache, key=lambda k: self._count_cache[k][1])
+ for k in oldest_keys[: len(oldest_keys) // 2]:
+ del self._count_cache[k]
+ self._count_cache[cache_key] = (count, now)
+ return count
+
+ async def _grep_fs(
+ self,
+ uri,
+ pattern,
+ exclude_uri,
+ case_insensitive,
+ node_limit,
+ level_limit,
+ ctx,
+ content_transform=None,
+ ):
+ """Filesystem grep path: prefer native agfs grep and fall back if unavailable."""
+ if content_transform is None:
+ try:
+ return await self._grep_with_agfs(
+ uri=uri,
+ pattern=pattern,
+ exclude_uri=exclude_uri,
+ case_insensitive=case_insensitive,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ )
+ except (AttributeError, AGFSNotSupportedError, NotImplementedError) as e:
+ logger.debug(f"agfs grep unavailable, falling back to VikingFS implementation: {e}")
+
+ return await self._grep_encrypted(
+ uri=uri,
+ pattern=pattern,
+ exclude_uri=exclude_uri,
+ case_insensitive=case_insensitive,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ content_transform=content_transform,
+ )
+
+ async def _grep_vikingdb_then_fs(
+ self,
+ uri,
+ pattern,
+ exclude_uri,
+ case_insensitive,
+ node_limit,
+ level_limit,
+ ctx,
+ ):
+ """VikingDB bm25 recall + local fs precise matching."""
+ vector_store = self._get_vector_store()
+
+ # Split regex alternation (e.g. "error|warning|fail") and join as a
+ # single query string for bm25 search. VikingDB's standard tokenizer
+ # will handle the tokenization of the query string.
+ query = " ".join(kw.strip() for kw in pattern.split("|") if kw.strip())
+ filter_expr = PathScope("uri", uri, depth=level_limit)
+ excluded_prefix = None
+ if exclude_uri:
+ excluded_prefix = exclude_uri.rstrip("/")
+ self._ensure_access(excluded_prefix, ctx)
+ filter_expr = And([
+ filter_expr,
+ RawDSL({
+ "op": "must_not",
+ "field": "uri",
+ "conds": [excluded_prefix],
+ "para": "-d=-1",
+ }),
+ ])
+
+ # Auto-adapt bm25 recall limit: recall up to 5x requested matches
+ # while capping at VikingDB's max limit. If node_limit is unset,
+ # use the maximum limit to avoid truncation.
+ remote_return_limit = min(node_limit * 5, 100000) if node_limit else 100000
+
+ # Step 1: vikingdb recall candidate files
+ try:
+ logger.debug(
+ "grep vikingdb search_by_keywords request: query=%r limit=%s filter=%r "
+ "output_fields=%s",
+ query,
+ remote_return_limit,
+ filter_expr,
+ ["uri"],
+ )
+ result = await vector_store.search_by_keywords(
+ query=query,
+ limit=remote_return_limit,
+ filter=filter_expr,
+ output_fields=["uri"],
+ ctx=ctx,
+ )
+ except Exception as e:
+ logger.warning(f"grep vikingdb step failed, falling back to fs: {e}")
+ return await self._grep_fs(
+ uri=uri,
+ pattern=pattern,
+ exclude_uri=exclude_uri,
+ case_insensitive=case_insensitive,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ )
+
+ candidate_uris = [r["uri"] for r in result if r.get("uri")]
+ if excluded_prefix:
+ candidate_uris = [
+ u
+ for u in candidate_uris
+ if u != excluded_prefix and not u.startswith(excluded_prefix + "/")
+ ]
+ if not candidate_uris:
+ # BM25 returned no candidates — the index confirms no matching content
+ return {"matches": [], "count": 0, "match_count": 0, "files_scanned": 0}
+
+ # Step 2: local fs precise matching on candidate files
+ return await self._grep_in_files(
+ candidate_uris,
+ pattern,
+ case_insensitive,
+ node_limit,
+ ctx,
+ )
+
+ async def _grep_in_files(
+ self,
+ file_uris: List[str],
+ pattern: str,
+ case_insensitive: bool,
+ node_limit: Optional[int],
+ ctx: Optional[RequestContext],
+ ) -> Dict:
+ """Execute regex matching in specified file list (vikingdb_then_fs Step 2)."""
+ flags = re.IGNORECASE if case_insensitive else 0
+ compiled = re.compile(pattern, flags)
+
+ results = []
+ files_scanned = 0
+
+ for file_uri in file_uris:
+ files_scanned += 1
+ try:
+ content_bytes = await self.read(file_uri, ctx=ctx)
+ content = content_bytes.decode("utf-8", errors="replace")
+ except Exception:
+ continue
+
+ for line_no, line in enumerate(content.splitlines(), 1):
+ if compiled.search(line):
+ results.append({"uri": file_uri, "line": line_no, "content": line})
+ if node_limit and len(results) >= node_limit:
+ return {
+ "matches": results,
+ "count": len(results),
+ "match_count": len(results),
+ "files_scanned": files_scanned,
+ }
+
+ return {
+ "matches": results,
+ "count": len(results),
+ "match_count": len(results),
+ "files_scanned": files_scanned,
+ }
+
+ async def _grep_with_agfs(
+ self,
+ uri: str,
+ pattern: str,
+ exclude_uri: Optional[str] = None,
+ case_insensitive: bool = False,
+ node_limit: Optional[int] = None,
+ level_limit: int = 10,
+ ctx: Optional[RequestContext] = None,
+ ) -> Dict:
+ """Grep using agfs native implementation.
+
+ This is the optimized path for non-encrypted files.
+ Uses agfs.grep() which performs matching on the server side.
+
+ Prefer pushing filters down to agfs backend:
+ - exclude_uri -> exclude_path
+ - level_limit -> level_limit
+
+ Args:
+ uri: Viking URI
+ pattern: Regular expression pattern to search for
+ exclude_uri: Optional URI prefix to exclude from search
+ case_insensitive: Whether to perform case-insensitive matching
+ node_limit: Maximum number of results to return
+ level_limit: Maximum depth level to traverse
+ ctx: Request context
+
+ Returns:
+ Dict with matches, count, match_count, files_scanned
+ """
+ path = self._uri_to_path(uri, ctx=ctx)
+
+ excluded_path = None
+ if exclude_uri:
+ normalized_excluded_uri = exclude_uri.rstrip("/")
+ self._ensure_access(normalized_excluded_uri, ctx)
+ excluded_path = self._uri_to_path(normalized_excluded_uri, ctx=ctx)
+
+ try:
+ result = await self._async_agfs.grep(
+ path=path,
+ pattern=pattern,
+ recursive=True,
+ case_insensitive=case_insensitive,
+ stream=False,
+ node_limit=node_limit,
+ exclude_path=excluded_path,
+ level_limit=level_limit,
+ )
+ except (AttributeError, AGFSNotSupportedError, NotImplementedError):
+ # Capability missing: let the outer caller fall back to the VikingFS implementation.
+ logger.warning("agfs grep unavailable, falling back to VikingFS implementation")
+ raise
+
+ matches = result.get("matches", [])
+ results = []
+ files_scanned_set = set()
+ real_ctx = self._ctx_or_default(ctx)
+
+ for match in matches:
+ match_file = match.get("file", "")
+ if not match_file:
+ continue
+
+ agfs_file_path = self._resolve_grep_match_agfs_path(path, match_file)
+
+ file_uri = self._path_to_uri(agfs_file_path, ctx=ctx)
+ if not self._is_accessible(file_uri, real_ctx):
+ continue
+
+ files_scanned_set.add(file_uri)
+
+ results.append({
+ "line": match.get("line", match.get("line_number", 0)),
+ "uri": file_uri,
+ "content": match.get("content", ""),
+ })
+
+ if node_limit and len(results) >= node_limit:
+ break
+
+ # Prefer backend-provided scanned file count if available; otherwise fall back to
+ # counting files that produced at least one match (best-effort).
+ backend_files_scanned = result.get("files_scanned")
+ if isinstance(backend_files_scanned, int) and backend_files_scanned >= 0:
+ files_scanned = (
+ len(files_scanned_set) if real_ctx.actor_peer_id else backend_files_scanned
+ )
+ else:
+ files_scanned = len(files_scanned_set)
+
+ return {
+ "matches": results,
+ "count": len(results),
+ "match_count": len(results),
+ "files_scanned": files_scanned,
+ }
+
+ async def _grep_encrypted(
+ self,
+ uri: str,
+ pattern: str,
+ exclude_uri: Optional[str] = None,
+ case_insensitive: bool = False,
+ node_limit: Optional[int] = None,
+ level_limit: int = 10,
+ ctx: Optional[RequestContext] = None,
+ content_transform: Optional[Callable[[str, str], str]] = None,
+ ) -> Dict:
+ """Grep implementation for encrypted files.
+
+ This implementation decrypts files at VikingFS layer before matching.
+ Used when encryption is enabled or when agfs.grep is not available.
+
+ Args:
+ uri: Viking URI
+ pattern: Regular expression pattern to search for
+ exclude_uri: Optional URI prefix to exclude from search
+ case_insensitive: Whether to perform case-insensitive matching
+ node_limit: Maximum number of results to return
+ level_limit: Maximum depth level to traverse (default: 10)
+ ctx: Request context
+
+ Returns:
+ Dict with matches, count, match_count, files_scanned
+ """
+ flags = re.IGNORECASE if case_insensitive else 0
+ compiled_pattern = re.compile(pattern, flags)
+ excluded_prefix = None
+ if exclude_uri:
+ excluded_prefix = exclude_uri.rstrip("/")
+ self._ensure_access(excluded_prefix, ctx)
+ file_uris = await self._collect_grep_files(
+ uri,
+ excluded_prefix=excluded_prefix,
+ level_limit=level_limit,
+ ctx=ctx,
+ )
+ results, files_scanned = await self._grep_files_parallel(
+ file_uris,
+ compiled_pattern=compiled_pattern,
+ node_limit=node_limit,
+ ctx=ctx,
+ content_transform=content_transform,
+ )
+
+ return {
+ "matches": results,
+ "count": len(results),
+ "match_count": len(results),
+ "files_scanned": files_scanned,
+ }
+
+ async def _collect_grep_files(
+ self,
+ uri: str,
+ excluded_prefix: Optional[str],
+ level_limit: int,
+ ctx: Optional[RequestContext] = None,
+ ) -> List[str]:
+ file_uris: List[str] = []
+
+ async def search_recursive(current_uri: str, current_depth: int) -> None:
+ if current_depth > level_limit:
+ return
+
+ normalized_current_uri = current_uri
+ if excluded_prefix and (
+ normalized_current_uri == excluded_prefix
+ or normalized_current_uri.startswith(excluded_prefix + "/")
+ ):
+ logger.debug(f"Skipping excluded uri during grep: {normalized_current_uri}")
+ return
+
+ try:
+ entries = await self.ls(normalized_current_uri, ctx=ctx)
+ except Exception:
+ return
+
+ for entry in entries:
+ entry_uri = f"{normalized_current_uri.rstrip('/')}/{entry['name']}"
+ if excluded_prefix and (
+ entry_uri == excluded_prefix or entry_uri.startswith(excluded_prefix + "/")
+ ):
+ logger.debug(f"Skipping excluded uri during grep: {entry_uri}")
+ continue
+
+ if entry.get("isDir"):
+ await search_recursive(entry_uri, current_depth + 1)
+ else:
+ file_uris.append(entry_uri)
+
+ normalized_uri = uri
+ if excluded_prefix and (
+ normalized_uri == excluded_prefix or normalized_uri.startswith(excluded_prefix + "/")
+ ):
+ logger.debug(f"Skipping excluded uri during grep: {normalized_uri}")
+ return file_uris
+ try:
+ root_stat = await self.stat(normalized_uri, ctx=ctx)
+ except Exception:
+ return file_uris
+ if not root_stat.get("isDir", False):
+ file_uris.append(normalized_uri)
+ return file_uris
+
+ await search_recursive(uri, 0)
+ return file_uris
+
+ async def _grep_files_parallel(
+ self,
+ file_uris: List[str],
+ compiled_pattern: re.Pattern,
+ node_limit: Optional[int],
+ ctx: Optional[RequestContext] = None,
+ content_transform: Optional[Callable[[str, str], str]] = None,
+ ) -> tuple[List[Dict[str, Any]], int]:
+ results: List[Dict[str, Any]] = []
+ files_scanned = 0
+ concurrency = _pkg()._DEFAULT_GREP_FILE_CONCURRENCY
+ for start in range(0, len(file_uris), concurrency):
+ batch_uris = file_uris[start : start + concurrency]
+ batch_jobs = [
+ self._grep_single_file(
+ entry_uri,
+ compiled_pattern,
+ ctx,
+ content_transform=content_transform,
+ )
+ for entry_uri in batch_uris
+ ]
+ batch_results = await asyncio.gather(*batch_jobs)
+ for matches, scanned_count in batch_results:
+ files_scanned += scanned_count
+ for match in matches:
+ results.append(match)
+ if node_limit and len(results) >= node_limit:
+ return results, files_scanned
+
+ return results, files_scanned
+
+ async def _grep_single_file(
+ self,
+ entry_uri: str,
+ compiled_pattern: re.Pattern,
+ ctx: Optional[RequestContext] = None,
+ content_transform: Optional[Callable[[str, str], str]] = None,
+ ) -> tuple[List[Dict[str, Any]], int]:
+ try:
+ content = await self.read(entry_uri, ctx=ctx)
+ if isinstance(content, bytes):
+ content = content.decode("utf-8", errors="replace")
+ if content_transform is not None:
+ content = content_transform(content, entry_uri)
+
+ matches: List[Dict[str, Any]] = []
+ lines = content.split("\n")
+ for line_num, line in enumerate(lines, 1):
+ if compiled_pattern.search(line):
+ matches.append({
+ "line": line_num,
+ "uri": entry_uri,
+ "content": line,
+ })
+ return matches, 1
+ except Exception as e:
+ logger.debug(f"Failed to grep {entry_uri}: {e}")
+ return [], 1
+
+ def _resolve_grep_match_agfs_path(self, base_path: str, match_file: str) -> str:
+ """Resolve a grep match path (relative to query root) into a full AGFS path."""
+ if match_file == ".":
+ return base_path
+ return f"{base_path.rstrip('/')}/{match_file.lstrip('/')}"
diff --git a/openviking/storage/viking_fs/_ops.py b/openviking/storage/viking_fs/_ops.py
new file mode 100644
index 000000000..f934c7c00
--- /dev/null
+++ b/openviking/storage/viking_fs/_ops.py
@@ -0,0 +1,1372 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Core filesystem operations mixin for VikingFS."""
+
+import asyncio
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
+
+from openviking.core.namespace import (
+ canonicalize_uri,
+ may_include_hidden_actor_peers,
+)
+from openviking.pyagfs.exceptions import (
+ AGFSDirectoryNotEmptyError,
+ AGFSHTTPError,
+ AGFSClientError,
+)
+from openviking.server.error_mapping import is_not_found_error, map_exception
+from openviking.server.identity import RequestContext
+from openviking.storage.expr import PathScope
+from openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMES
+from openviking.storage.viking_fs._base import (
+ _ABSTRACT_WORKER_COUNT,
+ _ensure_non_empty_search_query,
+ _is_directory_not_empty_error,
+ logger,
+ LS_ALL_NODES,
+)
+from openviking_cli.exceptions import (
+ FailedPreconditionError,
+ InvalidArgumentError,
+ NotFoundError,
+)
+from openviking_cli.utils.uri import VikingURI
+from openviking.utils.time_utils import format_iso8601, parse_iso_datetime
+
+if TYPE_CHECKING:
+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig
+
+
+class _OpsMixin:
+ """Core filesystem operations (read/write/mkdir/rm/mv/stat/glob/tree/ls/temp)."""
+
+ # ========== AGFS Basic Commands ==========
+
+ async def read(
+ self,
+ uri: str,
+ offset: int = 0,
+ size: int = -1,
+ ctx: Optional[RequestContext] = None,
+ ) -> bytes:
+ """Read file"""
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+
+ # Decryption + offset/size slicing now happen inside the ragfs encryption layer
+ # (when configured); the plaintext stack reads bytes directly. Either way, pass the
+ # offset/size through and let the Rust layer return the requested slice.
+ last_not_found: Optional[Exception] = None
+ for path in self._read_paths(uri, ctx=ctx):
+ if not await self._read_path_visible(uri, path, primary_path, real_ctx):
+ continue
+ try:
+ result = await self._async_agfs.read(path, offset, size)
+ break
+ except Exception as exc:
+ if is_not_found_error(exc):
+ last_not_found = exc
+ continue
+ raise
+ else:
+ raise NotFoundError(uri, "file") from last_not_found
+ if isinstance(result, bytes):
+ raw = result
+ elif result is not None and hasattr(result, "content"):
+ raw = result.content
+ else:
+ raw = b""
+
+ return raw
+
+ async def write(
+ self,
+ uri: str,
+ data: Union[bytes, str],
+ ctx: Optional[RequestContext] = None,
+ ) -> str:
+ """Write file"""
+ self._ensure_mutable_access(uri, ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ if isinstance(data, str):
+ data = data.encode("utf-8")
+
+ # Encryption (when configured) happens inside the ragfs layer keyed by account_id.
+ return await self._async_agfs.write(path, data)
+
+ async def mkdir(
+ self,
+ uri: str,
+ mode: str = "755",
+ exist_ok: bool = False,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> None:
+ """Create directory."""
+ self._ensure_mutable_access(uri, ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ # Always ensure parent directories exist before creating this directory
+ await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)
+ try:
+ await self._async_agfs.mkdir(path, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))
+ except Exception as exc:
+ message = str(exc).lower()
+ already_exists = "exist" in message or "already" in message
+ if exist_ok and already_exists:
+ return
+ raise
+
+ async def rm(
+ self,
+ uri: str,
+ recursive: bool = False,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> Dict[str, Any]:
+ """Delete file/directory + recursively update vector index.
+
+ This method is idempotent: deleting a non-existent file succeeds
+ after cleaning up any orphan index records.
+
+ Acquires a path lock, deletes VectorDB records, then FS files.
+ Raises ResourceBusyError when the target is locked by an ongoing
+ operation (e.g. semantic processing).
+
+ Returns:
+ Dict with 'estimated_deleted_count' indicating the estimated number
+ of nodes deleted from vector index.
+ """
+ from openviking.storage.errors import LockAcquisitionError, ResourceBusyError
+
+ self._ensure_delete_access(uri, ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ target_uri = self._path_to_uri(path, ctx=ctx)
+
+ async def _estimate_deleted_count(target_path: str, real_ctx: RequestContext) -> int:
+ """Estimate number of nodes to be deleted using vector index."""
+ vector_store = self._get_vector_store()
+ if not vector_store:
+ return 0
+ try:
+ target_canonical_uri = canonicalize_uri(
+ self._path_to_uri(target_path, ctx=real_ctx), real_ctx
+ )
+ filter_expr = PathScope("uri", target_canonical_uri, depth=-1)
+ return await vector_store.count(filter=filter_expr, ctx=real_ctx)
+ except Exception as e:
+ logger.warning(f"[VikingFS] Failed to count nodes before delete: {e}")
+ return 0
+
+ # Check existence and determine lock strategy
+ try:
+ stat = await self._async_agfs.stat(path)
+ is_dir = stat.get("isDir", False) if isinstance(stat, dict) else False
+ except Exception as exc:
+ if not is_not_found_error(exc):
+ mapped = map_exception(exc, resource=uri)
+ if mapped is not None:
+ raise mapped from exc
+ raise
+ # Path does not exist: clean up any orphan index records and return
+ uris_to_delete = await self._collect_uris(path, recursive, ctx=ctx)
+ uris_to_delete.append(target_uri)
+ real_ctx = self._ctx_or_default(ctx)
+ estimated_count = await _estimate_deleted_count(path, real_ctx)
+ await self._delete_from_vector_store(uris_to_delete, ctx=ctx)
+ logger.info(f"[VikingFS] rm target not found, cleaned orphan index: {uri}")
+ return {"estimated_deleted_count": estimated_count}
+
+ if is_dir:
+ if not recursive:
+ raise FailedPreconditionError(
+ f"Cannot remove directory without --recursive: {uri}",
+ details={"resource": uri, "expected_flag": "recursive"},
+ )
+ lock_method = self._async_agfs.pathlock_acquire_tree
+ else:
+ lock_method = self._async_agfs.pathlock_acquire_exact
+
+ lease = lease_ref
+ if lease is None:
+ try:
+ lease = await lock_method(path)
+ except LockAcquisitionError:
+ raise ResourceBusyError(f"Resource is being processed: {uri}", uri=uri)
+
+ try:
+ uris_to_delete = await self._collect_uris(path, recursive, ctx=ctx) if is_dir else []
+ uris_to_delete.append(target_uri)
+ real_ctx = self._ctx_or_default(ctx)
+ estimated_count = await _estimate_deleted_count(path, real_ctx)
+ await self._delete_from_vector_store(uris_to_delete, ctx=ctx)
+ try:
+ result = await self._async_agfs.rm(
+ path,
+ recursive=recursive,
+ fs_ctx=self._pathlock_fs_ctx(ctx, lease),
+ )
+ except AGFSDirectoryNotEmptyError:
+ raise FailedPreconditionError(
+ f"Directory not empty: {uri}. Use recursive=True to delete non-empty directories."
+ )
+ except RuntimeError as e:
+ # Fallback for older versions without typed exceptions
+ if _is_directory_not_empty_error(str(e)):
+ raise FailedPreconditionError(
+ f"Directory not empty: {uri}. Use recursive=True to delete non-empty directories."
+ )
+ raise
+ # Add estimated_deleted_count to the result
+ if isinstance(result, dict):
+ result["estimated_deleted_count"] = estimated_count
+ else:
+ result = {"estimated_deleted_count": estimated_count}
+ return result
+ finally:
+ if lease_ref is None:
+ await self._async_agfs.pathlock_release(lease)
+
+ async def mv(
+ self,
+ old_uri: str,
+ new_uri: str,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> Dict[str, Any]:
+ """Move file/directory while extending an optional outer pathlock lease.
+
+ Implemented as cp + rm to avoid lock files being carried by FS mv.
+ On VectorDB update failure the copy is cleaned up so the source stays intact.
+ """
+
+ self._ensure_mutable_access(old_uri, ctx)
+ # mv is implemented as copy + recursive rm of the source (see the
+ # ``rm(old_path, recursive=is_dir)`` below), so the source must also clear
+ # the delete guard. Without this, a protected account root such as
+ # ``viking://`` — which rm() rejects up front (#2873) — could still be
+ # destroyed via mv, since the write guard alone permits the bare root.
+ self._ensure_delete_access(old_uri, ctx)
+ self._ensure_mutable_access(new_uri, ctx)
+ old_path = self._uri_to_path(old_uri, ctx=ctx)
+ new_path = self._uri_to_path(new_uri, ctx=ctx)
+ target_uri = self._path_to_uri(old_path, ctx=ctx)
+
+ # Verify source exists and determine type before locking.
+ try:
+ stat = await self._async_agfs.stat(old_path)
+ is_dir = stat.get("isDir", False) if isinstance(stat, dict) else False
+ except Exception as exc:
+ if not is_not_found_error(exc):
+ mapped = map_exception(exc, resource=old_uri)
+ if mapped is not None:
+ raise mapped from exc
+ raise
+ raise FileNotFoundError(f"mv source not found: {old_uri}") from exc
+
+ if not is_dir:
+ if new_uri.rstrip("/") != new_uri:
+ raise InvalidArgumentError(
+ f"mv destination for a file must include the target file name: {new_uri}",
+ details={"from_uri": old_uri, "to_uri": new_uri},
+ )
+ try:
+ destination_stat = await self._async_agfs.stat(new_path)
+ except Exception as exc:
+ if not is_not_found_error(exc):
+ mapped = map_exception(exc, resource=new_uri)
+ if mapped is not None:
+ raise mapped from exc
+ raise
+ else:
+ if isinstance(destination_stat, dict) and destination_stat.get("isDir", False):
+ raise InvalidArgumentError(
+ f"mv destination for a file must include the target file name: {new_uri}",
+ details={"from_uri": old_uri, "to_uri": new_uri},
+ )
+
+ real_ctx = self._ctx_or_default(ctx)
+ old_uri = canonicalize_uri(old_uri, real_ctx)
+ new_uri = canonicalize_uri(new_uri, real_ctx)
+
+ if is_dir:
+ lease = await self._async_agfs.pathlock_acquire_batch(
+ [
+ {"path": old_path, "kind": "tree"},
+ {"path": new_path, "kind": "exact"},
+ ],
+ owner_lease_ref=lease_ref,
+ )
+ else:
+ lease = await self._async_agfs.pathlock_acquire_batch(
+ [
+ {"path": old_path, "kind": "exact"},
+ {"path": new_path, "kind": "exact"},
+ ],
+ owner_lease_ref=lease_ref,
+ )
+
+ try:
+ uris_to_move = (
+ await self._collect_uris(old_path, recursive=True, ctx=ctx) if is_dir else []
+ )
+ uris_to_move.append(target_uri)
+
+ # Check if it's temp directory (files already encrypted)
+ is_temp = old_uri.startswith("viking://temp/")
+
+ # Copy source to destination. Source must stay intact until vector updates succeed.
+ try:
+ await self._copy_for_mv(
+ old_uri=old_uri,
+ new_uri=new_uri,
+ old_path=old_path,
+ new_path=new_path,
+ is_dir=is_dir,
+ is_temp=is_temp,
+ ctx=ctx,
+ lease_ref=lease,
+ )
+ except Exception as e:
+ if "not found" in str(e).lower():
+ try:
+ await self._delete_from_vector_store(uris_to_move, ctx=ctx)
+ except Exception:
+ # Orphan cleanup is best effort here; preserve the copy error.
+ pass
+ else:
+ logger.info(
+ f"[VikingFS] mv source not found, cleaned orphan index: {old_uri}"
+ )
+ raise
+
+ # Update VectorDB URIs (on failure, clean up the copy)
+ try:
+ await self._update_vector_store_uris(uris_to_move, old_uri, new_uri, ctx=ctx)
+ except Exception:
+ try:
+ if is_dir:
+ cleanup_lease = await self._async_agfs.pathlock_acquire_tree(
+ new_path,
+ owner_lease_ref=lease,
+ )
+ try:
+ await self._async_agfs.rm(
+ new_path,
+ recursive=True,
+ fs_ctx=self._pathlock_fs_ctx(ctx, cleanup_lease),
+ )
+ finally:
+ await self._async_agfs.pathlock_release(cleanup_lease)
+ else:
+ await self._async_agfs.rm(
+ new_path,
+ fs_ctx=self._pathlock_fs_ctx(ctx, lease),
+ )
+ except Exception:
+ pass
+ raise
+
+ # Delete source
+ await self._async_agfs.rm(
+ old_path,
+ recursive=is_dir,
+ fs_ctx=self._pathlock_fs_ctx(ctx, lease),
+ )
+ return {}
+ finally:
+ await self._async_agfs.pathlock_release(lease)
+
+ async def _copy_for_mv(
+ self,
+ old_uri: str,
+ new_uri: str,
+ old_path: str,
+ new_path: str,
+ is_dir: bool,
+ is_temp: bool,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> None:
+ """Copy source to destination for mv without deleting source."""
+ if is_temp:
+ if is_dir:
+ await self._copy_temp_dir_with_exact_locks(
+ old_path,
+ new_path,
+ ctx=ctx,
+ lease_ref=lease_ref,
+ )
+ else:
+ await self._async_agfs.cp(
+ old_path,
+ new_path,
+ recursive=False,
+ fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),
+ )
+ return
+
+ if is_dir:
+ await self._copy_dir_through_vikingfs(old_uri, new_uri, ctx=ctx, lease_ref=lease_ref)
+ else:
+ await self._copy_file_through_vikingfs(old_uri, new_uri, ctx=ctx, lease_ref=lease_ref)
+
+ async def _copy_temp_dir_with_exact_locks(
+ self,
+ old_path: str,
+ new_path: str,
+ ctx: Optional[RequestContext],
+ lease_ref: Dict[str, Any] | None,
+ ) -> None:
+ """Copy an encrypted temp directory while locking every destination entry.
+
+ Args:
+ old_path: Source backend directory path.
+ new_path: Destination backend directory path.
+ ctx: Request context used for filesystem operations.
+ lease_ref: Exact lease covering the current destination directory.
+
+ Returns:
+ None.
+ """
+ if lease_ref is None:
+ raise ValueError("temp directory copy requires a pathlock lease")
+ fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)
+ await self._async_agfs.mkdir(new_path, fs_ctx=fs_ctx)
+ entries = await self._async_agfs.ls(old_path, fs_ctx=fs_ctx)
+ for entry in entries:
+ name = entry.get("name", "")
+ if not name or name in (".", ".."):
+ continue
+ old_child = f"{old_path.rstrip('/')}/{name}"
+ new_child = f"{new_path.rstrip('/')}/{name}"
+ child_lease = await self._async_agfs.pathlock_acquire_exact(
+ new_child,
+ owner_lease_ref=lease_ref,
+ )
+ try:
+ if entry.get("isDir", False):
+ await self._copy_temp_dir_with_exact_locks(
+ old_child,
+ new_child,
+ ctx=ctx,
+ lease_ref=child_lease,
+ )
+ else:
+ await self._async_agfs.cp(
+ old_child,
+ new_child,
+ recursive=False,
+ fs_ctx=self._pathlock_fs_ctx(ctx, child_lease),
+ )
+ finally:
+ await self._async_agfs.pathlock_release(child_lease)
+
+ async def _copy_dir_through_vikingfs(
+ self,
+ old_uri: str,
+ new_uri: str,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> None:
+ """Recursively copy a directory through VikingFS read/write hooks."""
+ await self.mkdir(new_uri, exist_ok=True, ctx=ctx, lease_ref=lease_ref)
+
+ entries = await self.ls(old_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx)
+ for entry in entries:
+ name = entry.get("name", "")
+ if not name or name in (".", ".."):
+ continue
+ old_child_uri = f"{old_uri.rstrip('/')}/{name}"
+ new_child_uri = f"{new_uri.rstrip('/')}/{name}"
+ if entry.get("isDir"):
+ await self._copy_dir_through_vikingfs(
+ old_child_uri,
+ new_child_uri,
+ ctx=ctx,
+ lease_ref=lease_ref,
+ )
+ else:
+ await self._copy_file_through_vikingfs(
+ old_child_uri,
+ new_child_uri,
+ ctx=ctx,
+ lease_ref=lease_ref,
+ )
+
+ async def _copy_file_through_vikingfs(
+ self,
+ from_uri: str,
+ to_uri: str,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> None:
+ """Copy one file through VikingFS read/write hooks without deleting source."""
+ content_bytes = await self.read_file_bytes(from_uri, ctx=ctx)
+ if lease_ref is None:
+ await self.write_file_bytes(to_uri, content_bytes, ctx=ctx)
+ return
+
+ child_path = self._uri_to_path(to_uri, ctx=ctx)
+ child_lease = await self._async_agfs.pathlock_acquire_exact(
+ child_path,
+ owner_lease_ref=lease_ref,
+ )
+ try:
+ await self.write_file_bytes(to_uri, content_bytes, ctx=ctx, lease_ref=child_lease)
+ finally:
+ await self._async_agfs.pathlock_release(child_lease)
+
+ async def stat(
+ self, uri: str, ctx: Optional[RequestContext] = None, skip_count: bool = False
+ ) -> Dict[str, Any]:
+ """
+ File/directory information.
+
+ example: {'name': 'resources', 'size': 128, 'mode': 2147484141, 'modTime': '2026-02-10T21:26:02.934376379+08:00', 'isDir': True, 'isLocked': False, 'count': 42, 'meta': {'Name': 'localfs', 'Type': 'local', 'Content': {'local_path': '...'}}}
+
+ Extra field:
+ isLocked (bool): Whether the path is currently held by a path lock
+ (either the path itself or any ancestor directory). Returns
+ False when the pathlock system is not enabled or the lookup
+ fails.
+ count (int): For directories, the number of nodes in the vector index
+ under this directory (including subdirectories). For files, this
+ field is not included.
+
+ Args:
+ uri: Viking URI
+ ctx: Request context
+ skip_count: If True, skip the vector_store.count() call for directories.
+ Use this when the count field is not needed (e.g. in grep) to avoid
+ an extra VikingDB API call.
+ """
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+ path = primary_path
+ last_not_found: Optional[Exception] = None
+ for candidate_path in self._read_paths(uri, ctx=ctx):
+ if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):
+ continue
+ try:
+ result = await self._async_agfs.stat(candidate_path)
+ path = candidate_path
+ break
+ except Exception as exc:
+ if is_not_found_error(exc):
+ last_not_found = exc
+ continue
+ raise
+ else:
+ if self._is_legacy_session_root_uri(uri):
+ now = datetime.now(timezone.utc).isoformat()
+ return {
+ "name": "session",
+ "size": 0,
+ "mode": 0o755,
+ "modTime": now,
+ "isDir": True,
+ "isLocked": False,
+ }
+ raise NotFoundError(uri, "file") from last_not_found
+ if isinstance(result, dict):
+ result["isLocked"] = await self._is_path_locked_async(path)
+ # Add count for directories if vector store available
+ if not skip_count and result.get("isDir", False):
+ try:
+ vector_store = self._get_vector_store()
+ if vector_store:
+ target_canonical_uri = canonicalize_uri(
+ self._path_to_uri(path, ctx=real_ctx), real_ctx
+ )
+ if not may_include_hidden_actor_peers(target_canonical_uri, real_ctx):
+ filter_expr = PathScope("uri", target_canonical_uri, depth=-1)
+ result["count"] = await vector_store.count(
+ filter=filter_expr,
+ ctx=real_ctx,
+ )
+ except Exception as e:
+ logger.warning(f"[VikingFS] Failed to count nodes for directory stat: {e}")
+ return result
+
+ async def exists(self, uri: str, ctx: Optional[RequestContext] = None) -> bool:
+ """Check if a URI exists.
+
+ Args:
+ uri: Viking URI
+ ctx: Request context
+
+ Returns:
+ bool: True if the URI exists, False otherwise
+ """
+ try:
+ await self.stat(uri, ctx=ctx)
+ return True
+ except Exception:
+ return False
+
+ async def glob(
+ self,
+ pattern: str,
+ uri: str = "viking://",
+ node_limit: Optional[int] = None,
+ ctx: Optional[RequestContext] = None,
+ ) -> Dict:
+ """File pattern matching, supports **/*.md recursive."""
+ _ensure_non_empty_search_query(pattern)
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+ path: Optional[str] = None
+ for candidate_path in self._read_paths(uri, ctx=ctx):
+ if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):
+ continue
+ if await self._agfs_path_exists(candidate_path):
+ path = candidate_path
+ break
+ if path is None:
+ if self._is_legacy_session_root_uri(uri):
+ return {"matches": [], "count": 0}
+ raise NotFoundError(uri, "directory")
+
+ page_size = self._glob_page_size(node_limit)
+ continuation_token: Optional[str] = None
+ matches = []
+ while True:
+ page = await self._async_agfs.glob_directory(
+ path,
+ pattern,
+ show_hidden=False,
+ page_size=page_size,
+ level_limit=None,
+ continuation_token=continuation_token,
+ )
+
+ for entry in page.get("entries", []):
+ if node_limit is not None and node_limit > 0 and len(matches) >= node_limit:
+ return {"matches": matches, "count": len(matches)}
+ if not self._is_path_entry_visible(
+ entry["path"],
+ entry.get("name") or entry["path"].rsplit("/", 1)[-1],
+ path,
+ real_ctx,
+ ):
+ continue
+ if not await self._read_path_visible(uri, entry["path"], primary_path, real_ctx):
+ continue
+ entry_uri = self._alias_uri_for_path(
+ request_uri=uri,
+ base_path=path,
+ entry_path=entry["path"],
+ ctx=ctx,
+ )
+ matches.append(entry_uri)
+
+ if node_limit is not None and node_limit > 0 and len(matches) >= node_limit:
+ return {"matches": matches, "count": len(matches)}
+ continuation_token = page.get("next_token")
+ if not continuation_token:
+ break
+ return {"matches": matches, "count": len(matches)}
+
+ async def _batch_fetch_abstracts(
+ self,
+ entries: List[Dict[str, Any]],
+ abs_limit: int,
+ ctx: Optional[RequestContext] = None,
+ ) -> None:
+ """Batch fetch abstracts for entries using a fixed-size worker pool.
+
+ Non-directory entries receive an empty abstract immediately.
+ Directory entries are processed concurrently via a worker pool,
+ using _read_abstract_for_known_dir to skip redundant stat() calls.
+
+ Args:
+ entries: List of entries to fetch abstracts for
+ abs_limit: Maximum length for abstract truncation
+ """
+ dir_jobs = []
+ for index, entry in enumerate(entries):
+ if not entry.get("isDir", False):
+ entry["abstract"] = ""
+ continue
+ dir_jobs.append((index, entry))
+
+ if not dir_jobs:
+ return
+
+ worker_count = min(_ABSTRACT_WORKER_COUNT, len(dir_jobs))
+
+ cursor = 0
+ cursor_lock = asyncio.Lock()
+ results: Dict[int, str] = {}
+
+ async def worker() -> None:
+ nonlocal cursor
+ while True:
+ async with cursor_lock:
+ if cursor >= len(dir_jobs):
+ return
+ index, entry = dir_jobs[cursor]
+ cursor += 1
+
+ try:
+ abstract = await self._read_abstract_for_known_dir(entry["uri"], ctx=ctx)
+ except Exception:
+ abstract = "[.abstract.md is not ready]"
+
+ results[index] = abstract
+
+ await asyncio.gather(*(worker() for _ in range(worker_count)))
+
+ for index, abstract in results.items():
+ if len(abstract) > abs_limit:
+ abstract = abstract[: abs_limit - 3] + "..."
+ entries[index]["abstract"] = abstract
+
+ async def tree(
+ self,
+ uri: str = "viking://",
+ output: str = "original",
+ abs_limit: int = 256,
+ show_all_hidden: bool = False,
+ node_limit: Optional[int] = 1000,
+ level_limit: Optional[int] = 3,
+ ctx: Optional[RequestContext] = None,
+ ) -> List[Dict[str, Any]]:
+ """
+ Recursively list all contents (includes rel_path).
+
+ Args:
+ uri: Viking URI
+ output: str = "original" or "agent"
+ abs_limit: int = 256 (for agent output abstract truncation)
+ show_all_hidden: bool = False (list all hidden files, like -a)
+ node_limit: int | None = 1000 (maximum number of nodes to list, None means unlimited)
+ level_limit: int | None = 3 (maximum depth level to traverse, None means unlimited)
+
+ output="original"
+ [{'name': '.abstract.md', 'size': 100, 'mode': 420, 'modTime': '2026-02-11T16:52:16.256334192+08:00', 'isDir': False, 'rel_path': '.abstract.md', 'uri': 'viking://resources...'}]
+
+ output="agent"
+ [{'uri': 'viking://resources...', 'size': 100, 'isDir': False, 'modTime': '2026-02-11T08:52:16.256Z', 'rel_path': '.abstract.md', 'abstract': "..."}]
+ """
+ self._ensure_access(uri, ctx)
+ if output == "original":
+ return await self._tree_original(uri, show_all_hidden, node_limit, level_limit, ctx=ctx)
+ elif output == "agent":
+ return await self._tree_agent(
+ uri, abs_limit, show_all_hidden, node_limit, level_limit, ctx=ctx
+ )
+ else:
+ raise ValueError(f"Invalid output format: {output}")
+
+ async def _tree_original(
+ self,
+ uri: str,
+ show_all_hidden: bool = False,
+ node_limit: Optional[int] = 1000,
+ level_limit: Optional[int] = 3,
+ ctx: Optional[RequestContext] = None,
+ ) -> List[Dict[str, Any]]:
+ """Recursively list all contents (original format)."""
+ result = []
+ async for entry, entry_uri in self._iter_visible_tree_entries(
+ uri,
+ show_all_hidden=show_all_hidden,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ ):
+ info = entry["info"]
+ new_entry = dict(entry.get("extra", {}))
+ new_entry.update({
+ "name": info["name"],
+ "size": info["size"],
+ "mode": info["mode"],
+ "modTime": info["modTime"],
+ "isDir": info["isDir"],
+ "rel_path": entry["rel_path"],
+ "uri": entry_uri,
+ })
+ result.append(new_entry)
+ return result
+
+ async def _tree_agent(
+ self,
+ uri: str,
+ abs_limit: int,
+ show_all_hidden: bool = False,
+ node_limit: Optional[int] = 1000,
+ level_limit: Optional[int] = 3,
+ ctx: Optional[RequestContext] = None,
+ ) -> List[Dict[str, Any]]:
+ """Recursively list all contents (agent format with abstracts)."""
+ result = []
+
+ async for entry, entry_uri in self._iter_visible_tree_entries(
+ uri,
+ show_all_hidden=show_all_hidden,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ ):
+ info = entry["info"]
+ is_dir = info["isDir"]
+ result.append({
+ "uri": entry_uri,
+ "size": 0 if is_dir else info["size"],
+ "isDir": is_dir,
+ "modTime": format_iso8601(parse_iso_datetime(info["modTime"])),
+ "rel_path": entry["rel_path"],
+ })
+
+ await self._batch_fetch_abstracts(result, abs_limit, ctx=ctx)
+
+ return result
+
+ # ========== Vector Sync Helper Methods ==========
+
+ async def _collect_uris(
+ self, path: str, recursive: bool, ctx: Optional[RequestContext] = None
+ ) -> List[str]:
+ """Recursively collect all URIs (for rm/mv), including directories."""
+ uris = []
+
+ async def _collect(p: str):
+ try:
+ entries = await self._ls_entries(p, ctx=ctx)
+ except Exception as exc:
+ if is_not_found_error(exc):
+ return
+ raise
+
+ for entry in entries:
+ name = entry.get("name", "")
+ if name in [".", ".."]:
+ continue
+ full_path = f"{p}/{name}".replace("//", "/")
+ if entry.get("isDir"):
+ uris.append(self._path_to_uri(full_path, ctx=ctx))
+ if recursive:
+ await _collect(full_path)
+ else:
+ uris.append(self._path_to_uri(full_path, ctx=ctx))
+
+ await _collect(path)
+ return uris
+
+ # ========== Parent Directory Creation ==========
+
+ async def _ensure_parent_dirs(
+ self,
+ path: str,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> None:
+ """Recursively create all parent directories."""
+ try:
+ await self._async_agfs.ensure_parent_dirs(
+ path,
+ fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),
+ )
+ except Exception as e:
+ logger.debug(f"Failed to ensure parent directories for {path}: {e}")
+ parent = path.rstrip("/").rsplit("/", 1)[0]
+ await self._mkdir_path_with_parents(parent, ctx=ctx, lease_ref=lease_ref)
+
+ async def _mkdir_path_with_parents(
+ self,
+ dir_path: str,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> None:
+ """Create a directory path segment-by-segment using the same fs context."""
+ parts = [part for part in dir_path.strip("/").split("/") if part]
+ current = ""
+ for part in parts:
+ current = f"{current}/{part}"
+ try:
+ await self._async_agfs.mkdir(
+ current,
+ fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),
+ )
+ except Exception as e:
+ message = str(e).lower()
+ if "exist" in message or "already" in message:
+ continue
+ logger.debug(f"Failed to create parent directory {current}: {e}")
+
+ # ========== Batch Read (backward compatible) ==========
+
+ async def read_batch(
+ self, uris: List[str], level: str = "l0", ctx: Optional[RequestContext] = None
+ ) -> Dict[str, str]:
+ """Batch read content from multiple URIs."""
+ results = {}
+ for uri in uris:
+ try:
+ content = ""
+ if level == "l0":
+ content = await self.abstract(uri, ctx=ctx)
+ elif level == "l1":
+ content = await self.overview(uri, ctx=ctx)
+ results[uri] = content
+ except Exception:
+ pass
+ return results
+
+ # ========== Other Preserved Methods ==========
+
+ async def write_file(
+ self,
+ uri: str,
+ content: Union[str, bytes],
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> None:
+ """Write file directly. Encryption lock handled internally by EncryptionWrappedFS."""
+ self._ensure_mutable_access(uri, ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)
+
+ if isinstance(content, str):
+ content = content.encode("utf-8")
+
+ await self._async_agfs.write(path, content, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))
+
+ async def read_file(
+ self,
+ uri: str,
+ offset: int = 0,
+ limit: int = -1,
+ ctx: Optional[RequestContext] = None,
+ ) -> str:
+ """Read single file, optionally sliced by line range.
+
+ Args:
+ uri: Viking URI
+ offset: Starting line number (0-indexed). Default 0.
+ limit: Number of lines to read. -1 means read to end. Default -1.
+
+ Raises:
+ FileNotFoundError: If the file does not exist.
+ """
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+ # Verify the file exists before reading, because AGFS read returns
+ # empty bytes for non-existent files instead of raising an error.
+ last_not_found: Optional[Exception] = None
+ for path in self._read_paths(uri, ctx=ctx):
+ if not await self._read_path_visible(uri, path, primary_path, real_ctx):
+ continue
+ try:
+ stat = await self._async_agfs.stat(path)
+ break
+ except Exception as exc:
+ if is_not_found_error(exc):
+ last_not_found = exc
+ continue
+ raise
+ else:
+ raise NotFoundError(uri, "file") from last_not_found
+ if isinstance(stat, dict) and stat.get("isDir", False):
+ raise InvalidArgumentError(
+ f"Cannot read directory as file: {uri}",
+ details={"resource": uri, "expected": "file", "actual": "directory"},
+ )
+ try:
+ content = await self._async_agfs.read(path)
+ if isinstance(content, bytes):
+ raw = content
+ elif content is not None and hasattr(content, "content"):
+ raw = content.content
+ else:
+ raw = b""
+
+ text = self._decode_bytes(raw)
+ except Exception:
+ raise NotFoundError(uri, "file")
+
+ if offset == 0 and limit == -1:
+ return text
+ lines = text.splitlines(keepends=True)
+ sliced = lines[offset:] if limit == -1 else lines[offset : offset + limit]
+ return "".join(sliced)
+
+ async def read_file_bytes(
+ self,
+ uri: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> bytes:
+ """Read single binary file."""
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+ last_not_found: Optional[Exception] = None
+ for path in self._read_paths(uri, ctx=ctx):
+ if not await self._read_path_visible(uri, path, primary_path, real_ctx):
+ continue
+ try:
+ stat = await self._async_agfs.stat(path)
+ break
+ except Exception as exc:
+ if is_not_found_error(exc):
+ last_not_found = exc
+ continue
+ raise
+ else:
+ raise NotFoundError(uri, "file") from last_not_found
+ if isinstance(stat, dict) and stat.get("isDir", False):
+ raise InvalidArgumentError(
+ f"Cannot read directory as file: {uri}",
+ details={"resource": uri, "expected": "file", "actual": "directory"},
+ )
+ try:
+ raw = self._handle_agfs_read(await self._async_agfs.read(path))
+ return raw
+ except Exception:
+ raise NotFoundError(uri, "file")
+
+ async def write_file_bytes(
+ self,
+ uri: str,
+ content: bytes,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> None:
+ """Write single binary file. Encryption lock handled internally by EncryptionWrappedFS."""
+ self._ensure_mutable_access(uri, ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)
+
+ await self._async_agfs.write(path, content, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))
+
+ async def append_file(
+ self,
+ uri: str,
+ content: str,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> None:
+ """Append content to file while holding one exact pathlock lease."""
+ self._ensure_mutable_access(uri, ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+
+ owned_lease = None
+ try:
+ await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)
+ lease = lease_ref
+ if lease is None:
+ lease = await self._async_agfs.pathlock_acquire_exact(path)
+ owned_lease = lease
+ fs_ctx = self._pathlock_fs_ctx(ctx, lease)
+
+ # Read old content and rewrite the whole file to avoid lost updates.
+ existing = ""
+ try:
+ existing_bytes = self._handle_agfs_read(
+ await self._async_agfs.read(path, fs_ctx=fs_ctx)
+ )
+ existing = self._decode_bytes(existing_bytes)
+ except FileNotFoundError:
+ pass
+ except AGFSHTTPError as e:
+ if e.status_code != 404:
+ raise
+ except AGFSClientError:
+ raise
+
+ final_content = (existing + content).encode("utf-8")
+ await self._async_agfs.write(
+ path,
+ final_content,
+ fs_ctx=fs_ctx,
+ )
+
+ except Exception as e:
+ logger.error(f"[VikingFS] Failed to append to file {uri}: {e}")
+ raise IOError(f"Failed to append to file {uri}: {e}")
+ finally:
+ if owned_lease is not None:
+ await self._async_agfs.pathlock_release(owned_lease)
+
+ async def ls(
+ self,
+ uri: str,
+ output: str = "original",
+ abs_limit: int = 256,
+ show_all_hidden: bool = False,
+ node_limit: int = 1000,
+ sort_by: Optional[str] = None,
+ sort_order: str = "asc",
+ ctx: Optional[RequestContext] = None,
+ ) -> List[Dict[str, Any]]:
+ """
+ List directory contents (URI version).
+
+ Args:
+ uri: Viking URI
+ output: str = "original"
+ abs_limit: int = 256
+ show_all_hidden: bool = False (list all hidden files, like -a)
+ node_limit: int = 1000 (maximum number of nodes to list)
+ sort_by: Optional sort field, "name" or "mtime"
+ sort_order: Sort direction, "asc" or "desc"
+
+ output="original"
+ [{'name': '.abstract.md', 'size': 100, 'mode': 420, 'modTime': '2026-02-11T16:52:16.256334192+08:00', 'isDir': False, 'meta': {'Name': 'localfs', 'Type': 'local', 'Content': None}, 'uri': 'viking://resources/.abstract.md'}]
+
+ output="agent"
+ [{'name': '.abstract.md', 'size': 100, 'modTime': '2026-02-11T08:52:16.256Z', 'isDir': False, 'uri': 'viking://resources/.abstract.md', 'abstract': "..."}]
+ """
+ self._ensure_access(uri, ctx)
+ if sort_by not in {None, "name", "mtime"}:
+ raise ValueError("sort_by must be 'name' or 'mtime'")
+ if sort_order not in {"asc", "desc"}:
+ raise ValueError("sort_order must be 'asc' or 'desc'")
+ if output == "original":
+ return await self._ls_original(
+ uri,
+ show_all_hidden,
+ node_limit,
+ sort_by=sort_by,
+ sort_order=sort_order,
+ ctx=ctx,
+ )
+ elif output == "agent":
+ return await self._ls_agent(
+ uri,
+ abs_limit,
+ show_all_hidden,
+ node_limit,
+ sort_by=sort_by,
+ sort_order=sort_order,
+ ctx=ctx,
+ )
+ else:
+ raise ValueError(f"Invalid output format: {output}")
+
+ @staticmethod
+ def _ls_entry_mtime(entry: Dict[str, Any]) -> Optional[float]:
+ raw_time = entry.get("modTime")
+ if isinstance(raw_time, (int, float)):
+ return float(raw_time)
+ if isinstance(raw_time, str) and raw_time:
+ try:
+ return parse_iso_datetime(raw_time).timestamp()
+ except (TypeError, ValueError, OverflowError):
+ return None
+
+ legacy_time = entry.get("mtime")
+ if isinstance(legacy_time, (int, float)):
+ return float(legacy_time)
+ return None
+
+ @classmethod
+ def _sort_ls_entry_items(
+ cls,
+ entry_items: List[tuple[Dict[str, Any], str]],
+ sort_by: Optional[str],
+ sort_order: str,
+ ) -> List[tuple[Dict[str, Any], str]]:
+ if sort_by is None:
+ return entry_items
+
+ descending = sort_order == "desc"
+ directories = [item for item in entry_items if item[0].get("isDir", False)]
+ files = [item for item in entry_items if not item[0].get("isDir", False)]
+
+ if sort_by == "name":
+
+ def name_key(item: tuple[Dict[str, Any], str]) -> tuple[str, str]:
+ name = str(item[0].get("name", ""))
+ return name.lower(), name
+
+ directories.sort(key=name_key, reverse=descending)
+ files.sort(key=name_key, reverse=descending)
+ return directories + files
+
+ def sort_by_mtime(
+ items: List[tuple[Dict[str, Any], str]],
+ ) -> List[tuple[Dict[str, Any], str]]:
+ timestamped = []
+ missing = []
+ for item in items:
+ timestamp = cls._ls_entry_mtime(item[0])
+ if timestamp is None:
+ missing.append(item)
+ else:
+ timestamped.append((timestamp, item))
+ timestamped.sort(
+ key=lambda pair: pair[0],
+ reverse=descending,
+ )
+ return [item for _, item in timestamped] + missing
+
+ return sort_by_mtime(directories) + sort_by_mtime(files)
+
+ async def _ls_agent(
+ self,
+ uri: str,
+ abs_limit: int,
+ show_all_hidden: bool,
+ node_limit: int = 1000,
+ sort_by: Optional[str] = None,
+ sort_order: str = "asc",
+ ctx: Optional[RequestContext] = None,
+ ) -> List[Dict[str, Any]]:
+ """List directory contents (URI version)."""
+ real_ctx = self._ctx_or_default(ctx)
+ entry_items = await self._list_read_path_items(uri, ctx=ctx)
+ entry_items = self._sort_ls_entry_items(entry_items, sort_by, sort_order)
+ # basic info
+ fallback_time = datetime.now(timezone.utc)
+ all_entries = []
+ for entry, entry_uri in entry_items:
+ if len(all_entries) >= node_limit:
+ break
+ name = entry.get("name", "")
+ raw_time = entry.get("modTime", "")
+ parsed_time = fallback_time
+ if isinstance(raw_time, (int, float)):
+ parsed_time = datetime.fromtimestamp(raw_time, tz=timezone.utc)
+ elif raw_time:
+ if len(raw_time) > 26 and "+" in raw_time:
+ parts = raw_time.split("+")
+ raw_time = parts[0][:26] + "+" + parts[1]
+ parsed_time = parse_iso_datetime(raw_time)
+ elif isinstance(entry.get("mtime"), (int, float)):
+ parsed_time = datetime.fromtimestamp(entry["mtime"], tz=timezone.utc)
+ is_dir = entry.get("isDir", False)
+ new_entry = {
+ "uri": entry_uri,
+ "size": 0 if is_dir else entry.get("size", 0),
+ "isDir": is_dir,
+ "modTime": format_iso8601(parsed_time),
+ }
+ if not self._is_accessible(new_entry["uri"], real_ctx):
+ continue
+ if is_dir:
+ all_entries.append(new_entry)
+ elif not name.startswith("."):
+ all_entries.append(new_entry)
+ elif show_all_hidden:
+ all_entries.append(new_entry)
+ await self._batch_fetch_abstracts(all_entries, abs_limit, ctx=ctx)
+ return all_entries
+
+ async def _ls_original(
+ self,
+ uri: str,
+ show_all_hidden: bool = False,
+ node_limit: int = 1000,
+ sort_by: Optional[str] = None,
+ sort_order: str = "asc",
+ ctx: Optional[RequestContext] = None,
+ ) -> List[Dict[str, Any]]:
+ """List directory contents (URI version)."""
+ real_ctx = self._ctx_or_default(ctx)
+ try:
+ entry_items = await self._list_read_path_items(uri, ctx=ctx)
+ entry_items = self._sort_ls_entry_items(entry_items, sort_by, sort_order)
+ # AGFS returns read-only structure, need to create new dict
+ all_entries = []
+ for entry, entry_uri in entry_items:
+ if len(all_entries) >= node_limit:
+ break
+ name = entry.get("name", "")
+ new_entry = dict(entry) # Copy original data
+ new_entry["uri"] = entry_uri
+ if not self._is_accessible(new_entry["uri"], real_ctx):
+ continue
+ if entry.get("isDir"):
+ all_entries.append(new_entry)
+ elif not name.startswith("."):
+ all_entries.append(new_entry)
+ elif show_all_hidden:
+ all_entries.append(new_entry)
+ return all_entries
+ except Exception:
+ raise NotFoundError(uri, "directory")
+
+ async def move_file(
+ self,
+ from_uri: str,
+ to_uri: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> None:
+ """Move file."""
+ self._ensure_mutable_access(from_uri, ctx)
+ self._ensure_mutable_access(to_uri, ctx)
+ from_path = self._uri_to_path(from_uri, ctx=ctx)
+
+ await self._copy_file_through_vikingfs(from_uri, to_uri, ctx=ctx)
+ await self._async_agfs.rm(from_path)
+
+ # ========== Temp File Operations (backward compatible) ==========
+
+ def create_temp_uri(self, ctx: Optional[RequestContext] = None) -> str:
+ """Create a temp directory URI.
+
+ - explicit ctx or bound request context -> user-scoped temp URI
+ - no explicit/bound context -> legacy temp URI shape for backward compatibility
+ """
+ real_ctx = ctx if ctx is not None else self._bound_ctx.get()
+ if real_ctx is None:
+ return VikingURI.create_temp_uri()
+ return VikingURI.create_temp_uri(space=real_ctx.user.user_space_name())
+
+ async def persist_temp_tree(
+ self,
+ temp_uri: str,
+ target_uri: str,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> None:
+ """Persist an already-encrypted temp tree without rewriting file bytes."""
+ self._ensure_access(temp_uri, ctx)
+ self._ensure_mutable_access(target_uri, ctx)
+ src_path = self._uri_to_path(temp_uri, ctx=ctx)
+ dst_path = self._uri_to_path(target_uri, ctx=ctx)
+ fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)
+ await self._ensure_parent_dirs(dst_path, ctx=ctx, lease_ref=lease_ref)
+ await self._async_agfs.cp(
+ src_path,
+ dst_path,
+ recursive=True,
+ fs_ctx=fs_ctx or {"account_id": self._ctx_or_default(ctx).account_id},
+ )
+
+ async def delete_temp(
+ self,
+ temp_uri: str,
+ ctx: Optional[RequestContext] = None,
+ lease_ref: Dict[str, Any] | None = None,
+ ) -> None:
+ """Delete temp directory and its contents."""
+ self._ensure_mutable_access(temp_uri, ctx)
+ path = self._uri_to_path(temp_uri, ctx=ctx)
+ fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)
+ try:
+ await self._async_agfs.rm(path, recursive=True, fs_ctx=fs_ctx)
+ except Exception as e:
+ logger.warning(f"[VikingFS] Failed to delete temp {temp_uri}: {e}")
+
+ async def _ls_entries(
+ self, path: str, ctx: Optional[RequestContext] = None
+ ) -> List[Dict[str, Any]]:
+ """List directory entries, filtering out internal directories.
+
+ At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.
+ At other levels, uses the shared storage internal-name blacklist.
+ """
+ entries = await self._async_agfs.ls(path)
+ parts = [p for p in path.strip("/").split("/") if p]
+ if len(parts) == 2 and parts[0] == "local":
+ return [e for e in entries if e.get("name") in VikingURI.LISTABLE_SCOPES]
+ return [e for e in entries if e.get("name") not in STORAGE_INTERNAL_ENTRY_NAMES]
diff --git a/openviking/storage/viking_fs/_semantic.py b/openviking/storage/viking_fs/_semantic.py
new file mode 100644
index 000000000..0ef6b7364
--- /dev/null
+++ b/openviking/storage/viking_fs/_semantic.py
@@ -0,0 +1,646 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Semantic retrieval and relation management mixin for VikingFS."""
+
+import asyncio
+import json
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+from openviking.core.namespace import canonicalize_uri
+from openviking.core.retrieval_targets import resolve_retrieval_targets
+from openviking.server.error_mapping import is_not_found_error, map_exception
+from openviking.server.identity import RequestContext
+from openviking.storage.viking_fs._base import (
+ RelationEntry,
+ _ensure_non_empty_search_query,
+ logger,
+)
+from openviking.telemetry import get_current_telemetry
+from openviking.utils.image_search import build_multimodal_embedding_input
+from openviking_cli.exceptions import NotFoundError
+
+if TYPE_CHECKING:
+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig
+
+
+class _SemanticMixin:
+ """Abstract/overview/relations/find/search/link (semantic retrieval layer)."""
+
+ # ========== VikingFS Specific Capabilities ==========
+
+ async def _read_abstract_file(
+ self,
+ path: str,
+ uri: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> str:
+ """Read and decrypt/decode .abstract.md from a known directory path.
+
+ Does NOT perform stat or isDir check -- caller is responsible for
+ ensuring the path points to a directory.
+ """
+ file_path = f"{path}/.abstract.md"
+ try:
+ content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))
+ except Exception as exc:
+ if not is_not_found_error(exc):
+ mapped = map_exception(exc, resource=uri)
+ if mapped is not None:
+ raise mapped from exc
+ raise
+ return f"# {uri} [Directory abstract is not ready]"
+
+ return self._decode_bytes(content_bytes)
+
+ async def _read_abstract_for_known_dir(
+ self,
+ uri: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> str:
+ """Read .abstract.md for a directory that is already known to be a directory.
+
+ Bypasses stat() and isDir check. Caller (i.e. _batch_fetch_abstracts)
+ must guarantee that the URI points to a directory.
+ """
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+ for path in self._read_paths(uri, ctx=ctx):
+ if not await self._read_path_visible(uri, path, primary_path, real_ctx):
+ continue
+ try:
+ if not await self._agfs_path_exists(path):
+ continue
+ return await self._read_abstract_file(path, uri, ctx=ctx)
+ except Exception as exc:
+ if is_not_found_error(exc):
+ continue
+ raise
+ return f"# {uri} [Directory abstract is not ready]"
+
+ async def abstract(
+ self,
+ uri: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> str:
+ """Read directory's L0 summary (.abstract.md).
+
+ If the caller points to a file, its parent directory is used instead so
+ the endpoint remains usable for both file and directory URIs.
+ """
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+ path = primary_path
+ last_exc: Optional[Exception] = None
+ for candidate_path in self._read_paths(uri, ctx=ctx):
+ if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):
+ continue
+ try:
+ info = await self._async_agfs.stat(candidate_path)
+ path = candidate_path
+ break
+ except Exception as exc:
+ if is_not_found_error(exc):
+ last_exc = exc
+ continue
+ mapped = map_exception(exc, resource=uri)
+ if mapped is not None:
+ raise mapped from exc
+ raise
+ else:
+ if last_exc is not None:
+ mapped = map_exception(last_exc, resource=uri)
+ if mapped is not None:
+ raise mapped from last_exc
+ raise NotFoundError(uri, "directory") from last_exc
+ if not info.get("isDir", info.get("is_dir")):
+ parent_path = path.rsplit("/", 1)[0] or "/"
+ parent_uri = self._path_to_uri(parent_path, ctx=ctx)
+ logger.info(
+ "content/abstract: %s is a file, falling back to parent directory %s",
+ uri,
+ parent_uri,
+ )
+ return await self.abstract(parent_uri, ctx=ctx)
+ return await self._read_abstract_file(path, uri, ctx=ctx)
+
+ async def overview(
+ self,
+ uri: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> str:
+ """Read directory's L1 overview (.overview.md).
+
+ If the caller points to a file, its parent directory is used instead so
+ the endpoint remains usable for both file and directory URIs.
+ """
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+ path = primary_path
+ last_exc: Optional[Exception] = None
+ for candidate_path in self._read_paths(uri, ctx=ctx):
+ if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):
+ continue
+ try:
+ info = await self._async_agfs.stat(candidate_path)
+ path = candidate_path
+ break
+ except Exception as exc:
+ if is_not_found_error(exc):
+ last_exc = exc
+ continue
+ mapped = map_exception(exc, resource=uri)
+ if mapped is not None:
+ raise mapped from exc
+ raise
+ else:
+ if last_exc is not None:
+ mapped = map_exception(last_exc, resource=uri)
+ if mapped is not None:
+ raise mapped from last_exc
+ raise NotFoundError(uri, "directory") from last_exc
+ if not info.get("isDir", info.get("is_dir")):
+ parent_path = path.rsplit("/", 1)[0] or "/"
+ parent_uri = self._path_to_uri(parent_path, ctx=ctx)
+ logger.info(
+ "content/overview: %s is a file, falling back to parent directory %s",
+ uri,
+ parent_uri,
+ )
+ return await self.overview(parent_uri, ctx=ctx)
+ file_path = f"{path}/.overview.md"
+ try:
+ content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))
+ except Exception as exc:
+ if not is_not_found_error(exc):
+ mapped = map_exception(exc, resource=uri)
+ if mapped is not None:
+ raise mapped from exc
+ raise
+ # Fallback to default if .overview.md doesn't exist
+ return f"# {uri}\n\n[Directory overview is not ready]"
+
+ return self._decode_bytes(content_bytes)
+
+ async def relations(
+ self,
+ uri: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> List[Dict[str, Any]]:
+ """Get relation list.
+
+ Returns: [{"uri": "...", "reason": "..."}, ...]
+ """
+ self._ensure_access(uri, ctx)
+ entries = await self.get_relation_table(uri, ctx=ctx)
+ result = []
+ for entry in entries:
+ for u in entry.uris:
+ if self._is_accessible(u, self._ctx_or_default(ctx)):
+ result.append({"uri": u, "reason": entry.reason})
+ return result
+
+ async def find(
+ self,
+ query: str,
+ target_uri: Union[str, List[str]] = "",
+ limit: int = 10,
+ score_threshold: Optional[float] = None,
+ filter: Optional[Dict] = None,
+ ctx: Optional[RequestContext] = None,
+ level: Optional[List[int]] = None,
+ image_url: Optional[str] = None,
+ ):
+ """Semantic search.
+
+ Args:
+ query: Search query
+ target_uri: Target directory URI(s), supports str or List[str]
+ limit: Return count
+ score_threshold: Score threshold
+ filter: Metadata filter
+
+ Returns:
+ FindResult
+ """
+ _ensure_non_empty_search_query(query, image_url)
+ telemetry = get_current_telemetry()
+ from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever
+ from openviking_cli.retrieve import (
+ ContextType,
+ FindResult,
+ TypedQuery,
+ )
+
+ real_ctx = self._ctx_or_default(ctx)
+ retrieval_targets = resolve_retrieval_targets(target_uri, real_ctx)
+
+ for target_dir in retrieval_targets.target_directories:
+ self._ensure_access(target_dir, ctx)
+
+ storage = self._get_vector_store()
+ if not storage:
+ raise RuntimeError("Vector store not initialized. Call OpenViking.initialize() first.")
+
+ embedder = self._get_embedder()
+ if not embedder:
+ raise RuntimeError("Embedder not configured.")
+
+ retriever = HierarchicalRetriever(
+ storage=storage,
+ embedder=embedder,
+ rerank_config=self.rerank_config,
+ retrieval_config=self.retrieval_config,
+ )
+
+ typed_query = TypedQuery(
+ query=query,
+ context_type=None,
+ intent="",
+ target_directories=retrieval_targets.target_directories,
+ embedding_input=(
+ build_multimodal_embedding_input(query, image_url) if image_url else None
+ ),
+ image_query=bool(image_url),
+ )
+
+ logger.debug(
+ "[VikingFS.find] Calling retriever.retrieve with "
+ f"ctx.account_id={real_ctx.account_id}, ctx.user={real_ctx.user}"
+ )
+
+ result = await retriever.retrieve(
+ typed_query,
+ ctx=real_ctx,
+ limit=limit,
+ score_threshold=score_threshold,
+ scope_dsl=filter,
+ level=level,
+ )
+
+ # Convert QueryResult to FindResult
+ memories, resources, skills = [], [], []
+ for ctx in result.matched_contexts:
+ if ctx.context_type == ContextType.MEMORY:
+ memories.append(ctx)
+ elif ctx.context_type == ContextType.RESOURCE:
+ resources.append(ctx)
+ elif ctx.context_type == ContextType.SKILL:
+ skills.append(ctx)
+
+ find_result = FindResult(
+ memories=memories,
+ resources=resources,
+ skills=skills,
+ )
+ telemetry.set("vector.returned", find_result.total)
+ return find_result
+
+ async def search(
+ self,
+ query: str,
+ target_uri: Union[str, List[str]] = "",
+ session_info: Optional[Dict] = None,
+ limit: int = 10,
+ score_threshold: Optional[float] = None,
+ filter: Optional[Dict] = None,
+ ctx: Optional[RequestContext] = None,
+ level: Optional[List[int]] = None,
+ image_url: Optional[str] = None,
+ ):
+ """Complex search with session context.
+
+ Args:
+ query: Search query
+ target_uri: Target directory URI(s), supports str or List[str]
+ session_info: Session information
+ limit: Return count
+ filter: Metadata filter
+
+ Returns:
+ FindResult
+ """
+ _ensure_non_empty_search_query(query, image_url)
+ telemetry = get_current_telemetry()
+ from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever
+ from openviking.retrieve.intent_analyzer import IntentAnalyzer
+ from openviking_cli.retrieve import (
+ ContextType,
+ FindResult,
+ QueryPlan,
+ TypedQuery,
+ )
+
+ real_ctx = self._ctx_or_default(ctx)
+ retrieval_targets = resolve_retrieval_targets(target_uri, real_ctx)
+ primary_target_uri = retrieval_targets.first_explicit_directory
+
+ session_summary = (
+ str(session_info.get("latest_archive_overview") or "") if session_info else ""
+ )
+ current_messages = session_info.get("current_messages") if session_info else None
+
+ query_plan: Optional[QueryPlan] = None
+ for target_dir in retrieval_targets.target_directories:
+ self._ensure_access(target_dir, ctx)
+
+ # When target_uri exists, read its abstract as optional query-planning context.
+ target_abstract = ""
+ if primary_target_uri:
+ try:
+ with telemetry.measure("search.target_abstract"):
+ target_abstract = await self.abstract(primary_target_uri, ctx=ctx)
+ except Exception:
+ target_abstract = ""
+
+ intent_enabled = (
+ bool(self.retrieval_config.enable_intent)
+ if self.retrieval_config is not None
+ else True
+ )
+
+ # With session context: optional intent analysis
+ if image_url:
+ typed_queries = [
+ TypedQuery(
+ query=query,
+ context_type=None,
+ intent="",
+ priority=1,
+ target_directories=retrieval_targets.target_directories,
+ embedding_input=build_multimodal_embedding_input(query, image_url),
+ image_query=True,
+ )
+ ]
+ elif intent_enabled and (session_summary or current_messages):
+ analyzer = IntentAnalyzer(max_recent_messages=5)
+ with telemetry.measure("search.intent_analysis"):
+ query_plan = await analyzer.analyze(
+ compression_summary=session_summary or "",
+ messages=current_messages or [],
+ current_message=query,
+ target_abstract=target_abstract,
+ )
+ typed_queries = query_plan.queries
+ for tq in typed_queries:
+ tq.target_directories = retrieval_targets.target_directories
+ else:
+ # No session context, or intent disabled: search with the raw query.
+ typed_queries = [
+ TypedQuery(
+ query=query,
+ context_type=None,
+ intent="",
+ priority=1,
+ target_directories=retrieval_targets.target_directories,
+ )
+ ]
+ telemetry.set("search.typed_queries_count", len(typed_queries))
+
+ # Concurrent execution
+ storage = self._get_vector_store()
+ embedder = self._get_embedder()
+ retriever = HierarchicalRetriever(
+ storage=storage,
+ embedder=embedder,
+ rerank_config=self.rerank_config,
+ retrieval_config=self.retrieval_config,
+ )
+
+ async def _execute(tq: TypedQuery):
+ real_ctx = self._ctx_or_default(ctx)
+ logger.debug(
+ "[VikingFS.search._execute] Calling retriever.retrieve with "
+ f"ctx.account_id={real_ctx.account_id}, ctx.user={real_ctx.user}"
+ )
+ return await retriever.retrieve(
+ tq,
+ ctx=real_ctx,
+ limit=limit,
+ score_threshold=score_threshold,
+ scope_dsl=filter,
+ level=level,
+ )
+
+ query_results = await asyncio.gather(*[_execute(tq) for tq in typed_queries])
+
+ # Aggregate results to FindResult
+ memories, resources, skills = [], [], []
+ for result in query_results:
+ for ctx in result.matched_contexts:
+ if ctx.context_type == ContextType.MEMORY:
+ memories.append(ctx)
+ elif ctx.context_type == ContextType.RESOURCE:
+ resources.append(ctx)
+ elif ctx.context_type == ContextType.SKILL:
+ skills.append(ctx)
+
+ find_result = FindResult(
+ memories=memories,
+ resources=resources,
+ skills=skills,
+ query_plan=query_plan,
+ query_results=query_results,
+ )
+ telemetry.set("vector.returned", find_result.total)
+ return find_result
+
+ # ========== Relation Management ==========
+
+ async def link(
+ self,
+ from_uri: str,
+ uris: Union[str, List[str]],
+ reason: str = "",
+ ctx: Optional[RequestContext] = None,
+ ) -> None:
+ """Create relation (maintained in .relations.json)."""
+ if isinstance(uris, str):
+ uris = [uris]
+ self._ensure_mutable_access(from_uri, ctx)
+ for uri in uris:
+ self._ensure_access(uri, ctx)
+
+ from_path = self._uri_to_path(from_uri, ctx=ctx)
+
+ entries = await self._read_relation_table(from_path, ctx=ctx)
+ existing_ids = {e.id for e in entries}
+
+ link_id = next(f"link_{i}" for i in range(1, 10000) if f"link_{i}" not in existing_ids)
+
+ entries.append(RelationEntry(id=link_id, uris=uris, reason=reason))
+
+ await self._write_relation_table(from_path, entries, ctx=ctx)
+ logger.debug(f"[VikingFS] Created link: {from_uri} -> {uris}")
+
+ async def unlink(
+ self,
+ from_uri: str,
+ uri: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> None:
+ """Delete relation."""
+ self._ensure_mutable_access(from_uri, ctx)
+ self._ensure_access(uri, ctx)
+ from_path = self._uri_to_path(from_uri, ctx=ctx)
+
+ try:
+ entries = await self._read_relation_table(from_path, ctx=ctx)
+
+ entry_to_modify = None
+ for entry in entries:
+ if uri in entry.uris:
+ entry_to_modify = entry
+ break
+
+ if not entry_to_modify:
+ logger.debug(f"[VikingFS] URI not found in relations: {uri}")
+ return
+
+ entry_to_modify.uris.remove(uri)
+
+ if not entry_to_modify.uris:
+ entries.remove(entry_to_modify)
+ logger.debug(f"[VikingFS] Removed empty entry: {entry_to_modify.id}")
+
+ await self._write_relation_table(from_path, entries, ctx=ctx)
+ logger.debug(f"[VikingFS] Removed link: {from_uri} -> {uri}")
+
+ except Exception as e:
+ logger.error(f"[VikingFS] Failed to unlink {from_uri} -> {uri}: {e}")
+ raise IOError(f"Failed to unlink: {e}")
+
+ async def get_relation_table(
+ self, uri: str, ctx: Optional[RequestContext] = None
+ ) -> List[RelationEntry]:
+ """Get relation table."""
+ self._ensure_access(uri, ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ return await self._read_relation_table(path, ctx=ctx)
+
+ # ========== Relation Table Internal Methods ==========
+
+ async def _read_relation_table(
+ self, dir_path: str, ctx: Optional[RequestContext] = None
+ ) -> List[RelationEntry]:
+ """Read .relations.json."""
+ table_path = f"{dir_path}/.relations.json"
+ try:
+ content = self._handle_agfs_read(await self._async_agfs.read(table_path))
+ data = json.loads(content.decode("utf-8"))
+ except FileNotFoundError:
+ return []
+ except Exception:
+ # logger.warning(f"[VikingFS] Failed to read relation table {table_path}: {e}")
+ return []
+
+ entries = []
+ # Compatible with old format (nested) and new format (flat)
+ if isinstance(data, list):
+ # New format: flat list
+ for entry_data in data:
+ entries.append(RelationEntry.from_dict(entry_data))
+ elif isinstance(data, dict):
+ # Old format: nested {namespace: {user: [entries]}}
+ for _namespace, user_dict in data.items():
+ for _user, entry_list in user_dict.items():
+ for entry_data in entry_list:
+ entries.append(RelationEntry.from_dict(entry_data))
+ return entries
+
+ async def _write_relation_table(
+ self, dir_path: str, entries: List[RelationEntry], ctx: Optional[RequestContext] = None
+ ) -> None:
+ """Write .relations.json."""
+ # Use flat list format
+ data = [entry.to_dict() for entry in entries]
+
+ content = json.dumps(data, ensure_ascii=False, indent=2)
+ table_path = f"{dir_path}/.relations.json"
+ if isinstance(content, str):
+ content = content.encode("utf-8")
+
+ await self._async_agfs.write(table_path, content)
+
+ async def get_relations(self, uri: str, ctx: Optional[RequestContext] = None) -> List[str]:
+ """Get all related URIs (backward compatible)."""
+ entries = await self.get_relation_table(uri, ctx=ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ all_uris = []
+ for entry in entries:
+ for related in entry.uris:
+ if self._is_accessible(related, real_ctx):
+ all_uris.append(related)
+ return all_uris
+
+ async def get_relations_with_content(
+ self,
+ uri: str,
+ include_l0: bool = True,
+ include_l1: bool = False,
+ ctx: Optional[RequestContext] = None,
+ ) -> List[Dict[str, Any]]:
+ """Get related URIs and their content (backward compatible)."""
+ relation_uris = await self.get_relations(uri, ctx=ctx)
+ if not relation_uris:
+ return []
+
+ results = []
+ abstracts = {}
+ overviews = {}
+ if include_l0:
+ abstracts = await self.read_batch(relation_uris, level="l0", ctx=ctx)
+ if include_l1:
+ overviews = await self.read_batch(relation_uris, level="l1", ctx=ctx)
+
+ for rel_uri in relation_uris:
+ info = {"uri": rel_uri}
+ if include_l0:
+ info["abstract"] = abstracts.get(rel_uri, "")
+ if include_l1:
+ info["overview"] = overviews.get(rel_uri, "")
+ results.append(info)
+
+ return results
+
+ async def write_context(
+ self,
+ uri: str,
+ content: Union[str, bytes] = "",
+ abstract: str = "",
+ overview: str = "",
+ content_filename: str = "content.md",
+ is_leaf: bool = False,
+ ctx: Optional[RequestContext] = None,
+ ) -> None:
+ """Write context to AGFS (L0/L1/L2)."""
+
+ self._ensure_mutable_access(uri, ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+
+ try:
+ await self._ensure_parent_dirs(path, ctx=ctx)
+ try:
+ await self._async_agfs.mkdir(path)
+ except Exception as e:
+ if "exist" not in str(e).lower():
+ raise
+
+ if content:
+ content_uri = f"{uri}/{content_filename}"
+ await self.write_file(content_uri, content, ctx=ctx)
+
+ if abstract:
+ abstract_uri = f"{uri}/.abstract.md"
+ await self.write_file(abstract_uri, abstract, ctx=ctx)
+
+ if overview:
+ overview_uri = f"{uri}/.overview.md"
+ await self.write_file(overview_uri, overview, ctx=ctx)
+
+ except Exception as e:
+ logger.error(f"[VikingFS] Failed to write {uri}: {e}")
+ raise IOError(f"Failed to write {uri}: {e}")
diff --git a/openviking/storage/viking_fs/_snapshot.py b/openviking/storage/viking_fs/_snapshot.py
new file mode 100644
index 000000000..938db3bad
--- /dev/null
+++ b/openviking/storage/viking_fs/_snapshot.py
@@ -0,0 +1,710 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Snapshot/git-like version control mixin for VikingFS."""
+
+import asyncio
+import sys
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+from openviking.core.context import ContextLevel
+from openviking.core.namespace import canonicalize_uri
+from openviking.pyagfs.exceptions import (
+ AGFSInvalidOperationError,
+ AGFSNotFoundError,
+ AGFSPathNotFoundError,
+ AGFSResourceExhaustedError,
+)
+from openviking.server.error_mapping import is_not_found_error, map_exception
+from openviking.server.identity import RequestContext
+from openviking.storage.viking_fs._base import (
+ _prepare_snapshot_diff,
+ logger,
+)
+from openviking_cli.exceptions import (
+ InvalidArgumentError,
+ NotFoundError,
+ ResourceExhaustedError,
+)
+
+
+def _pkg():
+ return sys.modules[__package__]
+
+if TYPE_CHECKING:
+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig
+
+
+class _SnapshotMixin:
+ """Snapshot/git-like version control (commit/restore/show/diff/log)."""
+
+ async def system_sync_status(
+ self, uri: str, ctx: Optional[RequestContext] = None
+ ) -> Dict[str, Any]:
+ """Return multi-write sync status for one Viking URI subtree."""
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ return await self._async_agfs.system_sync_status(
+ path,
+ fs_ctx={"account_id": real_ctx.account_id},
+ )
+
+ async def system_sync_retry(
+ self, uri: str, ctx: Optional[RequestContext] = None
+ ) -> Dict[str, Any]:
+ """Retry multi-write sync for one Viking URI subtree."""
+ self._ensure_mutable_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ return await self._async_agfs.system_sync_retry(
+ path,
+ fs_ctx={"account_id": real_ctx.account_id},
+ )
+
+ async def get_gitignore(self, ctx: Optional[RequestContext] = None) -> str:
+ """Return the account-level .ovgitignore content, or an empty string if absent.
+
+ Raises ``AGFSInvalidOperationError`` if the stored content is not valid
+ UTF-8, mirroring the Rust layer's reject-non-UTF-8 behavior at commit
+ time rather than leaking a raw ``UnicodeDecodeError``.
+ """
+ path = self._gitignore_agfs_path(ctx)
+ try:
+ raw = await self._async_agfs.read(path, 0, -1)
+ except Exception as exc:
+ if is_not_found_error(exc):
+ return ""
+ mapped = map_exception(exc, resource=path)
+ if mapped is not None:
+ raise mapped from exc
+ raise
+ if isinstance(raw, bytes):
+ data = raw
+ elif raw is not None and hasattr(raw, "content"):
+ data = raw.content
+ else:
+ data = b""
+ try:
+ return data.decode("utf-8")
+ except UnicodeDecodeError as exc:
+ raise AGFSInvalidOperationError(
+ f"invalid ignore file {path}: must be UTF-8: {exc}"
+ ) from exc
+
+ async def set_gitignore(
+ self,
+ content: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> None:
+ """Write the account-level .ovgitignore control file without semantic indexing.
+
+ Validates the size limit up front so a too-large file can never be
+ persisted only to make every later ``commit`` fail. Syntax (negation,
+ escaping) is still validated at commit time by the Rust layer.
+ """
+ if not isinstance(content, str):
+ raise TypeError("content must be a string")
+ path = self._gitignore_agfs_path(ctx)
+ data = content.encode("utf-8")
+ if len(data) > self._OVGITIGNORE_MAX_BYTES:
+ raise AGFSInvalidOperationError(
+ f"ignore file too large: {path} is {len(data)} bytes, "
+ f"limit {self._OVGITIGNORE_MAX_BYTES} bytes"
+ )
+ await self._ensure_parent_dirs(path, ctx=ctx)
+ await self._async_agfs.write(path, data)
+
+ async def delete_gitignore(self, ctx: Optional[RequestContext] = None) -> None:
+ """Delete the account-level .ovgitignore control file. Missing is success."""
+ path = self._gitignore_agfs_path(ctx)
+ try:
+ await self._async_agfs.rm(path, recursive=False)
+ except Exception as exc:
+ if is_not_found_error(exc):
+ return
+ mapped = map_exception(exc, resource=path)
+ if mapped is not None:
+ raise mapped from exc
+ raise
+
+ async def commit(
+ self,
+ *,
+ message: str,
+ paths: Optional[List[str]] = None,
+ branch: str = "main",
+ author_name: Optional[str] = None,
+ author_email: Optional[str] = None,
+ ctx: Optional[RequestContext] = None,
+ ) -> Dict[str, Any]:
+ """Create a git snapshot of the account's tree.
+
+ Args:
+ message: Commit message.
+ paths: Optional list of ``viking://`` URIs to scope the commit to;
+ entries may be files or directories. Directories are expanded
+ recursively with the snapshot pruning rules applied. ``None``
+ (default) enumerates the whole account tree. An empty list is
+ forwarded as an explicit empty path list (no-op commit). A
+ path that exists in neither the VFS nor the previous snapshot
+ logs a warning and is treated as a no-op deletion.
+ branch: Branch to advance. Defaults to ``"main"``.
+ author_name / author_email: Override the default bot author.
+ ctx: Request context (provides ``account_id``).
+
+ Returns:
+ Dict with ``result`` (``"created"`` / ``"noop"``) and ``commit_oid``.
+ When ``result == "created"`` it also includes ``changed`` (paths
+ added/updated/removed). Both variants include ``ignored``: the
+ number of candidate paths skipped by the account ``.ovgitignore``
+ rules (system pruning is not counted). Noop omits ``changed``.
+ """
+ real_ctx = self._ctx_or_default(ctx)
+ account = real_ctx.account_id
+ if paths is None:
+ tree_paths: Optional[List[str]] = None
+ else:
+ tree_paths = [self._uri_to_tree_path(p, ctx=real_ctx) for p in paths]
+ return await self._async_agfs.run(
+ "git_commit",
+ account=account,
+ branch=branch,
+ message=message,
+ paths=tree_paths,
+ author_name=author_name or self._DEFAULT_GIT_AUTHOR_NAME,
+ author_email=author_email or self._DEFAULT_GIT_AUTHOR_EMAIL,
+ )
+
+ async def restore(
+ self,
+ *,
+ project_dir: Optional[str] = None,
+ source_commit: str,
+ branch: str = "main",
+ dry_run: bool = False,
+ message: Optional[str] = None,
+ author_name: Optional[str] = None,
+ author_email: Optional[str] = None,
+ ctx: Optional[RequestContext] = None,
+ ) -> Dict[str, Any]:
+ """Restore a project subtree to the state at ``source_commit``.
+
+ Generates a new commit whose parent is the current HEAD (not the
+ source commit) and writes the diff through the VFS. Paths outside
+ ``project_dir`` are left untouched.
+
+ Args:
+ project_dir: ``viking://`` URI of the subtree, e.g.
+ ``"viking://resources/proj_a"``. May also be passed as a
+ short form like ``"resources/proj_a"``. Trailing slashes are
+ stripped. Internal scopes are rejected with ``ValueError``.
+ If None, restore the entire tree.
+ source_commit: 40-hex OID, branch name, or full ref path.
+ branch: Branch to advance. Defaults to ``"main"``.
+ dry_run: If True, returns the planned diff without writing.
+ message: Optional commit message; defaults to a generated string.
+ author_name / author_email: Override the default bot author.
+ ctx: Request context (provides ``account_id``).
+
+ Returns:
+ Dict containing ``result`` (``"applied"`` / ``"noop"`` / ``"dry_run"``)
+ and corresponding oid / diff fields. When an ``Applied`` result has
+ vector side-effects, a ``task_id`` is included for polling the
+ background reindex via ``GET /api/v1/tasks/{task_id}``.
+
+ After an ``Applied`` result, this method schedules background vector
+ maintenance for the affected paths via :class:`ReindexExecutor`:
+ directory markers (``.abstract.md`` / ``.overview.md``) recompute or
+ delete only that directory's L0/L1 vector; source files reindex (write)
+ or delete (removal) their DETAIL vector. ``.relations.json`` has no
+ vector side-effect. The rebuild is tracked as a single task
+ (``snapshot_restore_reindex``); per-path failures are logged and do not
+ block the return value. The task reaches ``completed`` only after each
+ affected vector has been written to (or deleted from) the index, so
+ polling ``task_id`` to ``completed`` guarantees subsequent ``find``
+ reads see the restored state.
+ """
+ real_ctx = self._ctx_or_default(ctx)
+ account = real_ctx.account_id
+ tree_dir: Optional[str]
+ if project_dir is None:
+ tree_dir = None
+ else:
+ tree_dir = self._uri_to_tree_path(project_dir, ctx=real_ctx).rstrip("/")
+ if not tree_dir:
+ raise ValueError(f"project_dir must not be empty: {project_dir!r}")
+ # Build kwargs dynamically, only include project_dir if it's not None
+ kwargs = {
+ "account": account,
+ "branch": branch,
+ "source_commit": source_commit,
+ "dry_run": dry_run,
+ "message": message,
+ "author_name": author_name or self._DEFAULT_GIT_AUTHOR_NAME,
+ "author_email": author_email or self._DEFAULT_GIT_AUTHOR_EMAIL,
+ }
+ if tree_dir is not None:
+ kwargs["project_dir"] = tree_dir
+ # dry_run only computes the diff; it never writes the VFS, so it needs
+ # no lock.
+ if dry_run:
+ return await self._async_agfs.run("git_restore", **kwargs)
+
+ from openviking.pyagfs.exceptions import GitRestoreWritebackPartialError
+ from openviking.storage.errors import LockAcquisitionError, ResourceBusyError
+
+ # Serialize the writeback against concurrent VFS mutations on the same
+ # subtree. A scoped restore tree-locks project_dir; a full restore
+ # (project_dir is None) locks the account root so only one restore runs
+ # per account at a time. The lock covers only the writeback — the
+ # background reindex is scheduled after release, otherwise its per-path
+ # child locks would conflict with this tree lock.
+ lock_path = (
+ self._uri_to_path(project_dir, ctx=real_ctx)
+ if project_dir is not None
+ else f"/local/{account}"
+ )
+ partial_exc: Optional[GitRestoreWritebackPartialError] = None
+ try:
+ lease = await self._async_agfs.pathlock_acquire_tree(lock_path)
+ except LockAcquisitionError:
+ raise ResourceBusyError(
+ f"Resource is being processed: {project_dir or '*'}",
+ uri=project_dir or "*",
+ )
+
+ try:
+ try:
+ result = await self._async_agfs.run("git_restore", **kwargs)
+ except GitRestoreWritebackPartialError as exc:
+ # The ref already advanced — capture the exception and
+ # finish the lock scope cleanly. Reindex scheduling and
+ # re-raise happen below, outside the tree lock.
+ partial_exc = exc
+ result = None
+ finally:
+ await self._async_agfs.pathlock_release(lease)
+
+ if partial_exc is not None:
+ # HEAD has moved forward but some VFS writes/deletes failed.
+ # Still schedule reindex for the paths that *did* reach the VFS so
+ # the vector index doesn't stay stale, then re-raise so the caller
+ # learns about the partial failure (and can inspect failed_writes
+ # / failed_deletes / task_id on the exception).
+ try:
+ partial_exc.task_id = await self._schedule_restore_reindex_for_paths(
+ written_paths=partial_exc.written_paths,
+ deleted_paths=partial_exc.deleted_paths,
+ project_dir=project_dir,
+ real_ctx=real_ctx,
+ )
+ except Exception:
+ logger.exception(
+ "[VikingFS] git restore partial: reindex scheduling failed; "
+ "HEAD advanced but reindex was not queued"
+ )
+ raise partial_exc
+
+ if result.get("result") != "applied":
+ return result
+
+ written = list(result.get("written_paths") or [])
+ deleted = list(result.get("deleted_paths") or [])
+ if written or deleted:
+ try:
+ task_id = await self._schedule_restore_reindex_for_paths(
+ written_paths=written,
+ deleted_paths=deleted,
+ project_dir=project_dir,
+ real_ctx=real_ctx,
+ )
+ if task_id is not None:
+ result["task_id"] = task_id
+ except Exception:
+ logger.exception(
+ "[VikingFS] git restore reindex task creation failed; "
+ "falling back to fire-and-forget rebuild"
+ )
+ self._schedule_vector_rebuild(written=written, deleted=deleted, ctx=real_ctx)
+ return result
+
+ async def _schedule_restore_reindex_for_paths(
+ self,
+ *,
+ written_paths: List[str],
+ deleted_paths: List[str],
+ project_dir: Optional[str],
+ real_ctx: RequestContext,
+ ) -> Optional[str]:
+ """Classify ``written``/``deleted`` paths into vector tasks and queue
+ them as a single tracked background rebuild. Returns the task id, or
+ ``None`` if there is nothing to do.
+
+ Shared by the applied path and the partial-writeback recovery path —
+ both schedule reindex for paths that actually reached the VFS.
+ """
+ if not written_paths and not deleted_paths:
+ return None
+ tasks = self._collect_restore_vector_tasks(written_paths, deleted_paths)
+ if not tasks:
+ return None
+
+ from openviking.service.task_tracker import get_task_tracker
+
+ tracker = get_task_tracker()
+ task = await tracker.create(
+ "snapshot_restore_reindex",
+ resource_id=project_dir or "*",
+ account_id=real_ctx.account_id,
+ user_id=real_ctx.user.user_id,
+ )
+ await tracker.update_stage(
+ task.task_id,
+ "queued",
+ account_id=real_ctx.account_id,
+ user_id=real_ctx.user.user_id,
+ )
+ background = asyncio.create_task(
+ self._run_restore_rebuild_tracked(task.task_id, tasks, real_ctx),
+ name=f"vikingfs-git-restore-reindex:{task.task_id}",
+ )
+ self._background_tasks.add(background)
+ background.add_done_callback(self._background_tasks.discard)
+ return task.task_id
+
+ async def show(
+ self,
+ target_ref: str,
+ *,
+ path: Optional[str] = None,
+ max_blob_bytes: Optional[int] = None,
+ ctx: Optional[RequestContext] = None,
+ ) -> Union[Dict[str, Any], bytes]:
+ """Read a commit's metadata or a single blob.
+
+ ``path=None`` returns the commit metadata dict (oid, tree, parents,
+ author, committer, message). ``path=str`` returns the blob bytes
+ directly, stripping the oid/size envelope returned by the binding.
+ """
+ real_ctx = self._ctx_or_default(ctx)
+ account = real_ctx.account_id
+ tree_path = self._uri_to_tree_path(path, ctx=real_ctx) if path else None
+ kwargs: Dict[str, Any] = {
+ "account": account,
+ "target_ref": target_ref,
+ "path": tree_path,
+ }
+ if max_blob_bytes is not None:
+ kwargs["max_blob_bytes"] = max_blob_bytes
+ resp = await self._async_agfs.run("git_show", **kwargs)
+ if path is not None and isinstance(resp, dict) and "bytes" in resp:
+ return resp["bytes"]
+ return resp
+
+ async def show_blob_raw(
+ self,
+ target_ref: str,
+ *,
+ path: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> Dict[str, Any]:
+ """Like ``show(target_ref, path=...)`` but returns the full envelope.
+
+ Returns ``{"oid": str, "size": int, "bytes": bytes}`` without
+ stripping. Used by the HTTP snapshot router to populate
+ ``X-Snapshot-Oid`` / ``X-Snapshot-Size`` response headers.
+ """
+ real_ctx = self._ctx_or_default(ctx)
+ account = real_ctx.account_id
+ tree_path = self._uri_to_tree_path(path, ctx=real_ctx)
+ resp = await self._async_agfs.run(
+ "git_show",
+ account=account,
+ target_ref=target_ref,
+ path=tree_path,
+ )
+ if not isinstance(resp, dict) or "bytes" not in resp:
+ raise TypeError(
+ f"git_show returned unexpected shape for blob path: {type(resp).__name__}"
+ )
+ return resp
+
+ async def diff(
+ self,
+ *,
+ path: str,
+ from_ref: Optional[str],
+ to_ref: str,
+ raw: bool = True,
+ ctx: Optional[RequestContext] = None,
+ ) -> Dict[str, Any]:
+ """Return a unified text diff for one path between two snapshots."""
+ real_ctx = self._ctx_or_default(ctx)
+ path = canonicalize_uri(path, ctx=real_ctx)
+ self._ensure_access(path, real_ctx)
+
+ from_meta: Optional[Dict[str, Any]] = None
+ if from_ref:
+ try:
+ from_meta = await self.show(from_ref, ctx=real_ctx)
+ except AGFSNotFoundError as exc:
+ raise NotFoundError(from_ref, "git_ref") from exc
+ try:
+ to_meta = await self.show(to_ref, ctx=real_ctx)
+ except AGFSNotFoundError as exc:
+ raise NotFoundError(to_ref, "git_ref") from exc
+
+ from_oid = str(from_meta.get("oid", from_ref)) if isinstance(from_meta, dict) else ""
+ to_oid = str(to_meta.get("oid", to_ref)) if isinstance(to_meta, dict) else to_ref
+
+ async def read_optional(ref: str) -> Optional[bytes]:
+ try:
+ value = await self.show(
+ ref,
+ path=path,
+ max_blob_bytes=_pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES,
+ ctx=real_ctx,
+ )
+ except AGFSPathNotFoundError:
+ return None
+ except AGFSResourceExhaustedError as exc:
+ raise ResourceExhaustedError(
+ f"snapshot diff file size limit exceeded "
+ f"({_pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES} bytes)",
+ details={"limit_bytes": _pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES, "path": path},
+ ) from exc
+ if not isinstance(value, bytes):
+ raise TypeError(f"git_show returned unexpected blob type: {type(value).__name__}")
+ return value
+
+ before_bytes = await read_optional(from_oid) if from_ref else None
+ after_bytes = await read_optional(to_oid)
+ if before_bytes is None and after_bytes is None:
+ raise NotFoundError(path, "git_blob")
+
+ for value in (before_bytes, after_bytes):
+ if value is not None and len(value) > _pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES:
+ raise ResourceExhaustedError(
+ f"snapshot diff file size limit exceeded ({_pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES} bytes)",
+ details={"limit_bytes": _pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES, "path": path},
+ )
+
+ if not raw:
+ from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils
+
+ def visible_content(value: Optional[bytes]) -> Optional[bytes]:
+ if value is None:
+ return None
+ try:
+ text = value.decode("utf-8")
+ except UnicodeDecodeError as exc:
+ raise InvalidArgumentError("raw=false requires UTF-8 snapshot content") from exc
+ return MemoryFileUtils.read(text, uri=path).content.encode("utf-8")
+
+ before_bytes = visible_content(before_bytes)
+ after_bytes = visible_content(after_bytes)
+
+ change_type, before, after = await asyncio.to_thread(
+ _prepare_snapshot_diff,
+ path=path,
+ before_bytes=before_bytes,
+ after_bytes=after_bytes,
+ max_lines=_pkg().SNAPSHOT_DIFF_MAX_LINES,
+ )
+ try:
+ diff_text = await self._async_agfs.run(
+ "git_diff_text",
+ before=before,
+ after=after,
+ fromfile=f"{path}@{from_oid}",
+ tofile=f"{path}@{to_oid}",
+ timeout_ms=_pkg().SNAPSHOT_DIFF_TIMEOUT_MS,
+ max_output_bytes=_pkg().SNAPSHOT_DIFF_MAX_OUTPUT_BYTES,
+ )
+ except AGFSResourceExhaustedError as exc:
+ raise ResourceExhaustedError(
+ f"snapshot diff output size limit exceeded "
+ f"({_pkg().SNAPSHOT_DIFF_MAX_OUTPUT_BYTES} bytes)",
+ details={"limit_bytes": _pkg().SNAPSHOT_DIFF_MAX_OUTPUT_BYTES, "path": path},
+ ) from exc
+ return {
+ "path": path,
+ "from_commit": from_oid,
+ "to_commit": to_oid,
+ "change_type": change_type,
+ "diff_text": diff_text,
+ }
+
+ async def log(
+ self,
+ *,
+ branch: str = "main",
+ limit: int = 20,
+ paths: Optional[List[str]] = None,
+ ctx: Optional[RequestContext] = None,
+ ) -> List[Dict[str, Any]]:
+ """Walk back from ``branch``'s HEAD along ``parents[0]`` up to ``limit`` commits.
+
+ Returns a list of commit metadata dicts (same shape as ``show(target_ref)``).
+ """
+ if limit <= 0:
+ return []
+ real_ctx = self._ctx_or_default(ctx)
+ account = real_ctx.account_id
+ tree_paths = (
+ None if not paths else [self._uri_to_tree_path(path, ctx=real_ctx) for path in paths]
+ )
+ return await self._async_agfs.run(
+ "git_log",
+ account=account,
+ branch=branch,
+ limit=limit,
+ paths=tree_paths,
+ )
+
+ def _collect_restore_vector_tasks(
+ self,
+ written: List[str],
+ deleted: List[str],
+ ) -> set[tuple]:
+ """Classify restore-affected paths into deduplicated ``(op, uri, level)`` tasks.
+
+ Tasks are deduplicated on the exact ``(op, uri, level)`` key (no ancestor
+ subsumption — each change is handled independently because no operation
+ recurses into descendants).
+ """
+ tasks: set[tuple] = set()
+ for tree_path in written:
+ try:
+ task = self._classify_restore_path(tree_path, deleted=False)
+ except ValueError:
+ continue
+ if task is not None:
+ tasks.add(task)
+ for tree_path in deleted:
+ try:
+ task = self._classify_restore_path(tree_path, deleted=True)
+ except ValueError:
+ continue
+ if task is not None:
+ tasks.add(task)
+ return tasks
+
+ def _schedule_vector_rebuild(
+ self,
+ *,
+ written: List[str],
+ deleted: List[str],
+ ctx: RequestContext,
+ ) -> None:
+ """Fire-and-forget precise vector maintenance for a git restore.
+
+ Each affected path is classified by :py:meth:`_classify_restore_path`
+ into a ``(op, uri, level)`` task and scheduled independently:
+
+ - ``reindex_marker`` — recompute only a directory's L0/L1 vector.
+ - ``reindex_file`` — recompute only a source file's DETAIL vector
+ (non-recursive ``execute(mode="vectors_only")``).
+ - ``delete`` — remove only the ``(uri, level)`` vector (directory
+ marker removal, or deleted source file).
+
+ Failures are logged and never propagate.
+ """
+ try:
+ from openviking.service.reindex_executor import get_reindex_executor
+ except Exception:
+ logger.exception("[VikingFS] ReindexExecutor import failed; skipping rebuild")
+ return
+
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ logger.warning("[VikingFS] git restore vector rebuild skipped: no running event loop")
+ return
+
+ tasks = self._collect_restore_vector_tasks(written, deleted)
+ if not tasks:
+ return
+
+ executor = get_reindex_executor()
+ for op, uri, level in tasks:
+ loop.create_task(
+ self._run_vector_rebuild(executor, op, uri, level, ctx),
+ name=f"vikingfs-git-{op}:{uri}:{int(level)}",
+ )
+
+ async def _run_restore_rebuild_tracked(
+ self,
+ task_id: str,
+ tasks: set[tuple],
+ ctx: RequestContext,
+ ) -> None:
+ """Background worker driving a tracked restore vector rebuild.
+
+ Runs all classified ``(op, uri, level)`` rebuild tasks concurrently and
+ drives the task through start → complete/fail. Per-task failures are
+ swallowed (and logged) inside :py:meth:`_run_vector_rebuild`, preserving
+ the "failures do not block" semantics.
+ """
+ from openviking.service.task_tracker import get_task_tracker
+ from openviking.service.task_work_index import bind_task_context
+
+ tracker = get_task_tracker()
+ tracker.register_running_task(task_id)
+ try:
+ await tracker.start(
+ task_id,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ stage="reindexing",
+ )
+ from openviking.service.reindex_executor import get_reindex_executor
+
+ executor = get_reindex_executor()
+ with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):
+ await asyncio.gather(
+ *[
+ self._run_vector_rebuild(executor, op, uri, level, ctx)
+ for (op, uri, level) in tasks
+ ]
+ )
+ await tracker.complete(
+ task_id,
+ {"status": "completed", "task_count": len(tasks)},
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ except asyncio.CancelledError:
+ # TaskWorkIndex finalizes after this active task and its queue work settle.
+ return
+ except Exception as exc:
+ await tracker.fail(
+ task_id,
+ str(exc),
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ finally:
+ await tracker.unregister_running_task(task_id)
+
+ async def _run_vector_rebuild(
+ self,
+ executor: Any,
+ op: str,
+ uri: str,
+ level: Any,
+ ctx: RequestContext,
+ ) -> None:
+ """Wrapper coroutine: dispatch one vector task and swallow errors."""
+ try:
+ if op == "reindex_marker":
+ await executor.reindex_directory_marker(dir_uri=uri, level=level, ctx=ctx)
+ elif op == "reindex_file":
+ await executor.execute(uri=uri, mode="vectors_only", wait=True, ctx=ctx)
+ elif op == "delete":
+ await executor.delete_uri_level(uri=uri, level=level, ctx=ctx)
+ else: # pragma: no cover - defensive
+ logger.warning("[VikingFS] unknown vector rebuild op %r for %s", op, uri)
+ except Exception:
+ logger.exception("[VikingFS] git restore vector task %s failed for %s", op, uri)
diff --git a/openviking/storage/viking_fs/_sync.py b/openviking/storage/viking_fs/_sync.py
new file mode 100644
index 000000000..a4eb7a935
--- /dev/null
+++ b/openviking/storage/viking_fs/_sync.py
@@ -0,0 +1,297 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Tree sync (diff + mv/rm merge) primitives for VikingFS."""
+
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple
+
+from openviking.storage.viking_fs._base import LS_ALL_NODES
+from openviking_cli.utils.uri import VikingURI
+
+if TYPE_CHECKING:
+ from openviking.server.identity import RequestContext
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class SyncDiff:
+ """Directory diff result for sync operations."""
+
+ added_files: List[str] = field(default_factory=list)
+ deleted_files: List[str] = field(default_factory=list)
+ updated_files: List[str] = field(default_factory=list)
+ added_dirs: List[str] = field(default_factory=list)
+ deleted_dirs: List[str] = field(default_factory=list)
+
+ def to_changes(self) -> Dict[str, List[str]]:
+ return {
+ "added": self.added_files + self.added_dirs,
+ "modified": self.updated_files,
+ "deleted": self.deleted_files + self.deleted_dirs,
+ }
+
+
+class _SyncMixin:
+ """Top-down recursive tree diff-and-merge primitive.
+
+ Moves (not copies) visible children of ``root_uri`` into ``target_uri``
+ so the target ends up mirroring the source, returning a :class:`SyncDiff`
+ describing what changed. Hidden files (``.``-prefixed) are skipped —
+ callers that need to carry over parser sidecars (e.g.
+ ``.image_mappings.json``) do so explicitly after the sync.
+ """
+
+ async def sync_tree(
+ self,
+ root_uri: str,
+ target_uri: str,
+ *,
+ ctx: Optional["RequestContext"] = None,
+ file_change_status: Optional[Dict[str, bool]] = None,
+ lease_ref: Optional[Dict[str, Any]] = None,
+ delete_temp_after: bool = False,
+ is_changed: Optional[Callable[[str, str, "RequestContext"], "bool"]] = None,
+ ) -> SyncDiff:
+ """Recursively merge ``root_uri`` into ``target_uri``.
+
+ Hidden entries (names starting with ``.``) are skipped on both sides.
+ Name conflicts (file vs. directory at the same name) are resolved by
+ deleting the target-side entry before moving the source in place.
+
+ Args:
+ root_uri: Source (typically a temp/staging tree). Must exist.
+ target_uri: Destination. Created (incl. parents) if missing.
+ ctx: Request context for ACL / tenancy checks.
+ file_change_status: Optional pre-computed map of source-URI →
+ ``True`` (changed) / ``False`` (unchanged). Used as a
+ performance hint to skip content comparison for files the
+ caller already knows about.
+ lease_ref: Optional pathlock lease passed through to every
+ ``mv``/``rm``/``mkdir`` so the whole sync is covered by one
+ lock.
+ delete_temp_after: If ``True``, call ``delete_temp(root_uri)``
+ after a successful sync into an existing target (the
+ whole-tree mv path already consumes the source).
+ is_changed: Optional async predicate ``(root_file, target_file,
+ ctx) -> bool`` used to decide if an existing file needs to
+ be replaced. Defaults to a stat-size shortcut followed by
+ byte-for-byte content comparison.
+
+ Returns:
+ A :class:`SyncDiff` describing the applied changes (URIs are
+ the *target* paths, suitable for feeding into incremental
+ reindex pipelines).
+ """
+ if not await self.exists(root_uri, ctx=ctx):
+ raise FileNotFoundError(
+ f"Sync source no longer exists; refusing to sync into {target_uri}: {root_uri}"
+ )
+
+ diff = SyncDiff()
+
+ async def list_children(dir_uri: str) -> Tuple[Dict[str, str], Dict[str, str]]:
+ files: Dict[str, str] = {}
+ dirs: Dict[str, str] = {}
+ entries = await self.ls(
+ dir_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx
+ )
+ for entry in entries:
+ name = entry.get("name", "")
+ if not name or name in [".", ".."]:
+ continue
+ if name.startswith("."):
+ continue
+ item_uri = VikingURI(dir_uri).join(name).uri
+ if entry.get("isDir", False):
+ dirs[name] = item_uri
+ else:
+ files[name] = item_uri
+ return files, dirs
+
+ async def default_is_changed(root_file: str, target_file: str) -> bool:
+ try:
+ current_stat = await self.stat(root_file, ctx=ctx)
+ target_stat = await self.stat(target_file, ctx=ctx)
+ current_size = current_stat.get("size") if isinstance(current_stat, dict) else None
+ target_size = target_stat.get("size") if isinstance(target_stat, dict) else None
+ if current_size is not None and target_size is not None and current_size != target_size:
+ return True
+ current_content = await self.read_file(root_file, ctx=ctx)
+ target_content = await self.read_file(target_file, ctx=ctx)
+ return current_content != target_content
+ except Exception:
+ return True
+
+ check_changed = is_changed if is_changed is not None else default_is_changed
+
+ async def sync_dir(root_dir: str, target_dir: str) -> None:
+ root_files, root_dirs = await list_children(root_dir)
+ target_files, target_dirs = await list_children(target_dir)
+
+ file_names = set(root_files.keys()) | set(target_files.keys())
+ for name in sorted(file_names):
+ root_file = root_files.get(name)
+ target_file = target_files.get(name)
+
+ if root_file and name in target_dirs:
+ target_conflict_dir = target_dirs[name]
+ try:
+ await self.rm(
+ target_conflict_dir,
+ recursive=True,
+ ctx=ctx,
+ lease_ref=lease_ref,
+ )
+ diff.deleted_dirs.append(target_conflict_dir)
+ target_dirs.pop(name, None)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to delete directory for file conflict: {target_conflict_dir}, error={e}"
+ )
+ target_file = None
+
+ if target_file and name in root_dirs and not root_file:
+ try:
+ await self.rm(target_file, ctx=ctx, lease_ref=lease_ref)
+ diff.deleted_files.append(target_file)
+ target_files.pop(name, None)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to delete file for dir conflict: {target_file}, error={e}"
+ )
+ continue
+
+ if target_file and not root_file:
+ try:
+ await self.rm(target_file, ctx=ctx, lease_ref=lease_ref)
+ diff.deleted_files.append(target_file)
+ except Exception as e:
+ logger.error(f"[SyncDiff] Failed to delete file: {target_file}, error={e}")
+ continue
+
+ if root_file and target_file:
+ changed = False
+ if file_change_status and root_file in file_change_status:
+ changed = file_change_status[root_file]
+ else:
+ try:
+ changed = await check_changed(root_file, target_file)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to compare file content for {root_file}: {e}, treating as unchanged"
+ )
+ changed = False
+ if changed:
+ diff.updated_files.append(target_file)
+ try:
+ await self.rm(target_file, ctx=ctx, lease_ref=lease_ref)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to remove old file before update: {target_file}, error={e}"
+ )
+ try:
+ await self.mv(
+ root_file,
+ target_file,
+ ctx=ctx,
+ lease_ref=lease_ref,
+ )
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to move updated file: {root_file} -> {target_file}, error={e}"
+ )
+ continue
+
+ if root_file and not target_file:
+ target_file_uri = VikingURI(target_dir).join(name).uri
+ diff.added_files.append(target_file_uri)
+ try:
+ await self.mv(
+ root_file,
+ target_file_uri,
+ ctx=ctx,
+ lease_ref=lease_ref,
+ )
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to move added file: {root_file} -> {target_file_uri}, error={e}"
+ )
+
+ dir_names = set(root_dirs.keys()) | set(target_dirs.keys())
+ for name in sorted(dir_names):
+ root_subdir = root_dirs.get(name)
+ target_subdir = target_dirs.get(name)
+
+ if root_subdir and name in target_files:
+ target_conflict_file = target_files[name]
+ try:
+ await self.rm(
+ target_conflict_file,
+ ctx=ctx,
+ lease_ref=lease_ref,
+ )
+ diff.deleted_files.append(target_conflict_file)
+ target_files.pop(name, None)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to delete file for dir conflict: {target_conflict_file}, error={e}"
+ )
+ target_subdir = None
+
+ if target_subdir and not root_subdir:
+ try:
+ await self.rm(
+ target_subdir,
+ recursive=True,
+ ctx=ctx,
+ lease_ref=lease_ref,
+ )
+ diff.deleted_dirs.append(target_subdir)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to delete directory: {target_subdir}, error={e}"
+ )
+ continue
+
+ if root_subdir and not target_subdir:
+ target_subdir_uri = VikingURI(target_dir).join(name).uri
+ diff.added_dirs.append(target_subdir_uri)
+ try:
+ await self.mv(
+ root_subdir,
+ target_subdir_uri,
+ ctx=ctx,
+ lease_ref=lease_ref,
+ )
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to move added directory: {root_subdir} -> {target_subdir_uri}, error={e}"
+ )
+ continue
+
+ if root_subdir and target_subdir:
+ await sync_dir(root_subdir, target_subdir)
+
+ target_exists = await self.exists(target_uri, ctx=ctx)
+ if not target_exists:
+ parent_uri = VikingURI(target_uri).parent
+ if parent_uri:
+ await self.mkdir(
+ parent_uri.uri,
+ exist_ok=True,
+ ctx=ctx,
+ lease_ref=lease_ref,
+ )
+ diff.added_dirs.append(target_uri)
+ await self.mv(root_uri, target_uri, ctx=ctx, lease_ref=lease_ref)
+ return diff
+
+ await sync_dir(root_uri, target_uri)
+ if delete_temp_after:
+ try:
+ await self.delete_temp(root_uri, ctx=ctx)
+ except Exception as e:
+ logger.error(f"[SyncDiff] Failed to delete temp root {root_uri}: {e}")
+ return diff
diff --git a/openviking/storage/viking_fs/_vector.py b/openviking/storage/viking_fs/_vector.py
new file mode 100644
index 000000000..e4176da5f
--- /dev/null
+++ b/openviking/storage/viking_fs/_vector.py
@@ -0,0 +1,72 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Vector store integration mixin for VikingFS."""
+
+from typing import TYPE_CHECKING, Any, List, Optional
+
+from openviking.server.identity import RequestContext
+from openviking.storage.viking_fs._base import logger
+
+if TYPE_CHECKING:
+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
+
+
+class _VectorMixin:
+ """Vector store integration: delete/update URIs, get store/embedder."""
+
+ async def _delete_from_vector_store(
+ self, uris: List[str], ctx: Optional[RequestContext] = None
+ ) -> None:
+ """Delete records with specified URIs from vector store.
+
+ Uses tenant-safe URI deletion semantics from vector store.
+ """
+ vector_store = self._get_vector_store()
+ if not vector_store:
+ return
+ real_ctx = self._ctx_or_default(ctx)
+
+ try:
+ await vector_store.delete_uris(real_ctx, uris)
+ for uri in uris:
+ logger.debug(f"[VikingFS] Deleted from vector store: {uri}")
+ except Exception as e:
+ logger.warning(f"[VikingFS] Failed to delete from vector store: {e}")
+ raise
+
+ async def _update_vector_store_uris(
+ self,
+ uris: List[str],
+ old_base: str,
+ new_base: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> None:
+ """Update URIs in vector store (when moving files).
+
+ Preserves vector data and updates URI-derived identifiers without regenerating embeddings.
+ """
+ vector_store = self._get_vector_store()
+ if not vector_store:
+ return
+
+ real_ctx = self._ctx_or_default(ctx)
+
+ for uri in uris:
+ try:
+ new_uri = new_base + uri[len(old_base) :]
+ if await vector_store.update_uri_mapping(
+ ctx=real_ctx,
+ uri=uri,
+ new_uri=new_uri,
+ ):
+ logger.debug(f"[VikingFS] Updated URI: {uri} -> {new_uri}")
+ except Exception as e:
+ logger.warning(f"[VikingFS] Failed to update {uri} in vector store: {e}")
+
+ def _get_vector_store(self) -> Optional["VikingVectorIndexBackend"]:
+ """Get vector store instance."""
+ return self.vector_store
+
+ def _get_embedder(self) -> Any:
+ """Get embedder instance."""
+ return self.query_embedder
diff --git a/tests/storage/test_ingest_ls_node_limit.py b/tests/storage/test_ingest_ls_node_limit.py
index 08cbb637f..c688b4ad7 100644
--- a/tests/storage/test_ingest_ls_node_limit.py
+++ b/tests/storage/test_ingest_ls_node_limit.py
@@ -2,11 +2,11 @@
at ``viking_fs.ls``'s default ``node_limit=1000``.
When a resource namespace already exists, re-ingesting a directory takes the
-incremental sync path (``_sync_topdown_recursive`` -> ``sync_dir`` ->
-``list_children``), which lists the temp root's children via ``viking_fs.ls``.
-``ls`` defaults ``node_limit=1000``, so an import of >1000 docs silently
-materialized only the first 1000 subdirectories into the target; the rest were
-dropped when temp was deleted.
+incremental sync path (``VikingFS.sync_tree`` -> ``sync_dir`` -> ``list_children``),
+which lists the temp root's children via ``viking_fs.ls`` with ``node_limit=LS_ALL_NODES``.
+``ls`` defaults ``node_limit=1000``, so an import of >1000 docs would silently
+materialize only the first 1000 subdirectories into the target if sync_tree
+forgot to pass the unlimited constant.
Observed in the wild: a 6,221-doc WixQA re-ingest (the namespace already held an
earlier 209-doc import, so the atomic-move fast path was skipped) produced
@@ -18,6 +18,7 @@ from __future__ import annotations
import pytest
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
+from openviking.storage.viking_fs import LS_ALL_NODES
class _TruncatingVikingFS:
@@ -34,13 +35,21 @@ class _TruncatingVikingFS:
def __init__(self, n_children: int):
self._children = [{"name": f"doc{i:05d}", "isDir": True} for i in range(n_children)]
self.moved: list[tuple[str, str]] = []
+ self.ls_node_limits: list[int] = []
async def exists(self, uri, ctx=None):
return uri in (self.TEMP, self.TARGET)
async def ls(self, uri, show_all_hidden=False, node_limit=1000, ctx=None):
+ self.ls_node_limits.append(node_limit)
entries = self._children if uri == self.TEMP else []
- return entries[:node_limit] # the real ls truncates here
+ return entries[:node_limit] if node_limit is not None else entries
+
+ async def read_file(self, uri, ctx=None):
+ return ""
+
+ async def stat(self, uri, ctx=None):
+ return {"size": 0}
async def mv(self, src, dst, ctx=None, lease_ref=None):
self.moved.append((src, dst))
@@ -54,6 +63,31 @@ class _TruncatingVikingFS:
async def delete_temp(self, uri, ctx=None, lease_ref=None):
pass
+ async def sync_tree(
+ self, root_uri, target_uri, *, ctx=None, file_change_status=None,
+ lease_ref=None, delete_temp_after=False, is_changed=None,
+ ):
+ from openviking.storage.viking_fs import SyncDiff
+ from openviking_cli.utils.uri import VikingURI
+
+ diff = SyncDiff()
+ # Mirror _SyncMixin.list_children: must pass node_limit=LS_ALL_NODES.
+ entries = await self.ls(
+ root_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx
+ )
+ for entry in entries:
+ name = entry.get("name", "")
+ if not name or name in (".", "..") or name.startswith("."):
+ continue
+ src = VikingURI(root_uri).join(name).uri
+ dst = VikingURI(target_uri).join(name).uri
+ if entry.get("isDir", False):
+ diff.added_dirs.append(dst)
+ else:
+ diff.added_files.append(dst)
+ await self.mv(src, dst, ctx=ctx, lease_ref=lease_ref)
+ return diff
+
@pytest.mark.asyncio
async def test_sync_materializes_all_children_above_default_node_limit(monkeypatch):
@@ -63,6 +97,10 @@ async def test_sync_materializes_all_children_above_default_node_limit(monkeypat
"openviking.storage.queuefs.semantic_processor.get_viking_fs",
lambda: fake,
)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.rewrite_image_uris",
+ lambda *a, **kw: None,
+ )
diff = await SemanticProcessor()._sync_topdown_recursive(
_TruncatingVikingFS.TEMP,
@@ -75,6 +113,9 @@ async def test_sync_materializes_all_children_above_default_node_limit(monkeypat
"sync was truncated at ls node_limit"
)
assert len(diff.added_dirs) == n_children
+ assert all(
+ lim == LS_ALL_NODES for lim in fake.ls_node_limits
+ ), f"sync_tree must pass LS_ALL_NODES to ls, got node_limits={fake.ls_node_limits}"
class _TruncatingLsFS:
diff --git a/tests/storage/test_semantic_processor_target_preexisting.py b/tests/storage/test_semantic_processor_target_preexisting.py
index f0b70bed4..aa913d25b 100644
--- a/tests/storage/test_semantic_processor_target_preexisting.py
+++ b/tests/storage/test_semantic_processor_target_preexisting.py
@@ -4,7 +4,8 @@ from unittest.mock import AsyncMock
import pytest
from openviking.storage.queuefs.semantic_msg import SemanticMsg
-from openviking.storage.queuefs.semantic_processor import DiffResult, SemanticProcessor
+from openviking.storage.queuefs.semantic_processor import SemanticProcessor
+from openviking.storage.viking_fs import SyncDiff
class _FakeVikingFS:
@@ -12,58 +13,52 @@ class _FakeVikingFS:
return True
-class _SyncVikingFS:
- def __init__(self):
- self.contents = {
- "viking://temp/import/a.md": "new",
- "viking://temp/import/b.md": "same",
- "viking://resources/root/a.md": "old",
- "viking://resources/root/b.md": "same",
- "viking://resources/root/.overview.md": "FILES:\n- b.md: old summary",
- "viking://resources/root/.abstract.md": "old abstract",
- }
- self.entries = {
- "viking://temp/import": [
- {"name": "a.md", "isDir": False},
- {"name": "b.md", "isDir": False},
- ],
- "viking://resources/root": [
- {"name": "a.md", "isDir": False},
- {"name": "b.md", "isDir": False},
- {"name": ".overview.md", "isDir": False},
- {"name": ".abstract.md", "isDir": False},
- ],
- }
+class _SyncWrapperVikingFS:
+ """Fake FS that verifies the wrapper calls sync_tree and post-processes correctly."""
+
+ def __init__(self, target_exists=True):
+ self.target_exists = target_exists
+ self.sync_tree_calls = []
self.deleted_temp = []
self.mutation_leases = []
+ self.sidecar_rewrite_calls = []
async def exists(self, uri, ctx=None):
- return uri in self.entries
+ if uri.startswith("viking://resources/root"):
+ return self.target_exists
+ return True
- async def ls(self, uri, show_all_hidden=False, node_limit=None, ctx=None):
- return self.entries.get(uri, [])
+ async def sync_tree(
+ self,
+ root_uri,
+ target_uri,
+ *,
+ ctx=None,
+ file_change_status=None,
+ lease_ref=None,
+ delete_temp_after=False,
+ ):
+ self.sync_tree_calls.append(
+ (root_uri, target_uri, file_change_status, lease_ref, delete_temp_after)
+ )
+ self.mutation_leases.append(("sync_tree", root_uri, target_uri, lease_ref))
+ return SyncDiff(updated_files=["viking://resources/root/a.md"])
- async def stat(self, uri, ctx=None):
- return {"size": len(self.contents.get(uri, ""))}
+ async def delete_temp(self, uri, ctx=None):
+ self.mutation_leases.append(("delete_temp", uri, None))
+ self.deleted_temp.append(uri)
async def read_file(self, uri, ctx=None):
- return self.contents.get(uri, "")
+ return "{}"
- async def rm(self, uri, recursive=False, ctx=None, lease_ref=None):
- self.mutation_leases.append(("rm", uri, lease_ref))
- self.contents.pop(uri, None)
+ async def stat(self, uri, ctx=None):
+ raise FileNotFoundError(uri)
- async def mv(self, src, dst, ctx=None, lease_ref=None):
- self.mutation_leases.append(("mv", dst, lease_ref))
- self.contents[dst] = self.contents.pop(src)
+ async def write_file(self, uri, content, ctx=None, lease_ref=None):
+ self.mutation_leases.append(("write_file", uri, lease_ref))
- async def mkdir(self, uri, exist_ok=False, ctx=None, lease_ref=None):
- self.mutation_leases.append(("mkdir", uri, lease_ref))
- self.entries.setdefault(uri, [])
-
- async def delete_temp(self, uri, ctx=None, lease_ref=None):
- self.mutation_leases.append(("delete_temp", uri, lease_ref))
- self.deleted_temp.append(uri)
+ async def glob(self, pattern, uri=None, ctx=None):
+ return {"matches": []}
class _FakeDagExecutor:
@@ -105,7 +100,7 @@ async def test_target_source_syncs_before_semantic_dag(monkeypatch):
processor = SemanticProcessor()
processor._enqueue_parent_refresh = AsyncMock()
processor._sync_topdown_recursive = AsyncMock(
- return_value=DiffResult(
+ return_value=SyncDiff(
updated_files=["viking://resources/org/repo/a.md"],
)
)
@@ -129,8 +124,9 @@ async def test_target_source_syncs_before_semantic_dag(monkeypatch):
@pytest.mark.asyncio
-async def test_sync_diff_reports_target_uris_and_preserves_sidecars(monkeypatch):
- fake_fs = _SyncVikingFS()
+async def test_sync_wrapper_delegates_to_sync_tree_and_cleans_temp(monkeypatch):
+ """The wrapper calls viking_fs.sync_tree and then deletes the temp tree."""
+ fake_fs = _SyncWrapperVikingFS(target_exists=True)
lease = {
"lease_ref": "outer-tree-ref",
"owner_id": "outer-tree-owner",
@@ -140,6 +136,10 @@ async def test_sync_diff_reports_target_uris_and_preserves_sidecars(monkeypatch)
"openviking.storage.queuefs.semantic_processor.get_viking_fs",
lambda: fake_fs,
)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.rewrite_image_uris",
+ AsyncMock(),
+ )
diff = await SemanticProcessor()._sync_topdown_recursive(
"viking://temp/import",
@@ -147,24 +147,41 @@ async def test_sync_diff_reports_target_uris_and_preserves_sidecars(monkeypatch)
lock=lease,
)
+ assert len(fake_fs.sync_tree_calls) == 1
+ root_uri, target_uri, fcs, lr, dta = fake_fs.sync_tree_calls[0]
+ assert root_uri == "viking://temp/import"
+ assert target_uri == "viking://resources/root"
+ assert lr is lease
+ assert dta is False
assert diff.to_changes() == {
"added": [],
"modified": ["viking://resources/root/a.md"],
"deleted": [],
}
- assert fake_fs.contents["viking://resources/root/a.md"] == "new"
- assert fake_fs.contents["viking://resources/root/.overview.md"] == (
- "FILES:\n- b.md: old summary"
- )
- assert fake_fs.contents["viking://resources/root/.abstract.md"] == "old abstract"
assert fake_fs.deleted_temp == ["viking://temp/import"]
- assert fake_fs.mutation_leases
- assert all(
- mutation_lease is lease
- for operation, _, mutation_lease in fake_fs.mutation_leases
- if operation != "delete_temp"
+
+
+async def test_sync_wrapper_whole_tree_mv_for_new_target(monkeypatch):
+ """When the target does not exist, sync_tree does a full mv and the wrapper does not delete (source is gone)."""
+ fake_fs = _SyncWrapperVikingFS(target_exists=False)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.get_viking_fs",
+ lambda: fake_fs,
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.rewrite_image_uris",
+ AsyncMock(),
)
- assert fake_fs.mutation_leases[-1] == ("delete_temp", "viking://temp/import", None)
+
+ diff = await SemanticProcessor()._sync_topdown_recursive(
+ "viking://temp/import",
+ "viking://resources/root",
+ lock=None,
+ )
+
+ assert len(fake_fs.sync_tree_calls) == 1
+ assert fake_fs.deleted_temp == [] # whole-tree mv already consumed the source
@pytest.mark.asyncio
@@ -183,5 +200,5 @@ async def test_sync_missing_source_never_touches_target(monkeypatch):
lock=None,
)
- fake_fs.ls.assert_not_awaited()
+ fake_fs.sync_tree.assert_not_awaited()
fake_fs.rm.assert_not_awaited()