Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
refactor: vikingfs.py (#3947)
MaojiaShengb84395d6dd6d
diff --git a/openviking/server/auth/__init__.py b/openviking/server/auth/__init__.pyindex 85d1af631..9d0e74518 100644--- a/openviking/server/auth/__init__.py+++ b/openviking/server/auth/__init__.py@@ -303,9 +303,8 @@ def require_auth_role(*allowed_roles: Role):"Admin API authentication failed: unable to resolve request context.")- plugin = _get_plugin(request)manager = getattr(request.app.state, "api_key_manager", None)- if manager is None and plugin.requires_api_key_manager():+ if manager is None:raise PermissionDeniedError(_DEV_MODE_ADMIN_API_MESSAGE)if ctx.role not in allowed_roles:@@ -345,11 +344,9 @@ def get_api_key_manager_or_raise(request: Request):"""Get APIKeyManager from app state or raise appropriate error.Raises:- PermissionDeniedError: When the current auth plugin requires an- APIKeyManager but none is available.+ PermissionDeniedError: When no APIKeyManager is available."""manager = getattr(request.app.state, "api_key_manager", None)- plugin = _get_plugin(request)- if manager is None and plugin.requires_api_key_manager():+ if manager is None:raise PermissionDeniedError(_DEV_MODE_ADMIN_API_MESSAGE)return managerdiff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex c73d6158c..dfc04ceb7 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -6,7 +6,6 @@ import asyncioimport reimport threadingfrom contextlib import nullcontext-from dataclasses import dataclass, fieldfrom typing import Any, Dict, List, Optional, Set, Tuplefrom openviking.observability.context import (@@ -40,7 +39,7 @@ from openviking.storage.queuefs.semantic_lock import SemanticLockScopefrom openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_coalesce_keyfrom openviking.storage.queuefs.semantic_queue import is_semantic_msg_stalefrom openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecars-from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs+from openviking.storage.viking_fs import LS_ALL_NODES, SyncDiff, get_viking_fsfrom openviking.telemetry import bind_telemetry, bind_telemetry_stage, resolve_telemetryfrom openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.telemetry.span_models import create_root_span_attributes@@ -59,24 +58,6 @@ from openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)-@dataclass-class DiffResult:- """Directory diff result for sync operations."""-- added_files: List[str] = field(default_factory=list)- deleted_files: List[str] = field(default_factory=list)- updated_files: List[str] = field(default_factory=list)- added_dirs: List[str] = field(default_factory=list)- deleted_dirs: List[str] = field(default_factory=list)-- def to_changes(self) -> Dict[str, List[str]]:- return {- "added": self.added_files + self.added_dirs,- "modified": self.updated_files,- "deleted": self.deleted_files + self.deleted_dirs,- }--class RequestQueueStats:processed: int = 0requeue_count: int = 0@@ -209,24 +190,6 @@ class SemanticProcessor(DequeueHandlerBase):# Default to otherreturn FILE_TYPE_OTHER- async def _check_file_content_changed(- self, file_path: str, target_file: str, ctx: Optional[RequestContext] = None- ) -> bool:- """Check if file content has changed compared to target file."""- viking_fs = get_viking_fs()- try:- current_stat = await viking_fs.stat(file_path, ctx=ctx)- target_stat = await viking_fs.stat(target_file, ctx=ctx)- current_size = current_stat.get("size") if isinstance(current_stat, dict) else None- target_size = target_stat.get("size") if isinstance(target_stat, dict) else None- if current_size is not None and target_size is not None and current_size != target_size:- return True- current_content = await viking_fs.read_file(file_path, ctx=ctx)- target_content = await viking_fs.read_file(target_file, ctx=ctx)- return current_content != target_content- except Exception:- return True-async def _reenqueue_semantic_msg(self, msg: SemanticMsg) -> None:"""Re-enqueue a semantic message for later processing.@@ -744,203 +707,39 @@ class SemanticProcessor(DequeueHandlerBase):ctx: Optional[RequestContext] = None,file_change_status: Optional[Dict[str, bool]] = None,lock: Optional[Dict[str, Any]] = None,- ) -> DiffResult:+ ) -> SyncDiff:+ """Merge a temp/staging source tree into the target via VikingFS.sync_tree.++ The pure filesystem diff-and-move is delegated to+ :py:meth:`VikingFS.sync_tree`; this wrapper carries over parser+ sidecars (``.image_mappings.json``) and rewrites markdown image+ references once the visible files have been moved into place.+ Hidden entries are skipped by ``sync_tree``, which is why the+ sidecar handling lives here rather than in the FS layer.+ """viking_fs = get_viking_fs()if not await viking_fs.exists(root_uri, ctx=ctx):raise FileNotFoundError(f"Semantic source no longer exists; refusing to sync into {target_uri}: {root_uri}")- diff = DiffResult()-- async def list_children(dir_uri: str) -> Tuple[Dict[str, str], Dict[str, str]]:- files: Dict[str, str] = {}- dirs: Dict[str, str] = {}- entries = await viking_fs.ls(- dir_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx- )-- for entry in entries:- name = entry.get("name", "")- if not name or name in [".", ".."]:- continue- if name.startswith("."):- continue- item_uri = VikingURI(dir_uri).join(name).uri- if entry.get("isDir", False):- dirs[name] = item_uri- else:- files[name] = item_uri- return files, dirs-- async def sync_dir(root_dir: str, target_dir: str) -> None:- root_files, root_dirs = await list_children(root_dir)- target_files, target_dirs = await list_children(target_dir)-- file_names = set(root_files.keys()) | set(target_files.keys())- for name in sorted(file_names):- root_file = root_files.get(name)- target_file = target_files.get(name)-- if root_file and name in target_dirs:- target_conflict_dir = target_dirs[name]- try:- await viking_fs.rm(- target_conflict_dir,- recursive=True,- ctx=ctx,- lease_ref=lock,- )- diff.deleted_dirs.append(target_conflict_dir)- target_dirs.pop(name, None)- except Exception as e:- logger.error(- f"[SyncDiff] Failed to delete directory for file conflict: {target_conflict_dir}, error={e}"- )- target_file = None-- if target_file and name in root_dirs and not root_file:- try:- await viking_fs.rm(target_file, ctx=ctx, lease_ref=lock)- diff.deleted_files.append(target_file)- target_files.pop(name, None)- except Exception as e:- logger.error(- f"[SyncDiff] Failed to delete file for dir conflict: {target_file}, error={e}"- )- continue-- if target_file and not root_file:- try:- await viking_fs.rm(target_file, ctx=ctx, lease_ref=lock)- diff.deleted_files.append(target_file)- except Exception as e:- logger.error(f"[SyncDiff] Failed to delete file: {target_file}, error={e}")- continue-- if root_file and target_file:- changed = False- if file_change_status and root_file in file_change_status:- changed = file_change_status[root_file]- else:- try:- changed = await self._check_file_content_changed(- root_file, target_file, ctx=ctx- )- except Exception as e:- logger.error(- f"[SyncDiff] Failed to compare file content for {root_file}: {e}, treating as unchanged"- )- changed = False- if changed:- diff.updated_files.append(target_file)- try:- await viking_fs.rm(target_file, ctx=ctx, lease_ref=lock)- except Exception as e:- logger.error(- f"[SyncDiff] Failed to remove old file before update: {target_file}, error={e}"- )- try:- await viking_fs.mv(- root_file,- target_file,- ctx=ctx,- lease_ref=lock,- )- except Exception as e:- logger.error(- f"[SyncDiff] Failed to move updated file: {root_file} -> {target_file}, error={e}"- )- continue-- if root_file and not target_file:- target_file_uri = VikingURI(target_dir).join(name).uri- diff.added_files.append(target_file_uri)- try:- await viking_fs.mv(- root_file,- target_file_uri,- ctx=ctx,- lease_ref=lock,- )- except Exception as e:- logger.error(- f"[SyncDiff] Failed to move added file: {root_file} -> {target_file_uri}, error={e}"- )-- dir_names = set(root_dirs.keys()) | set(target_dirs.keys())- for name in sorted(dir_names):- root_subdir = root_dirs.get(name)- target_subdir = target_dirs.get(name)-- if root_subdir and name in target_files:- target_conflict_file = target_files[name]- try:- await viking_fs.rm(- target_conflict_file,- ctx=ctx,- lease_ref=lock,- )- diff.deleted_files.append(target_conflict_file)- target_files.pop(name, None)- except Exception as e:- logger.error(- f"[SyncDiff] Failed to delete file for dir conflict: {target_conflict_file}, error={e}"- )- target_subdir = None-- if target_subdir and not root_subdir:- try:- await viking_fs.rm(- target_subdir,- recursive=True,- ctx=ctx,- lease_ref=lock,- )- diff.deleted_dirs.append(target_subdir)- except Exception as e:- logger.error(- f"[SyncDiff] Failed to delete directory: {target_subdir}, error={e}"- )- continue-- if root_subdir and not target_subdir:- target_subdir_uri = VikingURI(target_dir).join(name).uri- diff.added_dirs.append(target_subdir_uri)- try:- await viking_fs.mv(- root_subdir,- target_subdir_uri,- ctx=ctx,- lease_ref=lock,- )- except Exception as e:- logger.error(- f"[SyncDiff] Failed to move added directory: {root_subdir} -> {target_subdir_uri}, error={e}"- )- continue-- if root_subdir and target_subdir:- await sync_dir(root_subdir, target_subdir)target_exists = await viking_fs.exists(target_uri, ctx=ctx)+ diff = await viking_fs.sync_tree(+ root_uri,+ target_uri,+ ctx=ctx,+ file_change_status=file_change_status,+ lease_ref=lock,+ delete_temp_after=False,+ )+if not target_exists:- parent_uri = VikingURI(target_uri).parent- if parent_uri:- await viking_fs.mkdir(- parent_uri.uri,- exist_ok=True,- ctx=ctx,- lease_ref=lock,- )- diff.added_dirs.append(target_uri)- await viking_fs.mv(root_uri, target_uri, ctx=ctx, lease_ref=lock)# The whole temp tree (including the hidden .image_mappings.json# sidecar) was moved into the target; rewrite local image paths now.await self._rewrite_target_image_uris(root_uri, target_uri, ctx=ctx, lock=lock)return diff- await sync_dir(root_uri, target_uri)- # sync_dir skips hidden files, so the .image_mappings.json sidecar is+ # sync_tree skips hidden files, so the .image_mappings.json sidecar is# still at the temp root. Carry it over and rewrite the synced markdown# before the temp tree is deleted below.await self._rewrite_target_image_uris(root_uri, target_uri, ctx=ctx, lock=lock)diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pydeleted file mode 100644index 59c6347dc..000000000--- a/openviking/storage/viking_fs.py+++ /dev/null@@ -1,4513 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: AGPL-3.0-"""-VikingFS: OpenViking file system abstraction layer--Encapsulates the AGFS binding client, providing file operation interface based on Viking URI.-Responsibilities:-- URI conversion (viking:// <-> /local/)-- L0/L1 reading (.abstract.md, .overview.md)-- Relation management (.relations.json)-- Semantic search (vector retrieval + rerank)-- Vector sync (sync vector store on rm/mv)-"""--import asyncio-import contextvars-import hashlib-import json-import os-import re-import time-from contextlib import contextmanager-from dataclasses import dataclass, field-from datetime import datetime, timezone-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, TypeVar, Union--from openviking.core.context import ContextLevel-from openviking.core.namespace import (- canonicalize_uri,- is_hidden_by_actor_peer_view,- may_include_hidden_actor_peers,-)-from openviking.core.namespace import (- is_accessible as namespace_is_accessible,-)-from openviking.core.retrieval_targets import resolve_retrieval_targets-from openviking.pyagfs import AsyncAGFSClient-from openviking.pyagfs.exceptions import (- AGFSClientError,- AGFSDirectoryNotEmptyError,- AGFSHTTPError,- AGFSInvalidOperationError,- AGFSNotFoundError,- AGFSNotSupportedError,- AGFSPathNotFoundError,- AGFSResourceExhaustedError,-)-from openviking.resource.watch_storage import is_watch_task_control_uri-from openviking.server.error_mapping import is_not_found_error, map_exception-from openviking.server.identity import RequestContext, Role-from openviking.storage.expr import And, PathScope, RawDSL-from openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMES-from openviking.telemetry import get_current_telemetry-from openviking.utils.image_search import build_multimodal_embedding_input-from openviking.utils.time_utils import format_iso8601, get_current_timestamp, parse_iso_datetime-from openviking_cli.exceptions import (- FailedPreconditionError,- InvalidArgumentError,- NotFoundError,- PermissionDeniedError,- ResourceExhaustedError,-)-from openviking_cli.session.user_id import UserIdentifier-from openviking_cli.utils.config.grep_config import GrepEngine-from openviking_cli.utils.logger import get_logger-from openviking_cli.utils.uri import VikingURI--if TYPE_CHECKING:- from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend- from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig--logger = get_logger(__name__)--# Sentinel node_limit for internal callers that MUST enumerate an entire-# directory. ``ls()`` defaults to ``node_limit=1000`` to protect agent-facing-# context from being flooded, but internal system operations (parse merge,-# temp->final sync, summary DAG, vectorization) must see every child or they-# silently drop entries beyond the cap — e.g. a >1000-doc directory ingest only-# materializes its first 1000 subdirectories. Pass this explicitly at those-# call sites.-LS_ALL_NODES = 2**31 - 1-SNAPSHOT_DIFF_MAX_FILE_BYTES = 10 * 1024 * 1024-SNAPSHOT_DIFF_MAX_OUTPUT_BYTES = 20 * 1024 * 1024-SNAPSHOT_DIFF_MAX_LINES = 100_000-SNAPSHOT_DIFF_TIMEOUT_MS = 500-_T = TypeVar("_T")---def _snapshot_line_count(text: str) -> int:- if not text:- return 0- line_breaks = (- "\n",- "\r",- "\v",- "\f",- "\x1c",- "\x1d",- "\x1e",- "\x85",- "\u2028",- "\u2029",- )- count = sum(text.count(separator) for separator in line_breaks)- count -= text.count("\r\n")- return count if text.endswith(line_breaks) else count + 1---def _prepare_snapshot_diff(- *,- path: str,- before_bytes: Optional[bytes],- after_bytes: Optional[bytes],- max_lines: int,-) -> tuple[str, str, str]:- try:- before = before_bytes.decode("utf-8") if before_bytes is not None else ""- after = after_bytes.decode("utf-8") if after_bytes is not None else ""- except UnicodeDecodeError as exc:- raise InvalidArgumentError("snapshot diff only supports UTF-8 text files") from exc-- for text in (before, after):- line_count = _snapshot_line_count(text)- if line_count > max_lines:- raise ResourceExhaustedError(- f"snapshot diff line count limit exceeded ({max_lines} lines per file)",- details={"limit_lines": max_lines, "path": path},- )-- if before_bytes is None:- change_type = "added"- elif after_bytes is None:- change_type = "deleted"- elif before_bytes == after_bytes:- change_type = "unchanged"- else:- change_type = "modified"-- return change_type, before, after---def _ensure_non_empty_search_query(query: str, image_url: Optional[str] = None) -> None:- if not query.strip() and not image_url:- raise InvalidArgumentError("Search query or image_url must not be empty.")---def _is_directory_not_empty_error(message: str) -> bool:- """Check if an error message indicates a directory not empty error.-- Handles multiple possible error message formats from different backends.- """- msg = message.lower()- return any(- pattern in msg- for pattern in [- "directory not empty",- "dir not empty",- "directory is not empty",- ]- )---def _get_cpu_count() -> int:- """Return the number of CPUs available to this process.-- Tries process_cpu_count (Python 3.13+, cgroup-aware),- falls back to sched_getaffinity (Linux),- then os.cpu_count (may report host CPUs in containers).- """- if hasattr(os, "process_cpu_count"):- return os.process_cpu_count() or 1- try:- return len(os.sched_getaffinity(0))- except (AttributeError, NotImplementedError):- return os.cpu_count() or 1---def _get_abstract_worker_count() -> int:- default = max(4, min(12, min(32, _get_cpu_count() + 4) // 2))- env_val = os.getenv("OPENVIKING_FILE_OPS_CONCURRENCY")- if env_val is not None:- try:- return max(1, int(env_val))- except ValueError:- pass- return max(1, default)---_ABSTRACT_WORKER_COUNT = _get_abstract_worker_count()-_DEFAULT_GREP_FILE_CONCURRENCY = 32---# ========== Dataclass ==========---@dataclass-class RelationEntry:- """Relation table entry."""-- id: str- uris: List[str]- reason: str = ""- created_at: str = field(default_factory=get_current_timestamp)-- def to_dict(self) -> Dict[str, Any]:- return {- "id": self.id,- "uris": self.uris,- "reason": self.reason,- "created_at": self.created_at,- }-- @staticmethod- def from_dict(data: Dict[str, Any]) -> "RelationEntry":- return RelationEntry(**data)---# ========== Singleton Pattern ==========--_instance: Optional["VikingFS"] = None---def init_viking_fs(- agfs: Any,- query_embedder: Optional[Any] = None,- rerank_config: Optional["RerankConfig"] = None,- vector_store: Optional["VikingVectorIndexBackend"] = None,- retrieval_config: Optional["RetrievalConfig"] = None,- grep_config: Optional["GrepConfig"] = None,- timeout: int = 10,- enable_recorder: bool = False,- encryptor: Optional[Any] = None,-) -> "VikingFS":- """Initialize VikingFS singleton.-- Args:- agfs: Pre-initialized AGFS client (HTTP or Binding)- query_embedder: Embedder instance- rerank_config: Rerank configuration- retrieval_config: Retrieval ranking configuration- grep_config: Grep engine configuration- vector_store: Vector store instance- enable_recorder: Whether to enable IO recording- encryptor: FileEncryptor instance for encryption/decryption- """- global _instance-- _instance = VikingFS(- agfs=agfs,- query_embedder=query_embedder,- rerank_config=rerank_config,- vector_store=vector_store,- retrieval_config=retrieval_config,- grep_config=grep_config,- encryptor=encryptor,- )-- if enable_recorder:- _enable_viking_fs_recorder(_instance)-- return _instance---def _enable_viking_fs_recorder(viking_fs: "VikingFS") -> None:- """- Enable recorder for a VikingFS instance.-- This wraps the VikingFS instance with recording capabilities.- Called automatically when enable_recorder=True in init_viking_fs.-- Args:- viking_fs: VikingFS instance to enable recording for- """- from openviking.eval.recorder import RecordingVikingFS, get_recorder-- recorder = get_recorder()- if not recorder.enabled:- from openviking.eval.recorder import init_recorder-- init_recorder(enabled=True)-- global _instance- _instance = RecordingVikingFS(viking_fs)- logger.info("[VikingFS] IO Recorder enabled")---def enable_viking_fs_recorder() -> None:- """- Enable recorder for the global VikingFS singleton.-- This function wraps the existing VikingFS's AGFS client with recording.- Must be called after init_viking_fs().- """- global _instance- if _instance is None:- raise RuntimeError("VikingFS not initialized. Call init_viking_fs() first.")- _enable_viking_fs_recorder(_instance)---def get_viking_fs() -> "VikingFS":- """Get VikingFS singleton."""- if _instance is None:- raise RuntimeError("VikingFS not initialized. Call init_viking_fs() first.")- return _instance---# ========== VikingFS Main Class ==========---class VikingFS:- """RAGFS-based OpenViking file system.-- APIs are divided into two categories:- - RAGFS basic commands (direct forwarding): read, ls, write, mkdir, rm, mv, grep, stat- - VikingFS specific capabilities: abstract, overview, find, search, relations, link, unlink-- Uses Rust binding mode: Use RAGFSBindingClient to directly use RAGFS implementation- """-- def set_user_deletion_guard(self, guard: Optional[Callable[[str, str], bool]]) -> None:- self._user_deletion_guard = guard-- def __init__(- self,- agfs: Any,- query_embedder: Optional[Any] = None,- rerank_config: Optional["RerankConfig"] = None,- vector_store: Optional["VikingVectorIndexBackend"] = None,- retrieval_config: Optional["RetrievalConfig"] = None,- grep_config: Optional["GrepConfig"] = None,- timeout: int = 10,- encryptor: Optional[Any] = None,- ):- self.agfs = agfs- self._async_agfs = AsyncAGFSClient(agfs)- self.query_embedder = query_embedder- self.rerank_config = rerank_config- self.vector_store = vector_store- self.retrieval_config = retrieval_config- self.grep_config = grep_config- self._encryptor = encryptor- self._count_cache: Dict[str, tuple] = {} # cache_key → (count, timestamp)- self._count_cache_max_size = 1024- self._fulltext_available: Optional[bool] = None # cached result of _collection_has_fulltext- self._bound_ctx: contextvars.ContextVar[Optional[RequestContext]] = contextvars.ContextVar(- "vikingfs_bound_ctx", default=None- )- self._background_tasks: set = set()- self._user_deletion_guard: Optional[Callable[[str, str], bool]] = None-- @staticmethod- def _default_ctx() -> RequestContext:- return RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)-- def _ctx_or_default(self, ctx: Optional[RequestContext]) -> RequestContext:- if ctx is not None:- return ctx- bound = self._bound_ctx.get()- return bound or self._default_ctx()-- @contextmanager- def bind_request_context(self, ctx: RequestContext):- """Temporarily bind ctx for legacy internal call paths without explicit ctx param."""- token = self._bound_ctx.set(ctx)- try:- yield- finally:- self._bound_ctx.reset(token)-- @staticmethod- def _normalize_uri(uri: str) -> str:- """Normalize short-format URIs to the canonical viking:// form."""- if uri.startswith("viking://"):- return uri- return VikingURI.normalize(uri)-- @classmethod- def _normalized_uri_parts(cls, uri: str) -> tuple[str, List[str]]:- """Normalize a URI and reject ambiguous or platform-specific path traversal forms."""- normalized = cls._normalize_uri(uri)- parts = [p for p in normalized[len("viking://") :].strip("/").split("/") if p]-- for part in parts:- if part in {".", ".."}:- raise PermissionDeniedError(- f"Unsafe URI traversal segment '{part}' in {normalized}",- resource=normalized,- )- if "\\" in part:- raise PermissionDeniedError(- f"Unsafe URI path separator '\\\\' in component '{part}' of {normalized}",- resource=normalized,- )- if len(part) >= 2 and part[1] == ":" and part[0].isalpha():- raise PermissionDeniedError(- f"Unsafe URI drive-prefixed component '{part}' in {normalized}",- resource=normalized,- )-- return normalized, parts-- # TODO: Once pathlock moves down into ragfs, stop reconstructing the- # encrypted mount-relative path in Python and derive the lock target from- # the same backend-side source of truth.- def _encrypted_mount_relative_path(self, path: str) -> tuple[str, str]:- """Return the mount prefix and mount-relative path used by Rust encrypted writes."""- normalized = path if path.startswith("/") else f"/{path}"- parts = [part for part in normalized.split("/") if part]- if len(parts) < 2:- return "", normalized- return f"/{parts[0]}", f"/{'/'.join(parts[1:])}"-- def _encrypted_temp_path(self, path: str) -> str:- """Build the deterministic internal `.encrypt` temp-file path for a final file path."""- mount_prefix, relative_path = self._encrypted_mount_relative_path(path)- relative_parts = [part for part in relative_path.split("/") if part]- if len(relative_parts) >= 2 and relative_parts[0] == "local":- temp_root = f"/local/{relative_parts[1]}/temp/.encrypt_stage"- elif len(relative_parts) >= 2:- temp_root = f"/{relative_parts[0]}/temp/.encrypt_stage"- else:- temp_root = "/temp/.encrypt_stage"- digest = hashlib.sha256(relative_path.encode("utf-8")).hexdigest()- return f"{mount_prefix}{temp_root}/{digest}.encrypt"-- def _ensure_access(self, uri: str, ctx: Optional[RequestContext]) -> None:- real_ctx = self._ctx_or_default(ctx)- normalized_uri, _ = self._normalized_uri_parts(uri)- if not self._is_accessible(normalized_uri, real_ctx):- raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)-- def _ensure_mutable_access(self, uri: str, ctx: Optional[RequestContext]) -> None:- self._ensure_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- self._ensure_user_not_deleting(real_ctx)- normalized_uri, _ = self._normalized_uri_parts(uri)- if is_hidden_by_actor_peer_view(normalized_uri, real_ctx) or may_include_hidden_actor_peers(- normalized_uri, real_ctx- ):- raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)- self._ensure_supported_write_namespace(normalized_uri)- if real_ctx.role != Role.ROOT and normalized_uri.rstrip("/") == "viking://temp":- raise PermissionDeniedError(- "Temp root is read-only for non-root users",- resource=normalized_uri,- )-- def _ensure_delete_access(self, uri: str, ctx: Optional[RequestContext]) -> None:- self._ensure_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- self._ensure_user_not_deleting(real_ctx)- normalized_uri, _ = self._normalized_uri_parts(uri)- if is_hidden_by_actor_peer_view(normalized_uri, real_ctx) or may_include_hidden_actor_peers(- normalized_uri, real_ctx- ):- raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)- self._ensure_supported_delete_namespace(normalized_uri)- if real_ctx.role != Role.ROOT and normalized_uri.rstrip("/") == "viking://temp":- raise PermissionDeniedError(- "Temp root is read-only for non-root users",- resource=normalized_uri,- )-- def _ensure_user_not_deleting(self, ctx: RequestContext) -> None:- if (- ctx.role != Role.ROOT- and self._user_deletion_guard is not None- and self._user_deletion_guard(ctx.account_id, ctx.user.user_id)- ):- raise FailedPreconditionError("User deletion is in progress")-- def _ensure_supported_delete_namespace(self, normalized_uri: str) -> None:- parts = [p for p in normalized_uri[len("viking://") :].strip("/").split("/") if p]- if not parts:- raise PermissionDeniedError(- "Deleting viking:// is not supported; use a concrete scope instead.",- resource=normalized_uri,- )- if parts == ["user"]:- raise PermissionDeniedError(- "Deleting viking://user is not supported; use an explicit user namespace "- "or current-user content path instead.",- resource=normalized_uri,- )- if parts == ["agent"]:- # Parity with _ensure_supported_write_namespace, which forbids the- # account-shared viking://agent root: deleting it would recursively- # wipe every account's agent skills/endpoints/tools/payments. Concrete- # sub-paths (viking://agent/skills/...) remain deletable.- raise PermissionDeniedError(- "Deleting viking://agent root is not supported; use a concrete "- "agent sub-path (e.g. viking://agent/skills/...) instead.",- resource=normalized_uri,- )-- def _ensure_supported_write_namespace(self, normalized_uri: str) -> None:- parts = [p for p in normalized_uri[len("viking://") :].strip("/").split("/") if p]- if parts == ["user"]:- raise PermissionDeniedError(- "Writing viking://user is not supported; use an explicit user namespace "- "or current-user content path instead.",- resource=normalized_uri,- )- if parts and parts[0] == "session":- raise PermissionDeniedError(- f"Writing {normalized_uri} is not supported; use user-owned namespaces instead.",- resource=normalized_uri,- )- if parts and parts[0] == "agent":- if len(parts) >= 2 and parts[1] not in {"skills", "endpoints", "tools", "payments"}:- raise PermissionDeniedError(- "viking://agent/{agent_id} is deprecated. Use viking://user/.../peers/{agent_id} instead.",- resource=normalized_uri,- )- if len(parts) < 2:- raise PermissionDeniedError(- "Writing to viking://agent root is not supported.",- resource=normalized_uri,- )-- # ========== AGFS Basic Commands ==========-- async def read(- self,- uri: str,- offset: int = 0,- size: int = -1,- ctx: Optional[RequestContext] = None,- ) -> bytes:- """Read file"""- self._ensure_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- primary_path = self._uri_to_path(uri, ctx=ctx)-- # Decryption + offset/size slicing now happen inside the ragfs encryption layer- # (when configured); the plaintext stack reads bytes directly. Either way, pass the- # offset/size through and let the Rust layer return the requested slice.- last_not_found: Optional[Exception] = None- for path in self._read_paths(uri, ctx=ctx):- if not await self._read_path_visible(uri, path, primary_path, real_ctx):- continue- try:- result = await self._async_agfs.read(path, offset, size)- break- except Exception as exc:- if is_not_found_error(exc):- last_not_found = exc- continue- raise- else:- raise NotFoundError(uri, "file") from last_not_found- if isinstance(result, bytes):- raw = result- elif result is not None and hasattr(result, "content"):- raw = result.content- else:- raw = b""-- return raw-- async def write(- self,- uri: str,- data: Union[bytes, str],- ctx: Optional[RequestContext] = None,- ) -> str:- """Write file"""- self._ensure_mutable_access(uri, ctx)- path = self._uri_to_path(uri, ctx=ctx)- if isinstance(data, str):- data = data.encode("utf-8")-- # Encryption (when configured) happens inside the ragfs layer keyed by account_id.- return await self._async_agfs.write(path, data)-- def _pathlock_fs_ctx(- self,- ctx: Optional[RequestContext],- lease_ref: Optional[Dict[str, Any] | str],- ) -> Dict[str, str] | None:- """Build an AGFS fs_ctx carrying account_id and an opaque pathlock lease_ref."""- if lease_ref is None:- return None- if isinstance(lease_ref, str):- ref = lease_ref- elif isinstance(lease_ref, dict):- ref = lease_ref.get("lease_ref")- else:- raise ValueError("lease_ref must be a non-empty string or lease dictionary")- if not isinstance(ref, str) or not ref:- raise ValueError("lease_ref must contain a non-empty lease_ref")- return {"account_id": self._ctx_or_default(ctx).account_id, "lease_ref": ref}-- async def mkdir(- self,- uri: str,- mode: str = "755",- exist_ok: bool = False,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> None:- """Create directory."""- self._ensure_mutable_access(uri, ctx)- path = self._uri_to_path(uri, ctx=ctx)- # Always ensure parent directories exist before creating this directory- await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)- try:- await self._async_agfs.mkdir(path, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))- except Exception as exc:- message = str(exc).lower()- already_exists = "exist" in message or "already" in message- if exist_ok and already_exists:- return- raise-- async def rm(- self,- uri: str,- recursive: bool = False,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> Dict[str, Any]:- """Delete file/directory + recursively update vector index.-- This method is idempotent: deleting a non-existent file succeeds- after cleaning up any orphan index records.-- Acquires a path lock, deletes VectorDB records, then FS files.- Raises ResourceBusyError when the target is locked by an ongoing- operation (e.g. semantic processing).-- Returns:- Dict with 'estimated_deleted_count' indicating the estimated number- of nodes deleted from vector index.- """- from openviking.storage.errors import LockAcquisitionError, ResourceBusyError-- self._ensure_delete_access(uri, ctx)- path = self._uri_to_path(uri, ctx=ctx)- target_uri = self._path_to_uri(path, ctx=ctx)-- async def _estimate_deleted_count(target_path: str, real_ctx: RequestContext) -> int:- """Estimate number of nodes to be deleted using vector index."""- vector_store = self._get_vector_store()- if not vector_store:- return 0- try:- target_canonical_uri = canonicalize_uri(- self._path_to_uri(target_path, ctx=real_ctx), real_ctx- )- filter_expr = PathScope("uri", target_canonical_uri, depth=-1)- return await vector_store.count(filter=filter_expr, ctx=real_ctx)- except Exception as e:- logger.warning(f"[VikingFS] Failed to count nodes before delete: {e}")- return 0-- # Check existence and determine lock strategy- try:- stat = await self._async_agfs.stat(path)- is_dir = stat.get("isDir", False) if isinstance(stat, dict) else False- except Exception as exc:- if not is_not_found_error(exc):- mapped = map_exception(exc, resource=uri)- if mapped is not None:- raise mapped from exc- raise- # Path does not exist: clean up any orphan index records and return- uris_to_delete = await self._collect_uris(path, recursive, ctx=ctx)- uris_to_delete.append(target_uri)- real_ctx = self._ctx_or_default(ctx)- estimated_count = await _estimate_deleted_count(path, real_ctx)- await self._delete_from_vector_store(uris_to_delete, ctx=ctx)- logger.info(f"[VikingFS] rm target not found, cleaned orphan index: {uri}")- return {"estimated_deleted_count": estimated_count}-- if is_dir:- if not recursive:- raise FailedPreconditionError(- f"Cannot remove directory without --recursive: {uri}",- details={"resource": uri, "expected_flag": "recursive"},- )- lock_method = self._async_agfs.pathlock_acquire_tree- else:- lock_method = self._async_agfs.pathlock_acquire_exact-- lease = lease_ref- if lease is None:- try:- lease = await lock_method(path)- except LockAcquisitionError:- raise ResourceBusyError(f"Resource is being processed: {uri}", uri=uri)-- try:- uris_to_delete = await self._collect_uris(path, recursive, ctx=ctx) if is_dir else []- uris_to_delete.append(target_uri)- real_ctx = self._ctx_or_default(ctx)- estimated_count = await _estimate_deleted_count(path, real_ctx)- await self._delete_from_vector_store(uris_to_delete, ctx=ctx)- try:- result = await self._async_agfs.rm(- path,- recursive=recursive,- fs_ctx=self._pathlock_fs_ctx(ctx, lease),- )- except AGFSDirectoryNotEmptyError:- raise FailedPreconditionError(- f"Directory not empty: {uri}. Use recursive=True to delete non-empty directories."- )- except RuntimeError as e:- # Fallback for older versions without typed exceptions- if _is_directory_not_empty_error(str(e)):- raise FailedPreconditionError(- f"Directory not empty: {uri}. Use recursive=True to delete non-empty directories."- )- raise- # Add estimated_deleted_count to the result- if isinstance(result, dict):- result["estimated_deleted_count"] = estimated_count- else:- result = {"estimated_deleted_count": estimated_count}- return result- finally:- if lease_ref is None:- await self._async_agfs.pathlock_release(lease)-- async def mv(- self,- old_uri: str,- new_uri: str,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> Dict[str, Any]:- """Move file/directory while extending an optional outer pathlock lease.-- Implemented as cp + rm to avoid lock files being carried by FS mv.- On VectorDB update failure the copy is cleaned up so the source stays intact.- """-- self._ensure_mutable_access(old_uri, ctx)- # mv is implemented as copy + recursive rm of the source (see the- # ``rm(old_path, recursive=is_dir)`` below), so the source must also clear- # the delete guard. Without this, a protected account root such as- # ``viking://`` — which rm() rejects up front (#2873) — could still be- # destroyed via mv, since the write guard alone permits the bare root.- self._ensure_delete_access(old_uri, ctx)- self._ensure_mutable_access(new_uri, ctx)- old_path = self._uri_to_path(old_uri, ctx=ctx)- new_path = self._uri_to_path(new_uri, ctx=ctx)- target_uri = self._path_to_uri(old_path, ctx=ctx)-- # Verify source exists and determine type before locking.- try:- stat = await self._async_agfs.stat(old_path)- is_dir = stat.get("isDir", False) if isinstance(stat, dict) else False- except Exception as exc:- if not is_not_found_error(exc):- mapped = map_exception(exc, resource=old_uri)- if mapped is not None:- raise mapped from exc- raise- raise FileNotFoundError(f"mv source not found: {old_uri}") from exc-- if not is_dir:- if new_uri.rstrip("/") != new_uri:- raise InvalidArgumentError(- f"mv destination for a file must include the target file name: {new_uri}",- details={"from_uri": old_uri, "to_uri": new_uri},- )- try:- destination_stat = await self._async_agfs.stat(new_path)- except Exception as exc:- if not is_not_found_error(exc):- mapped = map_exception(exc, resource=new_uri)- if mapped is not None:- raise mapped from exc- raise- else:- if isinstance(destination_stat, dict) and destination_stat.get("isDir", False):- raise InvalidArgumentError(- f"mv destination for a file must include the target file name: {new_uri}",- details={"from_uri": old_uri, "to_uri": new_uri},- )-- real_ctx = self._ctx_or_default(ctx)- old_uri = canonicalize_uri(old_uri, real_ctx)- new_uri = canonicalize_uri(new_uri, real_ctx)-- if is_dir:- lease = await self._async_agfs.pathlock_acquire_batch(- [- {"path": old_path, "kind": "tree"},- {"path": new_path, "kind": "exact"},- ],- owner_lease_ref=lease_ref,- )- else:- lease = await self._async_agfs.pathlock_acquire_batch(- [- {"path": old_path, "kind": "exact"},- {"path": new_path, "kind": "exact"},- ],- owner_lease_ref=lease_ref,- )-- try:- uris_to_move = (- await self._collect_uris(old_path, recursive=True, ctx=ctx) if is_dir else []- )- uris_to_move.append(target_uri)-- # Check if it's temp directory (files already encrypted)- is_temp = old_uri.startswith("viking://temp/")-- # Copy source to destination. Source must stay intact until vector updates succeed.- try:- await self._copy_for_mv(- old_uri=old_uri,- new_uri=new_uri,- old_path=old_path,- new_path=new_path,- is_dir=is_dir,- is_temp=is_temp,- ctx=ctx,- lease_ref=lease,- )- except Exception as e:- if "not found" in str(e).lower():- try:- await self._delete_from_vector_store(uris_to_move, ctx=ctx)- except Exception:- # Orphan cleanup is best effort here; preserve the copy error.- pass- else:- logger.info(- f"[VikingFS] mv source not found, cleaned orphan index: {old_uri}"- )- raise-- # Update VectorDB URIs (on failure, clean up the copy)- try:- await self._update_vector_store_uris(uris_to_move, old_uri, new_uri, ctx=ctx)- except Exception:- try:- if is_dir:- cleanup_lease = await self._async_agfs.pathlock_acquire_tree(- new_path,- owner_lease_ref=lease,- )- try:- await self._async_agfs.rm(- new_path,- recursive=True,- fs_ctx=self._pathlock_fs_ctx(ctx, cleanup_lease),- )- finally:- await self._async_agfs.pathlock_release(cleanup_lease)- else:- await self._async_agfs.rm(- new_path,- fs_ctx=self._pathlock_fs_ctx(ctx, lease),- )- except Exception:- pass- raise-- # Delete source- await self._async_agfs.rm(- old_path,- recursive=is_dir,- fs_ctx=self._pathlock_fs_ctx(ctx, lease),- )- return {}- finally:- await self._async_agfs.pathlock_release(lease)-- async def system_sync_status(- self, uri: str, ctx: Optional[RequestContext] = None- ) -> Dict[str, Any]:- """Return multi-write sync status for one Viking URI subtree."""- self._ensure_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- path = self._uri_to_path(uri, ctx=ctx)- return await self._async_agfs.system_sync_status(- path,- fs_ctx={"account_id": real_ctx.account_id},- )-- async def system_sync_retry(- self, uri: str, ctx: Optional[RequestContext] = None- ) -> Dict[str, Any]:- """Retry multi-write sync for one Viking URI subtree."""- self._ensure_mutable_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- path = self._uri_to_path(uri, ctx=ctx)- return await self._async_agfs.system_sync_retry(- path,- fs_ctx={"account_id": real_ctx.account_id},- )-- async def _copy_for_mv(- self,- old_uri: str,- new_uri: str,- old_path: str,- new_path: str,- is_dir: bool,- is_temp: bool,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> None:- """Copy source to destination for mv without deleting source."""- if is_temp:- if is_dir:- await self._copy_temp_dir_with_exact_locks(- old_path,- new_path,- ctx=ctx,- lease_ref=lease_ref,- )- else:- await self._async_agfs.cp(- old_path,- new_path,- recursive=False,- fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),- )- return-- if is_dir:- await self._copy_dir_through_vikingfs(old_uri, new_uri, ctx=ctx, lease_ref=lease_ref)- else:- await self._copy_file_through_vikingfs(old_uri, new_uri, ctx=ctx, lease_ref=lease_ref)-- async def _copy_temp_dir_with_exact_locks(- self,- old_path: str,- new_path: str,- ctx: Optional[RequestContext],- lease_ref: Dict[str, Any] | None,- ) -> None:- """Copy an encrypted temp directory while locking every destination entry.-- Args:- old_path: Source backend directory path.- new_path: Destination backend directory path.- ctx: Request context used for filesystem operations.- lease_ref: Exact lease covering the current destination directory.-- Returns:- None.- """- if lease_ref is None:- raise ValueError("temp directory copy requires a pathlock lease")- fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)- await self._async_agfs.mkdir(new_path, fs_ctx=fs_ctx)- entries = await self._async_agfs.ls(old_path, fs_ctx=fs_ctx)- for entry in entries:- name = entry.get("name", "")- if not name or name in (".", ".."):- continue- old_child = f"{old_path.rstrip('/')}/{name}"- new_child = f"{new_path.rstrip('/')}/{name}"- child_lease = await self._async_agfs.pathlock_acquire_exact(- new_child,- owner_lease_ref=lease_ref,- )- try:- if entry.get("isDir", False):- await self._copy_temp_dir_with_exact_locks(- old_child,- new_child,- ctx=ctx,- lease_ref=child_lease,- )- else:- await self._async_agfs.cp(- old_child,- new_child,- recursive=False,- fs_ctx=self._pathlock_fs_ctx(ctx, child_lease),- )- finally:- await self._async_agfs.pathlock_release(child_lease)-- async def _copy_dir_through_vikingfs(- self,- old_uri: str,- new_uri: str,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> None:- """Recursively copy a directory through VikingFS read/write hooks."""- await self.mkdir(new_uri, exist_ok=True, ctx=ctx, lease_ref=lease_ref)-- entries = await self.ls(old_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx)- for entry in entries:- name = entry.get("name", "")- if not name or name in (".", ".."):- continue- old_child_uri = f"{old_uri.rstrip('/')}/{name}"- new_child_uri = f"{new_uri.rstrip('/')}/{name}"- if entry.get("isDir"):- await self._copy_dir_through_vikingfs(- old_child_uri,- new_child_uri,- ctx=ctx,- lease_ref=lease_ref,- )- else:- await self._copy_file_through_vikingfs(- old_child_uri,- new_child_uri,- ctx=ctx,- lease_ref=lease_ref,- )-- async def _copy_file_through_vikingfs(- self,- from_uri: str,- to_uri: str,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> None:- """Copy one file through VikingFS read/write hooks without deleting source."""- content_bytes = await self.read_file_bytes(from_uri, ctx=ctx)- if lease_ref is None:- await self.write_file_bytes(to_uri, content_bytes, ctx=ctx)- return-- child_path = self._uri_to_path(to_uri, ctx=ctx)- child_lease = await self._async_agfs.pathlock_acquire_exact(- child_path,- owner_lease_ref=lease_ref,- )- try:- await self.write_file_bytes(to_uri, content_bytes, ctx=ctx, lease_ref=child_lease)- finally:- await self._async_agfs.pathlock_release(child_lease)-- async def grep(- self,- uri: str,- pattern: str,- exclude_uri: Optional[str] = None,- case_insensitive: bool = False,- node_limit: Optional[int] = None,- level_limit: int = 10,- ctx: Optional[RequestContext] = None,- content_transform: Optional[Callable[[str, str], str]] = None,- ) -> Dict:- """Content search by pattern or keywords.-- Optimized implementation that uses agfs native grep when possible.- The ragfs layer greps transparently over encrypted and plaintext files- (it decrypts via account_id when an encryption layer is configured).- Falls back to VikingFS layer implementation if native grep is unavailable.- When engine="auto" and vikingdb is available with sufficient data,- uses vikingdb bm25 recall + local fs precise matching.-- Args:- uri: Viking URI- pattern: Regular expression pattern to search for- exclude_uri: Optional URI prefix to exclude from search- case_insensitive: Whether to perform case-insensitive matching- node_limit: Maximum number of results to return- level_limit: Maximum depth level to traverse (default: 10)- ctx: Request context- content_transform: Optional projection applied before regex matching.- Internal bm25 recall limit is auto-adapted from node_limit as- min(node_limit * 5, 100000); when node_limit is unset, use 100000.-- Returns:- Dict with matches, count, match_count, files_scanned- """- real_ctx = self._ctx_or_default(ctx)- uri = canonicalize_uri(uri, real_ctx)- if exclude_uri:- exclude_uri = canonicalize_uri(exclude_uri, real_ctx)- self._ensure_access(uri, ctx)- # Skip vector_store.count() — the count field is not needed for grep,- # and avoiding it saves one VikingDB API call.- await self.stat(uri, ctx=ctx, skip_count=True)-- # Read engine and threshold from grep_config (ov.conf)- engine = self.grep_config.engine if self.grep_config else "auto"- switch_to_remote_threshold = (- self.grep_config.switch_to_remote_threshold if self.grep_config else 10000- )-- # A projection must run before matching. The remote BM25 index contains- # persisted raw content, so it cannot safely recall projected results.- resolved_engine = (- "fs"- if content_transform is not None- else await self._resolve_grep_engine(engine, uri, ctx, switch_to_remote_threshold)- )-- if resolved_engine == "fs":- return await self._grep_fs(- uri=uri,- pattern=pattern,- exclude_uri=exclude_uri,- case_insensitive=case_insensitive,- node_limit=node_limit,- level_limit=level_limit,- ctx=ctx,- content_transform=content_transform,- )- else: # "vikingdb_then_fs"- return await self._grep_vikingdb_then_fs(- uri=uri,- pattern=pattern,- exclude_uri=exclude_uri,- case_insensitive=case_insensitive,- node_limit=node_limit,- level_limit=level_limit,- ctx=ctx,- )-- async def _resolve_grep_engine(- self, engine: GrepEngine, uri: str, ctx, switch_to_remote_threshold: int = 10000- ) -> str:- """Resolve the actual grep engine to use."""- if engine == "fs":- return "fs"-- # auto mode: check vikingdb availability- vector_store = self._get_vector_store()- if not vector_store:- return "fs"-- backend_type = getattr(vector_store, "_backend_type", "unknown")- # Keep this set consistent with ``CollectionAdapter.USE_CONTENT_FIELD``:- # only these backends store the ``content`` field required for full-text grep.- if backend_type not in ("volcengine", "vikingdb"):- return "fs"-- # Check collection has content field and FullText config- if not await self._collection_has_fulltext(vector_store, ctx):- return "fs"-- # switch_to_remote_threshold=0 means always use vikingdb- if switch_to_remote_threshold == 0:- return "vikingdb_then_fs"-- # Check data volume threshold- try:- count = await self._get_cached_count(uri, ctx)- if count < switch_to_remote_threshold:- return "fs"- except Exception:- logger.debug(- "grep engine=auto: count() check failed, falling back to fs", exc_info=True- )- return "fs"-- return "vikingdb_then_fs"-- async def _collection_has_fulltext(self, vector_store, ctx) -> bool:- """Check if collection has content field and FullText config.-- Result is cached on the VikingFS instance since collection schema- does not change at runtime.- """- if self._fulltext_available is not None:- return self._fulltext_available- try:- meta = None- if hasattr(vector_store, "get_collection_meta"):- meta = await vector_store.get_collection_meta(ctx=ctx)- if not meta:- self._fulltext_available = False- return False- fields = meta.get("Fields", [])- has_content = any(- f.get("FieldName") == "content" and f.get("FieldType") == "text" for f in fields- )- fulltext = meta.get("FullText") or []- has_content_fulltext = any(ft.get("Field") == "content" for ft in fulltext)- result = has_content and has_content_fulltext- self._fulltext_available = result- return result- except Exception:- logger.debug(- "Failed to check collection fulltext config, assuming no fulltext", exc_info=True- )- return False-- async def _get_cached_count(self, uri: str, ctx) -> int:- """Get cached count of records for a URI (TTL=1h)."""- _COUNT_CACHE_TTL = 3600- vector_store = self._get_vector_store()-- # Include account_id in cache key for multi-tenant safety- account_id = getattr(ctx, "account_id", None) if ctx else None- cache_key = f"{account_id}:{uri}" if account_id else uri-- now = time.time()- cached = self._count_cache.get(cache_key)- if cached and (now - cached[1]) < _COUNT_CACHE_TTL:- return cached[0]-- count = await vector_store.count(filter=PathScope("uri", uri, depth=-1), ctx=ctx)- # Evict oldest entries if cache exceeds max size- if len(self._count_cache) >= self._count_cache_max_size:- oldest_keys = sorted(self._count_cache, key=lambda k: self._count_cache[k][1])- for k in oldest_keys[: len(oldest_keys) // 2]:- del self._count_cache[k]- self._count_cache[cache_key] = (count, now)- return count-- async def _grep_fs(- self,- uri,- pattern,- exclude_uri,- case_insensitive,- node_limit,- level_limit,- ctx,- content_transform=None,- ):- """Filesystem grep path: prefer native agfs grep and fall back if unavailable."""- if content_transform is None:- try:- return await self._grep_with_agfs(- uri=uri,- pattern=pattern,- exclude_uri=exclude_uri,- case_insensitive=case_insensitive,- node_limit=node_limit,- level_limit=level_limit,- ctx=ctx,- )- except (AttributeError, AGFSNotSupportedError, NotImplementedError) as e:- logger.debug(f"agfs grep unavailable, falling back to VikingFS implementation: {e}")-- return await self._grep_encrypted(- uri=uri,- pattern=pattern,- exclude_uri=exclude_uri,- case_insensitive=case_insensitive,- node_limit=node_limit,- level_limit=level_limit,- ctx=ctx,- content_transform=content_transform,- )-- async def _grep_vikingdb_then_fs(- self,- uri,- pattern,- exclude_uri,- case_insensitive,- node_limit,- level_limit,- ctx,- ):- """VikingDB bm25 recall + local fs precise matching."""- vector_store = self._get_vector_store()-- # Split regex alternation (e.g. "error|warning|fail") and join as a- # single query string for bm25 search. VikingDB's standard tokenizer- # will handle the tokenization of the query string.- query = " ".join(kw.strip() for kw in pattern.split("|") if kw.strip())- filter_expr = PathScope("uri", uri, depth=level_limit)- excluded_prefix = None- if exclude_uri:- excluded_prefix = exclude_uri.rstrip("/")- self._ensure_access(excluded_prefix, ctx)- filter_expr = And([- filter_expr,- RawDSL({- "op": "must_not",- "field": "uri",- "conds": [excluded_prefix],- "para": "-d=-1",- }),- ])-- # Auto-adapt bm25 recall limit: recall up to 5x requested matches- # while capping at VikingDB's max limit. If node_limit is unset,- # use the maximum limit to avoid truncation.- remote_return_limit = min(node_limit * 5, 100000) if node_limit else 100000-- # Step 1: vikingdb recall candidate files- try:- logger.debug(- "grep vikingdb search_by_keywords request: query=%r limit=%s filter=%r "- "output_fields=%s",- query,- remote_return_limit,- filter_expr,- ["uri"],- )- result = await vector_store.search_by_keywords(- query=query,- limit=remote_return_limit,- filter=filter_expr,- output_fields=["uri"],- ctx=ctx,- )- except Exception as e:- logger.warning(f"grep vikingdb step failed, falling back to fs: {e}")- return await self._grep_fs(- uri=uri,- pattern=pattern,- exclude_uri=exclude_uri,- case_insensitive=case_insensitive,- node_limit=node_limit,- level_limit=level_limit,- ctx=ctx,- )-- candidate_uris = [r["uri"] for r in result if r.get("uri")]- if excluded_prefix:- candidate_uris = [- u- for u in candidate_uris- if u != excluded_prefix and not u.startswith(excluded_prefix + "/")- ]- if not candidate_uris:- # BM25 returned no candidates — the index confirms no matching content- return {"matches": [], "count": 0, "match_count": 0, "files_scanned": 0}-- # Step 2: local fs precise matching on candidate files- return await self._grep_in_files(- candidate_uris,- pattern,- case_insensitive,- node_limit,- ctx,- )-- async def _grep_in_files(- self,- file_uris: List[str],- pattern: str,- case_insensitive: bool,- node_limit: Optional[int],- ctx: Optional[RequestContext],- ) -> Dict:- """Execute regex matching in specified file list (vikingdb_then_fs Step 2)."""- flags = re.IGNORECASE if case_insensitive else 0- compiled = re.compile(pattern, flags)-- results = []- files_scanned = 0-- for file_uri in file_uris:- files_scanned += 1- try:- content_bytes = await self.read(file_uri, ctx=ctx)- content = content_bytes.decode("utf-8", errors="replace")- except Exception:- continue-- for line_no, line in enumerate(content.splitlines(), 1):- if compiled.search(line):- results.append({"uri": file_uri, "line": line_no, "content": line})- if node_limit and len(results) >= node_limit:- return {- "matches": results,- "count": len(results),- "match_count": len(results),- "files_scanned": files_scanned,- }-- return {- "matches": results,- "count": len(results),- "match_count": len(results),- "files_scanned": files_scanned,- }-- async def _grep_with_agfs(- self,- uri: str,- pattern: str,- exclude_uri: Optional[str] = None,- case_insensitive: bool = False,- node_limit: Optional[int] = None,- level_limit: int = 10,- ctx: Optional[RequestContext] = None,- ) -> Dict:- """Grep using agfs native implementation.-- This is the optimized path for non-encrypted files.- Uses agfs.grep() which performs matching on the server side.-- Prefer pushing filters down to agfs backend:- - exclude_uri -> exclude_path- - level_limit -> level_limit-- Args:- uri: Viking URI- pattern: Regular expression pattern to search for- exclude_uri: Optional URI prefix to exclude from search- case_insensitive: Whether to perform case-insensitive matching- node_limit: Maximum number of results to return- level_limit: Maximum depth level to traverse- ctx: Request context-- Returns:- Dict with matches, count, match_count, files_scanned- """- path = self._uri_to_path(uri, ctx=ctx)-- excluded_path = None- if exclude_uri:- normalized_excluded_uri = exclude_uri.rstrip("/")- self._ensure_access(normalized_excluded_uri, ctx)- excluded_path = self._uri_to_path(normalized_excluded_uri, ctx=ctx)-- try:- result = await self._async_agfs.grep(- path=path,- pattern=pattern,- recursive=True,- case_insensitive=case_insensitive,- stream=False,- node_limit=node_limit,- exclude_path=excluded_path,- level_limit=level_limit,- )- except (AttributeError, AGFSNotSupportedError, NotImplementedError):- # Capability missing: let the outer caller fall back to the VikingFS implementation.- logger.warning("agfs grep unavailable, falling back to VikingFS implementation")- raise-- matches = result.get("matches", [])- results = []- files_scanned_set = set()- real_ctx = self._ctx_or_default(ctx)-- for match in matches:- match_file = match.get("file", "")- if not match_file:- continue-- agfs_file_path = self._resolve_grep_match_agfs_path(path, match_file)-- file_uri = self._path_to_uri(agfs_file_path, ctx=ctx)- if not self._is_accessible(file_uri, real_ctx):- continue-- files_scanned_set.add(file_uri)-- results.append({- "line": match.get("line", match.get("line_number", 0)),- "uri": file_uri,- "content": match.get("content", ""),- })-- if node_limit and len(results) >= node_limit:- break-- # Prefer backend-provided scanned file count if available; otherwise fall back to- # counting files that produced at least one match (best-effort).- backend_files_scanned = result.get("files_scanned")- if isinstance(backend_files_scanned, int) and backend_files_scanned >= 0:- files_scanned = (- len(files_scanned_set) if real_ctx.actor_peer_id else backend_files_scanned- )- else:- files_scanned = len(files_scanned_set)-- return {- "matches": results,- "count": len(results),- "match_count": len(results),- "files_scanned": files_scanned,- }-- async def _grep_encrypted(- self,- uri: str,- pattern: str,- exclude_uri: Optional[str] = None,- case_insensitive: bool = False,- node_limit: Optional[int] = None,- level_limit: int = 10,- ctx: Optional[RequestContext] = None,- content_transform: Optional[Callable[[str, str], str]] = None,- ) -> Dict:- """Grep implementation for encrypted files.-- This implementation decrypts files at VikingFS layer before matching.- Used when encryption is enabled or when agfs.grep is not available.-- Args:- uri: Viking URI- pattern: Regular expression pattern to search for- exclude_uri: Optional URI prefix to exclude from search- case_insensitive: Whether to perform case-insensitive matching- node_limit: Maximum number of results to return- level_limit: Maximum depth level to traverse (default: 10)- ctx: Request context-- Returns:- Dict with matches, count, match_count, files_scanned- """- flags = re.IGNORECASE if case_insensitive else 0- compiled_pattern = re.compile(pattern, flags)- excluded_prefix = None- if exclude_uri:- excluded_prefix = exclude_uri.rstrip("/")- self._ensure_access(excluded_prefix, ctx)- file_uris = await self._collect_grep_files(- uri,- excluded_prefix=excluded_prefix,- level_limit=level_limit,- ctx=ctx,- )- results, files_scanned = await self._grep_files_parallel(- file_uris,- compiled_pattern=compiled_pattern,- node_limit=node_limit,- ctx=ctx,- content_transform=content_transform,- )-- return {- "matches": results,- "count": len(results),- "match_count": len(results),- "files_scanned": files_scanned,- }-- async def _collect_grep_files(- self,- uri: str,- excluded_prefix: Optional[str],- level_limit: int,- ctx: Optional[RequestContext] = None,- ) -> List[str]:- file_uris: List[str] = []-- async def search_recursive(current_uri: str, current_depth: int) -> None:- if current_depth > level_limit:- return-- normalized_current_uri = current_uri- if excluded_prefix and (- normalized_current_uri == excluded_prefix- or normalized_current_uri.startswith(excluded_prefix + "/")- ):- logger.debug(f"Skipping excluded uri during grep: {normalized_current_uri}")- return-- try:- entries = await self.ls(normalized_current_uri, ctx=ctx)- except Exception:- return-- for entry in entries:- entry_uri = f"{normalized_current_uri.rstrip('/')}/{entry['name']}"- if excluded_prefix and (- entry_uri == excluded_prefix or entry_uri.startswith(excluded_prefix + "/")- ):- logger.debug(f"Skipping excluded uri during grep: {entry_uri}")- continue-- if entry.get("isDir"):- await search_recursive(entry_uri, current_depth + 1)- else:- file_uris.append(entry_uri)-- normalized_uri = uri- if excluded_prefix and (- normalized_uri == excluded_prefix or normalized_uri.startswith(excluded_prefix + "/")- ):- logger.debug(f"Skipping excluded uri during grep: {normalized_uri}")- return file_uris- try:- root_stat = await self.stat(normalized_uri, ctx=ctx)- except Exception:- return file_uris- if not root_stat.get("isDir", False):- file_uris.append(normalized_uri)- return file_uris-- await search_recursive(uri, 0)- return file_uris-- async def _grep_files_parallel(- self,- file_uris: List[str],- compiled_pattern: re.Pattern,- node_limit: Optional[int],- ctx: Optional[RequestContext] = None,- content_transform: Optional[Callable[[str, str], str]] = None,- ) -> tuple[List[Dict[str, Any]], int]:- results: List[Dict[str, Any]] = []- files_scanned = 0- for start in range(0, len(file_uris), _DEFAULT_GREP_FILE_CONCURRENCY):- batch_uris = file_uris[start : start + _DEFAULT_GREP_FILE_CONCURRENCY]- batch_jobs = [- self._grep_single_file(- entry_uri,- compiled_pattern,- ctx,- content_transform=content_transform,- )- for entry_uri in batch_uris- ]- batch_results = await asyncio.gather(*batch_jobs)- for matches, scanned_count in batch_results:- files_scanned += scanned_count- for match in matches:- results.append(match)- if node_limit and len(results) >= node_limit:- return results, files_scanned-- return results, files_scanned-- async def _grep_single_file(- self,- entry_uri: str,- compiled_pattern: re.Pattern,- ctx: Optional[RequestContext] = None,- content_transform: Optional[Callable[[str, str], str]] = None,- ) -> tuple[List[Dict[str, Any]], int]:- try:- content = await self.read(entry_uri, ctx=ctx)- if isinstance(content, bytes):- content = content.decode("utf-8", errors="replace")- if content_transform is not None:- content = content_transform(content, entry_uri)-- matches: List[Dict[str, Any]] = []- lines = content.split("\n")- for line_num, line in enumerate(lines, 1):- if compiled_pattern.search(line):- matches.append({- "line": line_num,- "uri": entry_uri,- "content": line,- })- return matches, 1- except Exception as e:- logger.debug(f"Failed to grep {entry_uri}: {e}")- return [], 1-- def _resolve_grep_match_agfs_path(self, base_path: str, match_file: str) -> str:- """Resolve a grep match path (relative to query root) into a full AGFS path."""- if match_file == ".":- return base_path- return f"{base_path.rstrip('/')}/{match_file.lstrip('/')}"-- async def stat(- self, uri: str, ctx: Optional[RequestContext] = None, skip_count: bool = False- ) -> Dict[str, Any]:- """- File/directory information.-- example: {'name': 'resources', 'size': 128, 'mode': 2147484141, 'modTime': '2026-02-10T21:26:02.934376379+08:00', 'isDir': True, 'isLocked': False, 'count': 42, 'meta': {'Name': 'localfs', 'Type': 'local', 'Content': {'local_path': '...'}}}-- Extra field:- isLocked (bool): Whether the path is currently held by a path lock- (either the path itself or any ancestor directory). Returns- False when the pathlock system is not enabled or the lookup- fails.- count (int): For directories, the number of nodes in the vector index- under this directory (including subdirectories). For files, this- field is not included.-- Args:- uri: Viking URI- ctx: Request context- skip_count: If True, skip the vector_store.count() call for directories.- Use this when the count field is not needed (e.g. in grep) to avoid- an extra VikingDB API call.- """- self._ensure_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- primary_path = self._uri_to_path(uri, ctx=ctx)- path = primary_path- last_not_found: Optional[Exception] = None- for candidate_path in self._read_paths(uri, ctx=ctx):- if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):- continue- try:- result = await self._async_agfs.stat(candidate_path)- path = candidate_path- break- except Exception as exc:- if is_not_found_error(exc):- last_not_found = exc- continue- raise- else:- if self._is_legacy_session_root_uri(uri):- now = datetime.now(timezone.utc).isoformat()- return {- "name": "session",- "size": 0,- "mode": 0o755,- "modTime": now,- "isDir": True,- "isLocked": False,- }- raise NotFoundError(uri, "file") from last_not_found- if isinstance(result, dict):- result["isLocked"] = await self._is_path_locked_async(path)- # Add count for directories if vector store available- if not skip_count and result.get("isDir", False):- try:- vector_store = self._get_vector_store()- if vector_store:- target_canonical_uri = canonicalize_uri(- self._path_to_uri(path, ctx=real_ctx), real_ctx- )- if not may_include_hidden_actor_peers(target_canonical_uri, real_ctx):- filter_expr = PathScope("uri", target_canonical_uri, depth=-1)- result["count"] = await vector_store.count(- filter=filter_expr,- ctx=real_ctx,- )- except Exception as e:- logger.warning(f"[VikingFS] Failed to count nodes for directory stat: {e}")- return result-- async def _is_path_locked_async(self, path: str) -> bool:- """Best-effort async path-lock lookup; returns False on any error."""- try:- return await self._async_agfs.pathlock_is_locked(path)- except Exception:- return False-- async def exists(self, uri: str, ctx: Optional[RequestContext] = None) -> bool:- """Check if a URI exists.-- Args:- uri: Viking URI- ctx: Request context-- Returns:- bool: True if the URI exists, False otherwise- """- try:- await self.stat(uri, ctx=ctx)- return True- except Exception:- return False-- async def glob(- self,- pattern: str,- uri: str = "viking://",- node_limit: Optional[int] = None,- ctx: Optional[RequestContext] = None,- ) -> Dict:- """File pattern matching, supports **/*.md recursive."""- _ensure_non_empty_search_query(pattern)- self._ensure_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- primary_path = self._uri_to_path(uri, ctx=ctx)- path: Optional[str] = None- for candidate_path in self._read_paths(uri, ctx=ctx):- if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):- continue- if await self._agfs_path_exists(candidate_path):- path = candidate_path- break- if path is None:- if self._is_legacy_session_root_uri(uri):- return {"matches": [], "count": 0}- raise NotFoundError(uri, "directory")-- page_size = self._glob_page_size(node_limit)- continuation_token: Optional[str] = None- matches = []- while True:- page = await self._async_agfs.glob_directory(- path,- pattern,- show_hidden=False,- page_size=page_size,- level_limit=None,- continuation_token=continuation_token,- )-- for entry in page.get("entries", []):- if node_limit is not None and node_limit > 0 and len(matches) >= node_limit:- return {"matches": matches, "count": len(matches)}- if not self._is_path_entry_visible(- entry["path"],- entry.get("name") or entry["path"].rsplit("/", 1)[-1],- path,- real_ctx,- ):- continue- if not await self._read_path_visible(uri, entry["path"], primary_path, real_ctx):- continue- entry_uri = self._alias_uri_for_path(- request_uri=uri,- base_path=path,- entry_path=entry["path"],- ctx=ctx,- )- matches.append(entry_uri)-- if node_limit is not None and node_limit > 0 and len(matches) >= node_limit:- return {"matches": matches, "count": len(matches)}- continuation_token = page.get("next_token")- if not continuation_token:- break- return {"matches": matches, "count": len(matches)}-- async def _batch_fetch_abstracts(- self,- entries: List[Dict[str, Any]],- abs_limit: int,- ctx: Optional[RequestContext] = None,- ) -> None:- """Batch fetch abstracts for entries using a fixed-size worker pool.-- Non-directory entries receive an empty abstract immediately.- Directory entries are processed concurrently via a worker pool,- using _read_abstract_for_known_dir to skip redundant stat() calls.-- Args:- entries: List of entries to fetch abstracts for- abs_limit: Maximum length for abstract truncation- """- dir_jobs = []- for index, entry in enumerate(entries):- if not entry.get("isDir", False):- entry["abstract"] = ""- continue- dir_jobs.append((index, entry))-- if not dir_jobs:- return-- worker_count = min(_ABSTRACT_WORKER_COUNT, len(dir_jobs))-- cursor = 0- cursor_lock = asyncio.Lock()- results: Dict[int, str] = {}-- async def worker() -> None:- nonlocal cursor- while True:- async with cursor_lock:- if cursor >= len(dir_jobs):- return- index, entry = dir_jobs[cursor]- cursor += 1-- try:- abstract = await self._read_abstract_for_known_dir(entry["uri"], ctx=ctx)- except Exception:- abstract = "[.abstract.md is not ready]"-- results[index] = abstract-- await asyncio.gather(*(worker() for _ in range(worker_count)))-- for index, abstract in results.items():- if len(abstract) > abs_limit:- abstract = abstract[: abs_limit - 3] + "..."- entries[index]["abstract"] = abstract-- async def tree(- self,- uri: str = "viking://",- output: str = "original",- abs_limit: int = 256,- show_all_hidden: bool = False,- node_limit: Optional[int] = 1000,- level_limit: Optional[int] = 3,- ctx: Optional[RequestContext] = None,- ) -> List[Dict[str, Any]]:- """- Recursively list all contents (includes rel_path).-- Args:- uri: Viking URI- output: str = "original" or "agent"- abs_limit: int = 256 (for agent output abstract truncation)- show_all_hidden: bool = False (list all hidden files, like -a)- node_limit: int | None = 1000 (maximum number of nodes to list, None means unlimited)- level_limit: int | None = 3 (maximum depth level to traverse, None means unlimited)-- output="original"- [{'name': '.abstract.md', 'size': 100, 'mode': 420, 'modTime': '2026-02-11T16:52:16.256334192+08:00', 'isDir': False, 'rel_path': '.abstract.md', 'uri': 'viking://resources...'}]-- output="agent"- [{'uri': 'viking://resources...', 'size': 100, 'isDir': False, 'modTime': '2026-02-11T08:52:16.256Z', 'rel_path': '.abstract.md', 'abstract': "..."}]- """- self._ensure_access(uri, ctx)- if output == "original":- return await self._tree_original(uri, show_all_hidden, node_limit, level_limit, ctx=ctx)- elif output == "agent":- return await self._tree_agent(- uri, abs_limit, show_all_hidden, node_limit, level_limit, ctx=ctx- )- else:- raise ValueError(f"Invalid output format: {output}")-- async def _tree_original(- self,- uri: str,- show_all_hidden: bool = False,- node_limit: Optional[int] = 1000,- level_limit: Optional[int] = 3,- ctx: Optional[RequestContext] = None,- ) -> List[Dict[str, Any]]:- """Recursively list all contents (original format)."""- result = []- async for entry, entry_uri in self._iter_visible_tree_entries(- uri,- show_all_hidden=show_all_hidden,- node_limit=node_limit,- level_limit=level_limit,- ctx=ctx,- ):- info = entry["info"]- new_entry = dict(entry.get("extra", {}))- new_entry.update({- "name": info["name"],- "size": info["size"],- "mode": info["mode"],- "modTime": info["modTime"],- "isDir": info["isDir"],- "rel_path": entry["rel_path"],- "uri": entry_uri,- })- result.append(new_entry)- return result-- async def _tree_agent(- self,- uri: str,- abs_limit: int,- show_all_hidden: bool = False,- node_limit: Optional[int] = 1000,- level_limit: Optional[int] = 3,- ctx: Optional[RequestContext] = None,- ) -> List[Dict[str, Any]]:- """Recursively list all contents (agent format with abstracts)."""- result = []-- async for entry, entry_uri in self._iter_visible_tree_entries(- uri,- show_all_hidden=show_all_hidden,- node_limit=node_limit,- level_limit=level_limit,- ctx=ctx,- ):- info = entry["info"]- is_dir = info["isDir"]- result.append({- "uri": entry_uri,- "size": 0 if is_dir else info["size"],- "isDir": is_dir,- "modTime": format_iso8601(parse_iso_datetime(info["modTime"])),- "rel_path": entry["rel_path"],- })-- await self._batch_fetch_abstracts(result, abs_limit, ctx=ctx)-- return result-- # ========== VikingFS Specific Capabilities ==========-- async def _read_abstract_file(- self,- path: str,- uri: str,- ctx: Optional[RequestContext] = None,- ) -> str:- """Read and decrypt/decode .abstract.md from a known directory path.-- Does NOT perform stat or isDir check -- caller is responsible for- ensuring the path points to a directory.- """- file_path = f"{path}/.abstract.md"- try:- content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))- except Exception as exc:- if not is_not_found_error(exc):- mapped = map_exception(exc, resource=uri)- if mapped is not None:- raise mapped from exc- raise- return f"# {uri} [Directory abstract is not ready]"-- return self._decode_bytes(content_bytes)-- async def _read_abstract_for_known_dir(- self,- uri: str,- ctx: Optional[RequestContext] = None,- ) -> str:- """Read .abstract.md for a directory that is already known to be a directory.-- Bypasses stat() and isDir check. Caller (i.e. _batch_fetch_abstracts)- must guarantee that the URI points to a directory.- """- self._ensure_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- primary_path = self._uri_to_path(uri, ctx=ctx)- for path in self._read_paths(uri, ctx=ctx):- if not await self._read_path_visible(uri, path, primary_path, real_ctx):- continue- try:- if not await self._agfs_path_exists(path):- continue- return await self._read_abstract_file(path, uri, ctx=ctx)- except Exception as exc:- if is_not_found_error(exc):- continue- raise- return f"# {uri} [Directory abstract is not ready]"-- async def abstract(- self,- uri: str,- ctx: Optional[RequestContext] = None,- ) -> str:- """Read directory's L0 summary (.abstract.md).-- If the caller points to a file, its parent directory is used instead so- the endpoint remains usable for both file and directory URIs.- """- self._ensure_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- primary_path = self._uri_to_path(uri, ctx=ctx)- path = primary_path- last_exc: Optional[Exception] = None- for candidate_path in self._read_paths(uri, ctx=ctx):- if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):- continue- try:- info = await self._async_agfs.stat(candidate_path)- path = candidate_path- break- except Exception as exc:- if is_not_found_error(exc):- last_exc = exc- continue- mapped = map_exception(exc, resource=uri)- if mapped is not None:- raise mapped from exc- raise- else:- if last_exc is not None:- mapped = map_exception(last_exc, resource=uri)- if mapped is not None:- raise mapped from last_exc- raise NotFoundError(uri, "directory") from last_exc- if not info.get("isDir", info.get("is_dir")):- parent_path = path.rsplit("/", 1)[0] or "/"- parent_uri = self._path_to_uri(parent_path, ctx=ctx)- logger.info(- "content/abstract: %s is a file, falling back to parent directory %s",- uri,- parent_uri,- )- return await self.abstract(parent_uri, ctx=ctx)- return await self._read_abstract_file(path, uri, ctx=ctx)-- async def overview(- self,- uri: str,- ctx: Optional[RequestContext] = None,- ) -> str:- """Read directory's L1 overview (.overview.md).-- If the caller points to a file, its parent directory is used instead so- the endpoint remains usable for both file and directory URIs.- """- self._ensure_access(uri, ctx=ctx)- real_ctx = self._ctx_or_default(ctx)- primary_path = self._uri_to_path(uri, ctx=ctx)- path = primary_path- last_exc: Optional[Exception] = None- for candidate_path in self._read_paths(uri, ctx=ctx):- if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):- continue- try:- info = await self._async_agfs.stat(candidate_path)- path = candidate_path- break- except Exception as exc:- if is_not_found_error(exc):- last_exc = exc- continue- mapped = map_exception(exc, resource=uri)- if mapped is not None:- raise mapped from exc- raise- else:- if last_exc is not None:- mapped = map_exception(last_exc, resource=uri)- if mapped is not None:- raise mapped from last_exc- raise NotFoundError(uri, "directory") from last_exc- if not info.get("isDir", info.get("is_dir")):- parent_path = path.rsplit("/", 1)[0] or "/"- parent_uri = self._path_to_uri(parent_path, ctx=ctx)- logger.info(- "content/overview: %s is a file, falling back to parent directory %s",- uri,- parent_uri,- )- return await self.overview(parent_uri, ctx=ctx)- file_path = f"{path}/.overview.md"- try:- content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))- except Exception as exc:- if not is_not_found_error(exc):- mapped = map_exception(exc, resource=uri)- if mapped is not None:- raise mapped from exc- raise- # Fallback to default if .overview.md doesn't exist- return f"# {uri}\n\n[Directory overview is not ready]"-- return self._decode_bytes(content_bytes)-- async def relations(- self,- uri: str,- ctx: Optional[RequestContext] = None,- ) -> List[Dict[str, Any]]:- """Get relation list.-- Returns: [{"uri": "...", "reason": "..."}, ...]- """- self._ensure_access(uri, ctx)- entries = await self.get_relation_table(uri, ctx=ctx)- result = []- for entry in entries:- for u in entry.uris:- if self._is_accessible(u, self._ctx_or_default(ctx)):- result.append({"uri": u, "reason": entry.reason})- return result-- async def find(- self,- query: str,- target_uri: Union[str, List[str]] = "",- limit: int = 10,- score_threshold: Optional[float] = None,- filter: Optional[Dict] = None,- ctx: Optional[RequestContext] = None,- level: Optional[List[int]] = None,- image_url: Optional[str] = None,- ):- """Semantic search.-- Args:- query: Search query- target_uri: Target directory URI(s), supports str or List[str]- limit: Return count- score_threshold: Score threshold- filter: Metadata filter-- Returns:- FindResult- """- _ensure_non_empty_search_query(query, image_url)- telemetry = get_current_telemetry()- from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever- from openviking_cli.retrieve import (- ContextType,- FindResult,- TypedQuery,- )-- real_ctx = self._ctx_or_default(ctx)- retrieval_targets = resolve_retrieval_targets(target_uri, real_ctx)-- for target_dir in retrieval_targets.target_directories:- self._ensure_access(target_dir, ctx)-- storage = self._get_vector_store()- if not storage:- raise RuntimeError("Vector store not initialized. Call OpenViking.initialize() first.")-- embedder = self._get_embedder()- if not embedder:- raise RuntimeError("Embedder not configured.")-- retriever = HierarchicalRetriever(- storage=storage,- embedder=embedder,- rerank_config=self.rerank_config,- retrieval_config=self.retrieval_config,- )-- typed_query = TypedQuery(- query=query,- context_type=None,- intent="",- target_directories=retrieval_targets.target_directories,- embedding_input=(- build_multimodal_embedding_input(query, image_url) if image_url else None- ),- image_query=bool(image_url),- )-- logger.debug(- "[VikingFS.find] Calling retriever.retrieve with "- f"ctx.account_id={real_ctx.account_id}, ctx.user={real_ctx.user}"- )-- result = await retriever.retrieve(- typed_query,- ctx=real_ctx,- limit=limit,- score_threshold=score_threshold,- scope_dsl=filter,- level=level,- )-- # Convert QueryResult to FindResult- memories, resources, skills = [], [], []- for ctx in result.matched_contexts:- if ctx.context_type == ContextType.MEMORY:- memories.append(ctx)- elif ctx.context_type == ContextType.RESOURCE:- resources.append(ctx)- elif ctx.context_type == ContextType.SKILL:- skills.append(ctx)-- find_result = FindResult(- memories=memories,- resources=resources,- skills=skills,- )- telemetry.set("vector.returned", find_result.total)- return find_result-- async def search(- self,- query: str,- target_uri: Union[str, List[str]] = "",- session_info: Optional[Dict] = None,- limit: int = 10,- score_threshold: Optional[float] = None,- filter: Optional[Dict] = None,- ctx: Optional[RequestContext] = None,- level: Optional[List[int]] = None,- image_url: Optional[str] = None,- ):- """Complex search with session context.-- Args:- query: Search query- target_uri: Target directory URI(s), supports str or List[str]- session_info: Session information- limit: Return count- filter: Metadata filter-- Returns:- FindResult- """- _ensure_non_empty_search_query(query, image_url)- telemetry = get_current_telemetry()- from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever- from openviking.retrieve.intent_analyzer import IntentAnalyzer- from openviking_cli.retrieve import (- ContextType,- FindResult,- QueryPlan,- TypedQuery,- )-- real_ctx = self._ctx_or_default(ctx)- retrieval_targets = resolve_retrieval_targets(target_uri, real_ctx)- primary_target_uri = retrieval_targets.first_explicit_directory-- session_summary = (- str(session_info.get("latest_archive_overview") or "") if session_info else ""- )- current_messages = session_info.get("current_messages") if session_info else None-- query_plan: Optional[QueryPlan] = None- for target_dir in retrieval_targets.target_directories:- self._ensure_access(target_dir, ctx)-- # When target_uri exists, read its abstract as optional query-planning context.- target_abstract = ""- if primary_target_uri:- try:- with telemetry.measure("search.target_abstract"):- target_abstract = await self.abstract(primary_target_uri, ctx=ctx)- except Exception:- target_abstract = ""-- intent_enabled = (- bool(self.retrieval_config.enable_intent)- if self.retrieval_config is not None- else True- )-- # With session context: optional intent analysis- if image_url:- typed_queries = [- TypedQuery(- query=query,- context_type=None,- intent="",- priority=1,- target_directories=retrieval_targets.target_directories,- embedding_input=build_multimodal_embedding_input(query, image_url),- image_query=True,- )- ]- elif intent_enabled and (session_summary or current_messages):- analyzer = IntentAnalyzer(max_recent_messages=5)- with telemetry.measure("search.intent_analysis"):- query_plan = await analyzer.analyze(- compression_summary=session_summary or "",- messages=current_messages or [],- current_message=query,- target_abstract=target_abstract,- )- typed_queries = query_plan.queries- for tq in typed_queries:- tq.target_directories = retrieval_targets.target_directories- else:- # No session context, or intent disabled: search with the raw query.- typed_queries = [- TypedQuery(- query=query,- context_type=None,- intent="",- priority=1,- target_directories=retrieval_targets.target_directories,- )- ]- telemetry.set("search.typed_queries_count", len(typed_queries))-- # Concurrent execution- storage = self._get_vector_store()- embedder = self._get_embedder()- retriever = HierarchicalRetriever(- storage=storage,- embedder=embedder,- rerank_config=self.rerank_config,- retrieval_config=self.retrieval_config,- )-- async def _execute(tq: TypedQuery):- real_ctx = self._ctx_or_default(ctx)- logger.debug(- "[VikingFS.search._execute] Calling retriever.retrieve with "- f"ctx.account_id={real_ctx.account_id}, ctx.user={real_ctx.user}"- )- return await retriever.retrieve(- tq,- ctx=real_ctx,- limit=limit,- score_threshold=score_threshold,- scope_dsl=filter,- level=level,- )-- query_results = await asyncio.gather(*[_execute(tq) for tq in typed_queries])-- # Aggregate results to FindResult- memories, resources, skills = [], [], []- for result in query_results:- for ctx in result.matched_contexts:- if ctx.context_type == ContextType.MEMORY:- memories.append(ctx)- elif ctx.context_type == ContextType.RESOURCE:- resources.append(ctx)- elif ctx.context_type == ContextType.SKILL:- skills.append(ctx)-- find_result = FindResult(- memories=memories,- resources=resources,- skills=skills,- query_plan=query_plan,- query_results=query_results,- )- telemetry.set("vector.returned", find_result.total)- return find_result-- # ========== Relation Management ==========-- async def link(- self,- from_uri: str,- uris: Union[str, List[str]],- reason: str = "",- ctx: Optional[RequestContext] = None,- ) -> None:- """Create relation (maintained in .relations.json)."""- if isinstance(uris, str):- uris = [uris]- self._ensure_mutable_access(from_uri, ctx)- for uri in uris:- self._ensure_access(uri, ctx)-- from_path = self._uri_to_path(from_uri, ctx=ctx)-- entries = await self._read_relation_table(from_path, ctx=ctx)- existing_ids = {e.id for e in entries}-- link_id = next(f"link_{i}" for i in range(1, 10000) if f"link_{i}" not in existing_ids)-- entries.append(RelationEntry(id=link_id, uris=uris, reason=reason))-- await self._write_relation_table(from_path, entries, ctx=ctx)- logger.debug(f"[VikingFS] Created link: {from_uri} -> {uris}")-- async def unlink(- self,- from_uri: str,- uri: str,- ctx: Optional[RequestContext] = None,- ) -> None:- """Delete relation."""- self._ensure_mutable_access(from_uri, ctx)- self._ensure_access(uri, ctx)- from_path = self._uri_to_path(from_uri, ctx=ctx)-- try:- entries = await self._read_relation_table(from_path, ctx=ctx)-- entry_to_modify = None- for entry in entries:- if uri in entry.uris:- entry_to_modify = entry- break-- if not entry_to_modify:- logger.debug(f"[VikingFS] URI not found in relations: {uri}")- return-- entry_to_modify.uris.remove(uri)-- if not entry_to_modify.uris:- entries.remove(entry_to_modify)- logger.debug(f"[VikingFS] Removed empty entry: {entry_to_modify.id}")-- await self._write_relation_table(from_path, entries, ctx=ctx)- logger.debug(f"[VikingFS] Removed link: {from_uri} -> {uri}")-- except Exception as e:- logger.error(f"[VikingFS] Failed to unlink {from_uri} -> {uri}: {e}")- raise IOError(f"Failed to unlink: {e}")-- async def get_relation_table(- self, uri: str, ctx: Optional[RequestContext] = None- ) -> List[RelationEntry]:- """Get relation table."""- self._ensure_access(uri, ctx)- path = self._uri_to_path(uri, ctx=ctx)- return await self._read_relation_table(path, ctx=ctx)-- # ========== Tree Traversal (Refactored) ==========-- def _is_name_visible_at_path(self, name: str, parent_path: str) -> bool:- """Check if name would appear in _ls_entries(parent_path).-- At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.- At other levels, uses the shared storage internal-name blacklist.- """- parts = [p for p in parent_path.strip("/").split("/") if p]- if len(parts) == 2 and parts[0] == "local":- return name in VikingURI.LISTABLE_SCOPES- return name not in STORAGE_INTERNAL_ENTRY_NAMES-- def _ancestor_is_filtered(self, entry_path: str, base_path: str) -> bool:- """Check if any ancestor directory of entry_path would be filtered by _ls_entries.-- Walks from base_path (exclusive) to entry's parent directory (exclusive),- checking each component against _is_name_visible_at_path.- """- base_parts = [p for p in base_path.strip("/").split("/") if p]- entry_parts = [p for p in entry_path.strip("/").split("/") if p]-- for i in range(len(base_parts), len(entry_parts) - 1):- name = entry_parts[i]- parent_parts = entry_parts[:i]- parent_path = "/" + "/".join(parent_parts) if parent_parts else "/"- if not self._is_name_visible_at_path(name, parent_path):- return True- return False-- def _is_path_entry_visible(- self, entry_path: str, name: str, base_path: str, ctx: RequestContext- ) -> bool:- """Check visibility for one flattened path entry returned by Rust."""- if self._ancestor_is_filtered(entry_path, base_path):- return False-- entry_parts = [p for p in entry_path.strip("/").split("/") if p]- if entry_parts:- parent_parts = entry_parts[:-1]- parent_path = "/" + "/".join(parent_parts) if parent_parts else "/"- if not self._is_name_visible_at_path(name, parent_path):- return False-- uri = self._path_to_uri(entry_path, ctx=ctx)- if not self._is_accessible(uri, ctx):- return False-- return True-- def _is_tree_entry_visible(- self, entry: Dict[str, Any], base_path: str, ctx: RequestContext- ) -> bool:- """Check visibility for a single TreeEntry returned by Rust tree_directory."""- entry_path = entry["path"]- entry_info = entry.get("info", {})- name = entry_info.get("name") or entry_path.rstrip("/").rsplit("/", 1)[-1]- return self._is_path_entry_visible(entry_path, name, base_path, ctx)-- # Over-fetch multiplier for bounded tree traversal. When a node_limit is- # set, we push down node_limit * this factor as the raw-node limit to Rust,- # leaving headroom for ACL/internal-name filtering before re-fetching.- _TREE_OVERFETCH_FACTOR = 4- _GLOB_PAGE_SIZE_DEFAULT = 1024-- def _glob_page_size(self, node_limit: Optional[int]) -> int:- """Return the backend page size used by glob_directory."""- if node_limit is None or node_limit <= 0:- return self._GLOB_PAGE_SIZE_DEFAULT- return node_limit-- async def _iter_visible_tree_entries(- self,- uri: str,- show_all_hidden: bool = False,- node_limit: Optional[int] = None,- level_limit: Optional[int] = None,- ctx: Optional[RequestContext] = None,- ):- """Shared generator: fetch raw TreeEntry list from Rust, yield (entry, uri) tuples.-- node_limit counts ACL-visible entries (see design §6.5), so the user's- node_limit cannot be pushed directly to Rust — doing so would truncate- before filtering and drop entries that should be visible.-- To keep memory bounded without changing that semantic, we push down an- *amplified* raw-node limit (node_limit * _TREE_OVERFETCH_FACTOR). If ACL- filtering leaves fewer than node_limit visible entries while Rust still- returned a full page (i.e. more raw nodes may exist), we double the raw- limit and re-fetch. Because Rust truncates a deterministic sorted prefix,- this yields exactly the same result as an unbounded fetch, while avoiding- materializing the entire prefix in the common case.-- When node_limit is None (full-tree callers), no limit is pushed down.- level_limit IS always passed to Rust.- """- real_ctx = self._ctx_or_default(ctx)- primary_path = self._uri_to_path(uri, ctx=ctx)- path: Optional[str] = None- for candidate_path in self._read_paths(uri, ctx=ctx):- if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):- continue- if await self._agfs_path_exists(candidate_path):- path = candidate_path- break- if path is None:- if self._is_legacy_session_root_uri(uri):- return- raise NotFoundError(uri, "directory")-- if node_limit is None:- raw_limit: Optional[int] = None- else:- raw_limit = max(node_limit * self._TREE_OVERFETCH_FACTOR, node_limit)-- while True:- raw_entries = await self._async_agfs.tree_directory(- path,- show_hidden=show_all_hidden,- node_limit=raw_limit,- level_limit=level_limit,- )-- visible: List[tuple] = []- for entry in raw_entries:- if node_limit is not None and len(visible) >= node_limit:- break- if not self._is_tree_entry_visible(entry, path, real_ctx):- continue- if not await self._read_path_visible(uri, entry["path"], primary_path, real_ctx):- continue- entry_uri = self._alias_uri_for_path(- request_uri=uri,- base_path=path,- entry_path=entry["path"],- ctx=ctx,- )- visible.append((entry, entry_uri))-- # If we still lack enough visible entries but Rust returned a full- # page (raw_limit reached), more raw nodes may exist — re-fetch with- # a doubled limit. Otherwise Rust is exhausted and we yield as-is.- need_more = (- node_limit is not None- and len(visible) < node_limit- and raw_limit is not None- and len(raw_entries) >= raw_limit- )- if need_more:- raw_limit *= 2- continue-- for item in visible:- yield item- return-- # ========== URI Conversion ==========-- # Maximum bytes for a single filename component (filesystem limit is typically 255)- _MAX_FILENAME_BYTES = 255-- @staticmethod- def _shorten_component(component: str, max_bytes: int = 255) -> str:- """Shorten a path component if its UTF-8 encoding exceeds max_bytes."""- if len(component.encode("utf-8")) <= max_bytes:- return component- hash_suffix = hashlib.sha256(component.encode("utf-8")).hexdigest()[:8]- # Trim to fit within max_bytes after adding hash suffix- prefix = component- target = max_bytes - len(f"_{hash_suffix}".encode("utf-8"))- while len(prefix.encode("utf-8")) > target and prefix:- prefix = prefix[:-1]- return f"{prefix}_{hash_suffix}"-- def _uri_to_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:- """Map virtual URI to account-isolated AGFS path.-- Pure prefix replacement: viking://{remainder} -> /local/{account_id}/{remainder}.- No implicit space injection — URIs must include space segments explicitly.- """- real_ctx = self._ctx_or_default(ctx)- account_id = real_ctx.account_id- normalized_uri, legacy_parts = self._normalized_uri_parts(uri)- if legacy_parts and legacy_parts[0] == "agent" and self._is_legacy_agent_id_uri(uri):- # Old format: viking://agent/{agent_id}/... — direct mapping for read-only compat- safe_parts = [- self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in legacy_parts- ]- return f"/local/{account_id}/{'/'.join(safe_parts)}"- canonical_uri = canonicalize_uri(uri, real_ctx)- _, parts = self._normalized_uri_parts(canonical_uri)- if not parts:- return f"/local/{account_id}"-- safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]- return f"/local/{account_id}/{'/'.join(safe_parts)}"-- def _legacy_session_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:- """Map a legacy viking://session URI to its pre-user-namespace path."""- real_ctx = self._ctx_or_default(ctx)- _, parts = self._normalized_uri_parts(uri)- safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]- return f"/local/{real_ctx.account_id}/{'/'.join(safe_parts)}"-- def _legacy_current_user_session_path(- self, uri: str, ctx: Optional[RequestContext] = None- ) -> Optional[str]:- """Return the legacy nested /session/{user_id}/{session_id} candidate."""- real_ctx = self._ctx_or_default(ctx)- _, parts = self._normalized_uri_parts(uri)- if len(parts) <= 1 or parts[0] != "session":- return None- nested_parts = ["session", real_ctx.user.user_id, *parts[1:]]- safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in nested_parts]- return f"/local/{real_ctx.account_id}/{'/'.join(safe_parts)}"-- def _is_legacy_session_uri(self, uri: str) -> bool:- _, parts = self._normalized_uri_parts(uri)- return bool(parts and parts[0] == "session")-- def _is_legacy_session_root_uri(self, uri: str) -> bool:- _, parts = self._normalized_uri_parts(uri)- return parts == ["session"]-- def _is_legacy_agent_id_uri(self, uri: str) -> bool:- _, parts = self._normalized_uri_parts(uri)- return bool(- parts- and parts[0] == "agent"- and len(parts) >= 2- and parts[1] not in {"skills", "endpoints", "tools", "payments"}- )-- def _read_paths(self, uri: str, ctx: Optional[RequestContext] = None) -> List[str]:- """Return read candidates for a URI, including legacy alias fallbacks."""- paths = [self._uri_to_path(uri, ctx=ctx)]-- if self._is_legacy_session_uri(uri):- for candidate in (- self._legacy_session_path(uri, ctx=ctx),- self._legacy_current_user_session_path(uri, ctx=ctx),- ):- if candidate and candidate not in paths:- paths.append(candidate)- return paths-- async def _read_path_visible(- self,- request_uri: str,- path: str,- primary_path: str,- ctx: RequestContext,- ) -> bool:- if path == primary_path:- return True- if self._is_legacy_session_uri(request_uri):- return await self._legacy_session_path_visible(path, ctx)- return True-- def _alias_uri_for_path(- self,- *,- request_uri: str,- base_path: str,- entry_path: str,- ctx: Optional[RequestContext],- ) -> str:- base = base_path.rstrip("/")- normalized_request, request_parts = self._normalized_uri_parts(request_uri)- request_root = (- normalized_request- if normalized_request == "viking://"- else normalized_request.rstrip("/")- )- preserve_request_alias = normalized_request in {"viking://", "viking://user"} or bool(- request_parts- and (- request_parts[0] == "session"- or (request_parts[0] == "agent" and self._is_legacy_agent_id_uri(request_uri))- )- )- rel_path = entry_path[len(base) :].strip("/") if entry_path.startswith(base) else ""- if entry_path.startswith(base):- separator = "" if request_root.endswith("://") else "/"- candidate_uri = request_root if not rel_path else f"{request_root}{separator}{rel_path}"- if preserve_request_alias:- return candidate_uri- try:- if self._uri_to_path(candidate_uri, ctx=ctx) == entry_path:- return candidate_uri- except Exception:- pass- return self._path_to_uri(entry_path, ctx=ctx)-- async def _agfs_path_exists(self, path: str) -> bool:- try:- await self._async_agfs.stat(path)- return True- except Exception as exc:- if is_not_found_error(exc):- return False- raise-- async def _looks_like_legacy_session_dir(self, path: str) -> bool:- for leaf in (".meta.json", "messages.jsonl", "history", "tool-results", "tools"):- if await self._agfs_path_exists(f"{path}/{leaf}"):- return True- return False-- async def _legacy_session_owner(self, session_root_path: str) -> str:- try:- raw = self._handle_agfs_read(- await self._async_agfs.read(f"{session_root_path}/.meta.json")- )- except Exception as exc:- if is_not_found_error(exc):- return ""- raise- try:- data = json.loads(self._decode_bytes(raw))- except json.JSONDecodeError:- return ""- if not isinstance(data, dict):- return ""- for key in ("created_by_user_id", "user_id", "owner_user_id", "created_by"):- value = data.get(key)- if isinstance(value, str) and value.strip():- return value.strip()- return ""-- async def _legacy_session_visible(- self,- session_root_path: str,- ctx: RequestContext,- *,- owner_hint: Optional[str] = None,- ) -> bool:- if ctx.role == Role.ROOT:- return True- owner = await self._legacy_session_owner(session_root_path)- if owner:- return owner == ctx.user.user_id- if owner_hint:- return owner_hint == ctx.user.user_id- return True-- async def _legacy_session_path_visible(self, path: str, ctx: RequestContext) -> bool:- parts = [p for p in path.strip("/").split("/") if p]- try:- session_index = parts.index("session")- except ValueError:- return True- suffix = parts[session_index + 1 :]- if not suffix:- return True-- root_prefix = "/" + "/".join(parts[: session_index + 1])- direct_root = f"{root_prefix}/{suffix[0]}"- if await self._looks_like_legacy_session_dir(direct_root):- return await self._legacy_session_visible(direct_root, ctx)-- if len(suffix) >= 2:- nested_root = f"{root_prefix}/{suffix[0]}/{suffix[1]}"- if await self._looks_like_legacy_session_dir(nested_root):- return await self._legacy_session_visible(- nested_root,- ctx,- owner_hint=suffix[0],- )- return True-- async def _legacy_session_root_items(- self,- path: str,- ctx: RequestContext,- ) -> List[tuple[Dict[str, Any], str]]:- try:- entries = await self._ls_entries(path)- except Exception as exc:- if is_not_found_error(exc):- return []- raise-- items: List[tuple[Dict[str, Any], str]] = []- for entry in entries:- name = entry.get("name", "")- if not name or name in {".", ".."} or not entry.get("isDir"):- continue- child_path = f"{path.rstrip('/')}/{name}"- if await self._looks_like_legacy_session_dir(child_path):- if await self._legacy_session_visible(child_path, ctx):- items.append((entry, f"viking://session/{name}"))- continue-- if ctx.role != Role.ROOT and name != ctx.user.user_id:- continue- try:- nested_entries = await self._ls_entries(child_path)- except Exception as exc:- if is_not_found_error(exc):- continue- raise- for nested in nested_entries:- nested_name = nested.get("name", "")- if not nested_name or nested_name in {".", ".."} or not nested.get("isDir"):- continue- nested_path = f"{child_path.rstrip('/')}/{nested_name}"- if not await self._looks_like_legacy_session_dir(nested_path):- continue- if await self._legacy_session_visible(nested_path, ctx, owner_hint=name):- items.append((nested, f"viking://session/{nested_name}"))- return items-- async def _session_root_items(- self,- uri: str,- ctx: RequestContext,- ) -> List[tuple[Dict[str, Any], str]]:- primary_path = self._uri_to_path(uri, ctx=ctx)- by_name: Dict[str, tuple[Dict[str, Any], str]] = {}- try:- for entry in await self._ls_entries(primary_path, ctx=ctx):- name = entry.get("name", "")- if not name or name in {".", ".."}:- continue- by_name[name] = (entry, f"viking://session/{name}")- except Exception as exc:- if not is_not_found_error(exc):- raise-- legacy_path = self._legacy_session_path(uri, ctx=ctx)- for entry, entry_uri in await self._legacy_session_root_items(legacy_path, ctx):- name = entry.get("name", "")- if name and name not in by_name:- by_name[name] = (entry, entry_uri)- return list(by_name.values())-- async def _list_read_path_items(- self,- uri: str,- ctx: Optional[RequestContext] = None,- ) -> List[tuple[Dict[str, Any], str]]:- real_ctx = self._ctx_or_default(ctx)- if self._is_legacy_session_root_uri(uri):- return await self._session_root_items(uri, real_ctx)-- primary_path = self._uri_to_path(uri, ctx=ctx)- merge_paths = self._is_legacy_session_uri(uri)- found_path = False- last_not_found: Optional[Exception] = None- by_uri: Dict[str, tuple[Dict[str, Any], str]] = {}-- for path in self._read_paths(uri, ctx=ctx):- if not await self._read_path_visible(uri, path, primary_path, real_ctx):- continue- try:- entries = await self._ls_entries(path, ctx=ctx)- except Exception as exc:- if is_not_found_error(exc):- last_not_found = exc- continue- raise-- found_path = True- for entry in entries:- entry_uri = self._alias_uri_for_path(- request_uri=uri,- base_path=path,- entry_path=f"{path.rstrip('/')}/{entry.get('name', '')}",- ctx=ctx,- )- by_uri.setdefault(entry_uri, (entry, entry_uri))- if not merge_paths:- break-- if found_path:- return list(by_uri.values())- raise NotFoundError(uri, "directory") from last_not_found-- _ROOT_PATH = "/local"-- async def _ls_entries(- self, path: str, ctx: Optional[RequestContext] = None- ) -> List[Dict[str, Any]]:- """List directory entries, filtering out internal directories.-- At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.- At other levels, uses the shared storage internal-name blacklist.- """- entries = await self._async_agfs.ls(path)- parts = [p for p in path.strip("/").split("/") if p]- if len(parts) == 2 and parts[0] == "local":- return [e for e in entries if e.get("name") in VikingURI.LISTABLE_SCOPES]- return [e for e in entries if e.get("name") not in STORAGE_INTERNAL_ENTRY_NAMES]-- def _path_to_uri(self, path: str, ctx: Optional[RequestContext] = None) -> str:- """/local/{account}/... -> viking://...-- Pure prefix replacement: strips /local/{account_id}/ and prepends viking://.- No implicit space stripping.- """- if path.startswith("viking://"):- return path- elif path.startswith("/local/"):- inner = path[7:].strip("/")- if not inner:- return "viking://"- real_ctx = self._ctx_or_default(ctx)- parts = [p for p in inner.split("/") if p]- if parts and parts[0] == real_ctx.account_id:- parts = parts[1:]- if not parts:- return "viking://"- return f"viking://{'/'.join(parts)}"- elif path.startswith("/"):- return f"viking:/{path}"- else:- return f"viking://{path}"-- def _looks_like_legacy_temp_leaf(self, value: str) -> bool:- return bool(re.match(r"^\d{8}_[0-9a-f]{6}$", value or ""))-- def _is_legacy_temp_uri_parts(self, parts: List[str]) -> bool:- if len(parts) < 2 or parts[0] != "temp" or not self._looks_like_legacy_temp_leaf(parts[1]):- return False- if len(parts) == 2:- return True- return not self._looks_like_legacy_temp_leaf(parts[2])-- def _is_accessible(self, uri: str, ctx: RequestContext) -> bool:- """Check whether a URI is visible/accessible under current request context."""- normalized_uri, parts = self._normalized_uri_parts(uri)- if ctx.role == Role.ROOT:- return True- if is_hidden_by_actor_peer_view(normalized_uri, ctx):- return False- if not parts:- return True- if is_watch_task_control_uri(normalized_uri):- return False-- scope = parts[0]- if scope == "resources":- return True- if scope == "temp":- if len(parts) == 1:- return True- if parts[1] == ctx.user.user_space_name():- return True- return self._is_legacy_temp_uri_parts(parts)- if scope == "upload":- return ctx.role == Role.ROOT- if scope == "_system":- return False- if scope == "agent":- # New format: agent/skills/..., agent/endpoints/... — globally readable (account scope)- if len(parts) >= 2 and parts[1] in {"skills", "endpoints", "tools", "payments"}:- return True- # Old format: agent/{agent_id}/... — actor_peer_id match for read-only access- if not ctx.actor_peer_id or len(parts) < 2:- return True- return parts[1] == ctx.actor_peer_id- return namespace_is_accessible(normalized_uri, ctx)-- def _handle_agfs_read(self, result: Union[bytes, Any, None]) -> bytes:- """Handle AGFSClient read return types consistently."""- if isinstance(result, bytes):- return result- elif result is None:- return b""- elif hasattr(result, "content") and result.content is not None:- return result.content- else:- # Try to convert to bytes- try:- return str(result).encode("utf-8")- except Exception:- return b""-- def _decode_bytes(self, data: bytes) -> str:- """Robustly decode bytes to string."""- if not data:- return ""- try:- return data.decode("utf-8")- except UnicodeDecodeError:- try:- # Try common encoding for Windows/legacy files in China- return data.decode("gbk")- except UnicodeDecodeError:- try:- return data.decode("latin-1")- except UnicodeDecodeError:- return data.decode("utf-8", errors="replace")-- # ========== Vector Sync Helper Methods ==========-- async def _collect_uris(- self, path: str, recursive: bool, ctx: Optional[RequestContext] = None- ) -> List[str]:- """Recursively collect all URIs (for rm/mv), including directories."""- uris = []-- async def _collect(p: str):- try:- entries = await self._ls_entries(p, ctx=ctx)- except Exception as exc:- if is_not_found_error(exc):- return- raise-- for entry in entries:- name = entry.get("name", "")- if name in [".", ".."]:- continue- full_path = f"{p}/{name}".replace("//", "/")- if entry.get("isDir"):- uris.append(self._path_to_uri(full_path, ctx=ctx))- if recursive:- await _collect(full_path)- else:- uris.append(self._path_to_uri(full_path, ctx=ctx))-- await _collect(path)- return uris-- async def _delete_from_vector_store(- self, uris: List[str], ctx: Optional[RequestContext] = None- ) -> None:- """Delete records with specified URIs from vector store.-- Uses tenant-safe URI deletion semantics from vector store.- """- vector_store = self._get_vector_store()- if not vector_store:- return- real_ctx = self._ctx_or_default(ctx)-- try:- await vector_store.delete_uris(real_ctx, uris)- for uri in uris:- logger.debug(f"[VikingFS] Deleted from vector store: {uri}")- except Exception as e:- logger.warning(f"[VikingFS] Failed to delete from vector store: {e}")- raise-- async def _update_vector_store_uris(- self,- uris: List[str],- old_base: str,- new_base: str,- ctx: Optional[RequestContext] = None,- ) -> None:- """Update URIs in vector store (when moving files).-- Preserves vector data and updates URI-derived identifiers without regenerating embeddings.- """- vector_store = self._get_vector_store()- if not vector_store:- return-- real_ctx = self._ctx_or_default(ctx)-- for uri in uris:- try:- new_uri = new_base + uri[len(old_base) :]- if await vector_store.update_uri_mapping(- ctx=real_ctx,- uri=uri,- new_uri=new_uri,- ):- logger.debug(f"[VikingFS] Updated URI: {uri} -> {new_uri}")- except Exception as e:- logger.warning(f"[VikingFS] Failed to update {uri} in vector store: {e}")-- def _get_vector_store(self) -> Optional["VikingVectorIndexBackend"]:- """Get vector store instance."""- return self.vector_store-- def _get_embedder(self) -> Any:- """Get embedder instance."""- return self.query_embedder-- # ========== Parent Directory Creation ==========-- async def _ensure_parent_dirs(- self,- path: str,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> None:- """Recursively create all parent directories."""- try:- await self._async_agfs.ensure_parent_dirs(- path,- fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),- )- except Exception as e:- logger.debug(f"Failed to ensure parent directories for {path}: {e}")- parent = path.rstrip("/").rsplit("/", 1)[0]- await self._mkdir_path_with_parents(parent, ctx=ctx, lease_ref=lease_ref)-- async def _mkdir_path_with_parents(- self,- dir_path: str,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> None:- """Create a directory path segment-by-segment using the same fs context."""- parts = [part for part in dir_path.strip("/").split("/") if part]- current = ""- for part in parts:- current = f"{current}/{part}"- try:- await self._async_agfs.mkdir(- current,- fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),- )- except Exception as e:- message = str(e).lower()- if "exist" in message or "already" in message:- continue- logger.debug(f"Failed to create parent directory {current}: {e}")-- # ========== Relation Table Internal Methods ==========-- async def _read_relation_table(- self, dir_path: str, ctx: Optional[RequestContext] = None- ) -> List[RelationEntry]:- """Read .relations.json."""- table_path = f"{dir_path}/.relations.json"- try:- content = self._handle_agfs_read(await self._async_agfs.read(table_path))- data = json.loads(content.decode("utf-8"))- except FileNotFoundError:- return []- except Exception:- # logger.warning(f"[VikingFS] Failed to read relation table {table_path}: {e}")- return []-- entries = []- # Compatible with old format (nested) and new format (flat)- if isinstance(data, list):- # New format: flat list- for entry_data in data:- entries.append(RelationEntry.from_dict(entry_data))- elif isinstance(data, dict):- # Old format: nested {namespace: {user: [entries]}}- for _namespace, user_dict in data.items():- for _user, entry_list in user_dict.items():- for entry_data in entry_list:- entries.append(RelationEntry.from_dict(entry_data))- return entries-- async def _write_relation_table(- self, dir_path: str, entries: List[RelationEntry], ctx: Optional[RequestContext] = None- ) -> None:- """Write .relations.json."""- # Use flat list format- data = [entry.to_dict() for entry in entries]-- content = json.dumps(data, ensure_ascii=False, indent=2)- table_path = f"{dir_path}/.relations.json"- if isinstance(content, str):- content = content.encode("utf-8")-- await self._async_agfs.write(table_path, content)-- # ========== Batch Read (backward compatible) ==========-- async def read_batch(- self, uris: List[str], level: str = "l0", ctx: Optional[RequestContext] = None- ) -> Dict[str, str]:- """Batch read content from multiple URIs."""- results = {}- for uri in uris:- try:- content = ""- if level == "l0":- content = await self.abstract(uri, ctx=ctx)- elif level == "l1":- content = await self.overview(uri, ctx=ctx)- results[uri] = content- except Exception:- pass- return results-- # ========== Other Preserved Methods ==========-- async def write_file(- self,- uri: str,- content: Union[str, bytes],- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> None:- """Write file directly. Encryption lock handled internally by EncryptionWrappedFS."""- self._ensure_mutable_access(uri, ctx)- path = self._uri_to_path(uri, ctx=ctx)- await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)-- if isinstance(content, str):- content = content.encode("utf-8")-- await self._async_agfs.write(path, content, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))-- async def read_file(- self,- uri: str,- offset: int = 0,- limit: int = -1,- ctx: Optional[RequestContext] = None,- ) -> str:- """Read single file, optionally sliced by line range.-- Args:- uri: Viking URI- offset: Starting line number (0-indexed). Default 0.- limit: Number of lines to read. -1 means read to end. Default -1.-- Raises:- FileNotFoundError: If the file does not exist.- """- self._ensure_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- primary_path = self._uri_to_path(uri, ctx=ctx)- # Verify the file exists before reading, because AGFS read returns- # empty bytes for non-existent files instead of raising an error.- last_not_found: Optional[Exception] = None- for path in self._read_paths(uri, ctx=ctx):- if not await self._read_path_visible(uri, path, primary_path, real_ctx):- continue- try:- stat = await self._async_agfs.stat(path)- break- except Exception as exc:- if is_not_found_error(exc):- last_not_found = exc- continue- raise- else:- raise NotFoundError(uri, "file") from last_not_found- if isinstance(stat, dict) and stat.get("isDir", False):- raise InvalidArgumentError(- f"Cannot read directory as file: {uri}",- details={"resource": uri, "expected": "file", "actual": "directory"},- )- try:- content = await self._async_agfs.read(path)- if isinstance(content, bytes):- raw = content- elif content is not None and hasattr(content, "content"):- raw = content.content- else:- raw = b""-- text = self._decode_bytes(raw)- except Exception:- raise NotFoundError(uri, "file")-- if offset == 0 and limit == -1:- return text- lines = text.splitlines(keepends=True)- sliced = lines[offset:] if limit == -1 else lines[offset : offset + limit]- return "".join(sliced)-- async def read_file_bytes(- self,- uri: str,- ctx: Optional[RequestContext] = None,- ) -> bytes:- """Read single binary file."""- self._ensure_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- primary_path = self._uri_to_path(uri, ctx=ctx)- last_not_found: Optional[Exception] = None- for path in self._read_paths(uri, ctx=ctx):- if not await self._read_path_visible(uri, path, primary_path, real_ctx):- continue- try:- stat = await self._async_agfs.stat(path)- break- except Exception as exc:- if is_not_found_error(exc):- last_not_found = exc- continue- raise- else:- raise NotFoundError(uri, "file") from last_not_found- if isinstance(stat, dict) and stat.get("isDir", False):- raise InvalidArgumentError(- f"Cannot read directory as file: {uri}",- details={"resource": uri, "expected": "file", "actual": "directory"},- )- try:- raw = self._handle_agfs_read(await self._async_agfs.read(path))- return raw- except Exception:- raise NotFoundError(uri, "file")-- async def write_file_bytes(- self,- uri: str,- content: bytes,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> None:- """Write single binary file. Encryption lock handled internally by EncryptionWrappedFS."""- self._ensure_mutable_access(uri, ctx)- path = self._uri_to_path(uri, ctx=ctx)- await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)-- await self._async_agfs.write(path, content, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))-- async def append_file(- self,- uri: str,- content: str,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> None:- """Append content to file while holding one exact pathlock lease."""- self._ensure_mutable_access(uri, ctx)- path = self._uri_to_path(uri, ctx=ctx)-- owned_lease = None- try:- await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)- lease = lease_ref- if lease is None:- lease = await self._async_agfs.pathlock_acquire_exact(path)- owned_lease = lease- fs_ctx = self._pathlock_fs_ctx(ctx, lease)-- # Read old content and rewrite the whole file to avoid lost updates.- existing = ""- try:- existing_bytes = self._handle_agfs_read(- await self._async_agfs.read(path, fs_ctx=fs_ctx)- )- existing = self._decode_bytes(existing_bytes)- except FileNotFoundError:- pass- except AGFSHTTPError as e:- if e.status_code != 404:- raise- except AGFSClientError:- raise-- final_content = (existing + content).encode("utf-8")- await self._async_agfs.write(- path,- final_content,- fs_ctx=fs_ctx,- )-- except Exception as e:- logger.error(f"[VikingFS] Failed to append to file {uri}: {e}")- raise IOError(f"Failed to append to file {uri}: {e}")- finally:- if owned_lease is not None:- await self._async_agfs.pathlock_release(owned_lease)-- async def ls(- self,- uri: str,- output: str = "original",- abs_limit: int = 256,- show_all_hidden: bool = False,- node_limit: int = 1000,- sort_by: Optional[str] = None,- sort_order: str = "asc",- ctx: Optional[RequestContext] = None,- ) -> List[Dict[str, Any]]:- """- List directory contents (URI version).-- Args:- uri: Viking URI- output: str = "original"- abs_limit: int = 256- show_all_hidden: bool = False (list all hidden files, like -a)- node_limit: int = 1000 (maximum number of nodes to list)- sort_by: Optional sort field, "name" or "mtime"- sort_order: Sort direction, "asc" or "desc"-- output="original"- [{'name': '.abstract.md', 'size': 100, 'mode': 420, 'modTime': '2026-02-11T16:52:16.256334192+08:00', 'isDir': False, 'meta': {'Name': 'localfs', 'Type': 'local', 'Content': None}, 'uri': 'viking://resources/.abstract.md'}]-- output="agent"- [{'name': '.abstract.md', 'size': 100, 'modTime': '2026-02-11T08:52:16.256Z', 'isDir': False, 'uri': 'viking://resources/.abstract.md', 'abstract': "..."}]- """- self._ensure_access(uri, ctx)- if sort_by not in {None, "name", "mtime"}:- raise ValueError("sort_by must be 'name' or 'mtime'")- if sort_order not in {"asc", "desc"}:- raise ValueError("sort_order must be 'asc' or 'desc'")- if output == "original":- return await self._ls_original(- uri,- show_all_hidden,- node_limit,- sort_by=sort_by,- sort_order=sort_order,- ctx=ctx,- )- elif output == "agent":- return await self._ls_agent(- uri,- abs_limit,- show_all_hidden,- node_limit,- sort_by=sort_by,- sort_order=sort_order,- ctx=ctx,- )- else:- raise ValueError(f"Invalid output format: {output}")-- @staticmethod- def _ls_entry_mtime(entry: Dict[str, Any]) -> Optional[float]:- raw_time = entry.get("modTime")- if isinstance(raw_time, (int, float)):- return float(raw_time)- if isinstance(raw_time, str) and raw_time:- try:- return parse_iso_datetime(raw_time).timestamp()- except (TypeError, ValueError, OverflowError):- return None-- legacy_time = entry.get("mtime")- if isinstance(legacy_time, (int, float)):- return float(legacy_time)- return None-- @classmethod- def _sort_ls_entry_items(- cls,- entry_items: List[tuple[Dict[str, Any], str]],- sort_by: Optional[str],- sort_order: str,- ) -> List[tuple[Dict[str, Any], str]]:- if sort_by is None:- return entry_items-- descending = sort_order == "desc"- directories = [item for item in entry_items if item[0].get("isDir", False)]- files = [item for item in entry_items if not item[0].get("isDir", False)]-- if sort_by == "name":-- def name_key(item: tuple[Dict[str, Any], str]) -> tuple[str, str]:- name = str(item[0].get("name", ""))- return name.lower(), name-- directories.sort(key=name_key, reverse=descending)- files.sort(key=name_key, reverse=descending)- return directories + files-- def sort_by_mtime(- items: List[tuple[Dict[str, Any], str]],- ) -> List[tuple[Dict[str, Any], str]]:- timestamped = []- missing = []- for item in items:- timestamp = cls._ls_entry_mtime(item[0])- if timestamp is None:- missing.append(item)- else:- timestamped.append((timestamp, item))- timestamped.sort(- key=lambda pair: pair[0],- reverse=descending,- )- return [item for _, item in timestamped] + missing-- return sort_by_mtime(directories) + sort_by_mtime(files)-- async def _ls_agent(- self,- uri: str,- abs_limit: int,- show_all_hidden: bool,- node_limit: int = 1000,- sort_by: Optional[str] = None,- sort_order: str = "asc",- ctx: Optional[RequestContext] = None,- ) -> List[Dict[str, Any]]:- """List directory contents (URI version)."""- real_ctx = self._ctx_or_default(ctx)- entry_items = await self._list_read_path_items(uri, ctx=ctx)- entry_items = self._sort_ls_entry_items(entry_items, sort_by, sort_order)- # basic info- fallback_time = datetime.now(timezone.utc)- all_entries = []- for entry, entry_uri in entry_items:- if len(all_entries) >= node_limit:- break- name = entry.get("name", "")- raw_time = entry.get("modTime", "")- parsed_time = fallback_time- if isinstance(raw_time, (int, float)):- parsed_time = datetime.fromtimestamp(raw_time, tz=timezone.utc)- elif raw_time:- if len(raw_time) > 26 and "+" in raw_time:- parts = raw_time.split("+")- raw_time = parts[0][:26] + "+" + parts[1]- parsed_time = parse_iso_datetime(raw_time)- elif isinstance(entry.get("mtime"), (int, float)):- parsed_time = datetime.fromtimestamp(entry["mtime"], tz=timezone.utc)- is_dir = entry.get("isDir", False)- new_entry = {- "uri": entry_uri,- "size": 0 if is_dir else entry.get("size", 0),- "isDir": is_dir,- "modTime": format_iso8601(parsed_time),- }- if not self._is_accessible(new_entry["uri"], real_ctx):- continue- if is_dir:- all_entries.append(new_entry)- elif not name.startswith("."):- all_entries.append(new_entry)- elif show_all_hidden:- all_entries.append(new_entry)- await self._batch_fetch_abstracts(all_entries, abs_limit, ctx=ctx)- return all_entries-- async def _ls_original(- self,- uri: str,- show_all_hidden: bool = False,- node_limit: int = 1000,- sort_by: Optional[str] = None,- sort_order: str = "asc",- ctx: Optional[RequestContext] = None,- ) -> List[Dict[str, Any]]:- """List directory contents (URI version)."""- real_ctx = self._ctx_or_default(ctx)- try:- entry_items = await self._list_read_path_items(uri, ctx=ctx)- entry_items = self._sort_ls_entry_items(entry_items, sort_by, sort_order)- # AGFS returns read-only structure, need to create new dict- all_entries = []- for entry, entry_uri in entry_items:- if len(all_entries) >= node_limit:- break- name = entry.get("name", "")- new_entry = dict(entry) # Copy original data- new_entry["uri"] = entry_uri- if not self._is_accessible(new_entry["uri"], real_ctx):- continue- if entry.get("isDir"):- all_entries.append(new_entry)- elif not name.startswith("."):- all_entries.append(new_entry)- elif show_all_hidden:- all_entries.append(new_entry)- return all_entries- except Exception:- raise NotFoundError(uri, "directory")-- async def move_file(- self,- from_uri: str,- to_uri: str,- ctx: Optional[RequestContext] = None,- ) -> None:- """Move file."""- self._ensure_mutable_access(from_uri, ctx)- self._ensure_mutable_access(to_uri, ctx)- from_path = self._uri_to_path(from_uri, ctx=ctx)-- await self._copy_file_through_vikingfs(from_uri, to_uri, ctx=ctx)- await self._async_agfs.rm(from_path)-- # ========== Temp File Operations (backward compatible) ==========-- def create_temp_uri(self, ctx: Optional[RequestContext] = None) -> str:- """Create a temp directory URI.-- - explicit ctx or bound request context -> user-scoped temp URI- - no explicit/bound context -> legacy temp URI shape for backward compatibility- """- real_ctx = ctx if ctx is not None else self._bound_ctx.get()- if real_ctx is None:- return VikingURI.create_temp_uri()- return VikingURI.create_temp_uri(space=real_ctx.user.user_space_name())-- async def persist_temp_tree(- self,- temp_uri: str,- target_uri: str,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> None:- """Persist an already-encrypted temp tree without rewriting file bytes."""- self._ensure_access(temp_uri, ctx)- self._ensure_mutable_access(target_uri, ctx)- src_path = self._uri_to_path(temp_uri, ctx=ctx)- dst_path = self._uri_to_path(target_uri, ctx=ctx)- fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)- await self._ensure_parent_dirs(dst_path, ctx=ctx, lease_ref=lease_ref)- await self._async_agfs.cp(- src_path,- dst_path,- recursive=True,- fs_ctx=fs_ctx or {"account_id": self._ctx_or_default(ctx).account_id},- )-- async def delete_temp(- self,- temp_uri: str,- ctx: Optional[RequestContext] = None,- lease_ref: Dict[str, Any] | None = None,- ) -> None:- """Delete temp directory and its contents."""- self._ensure_mutable_access(temp_uri, ctx)- path = self._uri_to_path(temp_uri, ctx=ctx)- fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)- try:- await self._async_agfs.rm(path, recursive=True, fs_ctx=fs_ctx)- except Exception as e:- logger.warning(f"[VikingFS] Failed to delete temp {temp_uri}: {e}")-- async def get_relations(self, uri: str, ctx: Optional[RequestContext] = None) -> List[str]:- """Get all related URIs (backward compatible)."""- entries = await self.get_relation_table(uri, ctx=ctx)- real_ctx = self._ctx_or_default(ctx)- all_uris = []- for entry in entries:- for related in entry.uris:- if self._is_accessible(related, real_ctx):- all_uris.append(related)- return all_uris-- async def get_relations_with_content(- self,- uri: str,- include_l0: bool = True,- include_l1: bool = False,- ctx: Optional[RequestContext] = None,- ) -> List[Dict[str, Any]]:- """Get related URIs and their content (backward compatible)."""- relation_uris = await self.get_relations(uri, ctx=ctx)- if not relation_uris:- return []-- results = []- abstracts = {}- overviews = {}- if include_l0:- abstracts = await self.read_batch(relation_uris, level="l0", ctx=ctx)- if include_l1:- overviews = await self.read_batch(relation_uris, level="l1", ctx=ctx)-- for rel_uri in relation_uris:- info = {"uri": rel_uri}- if include_l0:- info["abstract"] = abstracts.get(rel_uri, "")- if include_l1:- info["overview"] = overviews.get(rel_uri, "")- results.append(info)-- return results-- async def write_context(- self,- uri: str,- content: Union[str, bytes] = "",- abstract: str = "",- overview: str = "",- content_filename: str = "content.md",- is_leaf: bool = False,- ctx: Optional[RequestContext] = None,- ) -> None:- """Write context to AGFS (L0/L1/L2)."""-- self._ensure_mutable_access(uri, ctx)- path = self._uri_to_path(uri, ctx=ctx)-- try:- await self._ensure_parent_dirs(path, ctx=ctx)- try:- await self._async_agfs.mkdir(path)- except Exception as e:- if "exist" not in str(e).lower():- raise-- if content:- content_uri = f"{uri}/{content_filename}"- await self.write_file(content_uri, content, ctx=ctx)-- if abstract:- abstract_uri = f"{uri}/.abstract.md"- await self.write_file(abstract_uri, abstract, ctx=ctx)-- if overview:- overview_uri = f"{uri}/.overview.md"- await self.write_file(overview_uri, overview, ctx=ctx)-- except Exception as e:- logger.error(f"[VikingFS] Failed to write {uri}: {e}")- raise IOError(f"Failed to write {uri}: {e}")-- # ------------------------------------------------------------------- # Git version control (commit / restore / show / log)- # -------------------------------------------------------------------- # First path segments that the Rust git enumerate.rs prunes from snapshots,- # plus the runtime lock name. Mirrors INTERNAL_FIRST_SEGMENTS in- # crates/ragfs/src/git/enumerate.rs and VikingFS._INTERNAL_NAMES so that- # callers fail fast in Python with a clear error rather than passing a- # path that the Rust side will silently drop.- _GIT_INTERNAL_FIRST_SEGMENTS = frozenset({- "_system",- "tasks",- "temp",- "queue",- "upload",- ".path.ovlock",- })-- _DEFAULT_GIT_AUTHOR_NAME = "viking-bot"- _DEFAULT_GIT_AUTHOR_EMAIL = "[email protected]"-- _OVGITIGNORE_TREE_PATH = ".ovgitignore"-- # Must stay in sync with `OVGITIGNORE_MAX_BYTES` in the Rust layer- # (`crates/ragfs/src/git/ignore.rs`); enforced here at write time so a bad- # file can never be persisted only to poison every later commit.- _OVGITIGNORE_MAX_BYTES = 64 * 1024-- def _gitignore_agfs_path(self, ctx: Optional[RequestContext] = None) -> str:- real_ctx = self._ctx_or_default(ctx)- return f"/local/{real_ctx.account_id}/{self._OVGITIGNORE_TREE_PATH}"-- def _uri_to_tree_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:- """Convert a viking:// URI to an account-relative git tree path.-- ``viking://resources/proj_a/docs/a.md`` -> ``resources/proj_a/docs/a.md``.-- Pure prefix stripping: removes the ``viking://`` scheme and any- ``/local/{account}/`` segment. Internal scopes that the Rust git layer- would prune (`_system`, `tasks`, `temp`, `queue`, `upload`) and the- runtime lock name (`.path.ovlock`) are rejected with ``ValueError``- — passing them through silently would result in a no-op commit and- confuse callers.- """- real_ctx = self._ctx_or_default(ctx)- canonical = canonicalize_uri(uri, real_ctx)- _, parts = self._normalized_uri_parts(canonical)- if not parts:- raise ValueError(f"git tree path cannot be the account root: {uri!r}")- first = parts[0]- if first in self._GIT_INTERNAL_FIRST_SEGMENTS:- raise ValueError(f"git tree path rejects internal scope/segment {first!r}: {uri!r}")- return "/".join(parts)-- def _tree_path_to_uri(self, tree_path: str) -> str:- """Convert an account-relative git tree path to a viking:// URI.-- Inverse of :py:meth:`_uri_to_tree_path` (without context canonicalization).- """- cleaned = tree_path.strip("/")- if not cleaned:- raise ValueError("tree path must not be empty")- return f"viking://{cleaned}"-- _DIR_MARKER_LEVELS = {- ".abstract.md": ContextLevel.ABSTRACT,- ".overview.md": ContextLevel.OVERVIEW,- }- _NO_VECTOR_DERIVED = frozenset({".relations.json", ".ovgitignore"})-- def _classify_restore_path(self, tree_path: str, *, deleted: bool) -> Optional[tuple]:- """Classify a restore-affected tree path into a vector maintenance task.-- Returns a ``(op, uri, level)`` triple, or ``None`` when the path has no- vector side-effect:-- - ``dir/.abstract.md`` / ``dir/.overview.md`` → recompute (write) or- delete (removal) ONLY that directory's L0/L1 vector:- ``("reindex_marker"|"delete", dir_uri, ABSTRACT|OVERVIEW)``.- - ``.relations.json`` → ``None`` (not a vector text source).- - anything else (a source file) → reindex (write) or delete (removal)- its DETAIL vector:- ``("reindex_file", file_uri, DETAIL)`` / ``("delete", file_uri, DETAIL)``.-- ``None`` is also returned for a directory marker at the account root- (no parent directory to scope an L0/L1 vector to).- """- parent, _, name = tree_path.rpartition("/")- if name in self._NO_VECTOR_DERIVED:- return None- level = self._DIR_MARKER_LEVELS.get(name)- if level is not None:- if not parent:- return None- dir_uri = self._tree_path_to_uri(parent)- op = "delete" if deleted else "reindex_marker"- return (op, dir_uri, level)- # Source file.- file_uri = self._tree_path_to_uri(tree_path)- op = "delete" if deleted else "reindex_file"- return (op, file_uri, ContextLevel.DETAIL)-- async def get_gitignore(self, ctx: Optional[RequestContext] = None) -> str:- """Return the account-level .ovgitignore content, or an empty string if absent.-- Raises ``AGFSInvalidOperationError`` if the stored content is not valid- UTF-8, mirroring the Rust layer's reject-non-UTF-8 behavior at commit- time rather than leaking a raw ``UnicodeDecodeError``.- """- path = self._gitignore_agfs_path(ctx)- try:- raw = await self._async_agfs.read(path, 0, -1)- except Exception as exc:- if is_not_found_error(exc):- return ""- mapped = map_exception(exc, resource=path)- if mapped is not None:- raise mapped from exc- raise- if isinstance(raw, bytes):- data = raw- elif raw is not None and hasattr(raw, "content"):- data = raw.content- else:- data = b""- try:- return data.decode("utf-8")- except UnicodeDecodeError as exc:- raise AGFSInvalidOperationError(- f"invalid ignore file {path}: must be UTF-8: {exc}"- ) from exc-- async def set_gitignore(- self,- content: str,- ctx: Optional[RequestContext] = None,- ) -> None:- """Write the account-level .ovgitignore control file without semantic indexing.-- Validates the size limit up front so a too-large file can never be- persisted only to make every later ``commit`` fail. Syntax (negation,- escaping) is still validated at commit time by the Rust layer.- """- if not isinstance(content, str):- raise TypeError("content must be a string")- path = self._gitignore_agfs_path(ctx)- data = content.encode("utf-8")- if len(data) > self._OVGITIGNORE_MAX_BYTES:- raise AGFSInvalidOperationError(- f"ignore file too large: {path} is {len(data)} bytes, "- f"limit {self._OVGITIGNORE_MAX_BYTES} bytes"- )- await self._ensure_parent_dirs(path, ctx=ctx)- await self._async_agfs.write(path, data)-- async def delete_gitignore(self, ctx: Optional[RequestContext] = None) -> None:- """Delete the account-level .ovgitignore control file. Missing is success."""- path = self._gitignore_agfs_path(ctx)- try:- await self._async_agfs.rm(path, recursive=False)- except Exception as exc:- if is_not_found_error(exc):- return- mapped = map_exception(exc, resource=path)- if mapped is not None:- raise mapped from exc- raise-- async def commit(- self,- *,- message: str,- paths: Optional[List[str]] = None,- branch: str = "main",- author_name: Optional[str] = None,- author_email: Optional[str] = None,- ctx: Optional[RequestContext] = None,- ) -> Dict[str, Any]:- """Create a git snapshot of the account's tree.-- Args:- message: Commit message.- paths: Optional list of ``viking://`` URIs to scope the commit to;- entries may be files or directories. Directories are expanded- recursively with the snapshot pruning rules applied. ``None``- (default) enumerates the whole account tree. An empty list is- forwarded as an explicit empty path list (no-op commit). A- path that exists in neither the VFS nor the previous snapshot- logs a warning and is treated as a no-op deletion.- branch: Branch to advance. Defaults to ``"main"``.- author_name / author_email: Override the default bot author.- ctx: Request context (provides ``account_id``).-- Returns:- Dict with ``result`` (``"created"`` / ``"noop"``) and ``commit_oid``.- When ``result == "created"`` it also includes ``changed`` (paths- added/updated/removed). Both variants include ``ignored``: the- number of candidate paths skipped by the account ``.ovgitignore``- rules (system pruning is not counted). Noop omits ``changed``.- """- real_ctx = self._ctx_or_default(ctx)- account = real_ctx.account_id- if paths is None:- tree_paths: Optional[List[str]] = None- else:- tree_paths = [self._uri_to_tree_path(p, ctx=real_ctx) for p in paths]- return await self._async_agfs.run(- "git_commit",- account=account,- branch=branch,- message=message,- paths=tree_paths,- author_name=author_name or self._DEFAULT_GIT_AUTHOR_NAME,- author_email=author_email or self._DEFAULT_GIT_AUTHOR_EMAIL,- )-- async def restore(- self,- *,- project_dir: Optional[str] = None,- source_commit: str,- branch: str = "main",- dry_run: bool = False,- message: Optional[str] = None,- author_name: Optional[str] = None,- author_email: Optional[str] = None,- ctx: Optional[RequestContext] = None,- ) -> Dict[str, Any]:- """Restore a project subtree to the state at ``source_commit``.-- Generates a new commit whose parent is the current HEAD (not the- source commit) and writes the diff through the VFS. Paths outside- ``project_dir`` are left untouched.-- Args:- project_dir: ``viking://`` URI of the subtree, e.g.- ``"viking://resources/proj_a"``. May also be passed as a- short form like ``"resources/proj_a"``. Trailing slashes are- stripped. Internal scopes are rejected with ``ValueError``.- If None, restore the entire tree.- source_commit: 40-hex OID, branch name, or full ref path.- branch: Branch to advance. Defaults to ``"main"``.- dry_run: If True, returns the planned diff without writing.- message: Optional commit message; defaults to a generated string.- author_name / author_email: Override the default bot author.- ctx: Request context (provides ``account_id``).-- Returns:- Dict containing ``result`` (``"applied"`` / ``"noop"`` / ``"dry_run"``)- and corresponding oid / diff fields. When an ``Applied`` result has- vector side-effects, a ``task_id`` is included for polling the- background reindex via ``GET /api/v1/tasks/{task_id}``.-- After an ``Applied`` result, this method schedules background vector- maintenance for the affected paths via :class:`ReindexExecutor`:- directory markers (``.abstract.md`` / ``.overview.md``) recompute or- delete only that directory's L0/L1 vector; source files reindex (write)- or delete (removal) their DETAIL vector. ``.relations.json`` has no- vector side-effect. The rebuild is tracked as a single task- (``snapshot_restore_reindex``); per-path failures are logged and do not- block the return value. The task reaches ``completed`` only after each- affected vector has been written to (or deleted from) the index, so- polling ``task_id`` to ``completed`` guarantees subsequent ``find``- reads see the restored state.- """- real_ctx = self._ctx_or_default(ctx)- account = real_ctx.account_id- tree_dir: Optional[str]- if project_dir is None:- tree_dir = None- else:- tree_dir = self._uri_to_tree_path(project_dir, ctx=real_ctx).rstrip("/")- if not tree_dir:- raise ValueError(f"project_dir must not be empty: {project_dir!r}")- # Build kwargs dynamically, only include project_dir if it's not None- kwargs = {- "account": account,- "branch": branch,- "source_commit": source_commit,- "dry_run": dry_run,- "message": message,- "author_name": author_name or self._DEFAULT_GIT_AUTHOR_NAME,- "author_email": author_email or self._DEFAULT_GIT_AUTHOR_EMAIL,- }- if tree_dir is not None:- kwargs["project_dir"] = tree_dir- # dry_run only computes the diff; it never writes the VFS, so it needs- # no lock.- if dry_run:- return await self._async_agfs.run("git_restore", **kwargs)-- from openviking.pyagfs.exceptions import GitRestoreWritebackPartialError- from openviking.storage.errors import LockAcquisitionError, ResourceBusyError-- # Serialize the writeback against concurrent VFS mutations on the same- # subtree. A scoped restore tree-locks project_dir; a full restore- # (project_dir is None) locks the account root so only one restore runs- # per account at a time. The lock covers only the writeback — the- # background reindex is scheduled after release, otherwise its per-path- # child locks would conflict with this tree lock.- lock_path = (- self._uri_to_path(project_dir, ctx=real_ctx)- if project_dir is not None- else f"/local/{account}"- )- partial_exc: Optional[GitRestoreWritebackPartialError] = None- try:- lease = await self._async_agfs.pathlock_acquire_tree(lock_path)- except LockAcquisitionError:- raise ResourceBusyError(- f"Resource is being processed: {project_dir or '*'}",- uri=project_dir or "*",- )-- try:- try:- result = await self._async_agfs.run("git_restore", **kwargs)- except GitRestoreWritebackPartialError as exc:- # The ref already advanced — capture the exception and- # finish the lock scope cleanly. Reindex scheduling and- # re-raise happen below, outside the tree lock.- partial_exc = exc- result = None- finally:- await self._async_agfs.pathlock_release(lease)-- if partial_exc is not None:- # HEAD has moved forward but some VFS writes/deletes failed.- # Still schedule reindex for the paths that *did* reach the VFS so- # the vector index doesn't stay stale, then re-raise so the caller- # learns about the partial failure (and can inspect failed_writes- # / failed_deletes / task_id on the exception).- try:- partial_exc.task_id = await self._schedule_restore_reindex_for_paths(- written_paths=partial_exc.written_paths,- deleted_paths=partial_exc.deleted_paths,- project_dir=project_dir,- real_ctx=real_ctx,- )- except Exception:- logger.exception(- "[VikingFS] git restore partial: reindex scheduling failed; "- "HEAD advanced but reindex was not queued"- )- raise partial_exc-- if result.get("result") != "applied":- return result-- written = list(result.get("written_paths") or [])- deleted = list(result.get("deleted_paths") or [])- if written or deleted:- try:- task_id = await self._schedule_restore_reindex_for_paths(- written_paths=written,- deleted_paths=deleted,- project_dir=project_dir,- real_ctx=real_ctx,- )- if task_id is not None:- result["task_id"] = task_id- except Exception:- logger.exception(- "[VikingFS] git restore reindex task creation failed; "- "falling back to fire-and-forget rebuild"- )- self._schedule_vector_rebuild(written=written, deleted=deleted, ctx=real_ctx)- return result-- async def _schedule_restore_reindex_for_paths(- self,- *,- written_paths: List[str],- deleted_paths: List[str],- project_dir: Optional[str],- real_ctx: RequestContext,- ) -> Optional[str]:- """Classify ``written``/``deleted`` paths into vector tasks and queue- them as a single tracked background rebuild. Returns the task id, or- ``None`` if there is nothing to do.-- Shared by the applied path and the partial-writeback recovery path —- both schedule reindex for paths that actually reached the VFS.- """- if not written_paths and not deleted_paths:- return None- tasks = self._collect_restore_vector_tasks(written_paths, deleted_paths)- if not tasks:- return None-- from openviking.service.task_tracker import get_task_tracker-- tracker = get_task_tracker()- task = await tracker.create(- "snapshot_restore_reindex",- resource_id=project_dir or "*",- account_id=real_ctx.account_id,- user_id=real_ctx.user.user_id,- )- await tracker.update_stage(- task.task_id,- "queued",- account_id=real_ctx.account_id,- user_id=real_ctx.user.user_id,- )- background = asyncio.create_task(- self._run_restore_rebuild_tracked(task.task_id, tasks, real_ctx),- name=f"vikingfs-git-restore-reindex:{task.task_id}",- )- self._background_tasks.add(background)- background.add_done_callback(self._background_tasks.discard)- return task.task_id-- async def show(- self,- target_ref: str,- *,- path: Optional[str] = None,- max_blob_bytes: Optional[int] = None,- ctx: Optional[RequestContext] = None,- ) -> Union[Dict[str, Any], bytes]:- """Read a commit's metadata or a single blob.-- ``path=None`` returns the commit metadata dict (oid, tree, parents,- author, committer, message). ``path=str`` returns the blob bytes- directly, stripping the oid/size envelope returned by the binding.- """- real_ctx = self._ctx_or_default(ctx)- account = real_ctx.account_id- tree_path = self._uri_to_tree_path(path, ctx=real_ctx) if path else None- kwargs: Dict[str, Any] = {- "account": account,- "target_ref": target_ref,- "path": tree_path,- }- if max_blob_bytes is not None:- kwargs["max_blob_bytes"] = max_blob_bytes- resp = await self._async_agfs.run("git_show", **kwargs)- if path is not None and isinstance(resp, dict) and "bytes" in resp:- return resp["bytes"]- return resp-- async def show_blob_raw(- self,- target_ref: str,- *,- path: str,- ctx: Optional[RequestContext] = None,- ) -> Dict[str, Any]:- """Like ``show(target_ref, path=...)`` but returns the full envelope.-- Returns ``{"oid": str, "size": int, "bytes": bytes}`` without- stripping. Used by the HTTP snapshot router to populate- ``X-Snapshot-Oid`` / ``X-Snapshot-Size`` response headers.- """- real_ctx = self._ctx_or_default(ctx)- account = real_ctx.account_id- tree_path = self._uri_to_tree_path(path, ctx=real_ctx)- resp = await self._async_agfs.run(- "git_show",- account=account,- target_ref=target_ref,- path=tree_path,- )- if not isinstance(resp, dict) or "bytes" not in resp:- raise TypeError(- f"git_show returned unexpected shape for blob path: {type(resp).__name__}"- )- return resp-- async def diff(- self,- *,- path: str,- from_ref: Optional[str],- to_ref: str,- raw: bool = True,- ctx: Optional[RequestContext] = None,- ) -> Dict[str, Any]:- """Return a unified text diff for one path between two snapshots."""- real_ctx = self._ctx_or_default(ctx)- path = canonicalize_uri(path, ctx=real_ctx)- self._ensure_access(path, real_ctx)-- from_meta: Optional[Dict[str, Any]] = None- if from_ref:- try:- from_meta = await self.show(from_ref, ctx=real_ctx)- except AGFSNotFoundError as exc:- raise NotFoundError(from_ref, "git_ref") from exc- try:- to_meta = await self.show(to_ref, ctx=real_ctx)- except AGFSNotFoundError as exc:- raise NotFoundError(to_ref, "git_ref") from exc-- from_oid = str(from_meta.get("oid", from_ref)) if isinstance(from_meta, dict) else ""- to_oid = str(to_meta.get("oid", to_ref)) if isinstance(to_meta, dict) else to_ref-- async def read_optional(ref: str) -> Optional[bytes]:- try:- value = await self.show(- ref,- path=path,- max_blob_bytes=SNAPSHOT_DIFF_MAX_FILE_BYTES,- ctx=real_ctx,- )- except AGFSPathNotFoundError:- return None- except AGFSResourceExhaustedError as exc:- raise ResourceExhaustedError(- f"snapshot diff file size limit exceeded "- f"({SNAPSHOT_DIFF_MAX_FILE_BYTES} bytes)",- details={"limit_bytes": SNAPSHOT_DIFF_MAX_FILE_BYTES, "path": path},- ) from exc- if not isinstance(value, bytes):- raise TypeError(f"git_show returned unexpected blob type: {type(value).__name__}")- return value-- before_bytes = await read_optional(from_oid) if from_ref else None- after_bytes = await read_optional(to_oid)- if before_bytes is None and after_bytes is None:- raise NotFoundError(path, "git_blob")-- for value in (before_bytes, after_bytes):- if value is not None and len(value) > SNAPSHOT_DIFF_MAX_FILE_BYTES:- raise ResourceExhaustedError(- f"snapshot diff file size limit exceeded ({SNAPSHOT_DIFF_MAX_FILE_BYTES} bytes)",- details={"limit_bytes": SNAPSHOT_DIFF_MAX_FILE_BYTES, "path": path},- )-- if not raw:- from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils-- def visible_content(value: Optional[bytes]) -> Optional[bytes]:- if value is None:- return None- try:- text = value.decode("utf-8")- except UnicodeDecodeError as exc:- raise InvalidArgumentError("raw=false requires UTF-8 snapshot content") from exc- return MemoryFileUtils.read(text, uri=path).content.encode("utf-8")-- before_bytes = visible_content(before_bytes)- after_bytes = visible_content(after_bytes)-- change_type, before, after = await asyncio.to_thread(- _prepare_snapshot_diff,- path=path,- before_bytes=before_bytes,- after_bytes=after_bytes,- max_lines=SNAPSHOT_DIFF_MAX_LINES,- )- try:- diff_text = await self._async_agfs.run(- "git_diff_text",- before=before,- after=after,- fromfile=f"{path}@{from_oid}",- tofile=f"{path}@{to_oid}",- timeout_ms=SNAPSHOT_DIFF_TIMEOUT_MS,- max_output_bytes=SNAPSHOT_DIFF_MAX_OUTPUT_BYTES,- )- except AGFSResourceExhaustedError as exc:- raise ResourceExhaustedError(- f"snapshot diff output size limit exceeded "- f"({SNAPSHOT_DIFF_MAX_OUTPUT_BYTES} bytes)",- details={"limit_bytes": SNAPSHOT_DIFF_MAX_OUTPUT_BYTES, "path": path},- ) from exc- return {- "path": path,- "from_commit": from_oid,- "to_commit": to_oid,- "change_type": change_type,- "diff_text": diff_text,- }-- async def log(- self,- *,- branch: str = "main",- limit: int = 20,- paths: Optional[List[str]] = None,- ctx: Optional[RequestContext] = None,- ) -> List[Dict[str, Any]]:- """Walk back from ``branch``'s HEAD along ``parents[0]`` up to ``limit`` commits.-- Returns a list of commit metadata dicts (same shape as ``show(target_ref)``).- """- if limit <= 0:- return []- real_ctx = self._ctx_or_default(ctx)- account = real_ctx.account_id- tree_paths = (- None if not paths else [self._uri_to_tree_path(path, ctx=real_ctx) for path in paths]- )- return await self._async_agfs.run(- "git_log",- account=account,- branch=branch,- limit=limit,- paths=tree_paths,- )-- def _collect_restore_vector_tasks(- self,- written: List[str],- deleted: List[str],- ) -> set[tuple]:- """Classify restore-affected paths into deduplicated ``(op, uri, level)`` tasks.-- Tasks are deduplicated on the exact ``(op, uri, level)`` key (no ancestor- subsumption — each change is handled independently because no operation- recurses into descendants).- """- tasks: set[tuple] = set()- for tree_path in written:- try:- task = self._classify_restore_path(tree_path, deleted=False)- except ValueError:- continue- if task is not None:- tasks.add(task)- for tree_path in deleted:- try:- task = self._classify_restore_path(tree_path, deleted=True)- except ValueError:- continue- if task is not None:- tasks.add(task)- return tasks-- def _schedule_vector_rebuild(- self,- *,- written: List[str],- deleted: List[str],- ctx: RequestContext,- ) -> None:- """Fire-and-forget precise vector maintenance for a git restore.-- Each affected path is classified by :py:meth:`_classify_restore_path`- into a ``(op, uri, level)`` task and scheduled independently:-- - ``reindex_marker`` — recompute only a directory's L0/L1 vector.- - ``reindex_file`` — recompute only a source file's DETAIL vector- (non-recursive ``execute(mode="vectors_only")``).- - ``delete`` — remove only the ``(uri, level)`` vector (directory- marker removal, or deleted source file).-- Failures are logged and never propagate.- """- try:- from openviking.service.reindex_executor import get_reindex_executor- except Exception:- logger.exception("[VikingFS] ReindexExecutor import failed; skipping rebuild")- return-- try:- loop = asyncio.get_running_loop()- except RuntimeError:- logger.warning("[VikingFS] git restore vector rebuild skipped: no running event loop")- return-- tasks = self._collect_restore_vector_tasks(written, deleted)- if not tasks:- return-- executor = get_reindex_executor()- for op, uri, level in tasks:- loop.create_task(- self._run_vector_rebuild(executor, op, uri, level, ctx),- name=f"vikingfs-git-{op}:{uri}:{int(level)}",- )-- async def _run_restore_rebuild_tracked(- self,- task_id: str,- tasks: set[tuple],- ctx: RequestContext,- ) -> None:- """Background worker driving a tracked restore vector rebuild.-- Runs all classified ``(op, uri, level)`` rebuild tasks concurrently and- drives the task through start → complete/fail. Per-task failures are- swallowed (and logged) inside :py:meth:`_run_vector_rebuild`, preserving- the "failures do not block" semantics.- """- from openviking.service.task_tracker import get_task_tracker- from openviking.service.task_work_index import bind_task_context-- tracker = get_task_tracker()- tracker.register_running_task(task_id)- try:- await tracker.start(- task_id,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- stage="reindexing",- )- from openviking.service.reindex_executor import get_reindex_executor-- executor = get_reindex_executor()- with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):- await asyncio.gather(- *[- self._run_vector_rebuild(executor, op, uri, level, ctx)- for (op, uri, level) in tasks- ]- )- await tracker.complete(- task_id,- {"status": "completed", "task_count": len(tasks)},- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- except asyncio.CancelledError:- # TaskWorkIndex finalizes after this active task and its queue work settle.- return- except Exception as exc:- await tracker.fail(- task_id,- str(exc),- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )- finally:- await tracker.unregister_running_task(task_id)-- async def _run_vector_rebuild(- self,- executor: Any,- op: str,- uri: str,- level: ContextLevel,- ctx: RequestContext,- ) -> None:- """Wrapper coroutine: dispatch one vector task and swallow errors."""- try:- if op == "reindex_marker":- await executor.reindex_directory_marker(dir_uri=uri, level=level, ctx=ctx)- elif op == "reindex_file":- await executor.execute(uri=uri, mode="vectors_only", wait=True, ctx=ctx)- elif op == "delete":- await executor.delete_uri_level(uri=uri, level=level, ctx=ctx)- else: # pragma: no cover - defensive- logger.warning("[VikingFS] unknown vector rebuild op %r for %s", op, uri)- except Exception:- logger.exception("[VikingFS] git restore vector task %s failed for %s", op, uri)diff --git a/openviking/storage/viking_fs/__init__.py b/openviking/storage/viking_fs/__init__.pynew file mode 100644index 000000000..d543ac689--- /dev/null+++ b/openviking/storage/viking_fs/__init__.py@@ -0,0 +1,205 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""+VikingFS: OpenViking file system abstraction layer++Encapsulates the AGFS binding client, providing file operation interface based on Viking URI.+Responsibilities:+- URI conversion (viking:// <-> /local/)+- L0/L1 reading (.abstract.md, .overview.md)+- Relation management (.relations.json)+- Semantic search (vector retrieval + rerank)+- Vector sync (sync vector store on rm/mv)++This package is assembled from mixin submodules. The ``VikingFS`` class+combines all mixins; shared constants, helpers, and singleton management+live in :mod:`openviking.storage.viking_fs._base`.+"""++import asyncio+import contextvars+import hashlib+import json+import os+import re+import time+from contextlib import contextmanager+from dataclasses import dataclass, field+from datetime import datetime, timezone+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, TypeVar, Union++from openviking.core.context import ContextLevel+from openviking.core.namespace import (+ canonicalize_uri,+ is_hidden_by_actor_peer_view,+ may_include_hidden_actor_peers,+)+from openviking.core.namespace import (+ is_accessible as namespace_is_accessible,+)+from openviking.core.retrieval_targets import resolve_retrieval_targets+from openviking.pyagfs import AsyncAGFSClient+from openviking.pyagfs.exceptions import (+ AGFSClientError,+ AGFSDirectoryNotEmptyError,+ AGFSHTTPError,+ AGFSInvalidOperationError,+ AGFSNotFoundError,+ AGFSNotSupportedError,+ AGFSPathNotFoundError,+ AGFSResourceExhaustedError,+)+from openviking.resource.watch_storage import is_watch_task_control_uri+from openviking.server.error_mapping import is_not_found_error, map_exception+from openviking.server.identity import RequestContext, Role+from openviking.storage.expr import And, PathScope, RawDSL+from openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMES+from openviking.telemetry import get_current_telemetry+from openviking.utils.image_search import build_multimodal_embedding_input+from openviking.utils.time_utils import format_iso8601, get_current_timestamp, parse_iso_datetime+from openviking_cli.exceptions import (+ FailedPreconditionError,+ InvalidArgumentError,+ NotFoundError,+ PermissionDeniedError,+ ResourceExhaustedError,+)+from openviking_cli.session.user_id import UserIdentifier+from openviking_cli.utils.config.grep_config import GrepEngine+from openviking_cli.utils.logger import get_logger+from openviking_cli.utils.uri import VikingURI++# Import mixins+from openviking.storage.viking_fs import _base as _base_mod+from openviking.storage.viking_fs._access import _AccessMixin+from openviking.storage.viking_fs._base import (+ LS_ALL_NODES,+ SNAPSHOT_DIFF_MAX_FILE_BYTES,+ SNAPSHOT_DIFF_MAX_LINES,+ SNAPSHOT_DIFF_MAX_OUTPUT_BYTES,+ SNAPSHOT_DIFF_TIMEOUT_MS,+ RelationEntry,+ _ABSTRACT_WORKER_COUNT,+ _DEFAULT_GREP_FILE_CONCURRENCY,+ _T,+ _ensure_non_empty_search_query,+ _get_abstract_worker_count,+ _get_cpu_count,+ _is_directory_not_empty_error,+ _prepare_snapshot_diff,+ _snapshot_line_count,+ enable_viking_fs_recorder,+ get_viking_fs,+ init_viking_fs,+ logger,+)+from openviking.storage.viking_fs._grep import _GrepMixin+from openviking.storage.viking_fs._ops import _OpsMixin+from openviking.storage.viking_fs._semantic import _SemanticMixin+from openviking.storage.viking_fs._snapshot import _SnapshotMixin+from openviking.storage.viking_fs._sync import SyncDiff, _SyncMixin+from openviking.storage.viking_fs._vector import _VectorMixin++if TYPE_CHECKING:+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig+++# ========== VikingFS Main Class ==========+++class VikingFS(+ _AccessMixin,+ _OpsMixin,+ _SyncMixin,+ _GrepMixin,+ _SemanticMixin,+ _SnapshotMixin,+ _VectorMixin,+):+ """RAGFS-based OpenViking file system.++ APIs are divided into two categories:+ - RAGFS basic commands (direct forwarding): read, ls, write, mkdir, rm, mv, grep, stat+ - VikingFS specific capabilities: abstract, overview, find, search, relations, link, unlink++ Uses Rust binding mode: Use RAGFSBindingClient to directly use RAGFS implementation+ """++ def __init__(+ self,+ agfs: Any,+ query_embedder: Optional[Any] = None,+ rerank_config: Optional["RerankConfig"] = None,+ vector_store: Optional["VikingVectorIndexBackend"] = None,+ retrieval_config: Optional["RetrievalConfig"] = None,+ grep_config: Optional["GrepConfig"] = None,+ timeout: int = 10,+ encryptor: Optional[Any] = None,+ ):+ self.agfs = agfs+ self._async_agfs = AsyncAGFSClient(agfs)+ self.query_embedder = query_embedder+ self.rerank_config = rerank_config+ self.vector_store = vector_store+ self.retrieval_config = retrieval_config+ self.grep_config = grep_config+ self._encryptor = encryptor+ self._count_cache: Dict[str, tuple] = {} # cache_key → (count, timestamp)+ self._count_cache_max_size = 1024+ self._fulltext_available: Optional[bool] = None # cached result of _collection_has_fulltext+ self._bound_ctx: contextvars.ContextVar[Optional[RequestContext]] = contextvars.ContextVar(+ "vikingfs_bound_ctx", default=None+ )+ self._background_tasks: set = set()+ self._user_deletion_guard: Optional[Callable[[str, str], bool]] = None+++VikingFS.__module__ = __name__++# Provide _instance on the package module that transparently proxies to+# _base._instance so that test fixtures (conftest.py) can save/restore the+# singleton across init_viking_fs boundaries. PEP 562 only gives us+# __getattr__; we also need to intercept writes so that assigning to+# ``openviking.storage.viking_fs._instance`` updates the real singleton in+# ``_base`` where ``get_viking_fs``/``init_viking_fs`` read/write it.+import sys as _sys+++class _ModuleProxy: # pragma: no cover - trivial proxy+ def __init__(self, wrapped):+ object.__setattr__(self, "_wrapped", wrapped)++ def __getattr__(self, name):+ if name == "_instance":+ return _base_mod._instance+ return getattr(object.__getattribute__(self, "_wrapped"), name)++ def __setattr__(self, name, value):+ if name == "_instance":+ _base_mod._instance = value+ return+ setattr(object.__getattribute__(self, "_wrapped"), name, value)++ def __delattr__(self, name):+ delattr(object.__getattribute__(self, "_wrapped"), name)++ def __dir__(self):+ return dir(object.__getattribute__(self, "_wrapped"))+++_sys.modules[__name__] = _ModuleProxy(_sys.modules[__name__]) # type: ignore[assignment]++__all__ = [+ "LS_ALL_NODES",+ "SNAPSHOT_DIFF_MAX_FILE_BYTES",+ "SNAPSHOT_DIFF_MAX_LINES",+ "SNAPSHOT_DIFF_MAX_OUTPUT_BYTES",+ "SNAPSHOT_DIFF_TIMEOUT_MS",+ "RelationEntry",+ "SyncDiff",+ "VikingFS",+ "enable_viking_fs_recorder",+ "get_viking_fs",+ "init_viking_fs",+]diff --git a/openviking/storage/viking_fs/_access.py b/openviking/storage/viking_fs/_access.pynew file mode 100644index 000000000..dc9cd3e6f--- /dev/null+++ b/openviking/storage/viking_fs/_access.py@@ -0,0 +1,917 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Access control, URI normalization, path conversion, and visibility mixin for VikingFS."""++import contextvars+import hashlib+import json+import re+from contextlib import contextmanager+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union++from openviking.core.context import ContextLevel+from openviking.core.namespace import (+ canonicalize_uri,+ is_hidden_by_actor_peer_view,+ may_include_hidden_actor_peers,+)+from openviking.core.namespace import (+ is_accessible as namespace_is_accessible,+)+from openviking.resource.watch_storage import is_watch_task_control_uri+from openviking.server.error_mapping import is_not_found_error, map_exception+from openviking.server.identity import RequestContext, Role+from openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMES+from openviking_cli.exceptions import (+ FailedPreconditionError,+ NotFoundError,+ PermissionDeniedError,+)+from openviking_cli.session.user_id import UserIdentifier+from openviking_cli.utils.uri import VikingURI++if TYPE_CHECKING:+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig+++class _AccessMixin:+ """URI normalization, access control, path conversion, and visibility helpers."""++ # Over-fetch multiplier for bounded tree traversal. When a node_limit is+ # set, we push down node_limit * this factor as the raw-node limit to Rust,+ # leaving headroom for ACL/internal-name filtering before re-fetching.+ _TREE_OVERFETCH_FACTOR = 4+ _GLOB_PAGE_SIZE_DEFAULT = 1024++ # Maximum bytes for a single filename component (filesystem limit is typically 255)+ _MAX_FILENAME_BYTES = 255++ _ROOT_PATH = "/local"++ # First path segments that the Rust git enumerate.rs prunes from snapshots,+ # plus the runtime lock name. Mirrors INTERNAL_FIRST_SEGMENTS in+ # crates/ragfs/src/git/enumerate.rs and VikingFS._INTERNAL_NAMES so that+ # callers fail fast in Python with a clear error rather than passing a+ # path that the Rust side will silently drop.+ _GIT_INTERNAL_FIRST_SEGMENTS = frozenset({+ "_system",+ "tasks",+ "temp",+ "queue",+ "upload",+ ".path.ovlock",+ })++ _DEFAULT_GIT_AUTHOR_NAME = "viking-bot"+ _DEFAULT_GIT_AUTHOR_EMAIL = "[email protected]"++ _OVGITIGNORE_TREE_PATH = ".ovgitignore"++ # Must stay in sync with `OVGITIGNORE_MAX_BYTES` in the Rust layer+ # (`crates/ragfs/src/git/ignore.rs`); enforced here at write time so a bad+ # file can never be persisted only to poison every later commit.+ _OVGITIGNORE_MAX_BYTES = 64 * 1024++ _DIR_MARKER_LEVELS = {+ ".abstract.md": ContextLevel.ABSTRACT,+ ".overview.md": ContextLevel.OVERVIEW,+ }+ _NO_VECTOR_DERIVED = frozenset({".relations.json", ".ovgitignore"})++ def set_user_deletion_guard(self, guard: Optional[Callable[[str, str], bool]]) -> None:+ self._user_deletion_guard = guard++ @staticmethod+ def _default_ctx() -> RequestContext:+ return RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)++ def _ctx_or_default(self, ctx: Optional[RequestContext]) -> RequestContext:+ if ctx is not None:+ return ctx+ bound = self._bound_ctx.get()+ return bound or self._default_ctx()++ @contextmanager+ def bind_request_context(self, ctx: RequestContext):+ """Temporarily bind ctx for legacy internal call paths without explicit ctx param."""+ token = self._bound_ctx.set(ctx)+ try:+ yield+ finally:+ self._bound_ctx.reset(token)++ @staticmethod+ def _normalize_uri(uri: str) -> str:+ """Normalize short-format URIs to the canonical viking:// form."""+ if uri.startswith("viking://"):+ return uri+ return VikingURI.normalize(uri)++ @classmethod+ def _normalized_uri_parts(cls, uri: str) -> tuple[str, List[str]]:+ """Normalize a URI and reject ambiguous or platform-specific path traversal forms."""+ normalized = cls._normalize_uri(uri)+ parts = [p for p in normalized[len("viking://") :].strip("/").split("/") if p]++ for part in parts:+ if part in {".", ".."}:+ raise PermissionDeniedError(+ f"Unsafe URI traversal segment '{part}' in {normalized}",+ resource=normalized,+ )+ if "\\" in part:+ raise PermissionDeniedError(+ f"Unsafe URI path separator '\\\\' in component '{part}' of {normalized}",+ resource=normalized,+ )+ if len(part) >= 2 and part[1] == ":" and part[0].isalpha():+ raise PermissionDeniedError(+ f"Unsafe URI drive-prefixed component '{part}' in {normalized}",+ resource=normalized,+ )++ return normalized, parts++ # TODO: Once pathlock moves down into ragfs, stop reconstructing the+ # encrypted mount-relative path in Python and derive the lock target from+ # the same backend-side source of truth.+ def _encrypted_mount_relative_path(self, path: str) -> tuple[str, str]:+ """Return the mount prefix and mount-relative path used by Rust encrypted writes."""+ normalized = path if path.startswith("/") else f"/{path}"+ parts = [part for part in normalized.split("/") if part]+ if len(parts) < 2:+ return "", normalized+ return f"/{parts[0]}", f"/{'/'.join(parts[1:])}"++ def _encrypted_temp_path(self, path: str) -> str:+ """Build the deterministic internal `.encrypt` temp-file path for a final file path."""+ mount_prefix, relative_path = self._encrypted_mount_relative_path(path)+ relative_parts = [part for part in relative_path.split("/") if part]+ if len(relative_parts) >= 2 and relative_parts[0] == "local":+ temp_root = f"/local/{relative_parts[1]}/temp/.encrypt_stage"+ elif len(relative_parts) >= 2:+ temp_root = f"/{relative_parts[0]}/temp/.encrypt_stage"+ else:+ temp_root = "/temp/.encrypt_stage"+ digest = hashlib.sha256(relative_path.encode("utf-8")).hexdigest()+ return f"{mount_prefix}{temp_root}/{digest}.encrypt"++ def _ensure_access(self, uri: str, ctx: Optional[RequestContext]) -> None:+ real_ctx = self._ctx_or_default(ctx)+ normalized_uri, _ = self._normalized_uri_parts(uri)+ if not self._is_accessible(normalized_uri, real_ctx):+ raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)++ def _ensure_mutable_access(self, uri: str, ctx: Optional[RequestContext]) -> None:+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ self._ensure_user_not_deleting(real_ctx)+ normalized_uri, _ = self._normalized_uri_parts(uri)+ if is_hidden_by_actor_peer_view(normalized_uri, real_ctx) or may_include_hidden_actor_peers(+ normalized_uri, real_ctx+ ):+ raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)+ self._ensure_supported_write_namespace(normalized_uri)+ if real_ctx.role != Role.ROOT and normalized_uri.rstrip("/") == "viking://temp":+ raise PermissionDeniedError(+ "Temp root is read-only for non-root users",+ resource=normalized_uri,+ )++ def _ensure_delete_access(self, uri: str, ctx: Optional[RequestContext]) -> None:+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ self._ensure_user_not_deleting(real_ctx)+ normalized_uri, _ = self._normalized_uri_parts(uri)+ if is_hidden_by_actor_peer_view(normalized_uri, real_ctx) or may_include_hidden_actor_peers(+ normalized_uri, real_ctx+ ):+ raise PermissionDeniedError(f"Access denied for {uri}", resource=normalized_uri)+ self._ensure_supported_delete_namespace(normalized_uri)+ if real_ctx.role != Role.ROOT and normalized_uri.rstrip("/") == "viking://temp":+ raise PermissionDeniedError(+ "Temp root is read-only for non-root users",+ resource=normalized_uri,+ )++ def _ensure_user_not_deleting(self, ctx: RequestContext) -> None:+ guard = getattr(self, "_user_deletion_guard", None)+ if (+ ctx.role != Role.ROOT+ and guard is not None+ and guard(ctx.account_id, ctx.user.user_id)+ ):+ raise FailedPreconditionError("User deletion is in progress")++ def _ensure_supported_delete_namespace(self, normalized_uri: str) -> None:+ parts = [p for p in normalized_uri[len("viking://") :].strip("/").split("/") if p]+ if not parts:+ raise PermissionDeniedError(+ "Deleting viking:// is not supported; use a concrete scope instead.",+ resource=normalized_uri,+ )+ if parts == ["user"]:+ raise PermissionDeniedError(+ "Deleting viking://user is not supported; use an explicit user namespace "+ "or current-user content path instead.",+ resource=normalized_uri,+ )+ if parts == ["agent"]:+ # Parity with _ensure_supported_write_namespace, which forbids the+ # account-shared viking://agent root: deleting it would recursively+ # wipe every account's agent skills/endpoints/tools/payments. Concrete+ # sub-paths (viking://agent/skills/...) remain deletable.+ raise PermissionDeniedError(+ "Deleting viking://agent root is not supported; use a concrete "+ "agent sub-path (e.g. viking://agent/skills/...) instead.",+ resource=normalized_uri,+ )++ def _ensure_supported_write_namespace(self, normalized_uri: str) -> None:+ parts = [p for p in normalized_uri[len("viking://") :].strip("/").split("/") if p]+ if parts == ["user"]:+ raise PermissionDeniedError(+ "Writing viking://user is not supported; use an explicit user namespace "+ "or current-user content path instead.",+ resource=normalized_uri,+ )+ if parts and parts[0] == "session":+ raise PermissionDeniedError(+ f"Writing {normalized_uri} is not supported; use user-owned namespaces instead.",+ resource=normalized_uri,+ )+ if parts and parts[0] == "agent":+ if len(parts) >= 2 and parts[1] not in {"skills", "endpoints", "tools", "payments"}:+ raise PermissionDeniedError(+ "viking://agent/{agent_id} is deprecated. Use viking://user/.../peers/{agent_id} instead.",+ resource=normalized_uri,+ )+ if len(parts) < 2:+ raise PermissionDeniedError(+ "Writing to viking://agent root is not supported.",+ resource=normalized_uri,+ )++ def _pathlock_fs_ctx(+ self,+ ctx: Optional[RequestContext],+ lease_ref: Optional[Dict[str, Any] | str],+ ) -> Dict[str, str] | None:+ """Build an AGFS fs_ctx carrying account_id and an opaque pathlock lease_ref."""+ if lease_ref is None:+ return None+ if isinstance(lease_ref, str):+ ref = lease_ref+ elif isinstance(lease_ref, dict):+ ref = lease_ref.get("lease_ref")+ else:+ raise ValueError("lease_ref must be a non-empty string or lease dictionary")+ if not isinstance(ref, str) or not ref:+ raise ValueError("lease_ref must contain a non-empty lease_ref")+ return {"account_id": self._ctx_or_default(ctx).account_id, "lease_ref": ref}++ # ========== Tree Traversal (Refactored) ==========++ def _is_name_visible_at_path(self, name: str, parent_path: str) -> bool:+ """Check if name would appear in _ls_entries(parent_path).++ At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.+ At other levels, uses the shared storage internal-name blacklist.+ """+ parts = [p for p in parent_path.strip("/").split("/") if p]+ if len(parts) == 2 and parts[0] == "local":+ return name in VikingURI.LISTABLE_SCOPES+ return name not in STORAGE_INTERNAL_ENTRY_NAMES++ def _ancestor_is_filtered(self, entry_path: str, base_path: str) -> bool:+ """Check if any ancestor directory of entry_path would be filtered by _ls_entries.++ Walks from base_path (exclusive) to entry's parent directory (exclusive),+ checking each component against _is_name_visible_at_path.+ """+ base_parts = [p for p in base_path.strip("/").split("/") if p]+ entry_parts = [p for p in entry_path.strip("/").split("/") if p]++ for i in range(len(base_parts), len(entry_parts) - 1):+ name = entry_parts[i]+ parent_parts = entry_parts[:i]+ parent_path = "/" + "/".join(parent_parts) if parent_parts else "/"+ if not self._is_name_visible_at_path(name, parent_path):+ return True+ return False++ def _is_path_entry_visible(+ self, entry_path: str, name: str, base_path: str, ctx: RequestContext+ ) -> bool:+ """Check visibility for one flattened path entry returned by Rust."""+ if self._ancestor_is_filtered(entry_path, base_path):+ return False++ entry_parts = [p for p in entry_path.strip("/").split("/") if p]+ if entry_parts:+ parent_parts = entry_parts[:-1]+ parent_path = "/" + "/".join(parent_parts) if parent_parts else "/"+ if not self._is_name_visible_at_path(name, parent_path):+ return False++ uri = self._path_to_uri(entry_path, ctx=ctx)+ if not self._is_accessible(uri, ctx):+ return False++ return True++ def _is_tree_entry_visible(+ self, entry: Dict[str, Any], base_path: str, ctx: RequestContext+ ) -> bool:+ """Check visibility for a single TreeEntry returned by Rust tree_directory."""+ entry_path = entry["path"]+ entry_info = entry.get("info", {})+ name = entry_info.get("name") or entry_path.rstrip("/").rsplit("/", 1)[-1]+ return self._is_path_entry_visible(entry_path, name, base_path, ctx)++ def _glob_page_size(self, node_limit: Optional[int]) -> int:+ """Return the backend page size used by glob_directory."""+ if node_limit is None or node_limit <= 0:+ return self._GLOB_PAGE_SIZE_DEFAULT+ return node_limit++ async def _iter_visible_tree_entries(+ self,+ uri: str,+ show_all_hidden: bool = False,+ node_limit: Optional[int] = None,+ level_limit: Optional[int] = None,+ ctx: Optional[RequestContext] = None,+ ):+ """Shared generator: fetch raw TreeEntry list from Rust, yield (entry, uri) tuples.++ node_limit counts ACL-visible entries (see design §6.5), so the user's+ node_limit cannot be pushed directly to Rust — doing so would truncate+ before filtering and drop entries that should be visible.++ To keep memory bounded without changing that semantic, we push down an+ *amplified* raw-node limit (node_limit * _TREE_OVERFETCH_FACTOR). If ACL+ filtering leaves fewer than node_limit visible entries while Rust still+ returned a full page (i.e. more raw nodes may exist), we double the raw+ limit and re-fetch. Because Rust truncates a deterministic sorted prefix,+ this yields exactly the same result as an unbounded fetch, while avoiding+ materializing the entire prefix in the common case.++ When node_limit is None (full-tree callers), no limit is pushed down.+ level_limit IS always passed to Rust.+ """+ real_ctx = self._ctx_or_default(ctx)+ primary_path = self._uri_to_path(uri, ctx=ctx)+ path: Optional[str] = None+ for candidate_path in self._read_paths(uri, ctx=ctx):+ if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):+ continue+ if await self._agfs_path_exists(candidate_path):+ path = candidate_path+ break+ if path is None:+ if self._is_legacy_session_root_uri(uri):+ return+ raise NotFoundError(uri, "directory")++ if node_limit is None:+ raw_limit: Optional[int] = None+ else:+ raw_limit = max(node_limit * self._TREE_OVERFETCH_FACTOR, node_limit)++ while True:+ raw_entries = await self._async_agfs.tree_directory(+ path,+ show_hidden=show_all_hidden,+ node_limit=raw_limit,+ level_limit=level_limit,+ )++ visible: List[tuple] = []+ for entry in raw_entries:+ if node_limit is not None and len(visible) >= node_limit:+ break+ if not self._is_tree_entry_visible(entry, path, real_ctx):+ continue+ if not await self._read_path_visible(uri, entry["path"], primary_path, real_ctx):+ continue+ entry_uri = self._alias_uri_for_path(+ request_uri=uri,+ base_path=path,+ entry_path=entry["path"],+ ctx=ctx,+ )+ visible.append((entry, entry_uri))++ # If we still lack enough visible entries but Rust returned a full+ # page (raw_limit reached), more raw nodes may exist — re-fetch with+ # a doubled limit. Otherwise Rust is exhausted and we yield as-is.+ need_more = (+ node_limit is not None+ and len(visible) < node_limit+ and raw_limit is not None+ and len(raw_entries) >= raw_limit+ )+ if need_more:+ raw_limit *= 2+ continue++ for item in visible:+ yield item+ return++ # ========== URI Conversion ==========++ @staticmethod+ def _shorten_component(component: str, max_bytes: int = 255) -> str:+ """Shorten a path component if its UTF-8 encoding exceeds max_bytes."""+ if len(component.encode("utf-8")) <= max_bytes:+ return component+ hash_suffix = hashlib.sha256(component.encode("utf-8")).hexdigest()[:8]+ # Trim to fit within max_bytes after adding hash suffix+ prefix = component+ target = max_bytes - len(f"_{hash_suffix}".encode("utf-8"))+ while len(prefix.encode("utf-8")) > target and prefix:+ prefix = prefix[:-1]+ return f"{prefix}_{hash_suffix}"++ def _uri_to_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:+ """Map virtual URI to account-isolated AGFS path.++ Pure prefix replacement: viking://{remainder} -> /local/{account_id}/{remainder}.+ No implicit space injection — URIs must include space segments explicitly.+ """+ real_ctx = self._ctx_or_default(ctx)+ account_id = real_ctx.account_id+ normalized_uri, legacy_parts = self._normalized_uri_parts(uri)+ if legacy_parts and legacy_parts[0] == "agent" and self._is_legacy_agent_id_uri(uri):+ # Old format: viking://agent/{agent_id}/... — direct mapping for read-only compat+ safe_parts = [+ self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in legacy_parts+ ]+ return f"/local/{account_id}/{'/'.join(safe_parts)}"+ canonical_uri = canonicalize_uri(uri, real_ctx)+ _, parts = self._normalized_uri_parts(canonical_uri)+ if not parts:+ return f"/local/{account_id}"++ safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]+ return f"/local/{account_id}/{'/'.join(safe_parts)}"++ def _legacy_session_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:+ """Map a legacy viking://session URI to its pre-user-namespace path."""+ real_ctx = self._ctx_or_default(ctx)+ _, parts = self._normalized_uri_parts(uri)+ safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]+ return f"/local/{real_ctx.account_id}/{'/'.join(safe_parts)}"++ def _legacy_current_user_session_path(+ self, uri: str, ctx: Optional[RequestContext] = None+ ) -> Optional[str]:+ """Return the legacy nested /session/{user_id}/{session_id} candidate."""+ real_ctx = self._ctx_or_default(ctx)+ _, parts = self._normalized_uri_parts(uri)+ if len(parts) <= 1 or parts[0] != "session":+ return None+ nested_parts = ["session", real_ctx.user.user_id, *parts[1:]]+ safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in nested_parts]+ return f"/local/{real_ctx.account_id}/{'/'.join(safe_parts)}"++ def _is_legacy_session_uri(self, uri: str) -> bool:+ _, parts = self._normalized_uri_parts(uri)+ return bool(parts and parts[0] == "session")++ def _is_legacy_session_root_uri(self, uri: str) -> bool:+ _, parts = self._normalized_uri_parts(uri)+ return parts == ["session"]++ def _is_legacy_agent_id_uri(self, uri: str) -> bool:+ _, parts = self._normalized_uri_parts(uri)+ return bool(+ parts+ and parts[0] == "agent"+ and len(parts) >= 2+ and parts[1] not in {"skills", "endpoints", "tools", "payments"}+ )++ def _read_paths(self, uri: str, ctx: Optional[RequestContext] = None) -> List[str]:+ """Return read candidates for a URI, including legacy alias fallbacks."""+ paths = [self._uri_to_path(uri, ctx=ctx)]++ if self._is_legacy_session_uri(uri):+ for candidate in (+ self._legacy_session_path(uri, ctx=ctx),+ self._legacy_current_user_session_path(uri, ctx=ctx),+ ):+ if candidate and candidate not in paths:+ paths.append(candidate)+ return paths++ async def _read_path_visible(+ self,+ request_uri: str,+ path: str,+ primary_path: str,+ ctx: RequestContext,+ ) -> bool:+ if path == primary_path:+ return True+ if self._is_legacy_session_uri(request_uri):+ return await self._legacy_session_path_visible(path, ctx)+ return True++ def _alias_uri_for_path(+ self,+ *,+ request_uri: str,+ base_path: str,+ entry_path: str,+ ctx: Optional[RequestContext],+ ) -> str:+ base = base_path.rstrip("/")+ normalized_request, request_parts = self._normalized_uri_parts(request_uri)+ request_root = (+ normalized_request+ if normalized_request == "viking://"+ else normalized_request.rstrip("/")+ )+ preserve_request_alias = normalized_request in {"viking://", "viking://user"} or bool(+ request_parts+ and (+ request_parts[0] == "session"+ or (request_parts[0] == "agent" and self._is_legacy_agent_id_uri(request_uri))+ )+ )+ rel_path = entry_path[len(base) :].strip("/") if entry_path.startswith(base) else ""+ if entry_path.startswith(base):+ separator = "" if request_root.endswith("://") else "/"+ candidate_uri = request_root if not rel_path else f"{request_root}{separator}{rel_path}"+ if preserve_request_alias:+ return candidate_uri+ try:+ if self._uri_to_path(candidate_uri, ctx=ctx) == entry_path:+ return candidate_uri+ except Exception:+ pass+ return self._path_to_uri(entry_path, ctx=ctx)++ async def _agfs_path_exists(self, path: str) -> bool:+ try:+ await self._async_agfs.stat(path)+ return True+ except Exception as exc:+ if is_not_found_error(exc):+ return False+ raise++ async def _looks_like_legacy_session_dir(self, path: str) -> bool:+ for leaf in (".meta.json", "messages.jsonl", "history", "tool-results", "tools"):+ if await self._agfs_path_exists(f"{path}/{leaf}"):+ return True+ return False++ async def _legacy_session_owner(self, session_root_path: str) -> str:+ try:+ raw = self._handle_agfs_read(+ await self._async_agfs.read(f"{session_root_path}/.meta.json")+ )+ except Exception as exc:+ if is_not_found_error(exc):+ return ""+ raise+ try:+ data = json.loads(self._decode_bytes(raw))+ except json.JSONDecodeError:+ return ""+ if not isinstance(data, dict):+ return ""+ for key in ("created_by_user_id", "user_id", "owner_user_id", "created_by"):+ value = data.get(key)+ if isinstance(value, str) and value.strip():+ return value.strip()+ return ""++ async def _legacy_session_visible(+ self,+ session_root_path: str,+ ctx: RequestContext,+ *,+ owner_hint: Optional[str] = None,+ ) -> bool:+ if ctx.role == Role.ROOT:+ return True+ owner = await self._legacy_session_owner(session_root_path)+ if owner:+ return owner == ctx.user.user_id+ if owner_hint:+ return owner_hint == ctx.user.user_id+ return True++ async def _legacy_session_path_visible(self, path: str, ctx: RequestContext) -> bool:+ parts = [p for p in path.strip("/").split("/") if p]+ try:+ session_index = parts.index("session")+ except ValueError:+ return True+ suffix = parts[session_index + 1 :]+ if not suffix:+ return True++ root_prefix = "/" + "/".join(parts[: session_index + 1])+ direct_root = f"{root_prefix}/{suffix[0]}"+ if await self._looks_like_legacy_session_dir(direct_root):+ return await self._legacy_session_visible(direct_root, ctx)++ if len(suffix) >= 2:+ nested_root = f"{root_prefix}/{suffix[0]}/{suffix[1]}"+ if await self._looks_like_legacy_session_dir(nested_root):+ return await self._legacy_session_visible(+ nested_root,+ ctx,+ owner_hint=suffix[0],+ )+ return True++ async def _legacy_session_root_items(+ self,+ path: str,+ ctx: RequestContext,+ ) -> List[tuple[Dict[str, Any], str]]:+ try:+ entries = await self._ls_entries(path)+ except Exception as exc:+ if is_not_found_error(exc):+ return []+ raise++ items: List[tuple[Dict[str, Any], str]] = []+ for entry in entries:+ name = entry.get("name", "")+ if not name or name in {".", ".."} or not entry.get("isDir"):+ continue+ child_path = f"{path.rstrip('/')}/{name}"+ if await self._looks_like_legacy_session_dir(child_path):+ if await self._legacy_session_visible(child_path, ctx):+ items.append((entry, f"viking://session/{name}"))+ continue++ if ctx.role != Role.ROOT and name != ctx.user.user_id:+ continue+ try:+ nested_entries = await self._ls_entries(child_path)+ except Exception as exc:+ if is_not_found_error(exc):+ continue+ raise+ for nested in nested_entries:+ nested_name = nested.get("name", "")+ if not nested_name or nested_name in {".", ".."} or not nested.get("isDir"):+ continue+ nested_path = f"{child_path.rstrip('/')}/{nested_name}"+ if not await self._looks_like_legacy_session_dir(nested_path):+ continue+ if await self._legacy_session_visible(nested_path, ctx, owner_hint=name):+ items.append((nested, f"viking://session/{nested_name}"))+ return items++ async def _session_root_items(+ self,+ uri: str,+ ctx: RequestContext,+ ) -> List[tuple[Dict[str, Any], str]]:+ primary_path = self._uri_to_path(uri, ctx=ctx)+ by_name: Dict[str, tuple[Dict[str, Any], str]] = {}+ try:+ for entry in await self._ls_entries(primary_path, ctx=ctx):+ name = entry.get("name", "")+ if not name or name in {".", ".."}:+ continue+ by_name[name] = (entry, f"viking://session/{name}")+ except Exception as exc:+ if not is_not_found_error(exc):+ raise++ legacy_path = self._legacy_session_path(uri, ctx=ctx)+ for entry, entry_uri in await self._legacy_session_root_items(legacy_path, ctx):+ name = entry.get("name", "")+ if name and name not in by_name:+ by_name[name] = (entry, entry_uri)+ return list(by_name.values())++ async def _list_read_path_items(+ self,+ uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> List[tuple[Dict[str, Any], str]]:+ real_ctx = self._ctx_or_default(ctx)+ if self._is_legacy_session_root_uri(uri):+ return await self._session_root_items(uri, real_ctx)++ primary_path = self._uri_to_path(uri, ctx=ctx)+ merge_paths = self._is_legacy_session_uri(uri)+ found_path = False+ last_not_found: Optional[Exception] = None+ by_uri: Dict[str, tuple[Dict[str, Any], str]] = {}++ for path in self._read_paths(uri, ctx=ctx):+ if not await self._read_path_visible(uri, path, primary_path, real_ctx):+ continue+ try:+ entries = await self._ls_entries(path, ctx=ctx)+ except Exception as exc:+ if is_not_found_error(exc):+ last_not_found = exc+ continue+ raise++ found_path = True+ for entry in entries:+ entry_uri = self._alias_uri_for_path(+ request_uri=uri,+ base_path=path,+ entry_path=f"{path.rstrip('/')}/{entry.get('name', '')}",+ ctx=ctx,+ )+ by_uri.setdefault(entry_uri, (entry, entry_uri))+ if not merge_paths:+ break++ if found_path:+ return list(by_uri.values())+ raise NotFoundError(uri, "directory") from last_not_found++ def _path_to_uri(self, path: str, ctx: Optional[RequestContext] = None) -> str:+ """/local/{account}/... -> viking://...++ Pure prefix replacement: strips /local/{account_id}/ and prepends viking://.+ No implicit space stripping.+ """+ if path.startswith("viking://"):+ return path+ elif path.startswith("/local/"):+ inner = path[7:].strip("/")+ if not inner:+ return "viking://"+ real_ctx = self._ctx_or_default(ctx)+ parts = [p for p in inner.split("/") if p]+ if parts and parts[0] == real_ctx.account_id:+ parts = parts[1:]+ if not parts:+ return "viking://"+ return f"viking://{'/'.join(parts)}"+ elif path.startswith("/"):+ return f"viking:/{path}"+ else:+ return f"viking://{path}"++ def _looks_like_legacy_temp_leaf(self, value: str) -> bool:+ return bool(re.match(r"^\d{8}_[0-9a-f]{6}$", value or ""))++ def _is_legacy_temp_uri_parts(self, parts: List[str]) -> bool:+ if len(parts) < 2 or parts[0] != "temp" or not self._looks_like_legacy_temp_leaf(parts[1]):+ return False+ if len(parts) == 2:+ return True+ return not self._looks_like_legacy_temp_leaf(parts[2])++ def _is_accessible(self, uri: str, ctx: RequestContext) -> bool:+ """Check whether a URI is visible/accessible under current request context."""+ normalized_uri, parts = self._normalized_uri_parts(uri)+ if ctx.role == Role.ROOT:+ return True+ if is_hidden_by_actor_peer_view(normalized_uri, ctx):+ return False+ if not parts:+ return True+ if is_watch_task_control_uri(normalized_uri):+ return False++ scope = parts[0]+ if scope == "resources":+ return True+ if scope == "temp":+ if len(parts) == 1:+ return True+ if parts[1] == ctx.user.user_space_name():+ return True+ return self._is_legacy_temp_uri_parts(parts)+ if scope == "upload":+ return ctx.role == Role.ROOT+ if scope == "_system":+ return False+ if scope == "agent":+ # New format: agent/skills/..., agent/endpoints/... — globally readable (account scope)+ if len(parts) >= 2 and parts[1] in {"skills", "endpoints", "tools", "payments"}:+ return True+ # Old format: agent/{agent_id}/... — actor_peer_id match for read-only access+ if not ctx.actor_peer_id or len(parts) < 2:+ return True+ return parts[1] == ctx.actor_peer_id+ return namespace_is_accessible(normalized_uri, ctx)++ def _handle_agfs_read(self, result: Union[bytes, Any, None]) -> bytes:+ """Handle AGFSClient read return types consistently."""+ if isinstance(result, bytes):+ return result+ elif result is None:+ return b""+ elif hasattr(result, "content") and result.content is not None:+ return result.content+ else:+ # Try to convert to bytes+ try:+ return str(result).encode("utf-8")+ except Exception:+ return b""++ def _decode_bytes(self, data: bytes) -> str:+ """Robustly decode bytes to string."""+ if not data:+ return ""+ try:+ return data.decode("utf-8")+ except UnicodeDecodeError:+ try:+ # Try common encoding for Windows/legacy files in China+ return data.decode("gbk")+ except UnicodeDecodeError:+ try:+ return data.decode("latin-1")+ except UnicodeDecodeError:+ return data.decode("utf-8", errors="replace")++ async def _is_path_locked_async(self, path: str) -> bool:+ """Best-effort async path-lock lookup; returns False on any error."""+ try:+ return await self._async_agfs.pathlock_is_locked(path)+ except Exception:+ return False++ def _gitignore_agfs_path(self, ctx: Optional[RequestContext] = None) -> str:+ real_ctx = self._ctx_or_default(ctx)+ return f"/local/{real_ctx.account_id}/{self._OVGITIGNORE_TREE_PATH}"++ def _uri_to_tree_path(self, uri: str, ctx: Optional[RequestContext] = None) -> str:+ """Convert a viking:// URI to an account-relative git tree path.++ ``viking://resources/proj_a/docs/a.md`` -> ``resources/proj_a/docs/a.md``.++ Pure prefix stripping: removes the ``viking://`` scheme and any+ ``/local/{account}/`` segment. Internal scopes that the Rust git layer+ would prune (`_system`, `tasks`, `temp`, `queue`, `upload`) and the+ runtime lock name (`.path.ovlock`) are rejected with ``ValueError``+ — passing them through silently would result in a no-op commit and+ confuse callers.+ """+ real_ctx = self._ctx_or_default(ctx)+ canonical = canonicalize_uri(uri, real_ctx)+ _, parts = self._normalized_uri_parts(canonical)+ if not parts:+ raise ValueError(f"git tree path cannot be the account root: {uri!r}")+ first = parts[0]+ if first in self._GIT_INTERNAL_FIRST_SEGMENTS:+ raise ValueError(f"git tree path rejects internal scope/segment {first!r}: {uri!r}")+ return "/".join(parts)++ def _tree_path_to_uri(self, tree_path: str) -> str:+ """Convert an account-relative git tree path to a viking:// URI.++ Inverse of :py:meth:`_uri_to_tree_path` (without context canonicalization).+ """+ cleaned = tree_path.strip("/")+ if not cleaned:+ raise ValueError("tree path must not be empty")+ return f"viking://{cleaned}"++ def _classify_restore_path(self, tree_path: str, *, deleted: bool) -> Optional[tuple]:+ """Classify a restore-affected tree path into a vector maintenance task.++ Returns a ``(op, uri, level)`` triple, or ``None`` when the path has no+ vector side-effect:++ - ``dir/.abstract.md`` / ``dir/.overview.md`` → recompute (write) or+ delete (removal) ONLY that directory's L0/L1 vector:+ ``("reindex_marker"|"delete", dir_uri, ABSTRACT|OVERVIEW)``.+ - ``.relations.json`` → ``None`` (not a vector text source).+ - anything else (a source file) → reindex (write) or delete (removal)+ its DETAIL vector:+ ``("reindex_file", file_uri, DETAIL)`` / ``("delete", file_uri, DETAIL)``.++ ``None`` is also returned for a directory marker at the account root+ (no parent directory to scope an L0/L1 vector to).+ """+ parent, _, name = tree_path.rpartition("/")+ if name in self._NO_VECTOR_DERIVED:+ return None+ level = self._DIR_MARKER_LEVELS.get(name)+ if level is not None:+ if not parent:+ return None+ dir_uri = self._tree_path_to_uri(parent)+ op = "delete" if deleted else "reindex_marker"+ return (op, dir_uri, level)+ # Source file.+ file_uri = self._tree_path_to_uri(tree_path)+ op = "delete" if deleted else "reindex_file"+ return (op, file_uri, ContextLevel.DETAIL)diff --git a/openviking/storage/viking_fs/_base.py b/openviking/storage/viking_fs/_base.pynew file mode 100644index 000000000..bde4fdf8e--- /dev/null+++ b/openviking/storage/viking_fs/_base.py@@ -0,0 +1,254 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Shared constants, helpers, dataclasses, and singleton management for VikingFS."""++import os+from dataclasses import dataclass, field+from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypeVar++from openviking.utils.time_utils import get_current_timestamp+from openviking_cli.exceptions import (+ InvalidArgumentError,+ ResourceExhaustedError,+)+from openviking_cli.utils.logger import get_logger++if TYPE_CHECKING:+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig++logger = get_logger(__name__)++# Sentinel node_limit for internal callers that MUST enumerate an entire+# directory. ``ls()`` defaults to ``node_limit=1000`` to protect agent-facing+# context from being flooded, but internal system operations (parse merge,+# temp->final sync, summary DAG, vectorization) must see every child or they+# silently drop entries beyond the cap — e.g. a >1000-doc directory ingest only+# materializes its first 1000 subdirectories. Pass this explicitly at those+# call sites.+LS_ALL_NODES = 2**31 - 1+SNAPSHOT_DIFF_MAX_FILE_BYTES = 10 * 1024 * 1024+SNAPSHOT_DIFF_MAX_OUTPUT_BYTES = 20 * 1024 * 1024+SNAPSHOT_DIFF_MAX_LINES = 100_000+SNAPSHOT_DIFF_TIMEOUT_MS = 500+_T = TypeVar("_T")+++def _snapshot_line_count(text: str) -> int:+ if not text:+ return 0+ line_breaks = (+ "\n",+ "\r",+ "\v",+ "\f",+ "\x1c",+ "\x1d",+ "\x1e",+ "\x85",+ "\u2028",+ "\u2029",+ )+ count = sum(text.count(separator) for separator in line_breaks)+ count -= text.count("\r\n")+ return count if text.endswith(line_breaks) else count + 1+++def _prepare_snapshot_diff(+ *,+ path: str,+ before_bytes: Optional[bytes],+ after_bytes: Optional[bytes],+ max_lines: int,+) -> tuple[str, str, str]:+ try:+ before = before_bytes.decode("utf-8") if before_bytes is not None else ""+ after = after_bytes.decode("utf-8") if after_bytes is not None else ""+ except UnicodeDecodeError as exc:+ raise InvalidArgumentError("snapshot diff only supports UTF-8 text files") from exc++ for text in (before, after):+ line_count = _snapshot_line_count(text)+ if line_count > max_lines:+ raise ResourceExhaustedError(+ f"snapshot diff line count limit exceeded ({max_lines} lines per file)",+ details={"limit_lines": max_lines, "path": path},+ )++ if before_bytes is None:+ change_type = "added"+ elif after_bytes is None:+ change_type = "deleted"+ elif before_bytes == after_bytes:+ change_type = "unchanged"+ else:+ change_type = "modified"++ return change_type, before, after+++def _ensure_non_empty_search_query(query: str, image_url: Optional[str] = None) -> None:+ if not query.strip() and not image_url:+ raise InvalidArgumentError("Search query or image_url must not be empty.")+++def _is_directory_not_empty_error(message: str) -> bool:+ """Check if an error message indicates a directory not empty error.++ Handles multiple possible error message formats from different backends.+ """+ msg = message.lower()+ return any(+ pattern in msg+ for pattern in [+ "directory not empty",+ "dir not empty",+ "directory is not empty",+ ]+ )+++def _get_cpu_count() -> int:+ """Return the number of CPUs available to this process.++ Tries process_cpu_count (Python 3.13+, cgroup-aware),+ falls back to sched_getaffinity (Linux),+ then os.cpu_count (may report host CPUs in containers).+ """+ if hasattr(os, "process_cpu_count"):+ return os.process_cpu_count() or 1+ try:+ return len(os.sched_getaffinity(0))+ except (AttributeError, NotImplementedError):+ return os.cpu_count() or 1+++def _get_abstract_worker_count() -> int:+ default = max(4, min(12, min(32, _get_cpu_count() + 4) // 2))+ env_val = os.getenv("OPENVIKING_FILE_OPS_CONCURRENCY")+ if env_val is not None:+ try:+ return max(1, int(env_val))+ except ValueError:+ pass+ return max(1, default)+++_ABSTRACT_WORKER_COUNT = _get_abstract_worker_count()+_DEFAULT_GREP_FILE_CONCURRENCY = 32+++# ========== Dataclass ==========+++@dataclass+class RelationEntry:+ """Relation table entry."""++ id: str+ uris: List[str]+ reason: str = ""+ created_at: str = field(default_factory=get_current_timestamp)++ def to_dict(self) -> Dict[str, Any]:+ return {+ "id": self.id,+ "uris": self.uris,+ "reason": self.reason,+ "created_at": self.created_at,+ }++ @staticmethod+ def from_dict(data: Dict[str, Any]) -> "RelationEntry":+ return RelationEntry(**data)+++# ========== Singleton Pattern ==========++_instance: Optional["Any"] = None+++def init_viking_fs(+ agfs: Any,+ query_embedder: Optional[Any] = None,+ rerank_config: Optional["RerankConfig"] = None,+ vector_store: Optional["VikingVectorIndexBackend"] = None,+ retrieval_config: Optional["RetrievalConfig"] = None,+ grep_config: Optional["GrepConfig"] = None,+ timeout: int = 10,+ enable_recorder: bool = False,+ encryptor: Optional[Any] = None,+):+ """Initialize VikingFS singleton.++ Args:+ agfs: Pre-initialized AGFS client (HTTP or Binding)+ query_embedder: Embedder instance+ rerank_config: Rerank configuration+ retrieval_config: Retrieval ranking configuration+ grep_config: Grep engine configuration+ vector_store: Vector store instance+ enable_recorder: Whether to enable IO recording+ encryptor: FileEncryptor instance for encryption/decryption+ """+ from openviking.storage.viking_fs import VikingFS++ global _instance++ _instance = VikingFS(+ agfs=agfs,+ query_embedder=query_embedder,+ rerank_config=rerank_config,+ vector_store=vector_store,+ retrieval_config=retrieval_config,+ grep_config=grep_config,+ encryptor=encryptor,+ )++ if enable_recorder:+ _enable_viking_fs_recorder(_instance)++ return _instance+++def _enable_viking_fs_recorder(viking_fs) -> None:+ """+ Enable recorder for a VikingFS instance.++ This wraps the VikingFS instance with recording capabilities.+ Called automatically when enable_recorder=True in init_viking_fs.++ Args:+ viking_fs: VikingFS instance to enable recording for+ """+ from openviking.eval.recorder import RecordingVikingFS, get_recorder++ recorder = get_recorder()+ if not recorder.enabled:+ from openviking.eval.recorder import init_recorder++ init_recorder(enabled=True)++ global _instance+ _instance = RecordingVikingFS(viking_fs)+ logger.info("[VikingFS] IO Recorder enabled")+++def enable_viking_fs_recorder() -> None:+ """+ Enable recorder for the global VikingFS singleton.++ This function wraps the existing VikingFS's AGFS client with recording.+ Must be called after init_viking_fs().+ """+ global _instance+ if _instance is None:+ raise RuntimeError("VikingFS not initialized. Call init_viking_fs() first.")+ _enable_viking_fs_recorder(_instance)+++def get_viking_fs():+ """Get VikingFS singleton."""+ if _instance is None:+ raise RuntimeError("VikingFS not initialized. Call init_viking_fs() first.")+ return _instancediff --git a/openviking/storage/viking_fs/_grep.py b/openviking/storage/viking_fs/_grep.pynew file mode 100644index 000000000..aa9b8ca44--- /dev/null+++ b/openviking/storage/viking_fs/_grep.py@@ -0,0 +1,641 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Grep and search-backend mixin for VikingFS."""++import asyncio+import re+import sys+import time+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional++from openviking.core.namespace import canonicalize_uri+from openviking.pyagfs.exceptions import AGFSNotSupportedError+from openviking.server.identity import RequestContext+from openviking.storage.expr import And, PathScope, RawDSL+from openviking.storage.viking_fs._base import logger+from openviking_cli.utils.config.grep_config import GrepEngine+++def _pkg():+ return sys.modules[__package__]+++if TYPE_CHECKING:+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig+++class _GrepMixin:+ """Grep and search-backend methods."""++ async def grep(+ self,+ uri: str,+ pattern: str,+ exclude_uri: Optional[str] = None,+ case_insensitive: bool = False,+ node_limit: Optional[int] = None,+ level_limit: int = 10,+ ctx: Optional[RequestContext] = None,+ content_transform: Optional[Callable[[str, str], str]] = None,+ ) -> Dict:+ """Content search by pattern or keywords.++ Optimized implementation that uses agfs native grep when possible.+ The ragfs layer greps transparently over encrypted and plaintext files+ (it decrypts via account_id when an encryption layer is configured).+ Falls back to VikingFS layer implementation if native grep is unavailable.+ When engine="auto" and vikingdb is available with sufficient data,+ uses vikingdb bm25 recall + local fs precise matching.++ Args:+ uri: Viking URI+ pattern: Regular expression pattern to search for+ exclude_uri: Optional URI prefix to exclude from search+ case_insensitive: Whether to perform case-insensitive matching+ node_limit: Maximum number of results to return+ level_limit: Maximum depth level to traverse (default: 10)+ ctx: Request context+ content_transform: Optional projection applied before regex matching.+ Internal bm25 recall limit is auto-adapted from node_limit as+ min(node_limit * 5, 100000); when node_limit is unset, use 100000.++ Returns:+ Dict with matches, count, match_count, files_scanned+ """+ real_ctx = self._ctx_or_default(ctx)+ uri = canonicalize_uri(uri, real_ctx)+ if exclude_uri:+ exclude_uri = canonicalize_uri(exclude_uri, real_ctx)+ self._ensure_access(uri, ctx)+ # Skip vector_store.count() — the count field is not needed for grep,+ # and avoiding it saves one VikingDB API call.+ await self.stat(uri, ctx=ctx, skip_count=True)++ # Read engine and threshold from grep_config (ov.conf)+ engine = self.grep_config.engine if self.grep_config else "auto"+ switch_to_remote_threshold = (+ self.grep_config.switch_to_remote_threshold if self.grep_config else 10000+ )++ # A projection must run before matching. The remote BM25 index contains+ # persisted raw content, so it cannot safely recall projected results.+ resolved_engine = (+ "fs"+ if content_transform is not None+ else await self._resolve_grep_engine(engine, uri, ctx, switch_to_remote_threshold)+ )++ if resolved_engine == "fs":+ return await self._grep_fs(+ uri=uri,+ pattern=pattern,+ exclude_uri=exclude_uri,+ case_insensitive=case_insensitive,+ node_limit=node_limit,+ level_limit=level_limit,+ ctx=ctx,+ content_transform=content_transform,+ )+ else: # "vikingdb_then_fs"+ return await self._grep_vikingdb_then_fs(+ uri=uri,+ pattern=pattern,+ exclude_uri=exclude_uri,+ case_insensitive=case_insensitive,+ node_limit=node_limit,+ level_limit=level_limit,+ ctx=ctx,+ )++ async def _resolve_grep_engine(+ self, engine: GrepEngine, uri: str, ctx, switch_to_remote_threshold: int = 10000+ ) -> str:+ """Resolve the actual grep engine to use."""+ if engine == "fs":+ return "fs"++ # auto mode: check vikingdb availability+ vector_store = self._get_vector_store()+ if not vector_store:+ return "fs"++ backend_type = getattr(vector_store, "_backend_type", "unknown")+ # Keep this set consistent with ``CollectionAdapter.USE_CONTENT_FIELD``:+ # only these backends store the ``content`` field required for full-text grep.+ if backend_type not in ("volcengine", "vikingdb"):+ return "fs"++ # Check collection has content field and FullText config+ if not await self._collection_has_fulltext(vector_store, ctx):+ return "fs"++ # switch_to_remote_threshold=0 means always use vikingdb+ if switch_to_remote_threshold == 0:+ return "vikingdb_then_fs"++ # Check data volume threshold+ try:+ count = await self._get_cached_count(uri, ctx)+ if count < switch_to_remote_threshold:+ return "fs"+ except Exception:+ logger.debug(+ "grep engine=auto: count() check failed, falling back to fs", exc_info=True+ )+ return "fs"++ return "vikingdb_then_fs"++ async def _collection_has_fulltext(self, vector_store, ctx) -> bool:+ """Check if collection has content field and FullText config.++ Result is cached on the VikingFS instance since collection schema+ does not change at runtime.+ """+ if self._fulltext_available is not None:+ return self._fulltext_available+ try:+ meta = None+ if hasattr(vector_store, "get_collection_meta"):+ meta = await vector_store.get_collection_meta(ctx=ctx)+ if not meta:+ self._fulltext_available = False+ return False+ fields = meta.get("Fields", [])+ has_content = any(+ f.get("FieldName") == "content" and f.get("FieldType") == "text" for f in fields+ )+ fulltext = meta.get("FullText") or []+ has_content_fulltext = any(ft.get("Field") == "content" for ft in fulltext)+ result = has_content and has_content_fulltext+ self._fulltext_available = result+ return result+ except Exception:+ logger.debug(+ "Failed to check collection fulltext config, assuming no fulltext", exc_info=True+ )+ return False++ async def _get_cached_count(self, uri: str, ctx) -> int:+ """Get cached count of records for a URI (TTL=1h)."""+ _COUNT_CACHE_TTL = 3600+ vector_store = self._get_vector_store()++ # Include account_id in cache key for multi-tenant safety+ account_id = getattr(ctx, "account_id", None) if ctx else None+ cache_key = f"{account_id}:{uri}" if account_id else uri++ now = time.time()+ cached = self._count_cache.get(cache_key)+ if cached and (now - cached[1]) < _COUNT_CACHE_TTL:+ return cached[0]++ count = await vector_store.count(filter=PathScope("uri", uri, depth=-1), ctx=ctx)+ # Evict oldest entries if cache exceeds max size+ if len(self._count_cache) >= self._count_cache_max_size:+ oldest_keys = sorted(self._count_cache, key=lambda k: self._count_cache[k][1])+ for k in oldest_keys[: len(oldest_keys) // 2]:+ del self._count_cache[k]+ self._count_cache[cache_key] = (count, now)+ return count++ async def _grep_fs(+ self,+ uri,+ pattern,+ exclude_uri,+ case_insensitive,+ node_limit,+ level_limit,+ ctx,+ content_transform=None,+ ):+ """Filesystem grep path: prefer native agfs grep and fall back if unavailable."""+ if content_transform is None:+ try:+ return await self._grep_with_agfs(+ uri=uri,+ pattern=pattern,+ exclude_uri=exclude_uri,+ case_insensitive=case_insensitive,+ node_limit=node_limit,+ level_limit=level_limit,+ ctx=ctx,+ )+ except (AttributeError, AGFSNotSupportedError, NotImplementedError) as e:+ logger.debug(f"agfs grep unavailable, falling back to VikingFS implementation: {e}")++ return await self._grep_encrypted(+ uri=uri,+ pattern=pattern,+ exclude_uri=exclude_uri,+ case_insensitive=case_insensitive,+ node_limit=node_limit,+ level_limit=level_limit,+ ctx=ctx,+ content_transform=content_transform,+ )++ async def _grep_vikingdb_then_fs(+ self,+ uri,+ pattern,+ exclude_uri,+ case_insensitive,+ node_limit,+ level_limit,+ ctx,+ ):+ """VikingDB bm25 recall + local fs precise matching."""+ vector_store = self._get_vector_store()++ # Split regex alternation (e.g. "error|warning|fail") and join as a+ # single query string for bm25 search. VikingDB's standard tokenizer+ # will handle the tokenization of the query string.+ query = " ".join(kw.strip() for kw in pattern.split("|") if kw.strip())+ filter_expr = PathScope("uri", uri, depth=level_limit)+ excluded_prefix = None+ if exclude_uri:+ excluded_prefix = exclude_uri.rstrip("/")+ self._ensure_access(excluded_prefix, ctx)+ filter_expr = And([+ filter_expr,+ RawDSL({+ "op": "must_not",+ "field": "uri",+ "conds": [excluded_prefix],+ "para": "-d=-1",+ }),+ ])++ # Auto-adapt bm25 recall limit: recall up to 5x requested matches+ # while capping at VikingDB's max limit. If node_limit is unset,+ # use the maximum limit to avoid truncation.+ remote_return_limit = min(node_limit * 5, 100000) if node_limit else 100000++ # Step 1: vikingdb recall candidate files+ try:+ logger.debug(+ "grep vikingdb search_by_keywords request: query=%r limit=%s filter=%r "+ "output_fields=%s",+ query,+ remote_return_limit,+ filter_expr,+ ["uri"],+ )+ result = await vector_store.search_by_keywords(+ query=query,+ limit=remote_return_limit,+ filter=filter_expr,+ output_fields=["uri"],+ ctx=ctx,+ )+ except Exception as e:+ logger.warning(f"grep vikingdb step failed, falling back to fs: {e}")+ return await self._grep_fs(+ uri=uri,+ pattern=pattern,+ exclude_uri=exclude_uri,+ case_insensitive=case_insensitive,+ node_limit=node_limit,+ level_limit=level_limit,+ ctx=ctx,+ )++ candidate_uris = [r["uri"] for r in result if r.get("uri")]+ if excluded_prefix:+ candidate_uris = [+ u+ for u in candidate_uris+ if u != excluded_prefix and not u.startswith(excluded_prefix + "/")+ ]+ if not candidate_uris:+ # BM25 returned no candidates — the index confirms no matching content+ return {"matches": [], "count": 0, "match_count": 0, "files_scanned": 0}++ # Step 2: local fs precise matching on candidate files+ return await self._grep_in_files(+ candidate_uris,+ pattern,+ case_insensitive,+ node_limit,+ ctx,+ )++ async def _grep_in_files(+ self,+ file_uris: List[str],+ pattern: str,+ case_insensitive: bool,+ node_limit: Optional[int],+ ctx: Optional[RequestContext],+ ) -> Dict:+ """Execute regex matching in specified file list (vikingdb_then_fs Step 2)."""+ flags = re.IGNORECASE if case_insensitive else 0+ compiled = re.compile(pattern, flags)++ results = []+ files_scanned = 0++ for file_uri in file_uris:+ files_scanned += 1+ try:+ content_bytes = await self.read(file_uri, ctx=ctx)+ content = content_bytes.decode("utf-8", errors="replace")+ except Exception:+ continue++ for line_no, line in enumerate(content.splitlines(), 1):+ if compiled.search(line):+ results.append({"uri": file_uri, "line": line_no, "content": line})+ if node_limit and len(results) >= node_limit:+ return {+ "matches": results,+ "count": len(results),+ "match_count": len(results),+ "files_scanned": files_scanned,+ }++ return {+ "matches": results,+ "count": len(results),+ "match_count": len(results),+ "files_scanned": files_scanned,+ }++ async def _grep_with_agfs(+ self,+ uri: str,+ pattern: str,+ exclude_uri: Optional[str] = None,+ case_insensitive: bool = False,+ node_limit: Optional[int] = None,+ level_limit: int = 10,+ ctx: Optional[RequestContext] = None,+ ) -> Dict:+ """Grep using agfs native implementation.++ This is the optimized path for non-encrypted files.+ Uses agfs.grep() which performs matching on the server side.++ Prefer pushing filters down to agfs backend:+ - exclude_uri -> exclude_path+ - level_limit -> level_limit++ Args:+ uri: Viking URI+ pattern: Regular expression pattern to search for+ exclude_uri: Optional URI prefix to exclude from search+ case_insensitive: Whether to perform case-insensitive matching+ node_limit: Maximum number of results to return+ level_limit: Maximum depth level to traverse+ ctx: Request context++ Returns:+ Dict with matches, count, match_count, files_scanned+ """+ path = self._uri_to_path(uri, ctx=ctx)++ excluded_path = None+ if exclude_uri:+ normalized_excluded_uri = exclude_uri.rstrip("/")+ self._ensure_access(normalized_excluded_uri, ctx)+ excluded_path = self._uri_to_path(normalized_excluded_uri, ctx=ctx)++ try:+ result = await self._async_agfs.grep(+ path=path,+ pattern=pattern,+ recursive=True,+ case_insensitive=case_insensitive,+ stream=False,+ node_limit=node_limit,+ exclude_path=excluded_path,+ level_limit=level_limit,+ )+ except (AttributeError, AGFSNotSupportedError, NotImplementedError):+ # Capability missing: let the outer caller fall back to the VikingFS implementation.+ logger.warning("agfs grep unavailable, falling back to VikingFS implementation")+ raise++ matches = result.get("matches", [])+ results = []+ files_scanned_set = set()+ real_ctx = self._ctx_or_default(ctx)++ for match in matches:+ match_file = match.get("file", "")+ if not match_file:+ continue++ agfs_file_path = self._resolve_grep_match_agfs_path(path, match_file)++ file_uri = self._path_to_uri(agfs_file_path, ctx=ctx)+ if not self._is_accessible(file_uri, real_ctx):+ continue++ files_scanned_set.add(file_uri)++ results.append({+ "line": match.get("line", match.get("line_number", 0)),+ "uri": file_uri,+ "content": match.get("content", ""),+ })++ if node_limit and len(results) >= node_limit:+ break++ # Prefer backend-provided scanned file count if available; otherwise fall back to+ # counting files that produced at least one match (best-effort).+ backend_files_scanned = result.get("files_scanned")+ if isinstance(backend_files_scanned, int) and backend_files_scanned >= 0:+ files_scanned = (+ len(files_scanned_set) if real_ctx.actor_peer_id else backend_files_scanned+ )+ else:+ files_scanned = len(files_scanned_set)++ return {+ "matches": results,+ "count": len(results),+ "match_count": len(results),+ "files_scanned": files_scanned,+ }++ async def _grep_encrypted(+ self,+ uri: str,+ pattern: str,+ exclude_uri: Optional[str] = None,+ case_insensitive: bool = False,+ node_limit: Optional[int] = None,+ level_limit: int = 10,+ ctx: Optional[RequestContext] = None,+ content_transform: Optional[Callable[[str, str], str]] = None,+ ) -> Dict:+ """Grep implementation for encrypted files.++ This implementation decrypts files at VikingFS layer before matching.+ Used when encryption is enabled or when agfs.grep is not available.++ Args:+ uri: Viking URI+ pattern: Regular expression pattern to search for+ exclude_uri: Optional URI prefix to exclude from search+ case_insensitive: Whether to perform case-insensitive matching+ node_limit: Maximum number of results to return+ level_limit: Maximum depth level to traverse (default: 10)+ ctx: Request context++ Returns:+ Dict with matches, count, match_count, files_scanned+ """+ flags = re.IGNORECASE if case_insensitive else 0+ compiled_pattern = re.compile(pattern, flags)+ excluded_prefix = None+ if exclude_uri:+ excluded_prefix = exclude_uri.rstrip("/")+ self._ensure_access(excluded_prefix, ctx)+ file_uris = await self._collect_grep_files(+ uri,+ excluded_prefix=excluded_prefix,+ level_limit=level_limit,+ ctx=ctx,+ )+ results, files_scanned = await self._grep_files_parallel(+ file_uris,+ compiled_pattern=compiled_pattern,+ node_limit=node_limit,+ ctx=ctx,+ content_transform=content_transform,+ )++ return {+ "matches": results,+ "count": len(results),+ "match_count": len(results),+ "files_scanned": files_scanned,+ }++ async def _collect_grep_files(+ self,+ uri: str,+ excluded_prefix: Optional[str],+ level_limit: int,+ ctx: Optional[RequestContext] = None,+ ) -> List[str]:+ file_uris: List[str] = []++ async def search_recursive(current_uri: str, current_depth: int) -> None:+ if current_depth > level_limit:+ return++ normalized_current_uri = current_uri+ if excluded_prefix and (+ normalized_current_uri == excluded_prefix+ or normalized_current_uri.startswith(excluded_prefix + "/")+ ):+ logger.debug(f"Skipping excluded uri during grep: {normalized_current_uri}")+ return++ try:+ entries = await self.ls(normalized_current_uri, ctx=ctx)+ except Exception:+ return++ for entry in entries:+ entry_uri = f"{normalized_current_uri.rstrip('/')}/{entry['name']}"+ if excluded_prefix and (+ entry_uri == excluded_prefix or entry_uri.startswith(excluded_prefix + "/")+ ):+ logger.debug(f"Skipping excluded uri during grep: {entry_uri}")+ continue++ if entry.get("isDir"):+ await search_recursive(entry_uri, current_depth + 1)+ else:+ file_uris.append(entry_uri)++ normalized_uri = uri+ if excluded_prefix and (+ normalized_uri == excluded_prefix or normalized_uri.startswith(excluded_prefix + "/")+ ):+ logger.debug(f"Skipping excluded uri during grep: {normalized_uri}")+ return file_uris+ try:+ root_stat = await self.stat(normalized_uri, ctx=ctx)+ except Exception:+ return file_uris+ if not root_stat.get("isDir", False):+ file_uris.append(normalized_uri)+ return file_uris++ await search_recursive(uri, 0)+ return file_uris++ async def _grep_files_parallel(+ self,+ file_uris: List[str],+ compiled_pattern: re.Pattern,+ node_limit: Optional[int],+ ctx: Optional[RequestContext] = None,+ content_transform: Optional[Callable[[str, str], str]] = None,+ ) -> tuple[List[Dict[str, Any]], int]:+ results: List[Dict[str, Any]] = []+ files_scanned = 0+ concurrency = _pkg()._DEFAULT_GREP_FILE_CONCURRENCY+ for start in range(0, len(file_uris), concurrency):+ batch_uris = file_uris[start : start + concurrency]+ batch_jobs = [+ self._grep_single_file(+ entry_uri,+ compiled_pattern,+ ctx,+ content_transform=content_transform,+ )+ for entry_uri in batch_uris+ ]+ batch_results = await asyncio.gather(*batch_jobs)+ for matches, scanned_count in batch_results:+ files_scanned += scanned_count+ for match in matches:+ results.append(match)+ if node_limit and len(results) >= node_limit:+ return results, files_scanned++ return results, files_scanned++ async def _grep_single_file(+ self,+ entry_uri: str,+ compiled_pattern: re.Pattern,+ ctx: Optional[RequestContext] = None,+ content_transform: Optional[Callable[[str, str], str]] = None,+ ) -> tuple[List[Dict[str, Any]], int]:+ try:+ content = await self.read(entry_uri, ctx=ctx)+ if isinstance(content, bytes):+ content = content.decode("utf-8", errors="replace")+ if content_transform is not None:+ content = content_transform(content, entry_uri)++ matches: List[Dict[str, Any]] = []+ lines = content.split("\n")+ for line_num, line in enumerate(lines, 1):+ if compiled_pattern.search(line):+ matches.append({+ "line": line_num,+ "uri": entry_uri,+ "content": line,+ })+ return matches, 1+ except Exception as e:+ logger.debug(f"Failed to grep {entry_uri}: {e}")+ return [], 1++ def _resolve_grep_match_agfs_path(self, base_path: str, match_file: str) -> str:+ """Resolve a grep match path (relative to query root) into a full AGFS path."""+ if match_file == ".":+ return base_path+ return f"{base_path.rstrip('/')}/{match_file.lstrip('/')}"diff --git a/openviking/storage/viking_fs/_ops.py b/openviking/storage/viking_fs/_ops.pynew file mode 100644index 000000000..f934c7c00--- /dev/null+++ b/openviking/storage/viking_fs/_ops.py@@ -0,0 +1,1372 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Core filesystem operations mixin for VikingFS."""++import asyncio+from datetime import datetime, timezone+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union++from openviking.core.namespace import (+ canonicalize_uri,+ may_include_hidden_actor_peers,+)+from openviking.pyagfs.exceptions import (+ AGFSDirectoryNotEmptyError,+ AGFSHTTPError,+ AGFSClientError,+)+from openviking.server.error_mapping import is_not_found_error, map_exception+from openviking.server.identity import RequestContext+from openviking.storage.expr import PathScope+from openviking.storage.internal_names import STORAGE_INTERNAL_ENTRY_NAMES+from openviking.storage.viking_fs._base import (+ _ABSTRACT_WORKER_COUNT,+ _ensure_non_empty_search_query,+ _is_directory_not_empty_error,+ logger,+ LS_ALL_NODES,+)+from openviking_cli.exceptions import (+ FailedPreconditionError,+ InvalidArgumentError,+ NotFoundError,+)+from openviking_cli.utils.uri import VikingURI+from openviking.utils.time_utils import format_iso8601, parse_iso_datetime++if TYPE_CHECKING:+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig+++class _OpsMixin:+ """Core filesystem operations (read/write/mkdir/rm/mv/stat/glob/tree/ls/temp)."""++ # ========== AGFS Basic Commands ==========++ async def read(+ self,+ uri: str,+ offset: int = 0,+ size: int = -1,+ ctx: Optional[RequestContext] = None,+ ) -> bytes:+ """Read file"""+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ primary_path = self._uri_to_path(uri, ctx=ctx)++ # Decryption + offset/size slicing now happen inside the ragfs encryption layer+ # (when configured); the plaintext stack reads bytes directly. Either way, pass the+ # offset/size through and let the Rust layer return the requested slice.+ last_not_found: Optional[Exception] = None+ for path in self._read_paths(uri, ctx=ctx):+ if not await self._read_path_visible(uri, path, primary_path, real_ctx):+ continue+ try:+ result = await self._async_agfs.read(path, offset, size)+ break+ except Exception as exc:+ if is_not_found_error(exc):+ last_not_found = exc+ continue+ raise+ else:+ raise NotFoundError(uri, "file") from last_not_found+ if isinstance(result, bytes):+ raw = result+ elif result is not None and hasattr(result, "content"):+ raw = result.content+ else:+ raw = b""++ return raw++ async def write(+ self,+ uri: str,+ data: Union[bytes, str],+ ctx: Optional[RequestContext] = None,+ ) -> str:+ """Write file"""+ self._ensure_mutable_access(uri, ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ if isinstance(data, str):+ data = data.encode("utf-8")++ # Encryption (when configured) happens inside the ragfs layer keyed by account_id.+ return await self._async_agfs.write(path, data)++ async def mkdir(+ self,+ uri: str,+ mode: str = "755",+ exist_ok: bool = False,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> None:+ """Create directory."""+ self._ensure_mutable_access(uri, ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ # Always ensure parent directories exist before creating this directory+ await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)+ try:+ await self._async_agfs.mkdir(path, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))+ except Exception as exc:+ message = str(exc).lower()+ already_exists = "exist" in message or "already" in message+ if exist_ok and already_exists:+ return+ raise++ async def rm(+ self,+ uri: str,+ recursive: bool = False,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> Dict[str, Any]:+ """Delete file/directory + recursively update vector index.++ This method is idempotent: deleting a non-existent file succeeds+ after cleaning up any orphan index records.++ Acquires a path lock, deletes VectorDB records, then FS files.+ Raises ResourceBusyError when the target is locked by an ongoing+ operation (e.g. semantic processing).++ Returns:+ Dict with 'estimated_deleted_count' indicating the estimated number+ of nodes deleted from vector index.+ """+ from openviking.storage.errors import LockAcquisitionError, ResourceBusyError++ self._ensure_delete_access(uri, ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ target_uri = self._path_to_uri(path, ctx=ctx)++ async def _estimate_deleted_count(target_path: str, real_ctx: RequestContext) -> int:+ """Estimate number of nodes to be deleted using vector index."""+ vector_store = self._get_vector_store()+ if not vector_store:+ return 0+ try:+ target_canonical_uri = canonicalize_uri(+ self._path_to_uri(target_path, ctx=real_ctx), real_ctx+ )+ filter_expr = PathScope("uri", target_canonical_uri, depth=-1)+ return await vector_store.count(filter=filter_expr, ctx=real_ctx)+ except Exception as e:+ logger.warning(f"[VikingFS] Failed to count nodes before delete: {e}")+ return 0++ # Check existence and determine lock strategy+ try:+ stat = await self._async_agfs.stat(path)+ is_dir = stat.get("isDir", False) if isinstance(stat, dict) else False+ except Exception as exc:+ if not is_not_found_error(exc):+ mapped = map_exception(exc, resource=uri)+ if mapped is not None:+ raise mapped from exc+ raise+ # Path does not exist: clean up any orphan index records and return+ uris_to_delete = await self._collect_uris(path, recursive, ctx=ctx)+ uris_to_delete.append(target_uri)+ real_ctx = self._ctx_or_default(ctx)+ estimated_count = await _estimate_deleted_count(path, real_ctx)+ await self._delete_from_vector_store(uris_to_delete, ctx=ctx)+ logger.info(f"[VikingFS] rm target not found, cleaned orphan index: {uri}")+ return {"estimated_deleted_count": estimated_count}++ if is_dir:+ if not recursive:+ raise FailedPreconditionError(+ f"Cannot remove directory without --recursive: {uri}",+ details={"resource": uri, "expected_flag": "recursive"},+ )+ lock_method = self._async_agfs.pathlock_acquire_tree+ else:+ lock_method = self._async_agfs.pathlock_acquire_exact++ lease = lease_ref+ if lease is None:+ try:+ lease = await lock_method(path)+ except LockAcquisitionError:+ raise ResourceBusyError(f"Resource is being processed: {uri}", uri=uri)++ try:+ uris_to_delete = await self._collect_uris(path, recursive, ctx=ctx) if is_dir else []+ uris_to_delete.append(target_uri)+ real_ctx = self._ctx_or_default(ctx)+ estimated_count = await _estimate_deleted_count(path, real_ctx)+ await self._delete_from_vector_store(uris_to_delete, ctx=ctx)+ try:+ result = await self._async_agfs.rm(+ path,+ recursive=recursive,+ fs_ctx=self._pathlock_fs_ctx(ctx, lease),+ )+ except AGFSDirectoryNotEmptyError:+ raise FailedPreconditionError(+ f"Directory not empty: {uri}. Use recursive=True to delete non-empty directories."+ )+ except RuntimeError as e:+ # Fallback for older versions without typed exceptions+ if _is_directory_not_empty_error(str(e)):+ raise FailedPreconditionError(+ f"Directory not empty: {uri}. Use recursive=True to delete non-empty directories."+ )+ raise+ # Add estimated_deleted_count to the result+ if isinstance(result, dict):+ result["estimated_deleted_count"] = estimated_count+ else:+ result = {"estimated_deleted_count": estimated_count}+ return result+ finally:+ if lease_ref is None:+ await self._async_agfs.pathlock_release(lease)++ async def mv(+ self,+ old_uri: str,+ new_uri: str,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> Dict[str, Any]:+ """Move file/directory while extending an optional outer pathlock lease.++ Implemented as cp + rm to avoid lock files being carried by FS mv.+ On VectorDB update failure the copy is cleaned up so the source stays intact.+ """++ self._ensure_mutable_access(old_uri, ctx)+ # mv is implemented as copy + recursive rm of the source (see the+ # ``rm(old_path, recursive=is_dir)`` below), so the source must also clear+ # the delete guard. Without this, a protected account root such as+ # ``viking://`` — which rm() rejects up front (#2873) — could still be+ # destroyed via mv, since the write guard alone permits the bare root.+ self._ensure_delete_access(old_uri, ctx)+ self._ensure_mutable_access(new_uri, ctx)+ old_path = self._uri_to_path(old_uri, ctx=ctx)+ new_path = self._uri_to_path(new_uri, ctx=ctx)+ target_uri = self._path_to_uri(old_path, ctx=ctx)++ # Verify source exists and determine type before locking.+ try:+ stat = await self._async_agfs.stat(old_path)+ is_dir = stat.get("isDir", False) if isinstance(stat, dict) else False+ except Exception as exc:+ if not is_not_found_error(exc):+ mapped = map_exception(exc, resource=old_uri)+ if mapped is not None:+ raise mapped from exc+ raise+ raise FileNotFoundError(f"mv source not found: {old_uri}") from exc++ if not is_dir:+ if new_uri.rstrip("/") != new_uri:+ raise InvalidArgumentError(+ f"mv destination for a file must include the target file name: {new_uri}",+ details={"from_uri": old_uri, "to_uri": new_uri},+ )+ try:+ destination_stat = await self._async_agfs.stat(new_path)+ except Exception as exc:+ if not is_not_found_error(exc):+ mapped = map_exception(exc, resource=new_uri)+ if mapped is not None:+ raise mapped from exc+ raise+ else:+ if isinstance(destination_stat, dict) and destination_stat.get("isDir", False):+ raise InvalidArgumentError(+ f"mv destination for a file must include the target file name: {new_uri}",+ details={"from_uri": old_uri, "to_uri": new_uri},+ )++ real_ctx = self._ctx_or_default(ctx)+ old_uri = canonicalize_uri(old_uri, real_ctx)+ new_uri = canonicalize_uri(new_uri, real_ctx)++ if is_dir:+ lease = await self._async_agfs.pathlock_acquire_batch(+ [+ {"path": old_path, "kind": "tree"},+ {"path": new_path, "kind": "exact"},+ ],+ owner_lease_ref=lease_ref,+ )+ else:+ lease = await self._async_agfs.pathlock_acquire_batch(+ [+ {"path": old_path, "kind": "exact"},+ {"path": new_path, "kind": "exact"},+ ],+ owner_lease_ref=lease_ref,+ )++ try:+ uris_to_move = (+ await self._collect_uris(old_path, recursive=True, ctx=ctx) if is_dir else []+ )+ uris_to_move.append(target_uri)++ # Check if it's temp directory (files already encrypted)+ is_temp = old_uri.startswith("viking://temp/")++ # Copy source to destination. Source must stay intact until vector updates succeed.+ try:+ await self._copy_for_mv(+ old_uri=old_uri,+ new_uri=new_uri,+ old_path=old_path,+ new_path=new_path,+ is_dir=is_dir,+ is_temp=is_temp,+ ctx=ctx,+ lease_ref=lease,+ )+ except Exception as e:+ if "not found" in str(e).lower():+ try:+ await self._delete_from_vector_store(uris_to_move, ctx=ctx)+ except Exception:+ # Orphan cleanup is best effort here; preserve the copy error.+ pass+ else:+ logger.info(+ f"[VikingFS] mv source not found, cleaned orphan index: {old_uri}"+ )+ raise++ # Update VectorDB URIs (on failure, clean up the copy)+ try:+ await self._update_vector_store_uris(uris_to_move, old_uri, new_uri, ctx=ctx)+ except Exception:+ try:+ if is_dir:+ cleanup_lease = await self._async_agfs.pathlock_acquire_tree(+ new_path,+ owner_lease_ref=lease,+ )+ try:+ await self._async_agfs.rm(+ new_path,+ recursive=True,+ fs_ctx=self._pathlock_fs_ctx(ctx, cleanup_lease),+ )+ finally:+ await self._async_agfs.pathlock_release(cleanup_lease)+ else:+ await self._async_agfs.rm(+ new_path,+ fs_ctx=self._pathlock_fs_ctx(ctx, lease),+ )+ except Exception:+ pass+ raise++ # Delete source+ await self._async_agfs.rm(+ old_path,+ recursive=is_dir,+ fs_ctx=self._pathlock_fs_ctx(ctx, lease),+ )+ return {}+ finally:+ await self._async_agfs.pathlock_release(lease)++ async def _copy_for_mv(+ self,+ old_uri: str,+ new_uri: str,+ old_path: str,+ new_path: str,+ is_dir: bool,+ is_temp: bool,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> None:+ """Copy source to destination for mv without deleting source."""+ if is_temp:+ if is_dir:+ await self._copy_temp_dir_with_exact_locks(+ old_path,+ new_path,+ ctx=ctx,+ lease_ref=lease_ref,+ )+ else:+ await self._async_agfs.cp(+ old_path,+ new_path,+ recursive=False,+ fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),+ )+ return++ if is_dir:+ await self._copy_dir_through_vikingfs(old_uri, new_uri, ctx=ctx, lease_ref=lease_ref)+ else:+ await self._copy_file_through_vikingfs(old_uri, new_uri, ctx=ctx, lease_ref=lease_ref)++ async def _copy_temp_dir_with_exact_locks(+ self,+ old_path: str,+ new_path: str,+ ctx: Optional[RequestContext],+ lease_ref: Dict[str, Any] | None,+ ) -> None:+ """Copy an encrypted temp directory while locking every destination entry.++ Args:+ old_path: Source backend directory path.+ new_path: Destination backend directory path.+ ctx: Request context used for filesystem operations.+ lease_ref: Exact lease covering the current destination directory.++ Returns:+ None.+ """+ if lease_ref is None:+ raise ValueError("temp directory copy requires a pathlock lease")+ fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)+ await self._async_agfs.mkdir(new_path, fs_ctx=fs_ctx)+ entries = await self._async_agfs.ls(old_path, fs_ctx=fs_ctx)+ for entry in entries:+ name = entry.get("name", "")+ if not name or name in (".", ".."):+ continue+ old_child = f"{old_path.rstrip('/')}/{name}"+ new_child = f"{new_path.rstrip('/')}/{name}"+ child_lease = await self._async_agfs.pathlock_acquire_exact(+ new_child,+ owner_lease_ref=lease_ref,+ )+ try:+ if entry.get("isDir", False):+ await self._copy_temp_dir_with_exact_locks(+ old_child,+ new_child,+ ctx=ctx,+ lease_ref=child_lease,+ )+ else:+ await self._async_agfs.cp(+ old_child,+ new_child,+ recursive=False,+ fs_ctx=self._pathlock_fs_ctx(ctx, child_lease),+ )+ finally:+ await self._async_agfs.pathlock_release(child_lease)++ async def _copy_dir_through_vikingfs(+ self,+ old_uri: str,+ new_uri: str,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> None:+ """Recursively copy a directory through VikingFS read/write hooks."""+ await self.mkdir(new_uri, exist_ok=True, ctx=ctx, lease_ref=lease_ref)++ entries = await self.ls(old_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx)+ for entry in entries:+ name = entry.get("name", "")+ if not name or name in (".", ".."):+ continue+ old_child_uri = f"{old_uri.rstrip('/')}/{name}"+ new_child_uri = f"{new_uri.rstrip('/')}/{name}"+ if entry.get("isDir"):+ await self._copy_dir_through_vikingfs(+ old_child_uri,+ new_child_uri,+ ctx=ctx,+ lease_ref=lease_ref,+ )+ else:+ await self._copy_file_through_vikingfs(+ old_child_uri,+ new_child_uri,+ ctx=ctx,+ lease_ref=lease_ref,+ )++ async def _copy_file_through_vikingfs(+ self,+ from_uri: str,+ to_uri: str,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> None:+ """Copy one file through VikingFS read/write hooks without deleting source."""+ content_bytes = await self.read_file_bytes(from_uri, ctx=ctx)+ if lease_ref is None:+ await self.write_file_bytes(to_uri, content_bytes, ctx=ctx)+ return++ child_path = self._uri_to_path(to_uri, ctx=ctx)+ child_lease = await self._async_agfs.pathlock_acquire_exact(+ child_path,+ owner_lease_ref=lease_ref,+ )+ try:+ await self.write_file_bytes(to_uri, content_bytes, ctx=ctx, lease_ref=child_lease)+ finally:+ await self._async_agfs.pathlock_release(child_lease)++ async def stat(+ self, uri: str, ctx: Optional[RequestContext] = None, skip_count: bool = False+ ) -> Dict[str, Any]:+ """+ File/directory information.++ example: {'name': 'resources', 'size': 128, 'mode': 2147484141, 'modTime': '2026-02-10T21:26:02.934376379+08:00', 'isDir': True, 'isLocked': False, 'count': 42, 'meta': {'Name': 'localfs', 'Type': 'local', 'Content': {'local_path': '...'}}}++ Extra field:+ isLocked (bool): Whether the path is currently held by a path lock+ (either the path itself or any ancestor directory). Returns+ False when the pathlock system is not enabled or the lookup+ fails.+ count (int): For directories, the number of nodes in the vector index+ under this directory (including subdirectories). For files, this+ field is not included.++ Args:+ uri: Viking URI+ ctx: Request context+ skip_count: If True, skip the vector_store.count() call for directories.+ Use this when the count field is not needed (e.g. in grep) to avoid+ an extra VikingDB API call.+ """+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ primary_path = self._uri_to_path(uri, ctx=ctx)+ path = primary_path+ last_not_found: Optional[Exception] = None+ for candidate_path in self._read_paths(uri, ctx=ctx):+ if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):+ continue+ try:+ result = await self._async_agfs.stat(candidate_path)+ path = candidate_path+ break+ except Exception as exc:+ if is_not_found_error(exc):+ last_not_found = exc+ continue+ raise+ else:+ if self._is_legacy_session_root_uri(uri):+ now = datetime.now(timezone.utc).isoformat()+ return {+ "name": "session",+ "size": 0,+ "mode": 0o755,+ "modTime": now,+ "isDir": True,+ "isLocked": False,+ }+ raise NotFoundError(uri, "file") from last_not_found+ if isinstance(result, dict):+ result["isLocked"] = await self._is_path_locked_async(path)+ # Add count for directories if vector store available+ if not skip_count and result.get("isDir", False):+ try:+ vector_store = self._get_vector_store()+ if vector_store:+ target_canonical_uri = canonicalize_uri(+ self._path_to_uri(path, ctx=real_ctx), real_ctx+ )+ if not may_include_hidden_actor_peers(target_canonical_uri, real_ctx):+ filter_expr = PathScope("uri", target_canonical_uri, depth=-1)+ result["count"] = await vector_store.count(+ filter=filter_expr,+ ctx=real_ctx,+ )+ except Exception as e:+ logger.warning(f"[VikingFS] Failed to count nodes for directory stat: {e}")+ return result++ async def exists(self, uri: str, ctx: Optional[RequestContext] = None) -> bool:+ """Check if a URI exists.++ Args:+ uri: Viking URI+ ctx: Request context++ Returns:+ bool: True if the URI exists, False otherwise+ """+ try:+ await self.stat(uri, ctx=ctx)+ return True+ except Exception:+ return False++ async def glob(+ self,+ pattern: str,+ uri: str = "viking://",+ node_limit: Optional[int] = None,+ ctx: Optional[RequestContext] = None,+ ) -> Dict:+ """File pattern matching, supports **/*.md recursive."""+ _ensure_non_empty_search_query(pattern)+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ primary_path = self._uri_to_path(uri, ctx=ctx)+ path: Optional[str] = None+ for candidate_path in self._read_paths(uri, ctx=ctx):+ if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):+ continue+ if await self._agfs_path_exists(candidate_path):+ path = candidate_path+ break+ if path is None:+ if self._is_legacy_session_root_uri(uri):+ return {"matches": [], "count": 0}+ raise NotFoundError(uri, "directory")++ page_size = self._glob_page_size(node_limit)+ continuation_token: Optional[str] = None+ matches = []+ while True:+ page = await self._async_agfs.glob_directory(+ path,+ pattern,+ show_hidden=False,+ page_size=page_size,+ level_limit=None,+ continuation_token=continuation_token,+ )++ for entry in page.get("entries", []):+ if node_limit is not None and node_limit > 0 and len(matches) >= node_limit:+ return {"matches": matches, "count": len(matches)}+ if not self._is_path_entry_visible(+ entry["path"],+ entry.get("name") or entry["path"].rsplit("/", 1)[-1],+ path,+ real_ctx,+ ):+ continue+ if not await self._read_path_visible(uri, entry["path"], primary_path, real_ctx):+ continue+ entry_uri = self._alias_uri_for_path(+ request_uri=uri,+ base_path=path,+ entry_path=entry["path"],+ ctx=ctx,+ )+ matches.append(entry_uri)++ if node_limit is not None and node_limit > 0 and len(matches) >= node_limit:+ return {"matches": matches, "count": len(matches)}+ continuation_token = page.get("next_token")+ if not continuation_token:+ break+ return {"matches": matches, "count": len(matches)}++ async def _batch_fetch_abstracts(+ self,+ entries: List[Dict[str, Any]],+ abs_limit: int,+ ctx: Optional[RequestContext] = None,+ ) -> None:+ """Batch fetch abstracts for entries using a fixed-size worker pool.++ Non-directory entries receive an empty abstract immediately.+ Directory entries are processed concurrently via a worker pool,+ using _read_abstract_for_known_dir to skip redundant stat() calls.++ Args:+ entries: List of entries to fetch abstracts for+ abs_limit: Maximum length for abstract truncation+ """+ dir_jobs = []+ for index, entry in enumerate(entries):+ if not entry.get("isDir", False):+ entry["abstract"] = ""+ continue+ dir_jobs.append((index, entry))++ if not dir_jobs:+ return++ worker_count = min(_ABSTRACT_WORKER_COUNT, len(dir_jobs))++ cursor = 0+ cursor_lock = asyncio.Lock()+ results: Dict[int, str] = {}++ async def worker() -> None:+ nonlocal cursor+ while True:+ async with cursor_lock:+ if cursor >= len(dir_jobs):+ return+ index, entry = dir_jobs[cursor]+ cursor += 1++ try:+ abstract = await self._read_abstract_for_known_dir(entry["uri"], ctx=ctx)+ except Exception:+ abstract = "[.abstract.md is not ready]"++ results[index] = abstract++ await asyncio.gather(*(worker() for _ in range(worker_count)))++ for index, abstract in results.items():+ if len(abstract) > abs_limit:+ abstract = abstract[: abs_limit - 3] + "..."+ entries[index]["abstract"] = abstract++ async def tree(+ self,+ uri: str = "viking://",+ output: str = "original",+ abs_limit: int = 256,+ show_all_hidden: bool = False,+ node_limit: Optional[int] = 1000,+ level_limit: Optional[int] = 3,+ ctx: Optional[RequestContext] = None,+ ) -> List[Dict[str, Any]]:+ """+ Recursively list all contents (includes rel_path).++ Args:+ uri: Viking URI+ output: str = "original" or "agent"+ abs_limit: int = 256 (for agent output abstract truncation)+ show_all_hidden: bool = False (list all hidden files, like -a)+ node_limit: int | None = 1000 (maximum number of nodes to list, None means unlimited)+ level_limit: int | None = 3 (maximum depth level to traverse, None means unlimited)++ output="original"+ [{'name': '.abstract.md', 'size': 100, 'mode': 420, 'modTime': '2026-02-11T16:52:16.256334192+08:00', 'isDir': False, 'rel_path': '.abstract.md', 'uri': 'viking://resources...'}]++ output="agent"+ [{'uri': 'viking://resources...', 'size': 100, 'isDir': False, 'modTime': '2026-02-11T08:52:16.256Z', 'rel_path': '.abstract.md', 'abstract': "..."}]+ """+ self._ensure_access(uri, ctx)+ if output == "original":+ return await self._tree_original(uri, show_all_hidden, node_limit, level_limit, ctx=ctx)+ elif output == "agent":+ return await self._tree_agent(+ uri, abs_limit, show_all_hidden, node_limit, level_limit, ctx=ctx+ )+ else:+ raise ValueError(f"Invalid output format: {output}")++ async def _tree_original(+ self,+ uri: str,+ show_all_hidden: bool = False,+ node_limit: Optional[int] = 1000,+ level_limit: Optional[int] = 3,+ ctx: Optional[RequestContext] = None,+ ) -> List[Dict[str, Any]]:+ """Recursively list all contents (original format)."""+ result = []+ async for entry, entry_uri in self._iter_visible_tree_entries(+ uri,+ show_all_hidden=show_all_hidden,+ node_limit=node_limit,+ level_limit=level_limit,+ ctx=ctx,+ ):+ info = entry["info"]+ new_entry = dict(entry.get("extra", {}))+ new_entry.update({+ "name": info["name"],+ "size": info["size"],+ "mode": info["mode"],+ "modTime": info["modTime"],+ "isDir": info["isDir"],+ "rel_path": entry["rel_path"],+ "uri": entry_uri,+ })+ result.append(new_entry)+ return result++ async def _tree_agent(+ self,+ uri: str,+ abs_limit: int,+ show_all_hidden: bool = False,+ node_limit: Optional[int] = 1000,+ level_limit: Optional[int] = 3,+ ctx: Optional[RequestContext] = None,+ ) -> List[Dict[str, Any]]:+ """Recursively list all contents (agent format with abstracts)."""+ result = []++ async for entry, entry_uri in self._iter_visible_tree_entries(+ uri,+ show_all_hidden=show_all_hidden,+ node_limit=node_limit,+ level_limit=level_limit,+ ctx=ctx,+ ):+ info = entry["info"]+ is_dir = info["isDir"]+ result.append({+ "uri": entry_uri,+ "size": 0 if is_dir else info["size"],+ "isDir": is_dir,+ "modTime": format_iso8601(parse_iso_datetime(info["modTime"])),+ "rel_path": entry["rel_path"],+ })++ await self._batch_fetch_abstracts(result, abs_limit, ctx=ctx)++ return result++ # ========== Vector Sync Helper Methods ==========++ async def _collect_uris(+ self, path: str, recursive: bool, ctx: Optional[RequestContext] = None+ ) -> List[str]:+ """Recursively collect all URIs (for rm/mv), including directories."""+ uris = []++ async def _collect(p: str):+ try:+ entries = await self._ls_entries(p, ctx=ctx)+ except Exception as exc:+ if is_not_found_error(exc):+ return+ raise++ for entry in entries:+ name = entry.get("name", "")+ if name in [".", ".."]:+ continue+ full_path = f"{p}/{name}".replace("//", "/")+ if entry.get("isDir"):+ uris.append(self._path_to_uri(full_path, ctx=ctx))+ if recursive:+ await _collect(full_path)+ else:+ uris.append(self._path_to_uri(full_path, ctx=ctx))++ await _collect(path)+ return uris++ # ========== Parent Directory Creation ==========++ async def _ensure_parent_dirs(+ self,+ path: str,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> None:+ """Recursively create all parent directories."""+ try:+ await self._async_agfs.ensure_parent_dirs(+ path,+ fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),+ )+ except Exception as e:+ logger.debug(f"Failed to ensure parent directories for {path}: {e}")+ parent = path.rstrip("/").rsplit("/", 1)[0]+ await self._mkdir_path_with_parents(parent, ctx=ctx, lease_ref=lease_ref)++ async def _mkdir_path_with_parents(+ self,+ dir_path: str,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> None:+ """Create a directory path segment-by-segment using the same fs context."""+ parts = [part for part in dir_path.strip("/").split("/") if part]+ current = ""+ for part in parts:+ current = f"{current}/{part}"+ try:+ await self._async_agfs.mkdir(+ current,+ fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref),+ )+ except Exception as e:+ message = str(e).lower()+ if "exist" in message or "already" in message:+ continue+ logger.debug(f"Failed to create parent directory {current}: {e}")++ # ========== Batch Read (backward compatible) ==========++ async def read_batch(+ self, uris: List[str], level: str = "l0", ctx: Optional[RequestContext] = None+ ) -> Dict[str, str]:+ """Batch read content from multiple URIs."""+ results = {}+ for uri in uris:+ try:+ content = ""+ if level == "l0":+ content = await self.abstract(uri, ctx=ctx)+ elif level == "l1":+ content = await self.overview(uri, ctx=ctx)+ results[uri] = content+ except Exception:+ pass+ return results++ # ========== Other Preserved Methods ==========++ async def write_file(+ self,+ uri: str,+ content: Union[str, bytes],+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> None:+ """Write file directly. Encryption lock handled internally by EncryptionWrappedFS."""+ self._ensure_mutable_access(uri, ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)++ if isinstance(content, str):+ content = content.encode("utf-8")++ await self._async_agfs.write(path, content, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))++ async def read_file(+ self,+ uri: str,+ offset: int = 0,+ limit: int = -1,+ ctx: Optional[RequestContext] = None,+ ) -> str:+ """Read single file, optionally sliced by line range.++ Args:+ uri: Viking URI+ offset: Starting line number (0-indexed). Default 0.+ limit: Number of lines to read. -1 means read to end. Default -1.++ Raises:+ FileNotFoundError: If the file does not exist.+ """+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ primary_path = self._uri_to_path(uri, ctx=ctx)+ # Verify the file exists before reading, because AGFS read returns+ # empty bytes for non-existent files instead of raising an error.+ last_not_found: Optional[Exception] = None+ for path in self._read_paths(uri, ctx=ctx):+ if not await self._read_path_visible(uri, path, primary_path, real_ctx):+ continue+ try:+ stat = await self._async_agfs.stat(path)+ break+ except Exception as exc:+ if is_not_found_error(exc):+ last_not_found = exc+ continue+ raise+ else:+ raise NotFoundError(uri, "file") from last_not_found+ if isinstance(stat, dict) and stat.get("isDir", False):+ raise InvalidArgumentError(+ f"Cannot read directory as file: {uri}",+ details={"resource": uri, "expected": "file", "actual": "directory"},+ )+ try:+ content = await self._async_agfs.read(path)+ if isinstance(content, bytes):+ raw = content+ elif content is not None and hasattr(content, "content"):+ raw = content.content+ else:+ raw = b""++ text = self._decode_bytes(raw)+ except Exception:+ raise NotFoundError(uri, "file")++ if offset == 0 and limit == -1:+ return text+ lines = text.splitlines(keepends=True)+ sliced = lines[offset:] if limit == -1 else lines[offset : offset + limit]+ return "".join(sliced)++ async def read_file_bytes(+ self,+ uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> bytes:+ """Read single binary file."""+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ primary_path = self._uri_to_path(uri, ctx=ctx)+ last_not_found: Optional[Exception] = None+ for path in self._read_paths(uri, ctx=ctx):+ if not await self._read_path_visible(uri, path, primary_path, real_ctx):+ continue+ try:+ stat = await self._async_agfs.stat(path)+ break+ except Exception as exc:+ if is_not_found_error(exc):+ last_not_found = exc+ continue+ raise+ else:+ raise NotFoundError(uri, "file") from last_not_found+ if isinstance(stat, dict) and stat.get("isDir", False):+ raise InvalidArgumentError(+ f"Cannot read directory as file: {uri}",+ details={"resource": uri, "expected": "file", "actual": "directory"},+ )+ try:+ raw = self._handle_agfs_read(await self._async_agfs.read(path))+ return raw+ except Exception:+ raise NotFoundError(uri, "file")++ async def write_file_bytes(+ self,+ uri: str,+ content: bytes,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> None:+ """Write single binary file. Encryption lock handled internally by EncryptionWrappedFS."""+ self._ensure_mutable_access(uri, ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)++ await self._async_agfs.write(path, content, fs_ctx=self._pathlock_fs_ctx(ctx, lease_ref))++ async def append_file(+ self,+ uri: str,+ content: str,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> None:+ """Append content to file while holding one exact pathlock lease."""+ self._ensure_mutable_access(uri, ctx)+ path = self._uri_to_path(uri, ctx=ctx)++ owned_lease = None+ try:+ await self._ensure_parent_dirs(path, ctx=ctx, lease_ref=lease_ref)+ lease = lease_ref+ if lease is None:+ lease = await self._async_agfs.pathlock_acquire_exact(path)+ owned_lease = lease+ fs_ctx = self._pathlock_fs_ctx(ctx, lease)++ # Read old content and rewrite the whole file to avoid lost updates.+ existing = ""+ try:+ existing_bytes = self._handle_agfs_read(+ await self._async_agfs.read(path, fs_ctx=fs_ctx)+ )+ existing = self._decode_bytes(existing_bytes)+ except FileNotFoundError:+ pass+ except AGFSHTTPError as e:+ if e.status_code != 404:+ raise+ except AGFSClientError:+ raise++ final_content = (existing + content).encode("utf-8")+ await self._async_agfs.write(+ path,+ final_content,+ fs_ctx=fs_ctx,+ )++ except Exception as e:+ logger.error(f"[VikingFS] Failed to append to file {uri}: {e}")+ raise IOError(f"Failed to append to file {uri}: {e}")+ finally:+ if owned_lease is not None:+ await self._async_agfs.pathlock_release(owned_lease)++ async def ls(+ self,+ uri: str,+ output: str = "original",+ abs_limit: int = 256,+ show_all_hidden: bool = False,+ node_limit: int = 1000,+ sort_by: Optional[str] = None,+ sort_order: str = "asc",+ ctx: Optional[RequestContext] = None,+ ) -> List[Dict[str, Any]]:+ """+ List directory contents (URI version).++ Args:+ uri: Viking URI+ output: str = "original"+ abs_limit: int = 256+ show_all_hidden: bool = False (list all hidden files, like -a)+ node_limit: int = 1000 (maximum number of nodes to list)+ sort_by: Optional sort field, "name" or "mtime"+ sort_order: Sort direction, "asc" or "desc"++ output="original"+ [{'name': '.abstract.md', 'size': 100, 'mode': 420, 'modTime': '2026-02-11T16:52:16.256334192+08:00', 'isDir': False, 'meta': {'Name': 'localfs', 'Type': 'local', 'Content': None}, 'uri': 'viking://resources/.abstract.md'}]++ output="agent"+ [{'name': '.abstract.md', 'size': 100, 'modTime': '2026-02-11T08:52:16.256Z', 'isDir': False, 'uri': 'viking://resources/.abstract.md', 'abstract': "..."}]+ """+ self._ensure_access(uri, ctx)+ if sort_by not in {None, "name", "mtime"}:+ raise ValueError("sort_by must be 'name' or 'mtime'")+ if sort_order not in {"asc", "desc"}:+ raise ValueError("sort_order must be 'asc' or 'desc'")+ if output == "original":+ return await self._ls_original(+ uri,+ show_all_hidden,+ node_limit,+ sort_by=sort_by,+ sort_order=sort_order,+ ctx=ctx,+ )+ elif output == "agent":+ return await self._ls_agent(+ uri,+ abs_limit,+ show_all_hidden,+ node_limit,+ sort_by=sort_by,+ sort_order=sort_order,+ ctx=ctx,+ )+ else:+ raise ValueError(f"Invalid output format: {output}")++ @staticmethod+ def _ls_entry_mtime(entry: Dict[str, Any]) -> Optional[float]:+ raw_time = entry.get("modTime")+ if isinstance(raw_time, (int, float)):+ return float(raw_time)+ if isinstance(raw_time, str) and raw_time:+ try:+ return parse_iso_datetime(raw_time).timestamp()+ except (TypeError, ValueError, OverflowError):+ return None++ legacy_time = entry.get("mtime")+ if isinstance(legacy_time, (int, float)):+ return float(legacy_time)+ return None++ @classmethod+ def _sort_ls_entry_items(+ cls,+ entry_items: List[tuple[Dict[str, Any], str]],+ sort_by: Optional[str],+ sort_order: str,+ ) -> List[tuple[Dict[str, Any], str]]:+ if sort_by is None:+ return entry_items++ descending = sort_order == "desc"+ directories = [item for item in entry_items if item[0].get("isDir", False)]+ files = [item for item in entry_items if not item[0].get("isDir", False)]++ if sort_by == "name":++ def name_key(item: tuple[Dict[str, Any], str]) -> tuple[str, str]:+ name = str(item[0].get("name", ""))+ return name.lower(), name++ directories.sort(key=name_key, reverse=descending)+ files.sort(key=name_key, reverse=descending)+ return directories + files++ def sort_by_mtime(+ items: List[tuple[Dict[str, Any], str]],+ ) -> List[tuple[Dict[str, Any], str]]:+ timestamped = []+ missing = []+ for item in items:+ timestamp = cls._ls_entry_mtime(item[0])+ if timestamp is None:+ missing.append(item)+ else:+ timestamped.append((timestamp, item))+ timestamped.sort(+ key=lambda pair: pair[0],+ reverse=descending,+ )+ return [item for _, item in timestamped] + missing++ return sort_by_mtime(directories) + sort_by_mtime(files)++ async def _ls_agent(+ self,+ uri: str,+ abs_limit: int,+ show_all_hidden: bool,+ node_limit: int = 1000,+ sort_by: Optional[str] = None,+ sort_order: str = "asc",+ ctx: Optional[RequestContext] = None,+ ) -> List[Dict[str, Any]]:+ """List directory contents (URI version)."""+ real_ctx = self._ctx_or_default(ctx)+ entry_items = await self._list_read_path_items(uri, ctx=ctx)+ entry_items = self._sort_ls_entry_items(entry_items, sort_by, sort_order)+ # basic info+ fallback_time = datetime.now(timezone.utc)+ all_entries = []+ for entry, entry_uri in entry_items:+ if len(all_entries) >= node_limit:+ break+ name = entry.get("name", "")+ raw_time = entry.get("modTime", "")+ parsed_time = fallback_time+ if isinstance(raw_time, (int, float)):+ parsed_time = datetime.fromtimestamp(raw_time, tz=timezone.utc)+ elif raw_time:+ if len(raw_time) > 26 and "+" in raw_time:+ parts = raw_time.split("+")+ raw_time = parts[0][:26] + "+" + parts[1]+ parsed_time = parse_iso_datetime(raw_time)+ elif isinstance(entry.get("mtime"), (int, float)):+ parsed_time = datetime.fromtimestamp(entry["mtime"], tz=timezone.utc)+ is_dir = entry.get("isDir", False)+ new_entry = {+ "uri": entry_uri,+ "size": 0 if is_dir else entry.get("size", 0),+ "isDir": is_dir,+ "modTime": format_iso8601(parsed_time),+ }+ if not self._is_accessible(new_entry["uri"], real_ctx):+ continue+ if is_dir:+ all_entries.append(new_entry)+ elif not name.startswith("."):+ all_entries.append(new_entry)+ elif show_all_hidden:+ all_entries.append(new_entry)+ await self._batch_fetch_abstracts(all_entries, abs_limit, ctx=ctx)+ return all_entries++ async def _ls_original(+ self,+ uri: str,+ show_all_hidden: bool = False,+ node_limit: int = 1000,+ sort_by: Optional[str] = None,+ sort_order: str = "asc",+ ctx: Optional[RequestContext] = None,+ ) -> List[Dict[str, Any]]:+ """List directory contents (URI version)."""+ real_ctx = self._ctx_or_default(ctx)+ try:+ entry_items = await self._list_read_path_items(uri, ctx=ctx)+ entry_items = self._sort_ls_entry_items(entry_items, sort_by, sort_order)+ # AGFS returns read-only structure, need to create new dict+ all_entries = []+ for entry, entry_uri in entry_items:+ if len(all_entries) >= node_limit:+ break+ name = entry.get("name", "")+ new_entry = dict(entry) # Copy original data+ new_entry["uri"] = entry_uri+ if not self._is_accessible(new_entry["uri"], real_ctx):+ continue+ if entry.get("isDir"):+ all_entries.append(new_entry)+ elif not name.startswith("."):+ all_entries.append(new_entry)+ elif show_all_hidden:+ all_entries.append(new_entry)+ return all_entries+ except Exception:+ raise NotFoundError(uri, "directory")++ async def move_file(+ self,+ from_uri: str,+ to_uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> None:+ """Move file."""+ self._ensure_mutable_access(from_uri, ctx)+ self._ensure_mutable_access(to_uri, ctx)+ from_path = self._uri_to_path(from_uri, ctx=ctx)++ await self._copy_file_through_vikingfs(from_uri, to_uri, ctx=ctx)+ await self._async_agfs.rm(from_path)++ # ========== Temp File Operations (backward compatible) ==========++ def create_temp_uri(self, ctx: Optional[RequestContext] = None) -> str:+ """Create a temp directory URI.++ - explicit ctx or bound request context -> user-scoped temp URI+ - no explicit/bound context -> legacy temp URI shape for backward compatibility+ """+ real_ctx = ctx if ctx is not None else self._bound_ctx.get()+ if real_ctx is None:+ return VikingURI.create_temp_uri()+ return VikingURI.create_temp_uri(space=real_ctx.user.user_space_name())++ async def persist_temp_tree(+ self,+ temp_uri: str,+ target_uri: str,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> None:+ """Persist an already-encrypted temp tree without rewriting file bytes."""+ self._ensure_access(temp_uri, ctx)+ self._ensure_mutable_access(target_uri, ctx)+ src_path = self._uri_to_path(temp_uri, ctx=ctx)+ dst_path = self._uri_to_path(target_uri, ctx=ctx)+ fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)+ await self._ensure_parent_dirs(dst_path, ctx=ctx, lease_ref=lease_ref)+ await self._async_agfs.cp(+ src_path,+ dst_path,+ recursive=True,+ fs_ctx=fs_ctx or {"account_id": self._ctx_or_default(ctx).account_id},+ )++ async def delete_temp(+ self,+ temp_uri: str,+ ctx: Optional[RequestContext] = None,+ lease_ref: Dict[str, Any] | None = None,+ ) -> None:+ """Delete temp directory and its contents."""+ self._ensure_mutable_access(temp_uri, ctx)+ path = self._uri_to_path(temp_uri, ctx=ctx)+ fs_ctx = self._pathlock_fs_ctx(ctx, lease_ref)+ try:+ await self._async_agfs.rm(path, recursive=True, fs_ctx=fs_ctx)+ except Exception as e:+ logger.warning(f"[VikingFS] Failed to delete temp {temp_uri}: {e}")++ async def _ls_entries(+ self, path: str, ctx: Optional[RequestContext] = None+ ) -> List[Dict[str, Any]]:+ """List directory entries, filtering out internal directories.++ At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.+ At other levels, uses the shared storage internal-name blacklist.+ """+ entries = await self._async_agfs.ls(path)+ parts = [p for p in path.strip("/").split("/") if p]+ if len(parts) == 2 and parts[0] == "local":+ return [e for e in entries if e.get("name") in VikingURI.LISTABLE_SCOPES]+ return [e for e in entries if e.get("name") not in STORAGE_INTERNAL_ENTRY_NAMES]diff --git a/openviking/storage/viking_fs/_semantic.py b/openviking/storage/viking_fs/_semantic.pynew file mode 100644index 000000000..0ef6b7364--- /dev/null+++ b/openviking/storage/viking_fs/_semantic.py@@ -0,0 +1,646 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Semantic retrieval and relation management mixin for VikingFS."""++import asyncio+import json+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union++from openviking.core.namespace import canonicalize_uri+from openviking.core.retrieval_targets import resolve_retrieval_targets+from openviking.server.error_mapping import is_not_found_error, map_exception+from openviking.server.identity import RequestContext+from openviking.storage.viking_fs._base import (+ RelationEntry,+ _ensure_non_empty_search_query,+ logger,+)+from openviking.telemetry import get_current_telemetry+from openviking.utils.image_search import build_multimodal_embedding_input+from openviking_cli.exceptions import NotFoundError++if TYPE_CHECKING:+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig+++class _SemanticMixin:+ """Abstract/overview/relations/find/search/link (semantic retrieval layer)."""++ # ========== VikingFS Specific Capabilities ==========++ async def _read_abstract_file(+ self,+ path: str,+ uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> str:+ """Read and decrypt/decode .abstract.md from a known directory path.++ Does NOT perform stat or isDir check -- caller is responsible for+ ensuring the path points to a directory.+ """+ file_path = f"{path}/.abstract.md"+ try:+ content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))+ except Exception as exc:+ if not is_not_found_error(exc):+ mapped = map_exception(exc, resource=uri)+ if mapped is not None:+ raise mapped from exc+ raise+ return f"# {uri} [Directory abstract is not ready]"++ return self._decode_bytes(content_bytes)++ async def _read_abstract_for_known_dir(+ self,+ uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> str:+ """Read .abstract.md for a directory that is already known to be a directory.++ Bypasses stat() and isDir check. Caller (i.e. _batch_fetch_abstracts)+ must guarantee that the URI points to a directory.+ """+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ primary_path = self._uri_to_path(uri, ctx=ctx)+ for path in self._read_paths(uri, ctx=ctx):+ if not await self._read_path_visible(uri, path, primary_path, real_ctx):+ continue+ try:+ if not await self._agfs_path_exists(path):+ continue+ return await self._read_abstract_file(path, uri, ctx=ctx)+ except Exception as exc:+ if is_not_found_error(exc):+ continue+ raise+ return f"# {uri} [Directory abstract is not ready]"++ async def abstract(+ self,+ uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> str:+ """Read directory's L0 summary (.abstract.md).++ If the caller points to a file, its parent directory is used instead so+ the endpoint remains usable for both file and directory URIs.+ """+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ primary_path = self._uri_to_path(uri, ctx=ctx)+ path = primary_path+ last_exc: Optional[Exception] = None+ for candidate_path in self._read_paths(uri, ctx=ctx):+ if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):+ continue+ try:+ info = await self._async_agfs.stat(candidate_path)+ path = candidate_path+ break+ except Exception as exc:+ if is_not_found_error(exc):+ last_exc = exc+ continue+ mapped = map_exception(exc, resource=uri)+ if mapped is not None:+ raise mapped from exc+ raise+ else:+ if last_exc is not None:+ mapped = map_exception(last_exc, resource=uri)+ if mapped is not None:+ raise mapped from last_exc+ raise NotFoundError(uri, "directory") from last_exc+ if not info.get("isDir", info.get("is_dir")):+ parent_path = path.rsplit("/", 1)[0] or "/"+ parent_uri = self._path_to_uri(parent_path, ctx=ctx)+ logger.info(+ "content/abstract: %s is a file, falling back to parent directory %s",+ uri,+ parent_uri,+ )+ return await self.abstract(parent_uri, ctx=ctx)+ return await self._read_abstract_file(path, uri, ctx=ctx)++ async def overview(+ self,+ uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> str:+ """Read directory's L1 overview (.overview.md).++ If the caller points to a file, its parent directory is used instead so+ the endpoint remains usable for both file and directory URIs.+ """+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ primary_path = self._uri_to_path(uri, ctx=ctx)+ path = primary_path+ last_exc: Optional[Exception] = None+ for candidate_path in self._read_paths(uri, ctx=ctx):+ if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):+ continue+ try:+ info = await self._async_agfs.stat(candidate_path)+ path = candidate_path+ break+ except Exception as exc:+ if is_not_found_error(exc):+ last_exc = exc+ continue+ mapped = map_exception(exc, resource=uri)+ if mapped is not None:+ raise mapped from exc+ raise+ else:+ if last_exc is not None:+ mapped = map_exception(last_exc, resource=uri)+ if mapped is not None:+ raise mapped from last_exc+ raise NotFoundError(uri, "directory") from last_exc+ if not info.get("isDir", info.get("is_dir")):+ parent_path = path.rsplit("/", 1)[0] or "/"+ parent_uri = self._path_to_uri(parent_path, ctx=ctx)+ logger.info(+ "content/overview: %s is a file, falling back to parent directory %s",+ uri,+ parent_uri,+ )+ return await self.overview(parent_uri, ctx=ctx)+ file_path = f"{path}/.overview.md"+ try:+ content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))+ except Exception as exc:+ if not is_not_found_error(exc):+ mapped = map_exception(exc, resource=uri)+ if mapped is not None:+ raise mapped from exc+ raise+ # Fallback to default if .overview.md doesn't exist+ return f"# {uri}\n\n[Directory overview is not ready]"++ return self._decode_bytes(content_bytes)++ async def relations(+ self,+ uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> List[Dict[str, Any]]:+ """Get relation list.++ Returns: [{"uri": "...", "reason": "..."}, ...]+ """+ self._ensure_access(uri, ctx)+ entries = await self.get_relation_table(uri, ctx=ctx)+ result = []+ for entry in entries:+ for u in entry.uris:+ if self._is_accessible(u, self._ctx_or_default(ctx)):+ result.append({"uri": u, "reason": entry.reason})+ return result++ async def find(+ self,+ query: str,+ target_uri: Union[str, List[str]] = "",+ limit: int = 10,+ score_threshold: Optional[float] = None,+ filter: Optional[Dict] = None,+ ctx: Optional[RequestContext] = None,+ level: Optional[List[int]] = None,+ image_url: Optional[str] = None,+ ):+ """Semantic search.++ Args:+ query: Search query+ target_uri: Target directory URI(s), supports str or List[str]+ limit: Return count+ score_threshold: Score threshold+ filter: Metadata filter++ Returns:+ FindResult+ """+ _ensure_non_empty_search_query(query, image_url)+ telemetry = get_current_telemetry()+ from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever+ from openviking_cli.retrieve import (+ ContextType,+ FindResult,+ TypedQuery,+ )++ real_ctx = self._ctx_or_default(ctx)+ retrieval_targets = resolve_retrieval_targets(target_uri, real_ctx)++ for target_dir in retrieval_targets.target_directories:+ self._ensure_access(target_dir, ctx)++ storage = self._get_vector_store()+ if not storage:+ raise RuntimeError("Vector store not initialized. Call OpenViking.initialize() first.")++ embedder = self._get_embedder()+ if not embedder:+ raise RuntimeError("Embedder not configured.")++ retriever = HierarchicalRetriever(+ storage=storage,+ embedder=embedder,+ rerank_config=self.rerank_config,+ retrieval_config=self.retrieval_config,+ )++ typed_query = TypedQuery(+ query=query,+ context_type=None,+ intent="",+ target_directories=retrieval_targets.target_directories,+ embedding_input=(+ build_multimodal_embedding_input(query, image_url) if image_url else None+ ),+ image_query=bool(image_url),+ )++ logger.debug(+ "[VikingFS.find] Calling retriever.retrieve with "+ f"ctx.account_id={real_ctx.account_id}, ctx.user={real_ctx.user}"+ )++ result = await retriever.retrieve(+ typed_query,+ ctx=real_ctx,+ limit=limit,+ score_threshold=score_threshold,+ scope_dsl=filter,+ level=level,+ )++ # Convert QueryResult to FindResult+ memories, resources, skills = [], [], []+ for ctx in result.matched_contexts:+ if ctx.context_type == ContextType.MEMORY:+ memories.append(ctx)+ elif ctx.context_type == ContextType.RESOURCE:+ resources.append(ctx)+ elif ctx.context_type == ContextType.SKILL:+ skills.append(ctx)++ find_result = FindResult(+ memories=memories,+ resources=resources,+ skills=skills,+ )+ telemetry.set("vector.returned", find_result.total)+ return find_result++ async def search(+ self,+ query: str,+ target_uri: Union[str, List[str]] = "",+ session_info: Optional[Dict] = None,+ limit: int = 10,+ score_threshold: Optional[float] = None,+ filter: Optional[Dict] = None,+ ctx: Optional[RequestContext] = None,+ level: Optional[List[int]] = None,+ image_url: Optional[str] = None,+ ):+ """Complex search with session context.++ Args:+ query: Search query+ target_uri: Target directory URI(s), supports str or List[str]+ session_info: Session information+ limit: Return count+ filter: Metadata filter++ Returns:+ FindResult+ """+ _ensure_non_empty_search_query(query, image_url)+ telemetry = get_current_telemetry()+ from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever+ from openviking.retrieve.intent_analyzer import IntentAnalyzer+ from openviking_cli.retrieve import (+ ContextType,+ FindResult,+ QueryPlan,+ TypedQuery,+ )++ real_ctx = self._ctx_or_default(ctx)+ retrieval_targets = resolve_retrieval_targets(target_uri, real_ctx)+ primary_target_uri = retrieval_targets.first_explicit_directory++ session_summary = (+ str(session_info.get("latest_archive_overview") or "") if session_info else ""+ )+ current_messages = session_info.get("current_messages") if session_info else None++ query_plan: Optional[QueryPlan] = None+ for target_dir in retrieval_targets.target_directories:+ self._ensure_access(target_dir, ctx)++ # When target_uri exists, read its abstract as optional query-planning context.+ target_abstract = ""+ if primary_target_uri:+ try:+ with telemetry.measure("search.target_abstract"):+ target_abstract = await self.abstract(primary_target_uri, ctx=ctx)+ except Exception:+ target_abstract = ""++ intent_enabled = (+ bool(self.retrieval_config.enable_intent)+ if self.retrieval_config is not None+ else True+ )++ # With session context: optional intent analysis+ if image_url:+ typed_queries = [+ TypedQuery(+ query=query,+ context_type=None,+ intent="",+ priority=1,+ target_directories=retrieval_targets.target_directories,+ embedding_input=build_multimodal_embedding_input(query, image_url),+ image_query=True,+ )+ ]+ elif intent_enabled and (session_summary or current_messages):+ analyzer = IntentAnalyzer(max_recent_messages=5)+ with telemetry.measure("search.intent_analysis"):+ query_plan = await analyzer.analyze(+ compression_summary=session_summary or "",+ messages=current_messages or [],+ current_message=query,+ target_abstract=target_abstract,+ )+ typed_queries = query_plan.queries+ for tq in typed_queries:+ tq.target_directories = retrieval_targets.target_directories+ else:+ # No session context, or intent disabled: search with the raw query.+ typed_queries = [+ TypedQuery(+ query=query,+ context_type=None,+ intent="",+ priority=1,+ target_directories=retrieval_targets.target_directories,+ )+ ]+ telemetry.set("search.typed_queries_count", len(typed_queries))++ # Concurrent execution+ storage = self._get_vector_store()+ embedder = self._get_embedder()+ retriever = HierarchicalRetriever(+ storage=storage,+ embedder=embedder,+ rerank_config=self.rerank_config,+ retrieval_config=self.retrieval_config,+ )++ async def _execute(tq: TypedQuery):+ real_ctx = self._ctx_or_default(ctx)+ logger.debug(+ "[VikingFS.search._execute] Calling retriever.retrieve with "+ f"ctx.account_id={real_ctx.account_id}, ctx.user={real_ctx.user}"+ )+ return await retriever.retrieve(+ tq,+ ctx=real_ctx,+ limit=limit,+ score_threshold=score_threshold,+ scope_dsl=filter,+ level=level,+ )++ query_results = await asyncio.gather(*[_execute(tq) for tq in typed_queries])++ # Aggregate results to FindResult+ memories, resources, skills = [], [], []+ for result in query_results:+ for ctx in result.matched_contexts:+ if ctx.context_type == ContextType.MEMORY:+ memories.append(ctx)+ elif ctx.context_type == ContextType.RESOURCE:+ resources.append(ctx)+ elif ctx.context_type == ContextType.SKILL:+ skills.append(ctx)++ find_result = FindResult(+ memories=memories,+ resources=resources,+ skills=skills,+ query_plan=query_plan,+ query_results=query_results,+ )+ telemetry.set("vector.returned", find_result.total)+ return find_result++ # ========== Relation Management ==========++ async def link(+ self,+ from_uri: str,+ uris: Union[str, List[str]],+ reason: str = "",+ ctx: Optional[RequestContext] = None,+ ) -> None:+ """Create relation (maintained in .relations.json)."""+ if isinstance(uris, str):+ uris = [uris]+ self._ensure_mutable_access(from_uri, ctx)+ for uri in uris:+ self._ensure_access(uri, ctx)++ from_path = self._uri_to_path(from_uri, ctx=ctx)++ entries = await self._read_relation_table(from_path, ctx=ctx)+ existing_ids = {e.id for e in entries}++ link_id = next(f"link_{i}" for i in range(1, 10000) if f"link_{i}" not in existing_ids)++ entries.append(RelationEntry(id=link_id, uris=uris, reason=reason))++ await self._write_relation_table(from_path, entries, ctx=ctx)+ logger.debug(f"[VikingFS] Created link: {from_uri} -> {uris}")++ async def unlink(+ self,+ from_uri: str,+ uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> None:+ """Delete relation."""+ self._ensure_mutable_access(from_uri, ctx)+ self._ensure_access(uri, ctx)+ from_path = self._uri_to_path(from_uri, ctx=ctx)++ try:+ entries = await self._read_relation_table(from_path, ctx=ctx)++ entry_to_modify = None+ for entry in entries:+ if uri in entry.uris:+ entry_to_modify = entry+ break++ if not entry_to_modify:+ logger.debug(f"[VikingFS] URI not found in relations: {uri}")+ return++ entry_to_modify.uris.remove(uri)++ if not entry_to_modify.uris:+ entries.remove(entry_to_modify)+ logger.debug(f"[VikingFS] Removed empty entry: {entry_to_modify.id}")++ await self._write_relation_table(from_path, entries, ctx=ctx)+ logger.debug(f"[VikingFS] Removed link: {from_uri} -> {uri}")++ except Exception as e:+ logger.error(f"[VikingFS] Failed to unlink {from_uri} -> {uri}: {e}")+ raise IOError(f"Failed to unlink: {e}")++ async def get_relation_table(+ self, uri: str, ctx: Optional[RequestContext] = None+ ) -> List[RelationEntry]:+ """Get relation table."""+ self._ensure_access(uri, ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ return await self._read_relation_table(path, ctx=ctx)++ # ========== Relation Table Internal Methods ==========++ async def _read_relation_table(+ self, dir_path: str, ctx: Optional[RequestContext] = None+ ) -> List[RelationEntry]:+ """Read .relations.json."""+ table_path = f"{dir_path}/.relations.json"+ try:+ content = self._handle_agfs_read(await self._async_agfs.read(table_path))+ data = json.loads(content.decode("utf-8"))+ except FileNotFoundError:+ return []+ except Exception:+ # logger.warning(f"[VikingFS] Failed to read relation table {table_path}: {e}")+ return []++ entries = []+ # Compatible with old format (nested) and new format (flat)+ if isinstance(data, list):+ # New format: flat list+ for entry_data in data:+ entries.append(RelationEntry.from_dict(entry_data))+ elif isinstance(data, dict):+ # Old format: nested {namespace: {user: [entries]}}+ for _namespace, user_dict in data.items():+ for _user, entry_list in user_dict.items():+ for entry_data in entry_list:+ entries.append(RelationEntry.from_dict(entry_data))+ return entries++ async def _write_relation_table(+ self, dir_path: str, entries: List[RelationEntry], ctx: Optional[RequestContext] = None+ ) -> None:+ """Write .relations.json."""+ # Use flat list format+ data = [entry.to_dict() for entry in entries]++ content = json.dumps(data, ensure_ascii=False, indent=2)+ table_path = f"{dir_path}/.relations.json"+ if isinstance(content, str):+ content = content.encode("utf-8")++ await self._async_agfs.write(table_path, content)++ async def get_relations(self, uri: str, ctx: Optional[RequestContext] = None) -> List[str]:+ """Get all related URIs (backward compatible)."""+ entries = await self.get_relation_table(uri, ctx=ctx)+ real_ctx = self._ctx_or_default(ctx)+ all_uris = []+ for entry in entries:+ for related in entry.uris:+ if self._is_accessible(related, real_ctx):+ all_uris.append(related)+ return all_uris++ async def get_relations_with_content(+ self,+ uri: str,+ include_l0: bool = True,+ include_l1: bool = False,+ ctx: Optional[RequestContext] = None,+ ) -> List[Dict[str, Any]]:+ """Get related URIs and their content (backward compatible)."""+ relation_uris = await self.get_relations(uri, ctx=ctx)+ if not relation_uris:+ return []++ results = []+ abstracts = {}+ overviews = {}+ if include_l0:+ abstracts = await self.read_batch(relation_uris, level="l0", ctx=ctx)+ if include_l1:+ overviews = await self.read_batch(relation_uris, level="l1", ctx=ctx)++ for rel_uri in relation_uris:+ info = {"uri": rel_uri}+ if include_l0:+ info["abstract"] = abstracts.get(rel_uri, "")+ if include_l1:+ info["overview"] = overviews.get(rel_uri, "")+ results.append(info)++ return results++ async def write_context(+ self,+ uri: str,+ content: Union[str, bytes] = "",+ abstract: str = "",+ overview: str = "",+ content_filename: str = "content.md",+ is_leaf: bool = False,+ ctx: Optional[RequestContext] = None,+ ) -> None:+ """Write context to AGFS (L0/L1/L2)."""++ self._ensure_mutable_access(uri, ctx)+ path = self._uri_to_path(uri, ctx=ctx)++ try:+ await self._ensure_parent_dirs(path, ctx=ctx)+ try:+ await self._async_agfs.mkdir(path)+ except Exception as e:+ if "exist" not in str(e).lower():+ raise++ if content:+ content_uri = f"{uri}/{content_filename}"+ await self.write_file(content_uri, content, ctx=ctx)++ if abstract:+ abstract_uri = f"{uri}/.abstract.md"+ await self.write_file(abstract_uri, abstract, ctx=ctx)++ if overview:+ overview_uri = f"{uri}/.overview.md"+ await self.write_file(overview_uri, overview, ctx=ctx)++ except Exception as e:+ logger.error(f"[VikingFS] Failed to write {uri}: {e}")+ raise IOError(f"Failed to write {uri}: {e}")diff --git a/openviking/storage/viking_fs/_snapshot.py b/openviking/storage/viking_fs/_snapshot.pynew file mode 100644index 000000000..938db3bad--- /dev/null+++ b/openviking/storage/viking_fs/_snapshot.py@@ -0,0 +1,710 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Snapshot/git-like version control mixin for VikingFS."""++import asyncio+import sys+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union++from openviking.core.context import ContextLevel+from openviking.core.namespace import canonicalize_uri+from openviking.pyagfs.exceptions import (+ AGFSInvalidOperationError,+ AGFSNotFoundError,+ AGFSPathNotFoundError,+ AGFSResourceExhaustedError,+)+from openviking.server.error_mapping import is_not_found_error, map_exception+from openviking.server.identity import RequestContext+from openviking.storage.viking_fs._base import (+ _prepare_snapshot_diff,+ logger,+)+from openviking_cli.exceptions import (+ InvalidArgumentError,+ NotFoundError,+ ResourceExhaustedError,+)+++def _pkg():+ return sys.modules[__package__]++if TYPE_CHECKING:+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend+ from openviking_cli.utils.config import GrepConfig, RerankConfig, RetrievalConfig+++class _SnapshotMixin:+ """Snapshot/git-like version control (commit/restore/show/diff/log)."""++ async def system_sync_status(+ self, uri: str, ctx: Optional[RequestContext] = None+ ) -> Dict[str, Any]:+ """Return multi-write sync status for one Viking URI subtree."""+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ return await self._async_agfs.system_sync_status(+ path,+ fs_ctx={"account_id": real_ctx.account_id},+ )++ async def system_sync_retry(+ self, uri: str, ctx: Optional[RequestContext] = None+ ) -> Dict[str, Any]:+ """Retry multi-write sync for one Viking URI subtree."""+ self._ensure_mutable_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ return await self._async_agfs.system_sync_retry(+ path,+ fs_ctx={"account_id": real_ctx.account_id},+ )++ async def get_gitignore(self, ctx: Optional[RequestContext] = None) -> str:+ """Return the account-level .ovgitignore content, or an empty string if absent.++ Raises ``AGFSInvalidOperationError`` if the stored content is not valid+ UTF-8, mirroring the Rust layer's reject-non-UTF-8 behavior at commit+ time rather than leaking a raw ``UnicodeDecodeError``.+ """+ path = self._gitignore_agfs_path(ctx)+ try:+ raw = await self._async_agfs.read(path, 0, -1)+ except Exception as exc:+ if is_not_found_error(exc):+ return ""+ mapped = map_exception(exc, resource=path)+ if mapped is not None:+ raise mapped from exc+ raise+ if isinstance(raw, bytes):+ data = raw+ elif raw is not None and hasattr(raw, "content"):+ data = raw.content+ else:+ data = b""+ try:+ return data.decode("utf-8")+ except UnicodeDecodeError as exc:+ raise AGFSInvalidOperationError(+ f"invalid ignore file {path}: must be UTF-8: {exc}"+ ) from exc++ async def set_gitignore(+ self,+ content: str,+ ctx: Optional[RequestContext] = None,+ ) -> None:+ """Write the account-level .ovgitignore control file without semantic indexing.++ Validates the size limit up front so a too-large file can never be+ persisted only to make every later ``commit`` fail. Syntax (negation,+ escaping) is still validated at commit time by the Rust layer.+ """+ if not isinstance(content, str):+ raise TypeError("content must be a string")+ path = self._gitignore_agfs_path(ctx)+ data = content.encode("utf-8")+ if len(data) > self._OVGITIGNORE_MAX_BYTES:+ raise AGFSInvalidOperationError(+ f"ignore file too large: {path} is {len(data)} bytes, "+ f"limit {self._OVGITIGNORE_MAX_BYTES} bytes"+ )+ await self._ensure_parent_dirs(path, ctx=ctx)+ await self._async_agfs.write(path, data)++ async def delete_gitignore(self, ctx: Optional[RequestContext] = None) -> None:+ """Delete the account-level .ovgitignore control file. Missing is success."""+ path = self._gitignore_agfs_path(ctx)+ try:+ await self._async_agfs.rm(path, recursive=False)+ except Exception as exc:+ if is_not_found_error(exc):+ return+ mapped = map_exception(exc, resource=path)+ if mapped is not None:+ raise mapped from exc+ raise++ async def commit(+ self,+ *,+ message: str,+ paths: Optional[List[str]] = None,+ branch: str = "main",+ author_name: Optional[str] = None,+ author_email: Optional[str] = None,+ ctx: Optional[RequestContext] = None,+ ) -> Dict[str, Any]:+ """Create a git snapshot of the account's tree.++ Args:+ message: Commit message.+ paths: Optional list of ``viking://`` URIs to scope the commit to;+ entries may be files or directories. Directories are expanded+ recursively with the snapshot pruning rules applied. ``None``+ (default) enumerates the whole account tree. An empty list is+ forwarded as an explicit empty path list (no-op commit). A+ path that exists in neither the VFS nor the previous snapshot+ logs a warning and is treated as a no-op deletion.+ branch: Branch to advance. Defaults to ``"main"``.+ author_name / author_email: Override the default bot author.+ ctx: Request context (provides ``account_id``).++ Returns:+ Dict with ``result`` (``"created"`` / ``"noop"``) and ``commit_oid``.+ When ``result == "created"`` it also includes ``changed`` (paths+ added/updated/removed). Both variants include ``ignored``: the+ number of candidate paths skipped by the account ``.ovgitignore``+ rules (system pruning is not counted). Noop omits ``changed``.+ """+ real_ctx = self._ctx_or_default(ctx)+ account = real_ctx.account_id+ if paths is None:+ tree_paths: Optional[List[str]] = None+ else:+ tree_paths = [self._uri_to_tree_path(p, ctx=real_ctx) for p in paths]+ return await self._async_agfs.run(+ "git_commit",+ account=account,+ branch=branch,+ message=message,+ paths=tree_paths,+ author_name=author_name or self._DEFAULT_GIT_AUTHOR_NAME,+ author_email=author_email or self._DEFAULT_GIT_AUTHOR_EMAIL,+ )++ async def restore(+ self,+ *,+ project_dir: Optional[str] = None,+ source_commit: str,+ branch: str = "main",+ dry_run: bool = False,+ message: Optional[str] = None,+ author_name: Optional[str] = None,+ author_email: Optional[str] = None,+ ctx: Optional[RequestContext] = None,+ ) -> Dict[str, Any]:+ """Restore a project subtree to the state at ``source_commit``.++ Generates a new commit whose parent is the current HEAD (not the+ source commit) and writes the diff through the VFS. Paths outside+ ``project_dir`` are left untouched.++ Args:+ project_dir: ``viking://`` URI of the subtree, e.g.+ ``"viking://resources/proj_a"``. May also be passed as a+ short form like ``"resources/proj_a"``. Trailing slashes are+ stripped. Internal scopes are rejected with ``ValueError``.+ If None, restore the entire tree.+ source_commit: 40-hex OID, branch name, or full ref path.+ branch: Branch to advance. Defaults to ``"main"``.+ dry_run: If True, returns the planned diff without writing.+ message: Optional commit message; defaults to a generated string.+ author_name / author_email: Override the default bot author.+ ctx: Request context (provides ``account_id``).++ Returns:+ Dict containing ``result`` (``"applied"`` / ``"noop"`` / ``"dry_run"``)+ and corresponding oid / diff fields. When an ``Applied`` result has+ vector side-effects, a ``task_id`` is included for polling the+ background reindex via ``GET /api/v1/tasks/{task_id}``.++ After an ``Applied`` result, this method schedules background vector+ maintenance for the affected paths via :class:`ReindexExecutor`:+ directory markers (``.abstract.md`` / ``.overview.md``) recompute or+ delete only that directory's L0/L1 vector; source files reindex (write)+ or delete (removal) their DETAIL vector. ``.relations.json`` has no+ vector side-effect. The rebuild is tracked as a single task+ (``snapshot_restore_reindex``); per-path failures are logged and do not+ block the return value. The task reaches ``completed`` only after each+ affected vector has been written to (or deleted from) the index, so+ polling ``task_id`` to ``completed`` guarantees subsequent ``find``+ reads see the restored state.+ """+ real_ctx = self._ctx_or_default(ctx)+ account = real_ctx.account_id+ tree_dir: Optional[str]+ if project_dir is None:+ tree_dir = None+ else:+ tree_dir = self._uri_to_tree_path(project_dir, ctx=real_ctx).rstrip("/")+ if not tree_dir:+ raise ValueError(f"project_dir must not be empty: {project_dir!r}")+ # Build kwargs dynamically, only include project_dir if it's not None+ kwargs = {+ "account": account,+ "branch": branch,+ "source_commit": source_commit,+ "dry_run": dry_run,+ "message": message,+ "author_name": author_name or self._DEFAULT_GIT_AUTHOR_NAME,+ "author_email": author_email or self._DEFAULT_GIT_AUTHOR_EMAIL,+ }+ if tree_dir is not None:+ kwargs["project_dir"] = tree_dir+ # dry_run only computes the diff; it never writes the VFS, so it needs+ # no lock.+ if dry_run:+ return await self._async_agfs.run("git_restore", **kwargs)++ from openviking.pyagfs.exceptions import GitRestoreWritebackPartialError+ from openviking.storage.errors import LockAcquisitionError, ResourceBusyError++ # Serialize the writeback against concurrent VFS mutations on the same+ # subtree. A scoped restore tree-locks project_dir; a full restore+ # (project_dir is None) locks the account root so only one restore runs+ # per account at a time. The lock covers only the writeback — the+ # background reindex is scheduled after release, otherwise its per-path+ # child locks would conflict with this tree lock.+ lock_path = (+ self._uri_to_path(project_dir, ctx=real_ctx)+ if project_dir is not None+ else f"/local/{account}"+ )+ partial_exc: Optional[GitRestoreWritebackPartialError] = None+ try:+ lease = await self._async_agfs.pathlock_acquire_tree(lock_path)+ except LockAcquisitionError:+ raise ResourceBusyError(+ f"Resource is being processed: {project_dir or '*'}",+ uri=project_dir or "*",+ )++ try:+ try:+ result = await self._async_agfs.run("git_restore", **kwargs)+ except GitRestoreWritebackPartialError as exc:+ # The ref already advanced — capture the exception and+ # finish the lock scope cleanly. Reindex scheduling and+ # re-raise happen below, outside the tree lock.+ partial_exc = exc+ result = None+ finally:+ await self._async_agfs.pathlock_release(lease)++ if partial_exc is not None:+ # HEAD has moved forward but some VFS writes/deletes failed.+ # Still schedule reindex for the paths that *did* reach the VFS so+ # the vector index doesn't stay stale, then re-raise so the caller+ # learns about the partial failure (and can inspect failed_writes+ # / failed_deletes / task_id on the exception).+ try:+ partial_exc.task_id = await self._schedule_restore_reindex_for_paths(+ written_paths=partial_exc.written_paths,+ deleted_paths=partial_exc.deleted_paths,+ project_dir=project_dir,+ real_ctx=real_ctx,+ )+ except Exception:+ logger.exception(+ "[VikingFS] git restore partial: reindex scheduling failed; "+ "HEAD advanced but reindex was not queued"+ )+ raise partial_exc++ if result.get("result") != "applied":+ return result++ written = list(result.get("written_paths") or [])+ deleted = list(result.get("deleted_paths") or [])+ if written or deleted:+ try:+ task_id = await self._schedule_restore_reindex_for_paths(+ written_paths=written,+ deleted_paths=deleted,+ project_dir=project_dir,+ real_ctx=real_ctx,+ )+ if task_id is not None:+ result["task_id"] = task_id+ except Exception:+ logger.exception(+ "[VikingFS] git restore reindex task creation failed; "+ "falling back to fire-and-forget rebuild"+ )+ self._schedule_vector_rebuild(written=written, deleted=deleted, ctx=real_ctx)+ return result++ async def _schedule_restore_reindex_for_paths(+ self,+ *,+ written_paths: List[str],+ deleted_paths: List[str],+ project_dir: Optional[str],+ real_ctx: RequestContext,+ ) -> Optional[str]:+ """Classify ``written``/``deleted`` paths into vector tasks and queue+ them as a single tracked background rebuild. Returns the task id, or+ ``None`` if there is nothing to do.++ Shared by the applied path and the partial-writeback recovery path —+ both schedule reindex for paths that actually reached the VFS.+ """+ if not written_paths and not deleted_paths:+ return None+ tasks = self._collect_restore_vector_tasks(written_paths, deleted_paths)+ if not tasks:+ return None++ from openviking.service.task_tracker import get_task_tracker++ tracker = get_task_tracker()+ task = await tracker.create(+ "snapshot_restore_reindex",+ resource_id=project_dir or "*",+ account_id=real_ctx.account_id,+ user_id=real_ctx.user.user_id,+ )+ await tracker.update_stage(+ task.task_id,+ "queued",+ account_id=real_ctx.account_id,+ user_id=real_ctx.user.user_id,+ )+ background = asyncio.create_task(+ self._run_restore_rebuild_tracked(task.task_id, tasks, real_ctx),+ name=f"vikingfs-git-restore-reindex:{task.task_id}",+ )+ self._background_tasks.add(background)+ background.add_done_callback(self._background_tasks.discard)+ return task.task_id++ async def show(+ self,+ target_ref: str,+ *,+ path: Optional[str] = None,+ max_blob_bytes: Optional[int] = None,+ ctx: Optional[RequestContext] = None,+ ) -> Union[Dict[str, Any], bytes]:+ """Read a commit's metadata or a single blob.++ ``path=None`` returns the commit metadata dict (oid, tree, parents,+ author, committer, message). ``path=str`` returns the blob bytes+ directly, stripping the oid/size envelope returned by the binding.+ """+ real_ctx = self._ctx_or_default(ctx)+ account = real_ctx.account_id+ tree_path = self._uri_to_tree_path(path, ctx=real_ctx) if path else None+ kwargs: Dict[str, Any] = {+ "account": account,+ "target_ref": target_ref,+ "path": tree_path,+ }+ if max_blob_bytes is not None:+ kwargs["max_blob_bytes"] = max_blob_bytes+ resp = await self._async_agfs.run("git_show", **kwargs)+ if path is not None and isinstance(resp, dict) and "bytes" in resp:+ return resp["bytes"]+ return resp++ async def show_blob_raw(+ self,+ target_ref: str,+ *,+ path: str,+ ctx: Optional[RequestContext] = None,+ ) -> Dict[str, Any]:+ """Like ``show(target_ref, path=...)`` but returns the full envelope.++ Returns ``{"oid": str, "size": int, "bytes": bytes}`` without+ stripping. Used by the HTTP snapshot router to populate+ ``X-Snapshot-Oid`` / ``X-Snapshot-Size`` response headers.+ """+ real_ctx = self._ctx_or_default(ctx)+ account = real_ctx.account_id+ tree_path = self._uri_to_tree_path(path, ctx=real_ctx)+ resp = await self._async_agfs.run(+ "git_show",+ account=account,+ target_ref=target_ref,+ path=tree_path,+ )+ if not isinstance(resp, dict) or "bytes" not in resp:+ raise TypeError(+ f"git_show returned unexpected shape for blob path: {type(resp).__name__}"+ )+ return resp++ async def diff(+ self,+ *,+ path: str,+ from_ref: Optional[str],+ to_ref: str,+ raw: bool = True,+ ctx: Optional[RequestContext] = None,+ ) -> Dict[str, Any]:+ """Return a unified text diff for one path between two snapshots."""+ real_ctx = self._ctx_or_default(ctx)+ path = canonicalize_uri(path, ctx=real_ctx)+ self._ensure_access(path, real_ctx)++ from_meta: Optional[Dict[str, Any]] = None+ if from_ref:+ try:+ from_meta = await self.show(from_ref, ctx=real_ctx)+ except AGFSNotFoundError as exc:+ raise NotFoundError(from_ref, "git_ref") from exc+ try:+ to_meta = await self.show(to_ref, ctx=real_ctx)+ except AGFSNotFoundError as exc:+ raise NotFoundError(to_ref, "git_ref") from exc++ from_oid = str(from_meta.get("oid", from_ref)) if isinstance(from_meta, dict) else ""+ to_oid = str(to_meta.get("oid", to_ref)) if isinstance(to_meta, dict) else to_ref++ async def read_optional(ref: str) -> Optional[bytes]:+ try:+ value = await self.show(+ ref,+ path=path,+ max_blob_bytes=_pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES,+ ctx=real_ctx,+ )+ except AGFSPathNotFoundError:+ return None+ except AGFSResourceExhaustedError as exc:+ raise ResourceExhaustedError(+ f"snapshot diff file size limit exceeded "+ f"({_pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES} bytes)",+ details={"limit_bytes": _pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES, "path": path},+ ) from exc+ if not isinstance(value, bytes):+ raise TypeError(f"git_show returned unexpected blob type: {type(value).__name__}")+ return value++ before_bytes = await read_optional(from_oid) if from_ref else None+ after_bytes = await read_optional(to_oid)+ if before_bytes is None and after_bytes is None:+ raise NotFoundError(path, "git_blob")++ for value in (before_bytes, after_bytes):+ if value is not None and len(value) > _pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES:+ raise ResourceExhaustedError(+ f"snapshot diff file size limit exceeded ({_pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES} bytes)",+ details={"limit_bytes": _pkg().SNAPSHOT_DIFF_MAX_FILE_BYTES, "path": path},+ )++ if not raw:+ from openviking.session.memory.utils.memory_file_utils import MemoryFileUtils++ def visible_content(value: Optional[bytes]) -> Optional[bytes]:+ if value is None:+ return None+ try:+ text = value.decode("utf-8")+ except UnicodeDecodeError as exc:+ raise InvalidArgumentError("raw=false requires UTF-8 snapshot content") from exc+ return MemoryFileUtils.read(text, uri=path).content.encode("utf-8")++ before_bytes = visible_content(before_bytes)+ after_bytes = visible_content(after_bytes)++ change_type, before, after = await asyncio.to_thread(+ _prepare_snapshot_diff,+ path=path,+ before_bytes=before_bytes,+ after_bytes=after_bytes,+ max_lines=_pkg().SNAPSHOT_DIFF_MAX_LINES,+ )+ try:+ diff_text = await self._async_agfs.run(+ "git_diff_text",+ before=before,+ after=after,+ fromfile=f"{path}@{from_oid}",+ tofile=f"{path}@{to_oid}",+ timeout_ms=_pkg().SNAPSHOT_DIFF_TIMEOUT_MS,+ max_output_bytes=_pkg().SNAPSHOT_DIFF_MAX_OUTPUT_BYTES,+ )+ except AGFSResourceExhaustedError as exc:+ raise ResourceExhaustedError(+ f"snapshot diff output size limit exceeded "+ f"({_pkg().SNAPSHOT_DIFF_MAX_OUTPUT_BYTES} bytes)",+ details={"limit_bytes": _pkg().SNAPSHOT_DIFF_MAX_OUTPUT_BYTES, "path": path},+ ) from exc+ return {+ "path": path,+ "from_commit": from_oid,+ "to_commit": to_oid,+ "change_type": change_type,+ "diff_text": diff_text,+ }++ async def log(+ self,+ *,+ branch: str = "main",+ limit: int = 20,+ paths: Optional[List[str]] = None,+ ctx: Optional[RequestContext] = None,+ ) -> List[Dict[str, Any]]:+ """Walk back from ``branch``'s HEAD along ``parents[0]`` up to ``limit`` commits.++ Returns a list of commit metadata dicts (same shape as ``show(target_ref)``).+ """+ if limit <= 0:+ return []+ real_ctx = self._ctx_or_default(ctx)+ account = real_ctx.account_id+ tree_paths = (+ None if not paths else [self._uri_to_tree_path(path, ctx=real_ctx) for path in paths]+ )+ return await self._async_agfs.run(+ "git_log",+ account=account,+ branch=branch,+ limit=limit,+ paths=tree_paths,+ )++ def _collect_restore_vector_tasks(+ self,+ written: List[str],+ deleted: List[str],+ ) -> set[tuple]:+ """Classify restore-affected paths into deduplicated ``(op, uri, level)`` tasks.++ Tasks are deduplicated on the exact ``(op, uri, level)`` key (no ancestor+ subsumption — each change is handled independently because no operation+ recurses into descendants).+ """+ tasks: set[tuple] = set()+ for tree_path in written:+ try:+ task = self._classify_restore_path(tree_path, deleted=False)+ except ValueError:+ continue+ if task is not None:+ tasks.add(task)+ for tree_path in deleted:+ try:+ task = self._classify_restore_path(tree_path, deleted=True)+ except ValueError:+ continue+ if task is not None:+ tasks.add(task)+ return tasks++ def _schedule_vector_rebuild(+ self,+ *,+ written: List[str],+ deleted: List[str],+ ctx: RequestContext,+ ) -> None:+ """Fire-and-forget precise vector maintenance for a git restore.++ Each affected path is classified by :py:meth:`_classify_restore_path`+ into a ``(op, uri, level)`` task and scheduled independently:++ - ``reindex_marker`` — recompute only a directory's L0/L1 vector.+ - ``reindex_file`` — recompute only a source file's DETAIL vector+ (non-recursive ``execute(mode="vectors_only")``).+ - ``delete`` — remove only the ``(uri, level)`` vector (directory+ marker removal, or deleted source file).++ Failures are logged and never propagate.+ """+ try:+ from openviking.service.reindex_executor import get_reindex_executor+ except Exception:+ logger.exception("[VikingFS] ReindexExecutor import failed; skipping rebuild")+ return++ try:+ loop = asyncio.get_running_loop()+ except RuntimeError:+ logger.warning("[VikingFS] git restore vector rebuild skipped: no running event loop")+ return++ tasks = self._collect_restore_vector_tasks(written, deleted)+ if not tasks:+ return++ executor = get_reindex_executor()+ for op, uri, level in tasks:+ loop.create_task(+ self._run_vector_rebuild(executor, op, uri, level, ctx),+ name=f"vikingfs-git-{op}:{uri}:{int(level)}",+ )++ async def _run_restore_rebuild_tracked(+ self,+ task_id: str,+ tasks: set[tuple],+ ctx: RequestContext,+ ) -> None:+ """Background worker driving a tracked restore vector rebuild.++ Runs all classified ``(op, uri, level)`` rebuild tasks concurrently and+ drives the task through start → complete/fail. Per-task failures are+ swallowed (and logged) inside :py:meth:`_run_vector_rebuild`, preserving+ the "failures do not block" semantics.+ """+ from openviking.service.task_tracker import get_task_tracker+ from openviking.service.task_work_index import bind_task_context++ tracker = get_task_tracker()+ tracker.register_running_task(task_id)+ try:+ await tracker.start(+ task_id,+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ stage="reindexing",+ )+ from openviking.service.reindex_executor import get_reindex_executor++ executor = get_reindex_executor()+ with bind_task_context(task_id, ctx.account_id, ctx.user.user_id):+ await asyncio.gather(+ *[+ self._run_vector_rebuild(executor, op, uri, level, ctx)+ for (op, uri, level) in tasks+ ]+ )+ await tracker.complete(+ task_id,+ {"status": "completed", "task_count": len(tasks)},+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ except asyncio.CancelledError:+ # TaskWorkIndex finalizes after this active task and its queue work settle.+ return+ except Exception as exc:+ await tracker.fail(+ task_id,+ str(exc),+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )+ finally:+ await tracker.unregister_running_task(task_id)++ async def _run_vector_rebuild(+ self,+ executor: Any,+ op: str,+ uri: str,+ level: Any,+ ctx: RequestContext,+ ) -> None:+ """Wrapper coroutine: dispatch one vector task and swallow errors."""+ try:+ if op == "reindex_marker":+ await executor.reindex_directory_marker(dir_uri=uri, level=level, ctx=ctx)+ elif op == "reindex_file":+ await executor.execute(uri=uri, mode="vectors_only", wait=True, ctx=ctx)+ elif op == "delete":+ await executor.delete_uri_level(uri=uri, level=level, ctx=ctx)+ else: # pragma: no cover - defensive+ logger.warning("[VikingFS] unknown vector rebuild op %r for %s", op, uri)+ except Exception:+ logger.exception("[VikingFS] git restore vector task %s failed for %s", op, uri)diff --git a/openviking/storage/viking_fs/_sync.py b/openviking/storage/viking_fs/_sync.pynew file mode 100644index 000000000..a4eb7a935--- /dev/null+++ b/openviking/storage/viking_fs/_sync.py@@ -0,0 +1,297 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Tree sync (diff + mv/rm merge) primitives for VikingFS."""++import logging+from dataclasses import dataclass, field+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple++from openviking.storage.viking_fs._base import LS_ALL_NODES+from openviking_cli.utils.uri import VikingURI++if TYPE_CHECKING:+ from openviking.server.identity import RequestContext++logger = logging.getLogger(__name__)+++@dataclass+class SyncDiff:+ """Directory diff result for sync operations."""++ added_files: List[str] = field(default_factory=list)+ deleted_files: List[str] = field(default_factory=list)+ updated_files: List[str] = field(default_factory=list)+ added_dirs: List[str] = field(default_factory=list)+ deleted_dirs: List[str] = field(default_factory=list)++ def to_changes(self) -> Dict[str, List[str]]:+ return {+ "added": self.added_files + self.added_dirs,+ "modified": self.updated_files,+ "deleted": self.deleted_files + self.deleted_dirs,+ }+++class _SyncMixin:+ """Top-down recursive tree diff-and-merge primitive.++ Moves (not copies) visible children of ``root_uri`` into ``target_uri``+ so the target ends up mirroring the source, returning a :class:`SyncDiff`+ describing what changed. Hidden files (``.``-prefixed) are skipped —+ callers that need to carry over parser sidecars (e.g.+ ``.image_mappings.json``) do so explicitly after the sync.+ """++ async def sync_tree(+ self,+ root_uri: str,+ target_uri: str,+ *,+ ctx: Optional["RequestContext"] = None,+ file_change_status: Optional[Dict[str, bool]] = None,+ lease_ref: Optional[Dict[str, Any]] = None,+ delete_temp_after: bool = False,+ is_changed: Optional[Callable[[str, str, "RequestContext"], "bool"]] = None,+ ) -> SyncDiff:+ """Recursively merge ``root_uri`` into ``target_uri``.++ Hidden entries (names starting with ``.``) are skipped on both sides.+ Name conflicts (file vs. directory at the same name) are resolved by+ deleting the target-side entry before moving the source in place.++ Args:+ root_uri: Source (typically a temp/staging tree). Must exist.+ target_uri: Destination. Created (incl. parents) if missing.+ ctx: Request context for ACL / tenancy checks.+ file_change_status: Optional pre-computed map of source-URI →+ ``True`` (changed) / ``False`` (unchanged). Used as a+ performance hint to skip content comparison for files the+ caller already knows about.+ lease_ref: Optional pathlock lease passed through to every+ ``mv``/``rm``/``mkdir`` so the whole sync is covered by one+ lock.+ delete_temp_after: If ``True``, call ``delete_temp(root_uri)``+ after a successful sync into an existing target (the+ whole-tree mv path already consumes the source).+ is_changed: Optional async predicate ``(root_file, target_file,+ ctx) -> bool`` used to decide if an existing file needs to+ be replaced. Defaults to a stat-size shortcut followed by+ byte-for-byte content comparison.++ Returns:+ A :class:`SyncDiff` describing the applied changes (URIs are+ the *target* paths, suitable for feeding into incremental+ reindex pipelines).+ """+ if not await self.exists(root_uri, ctx=ctx):+ raise FileNotFoundError(+ f"Sync source no longer exists; refusing to sync into {target_uri}: {root_uri}"+ )++ diff = SyncDiff()++ async def list_children(dir_uri: str) -> Tuple[Dict[str, str], Dict[str, str]]:+ files: Dict[str, str] = {}+ dirs: Dict[str, str] = {}+ entries = await self.ls(+ dir_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx+ )+ for entry in entries:+ name = entry.get("name", "")+ if not name or name in [".", ".."]:+ continue+ if name.startswith("."):+ continue+ item_uri = VikingURI(dir_uri).join(name).uri+ if entry.get("isDir", False):+ dirs[name] = item_uri+ else:+ files[name] = item_uri+ return files, dirs++ async def default_is_changed(root_file: str, target_file: str) -> bool:+ try:+ current_stat = await self.stat(root_file, ctx=ctx)+ target_stat = await self.stat(target_file, ctx=ctx)+ current_size = current_stat.get("size") if isinstance(current_stat, dict) else None+ target_size = target_stat.get("size") if isinstance(target_stat, dict) else None+ if current_size is not None and target_size is not None and current_size != target_size:+ return True+ current_content = await self.read_file(root_file, ctx=ctx)+ target_content = await self.read_file(target_file, ctx=ctx)+ return current_content != target_content+ except Exception:+ return True++ check_changed = is_changed if is_changed is not None else default_is_changed++ async def sync_dir(root_dir: str, target_dir: str) -> None:+ root_files, root_dirs = await list_children(root_dir)+ target_files, target_dirs = await list_children(target_dir)++ file_names = set(root_files.keys()) | set(target_files.keys())+ for name in sorted(file_names):+ root_file = root_files.get(name)+ target_file = target_files.get(name)++ if root_file and name in target_dirs:+ target_conflict_dir = target_dirs[name]+ try:+ await self.rm(+ target_conflict_dir,+ recursive=True,+ ctx=ctx,+ lease_ref=lease_ref,+ )+ diff.deleted_dirs.append(target_conflict_dir)+ target_dirs.pop(name, None)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to delete directory for file conflict: {target_conflict_dir}, error={e}"+ )+ target_file = None++ if target_file and name in root_dirs and not root_file:+ try:+ await self.rm(target_file, ctx=ctx, lease_ref=lease_ref)+ diff.deleted_files.append(target_file)+ target_files.pop(name, None)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to delete file for dir conflict: {target_file}, error={e}"+ )+ continue++ if target_file and not root_file:+ try:+ await self.rm(target_file, ctx=ctx, lease_ref=lease_ref)+ diff.deleted_files.append(target_file)+ except Exception as e:+ logger.error(f"[SyncDiff] Failed to delete file: {target_file}, error={e}")+ continue++ if root_file and target_file:+ changed = False+ if file_change_status and root_file in file_change_status:+ changed = file_change_status[root_file]+ else:+ try:+ changed = await check_changed(root_file, target_file)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to compare file content for {root_file}: {e}, treating as unchanged"+ )+ changed = False+ if changed:+ diff.updated_files.append(target_file)+ try:+ await self.rm(target_file, ctx=ctx, lease_ref=lease_ref)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to remove old file before update: {target_file}, error={e}"+ )+ try:+ await self.mv(+ root_file,+ target_file,+ ctx=ctx,+ lease_ref=lease_ref,+ )+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to move updated file: {root_file} -> {target_file}, error={e}"+ )+ continue++ if root_file and not target_file:+ target_file_uri = VikingURI(target_dir).join(name).uri+ diff.added_files.append(target_file_uri)+ try:+ await self.mv(+ root_file,+ target_file_uri,+ ctx=ctx,+ lease_ref=lease_ref,+ )+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to move added file: {root_file} -> {target_file_uri}, error={e}"+ )++ dir_names = set(root_dirs.keys()) | set(target_dirs.keys())+ for name in sorted(dir_names):+ root_subdir = root_dirs.get(name)+ target_subdir = target_dirs.get(name)++ if root_subdir and name in target_files:+ target_conflict_file = target_files[name]+ try:+ await self.rm(+ target_conflict_file,+ ctx=ctx,+ lease_ref=lease_ref,+ )+ diff.deleted_files.append(target_conflict_file)+ target_files.pop(name, None)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to delete file for dir conflict: {target_conflict_file}, error={e}"+ )+ target_subdir = None++ if target_subdir and not root_subdir:+ try:+ await self.rm(+ target_subdir,+ recursive=True,+ ctx=ctx,+ lease_ref=lease_ref,+ )+ diff.deleted_dirs.append(target_subdir)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to delete directory: {target_subdir}, error={e}"+ )+ continue++ if root_subdir and not target_subdir:+ target_subdir_uri = VikingURI(target_dir).join(name).uri+ diff.added_dirs.append(target_subdir_uri)+ try:+ await self.mv(+ root_subdir,+ target_subdir_uri,+ ctx=ctx,+ lease_ref=lease_ref,+ )+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to move added directory: {root_subdir} -> {target_subdir_uri}, error={e}"+ )+ continue++ if root_subdir and target_subdir:+ await sync_dir(root_subdir, target_subdir)++ target_exists = await self.exists(target_uri, ctx=ctx)+ if not target_exists:+ parent_uri = VikingURI(target_uri).parent+ if parent_uri:+ await self.mkdir(+ parent_uri.uri,+ exist_ok=True,+ ctx=ctx,+ lease_ref=lease_ref,+ )+ diff.added_dirs.append(target_uri)+ await self.mv(root_uri, target_uri, ctx=ctx, lease_ref=lease_ref)+ return diff++ await sync_dir(root_uri, target_uri)+ if delete_temp_after:+ try:+ await self.delete_temp(root_uri, ctx=ctx)+ except Exception as e:+ logger.error(f"[SyncDiff] Failed to delete temp root {root_uri}: {e}")+ return diffdiff --git a/openviking/storage/viking_fs/_vector.py b/openviking/storage/viking_fs/_vector.pynew file mode 100644index 000000000..e4176da5f--- /dev/null+++ b/openviking/storage/viking_fs/_vector.py@@ -0,0 +1,72 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Vector store integration mixin for VikingFS."""++from typing import TYPE_CHECKING, Any, List, Optional++from openviking.server.identity import RequestContext+from openviking.storage.viking_fs._base import logger++if TYPE_CHECKING:+ from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend+++class _VectorMixin:+ """Vector store integration: delete/update URIs, get store/embedder."""++ async def _delete_from_vector_store(+ self, uris: List[str], ctx: Optional[RequestContext] = None+ ) -> None:+ """Delete records with specified URIs from vector store.++ Uses tenant-safe URI deletion semantics from vector store.+ """+ vector_store = self._get_vector_store()+ if not vector_store:+ return+ real_ctx = self._ctx_or_default(ctx)++ try:+ await vector_store.delete_uris(real_ctx, uris)+ for uri in uris:+ logger.debug(f"[VikingFS] Deleted from vector store: {uri}")+ except Exception as e:+ logger.warning(f"[VikingFS] Failed to delete from vector store: {e}")+ raise++ async def _update_vector_store_uris(+ self,+ uris: List[str],+ old_base: str,+ new_base: str,+ ctx: Optional[RequestContext] = None,+ ) -> None:+ """Update URIs in vector store (when moving files).++ Preserves vector data and updates URI-derived identifiers without regenerating embeddings.+ """+ vector_store = self._get_vector_store()+ if not vector_store:+ return++ real_ctx = self._ctx_or_default(ctx)++ for uri in uris:+ try:+ new_uri = new_base + uri[len(old_base) :]+ if await vector_store.update_uri_mapping(+ ctx=real_ctx,+ uri=uri,+ new_uri=new_uri,+ ):+ logger.debug(f"[VikingFS] Updated URI: {uri} -> {new_uri}")+ except Exception as e:+ logger.warning(f"[VikingFS] Failed to update {uri} in vector store: {e}")++ def _get_vector_store(self) -> Optional["VikingVectorIndexBackend"]:+ """Get vector store instance."""+ return self.vector_store++ def _get_embedder(self) -> Any:+ """Get embedder instance."""+ return self.query_embedderdiff --git a/tests/storage/test_ingest_ls_node_limit.py b/tests/storage/test_ingest_ls_node_limit.pyindex 08cbb637f..c688b4ad7 100644--- a/tests/storage/test_ingest_ls_node_limit.py+++ b/tests/storage/test_ingest_ls_node_limit.py@@ -2,11 +2,11 @@at ``viking_fs.ls``'s default ``node_limit=1000``.When a resource namespace already exists, re-ingesting a directory takes the-incremental sync path (``_sync_topdown_recursive`` -> ``sync_dir`` ->-``list_children``), which lists the temp root's children via ``viking_fs.ls``.-``ls`` defaults ``node_limit=1000``, so an import of >1000 docs silently-materialized only the first 1000 subdirectories into the target; the rest were-dropped when temp was deleted.+incremental sync path (``VikingFS.sync_tree`` -> ``sync_dir`` -> ``list_children``),+which lists the temp root's children via ``viking_fs.ls`` with ``node_limit=LS_ALL_NODES``.+``ls`` defaults ``node_limit=1000``, so an import of >1000 docs would silently+materialize only the first 1000 subdirectories into the target if sync_tree+forgot to pass the unlimited constant.Observed in the wild: a 6,221-doc WixQA re-ingest (the namespace already held anearlier 209-doc import, so the atomic-move fast path was skipped) produced@@ -18,6 +18,7 @@ from __future__ import annotationsimport pytestfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor+from openviking.storage.viking_fs import LS_ALL_NODESclass _TruncatingVikingFS:@@ -34,13 +35,21 @@ class _TruncatingVikingFS:def __init__(self, n_children: int):self._children = [{"name": f"doc{i:05d}", "isDir": True} for i in range(n_children)]self.moved: list[tuple[str, str]] = []+ self.ls_node_limits: list[int] = []async def exists(self, uri, ctx=None):return uri in (self.TEMP, self.TARGET)async def ls(self, uri, show_all_hidden=False, node_limit=1000, ctx=None):+ self.ls_node_limits.append(node_limit)entries = self._children if uri == self.TEMP else []- return entries[:node_limit] # the real ls truncates here+ return entries[:node_limit] if node_limit is not None else entries++ async def read_file(self, uri, ctx=None):+ return ""++ async def stat(self, uri, ctx=None):+ return {"size": 0}async def mv(self, src, dst, ctx=None, lease_ref=None):self.moved.append((src, dst))@@ -54,6 +63,31 @@ class _TruncatingVikingFS:async def delete_temp(self, uri, ctx=None, lease_ref=None):pass+ async def sync_tree(+ self, root_uri, target_uri, *, ctx=None, file_change_status=None,+ lease_ref=None, delete_temp_after=False, is_changed=None,+ ):+ from openviking.storage.viking_fs import SyncDiff+ from openviking_cli.utils.uri import VikingURI++ diff = SyncDiff()+ # Mirror _SyncMixin.list_children: must pass node_limit=LS_ALL_NODES.+ entries = await self.ls(+ root_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx+ )+ for entry in entries:+ name = entry.get("name", "")+ if not name or name in (".", "..") or name.startswith("."):+ continue+ src = VikingURI(root_uri).join(name).uri+ dst = VikingURI(target_uri).join(name).uri+ if entry.get("isDir", False):+ diff.added_dirs.append(dst)+ else:+ diff.added_files.append(dst)+ await self.mv(src, dst, ctx=ctx, lease_ref=lease_ref)+ return diff+@pytest.mark.asyncioasync def test_sync_materializes_all_children_above_default_node_limit(monkeypatch):@@ -63,6 +97,10 @@ async def test_sync_materializes_all_children_above_default_node_limit(monkeypat"openviking.storage.queuefs.semantic_processor.get_viking_fs",lambda: fake,)+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_processor.rewrite_image_uris",+ lambda *a, **kw: None,+ )diff = await SemanticProcessor()._sync_topdown_recursive(_TruncatingVikingFS.TEMP,@@ -75,6 +113,9 @@ async def test_sync_materializes_all_children_above_default_node_limit(monkeypat"sync was truncated at ls node_limit")assert len(diff.added_dirs) == n_children+ assert all(+ lim == LS_ALL_NODES for lim in fake.ls_node_limits+ ), f"sync_tree must pass LS_ALL_NODES to ls, got node_limits={fake.ls_node_limits}"class _TruncatingLsFS:diff --git a/tests/storage/test_semantic_processor_target_preexisting.py b/tests/storage/test_semantic_processor_target_preexisting.pyindex f0b70bed4..aa913d25b 100644--- a/tests/storage/test_semantic_processor_target_preexisting.py+++ b/tests/storage/test_semantic_processor_target_preexisting.py@@ -4,7 +4,8 @@ from unittest.mock import AsyncMockimport pytestfrom openviking.storage.queuefs.semantic_msg import SemanticMsg-from openviking.storage.queuefs.semantic_processor import DiffResult, SemanticProcessor+from openviking.storage.queuefs.semantic_processor import SemanticProcessor+from openviking.storage.viking_fs import SyncDiffclass _FakeVikingFS:@@ -12,58 +13,52 @@ class _FakeVikingFS:return True-class _SyncVikingFS:- def __init__(self):- self.contents = {- "viking://temp/import/a.md": "new",- "viking://temp/import/b.md": "same",- "viking://resources/root/a.md": "old",- "viking://resources/root/b.md": "same",- "viking://resources/root/.overview.md": "FILES:\n- b.md: old summary",- "viking://resources/root/.abstract.md": "old abstract",- }- self.entries = {- "viking://temp/import": [- {"name": "a.md", "isDir": False},- {"name": "b.md", "isDir": False},- ],- "viking://resources/root": [- {"name": "a.md", "isDir": False},- {"name": "b.md", "isDir": False},- {"name": ".overview.md", "isDir": False},- {"name": ".abstract.md", "isDir": False},- ],- }+class _SyncWrapperVikingFS:+ """Fake FS that verifies the wrapper calls sync_tree and post-processes correctly."""++ def __init__(self, target_exists=True):+ self.target_exists = target_exists+ self.sync_tree_calls = []self.deleted_temp = []self.mutation_leases = []+ self.sidecar_rewrite_calls = []async def exists(self, uri, ctx=None):- return uri in self.entries+ if uri.startswith("viking://resources/root"):+ return self.target_exists+ return True- async def ls(self, uri, show_all_hidden=False, node_limit=None, ctx=None):- return self.entries.get(uri, [])+ async def sync_tree(+ self,+ root_uri,+ target_uri,+ *,+ ctx=None,+ file_change_status=None,+ lease_ref=None,+ delete_temp_after=False,+ ):+ self.sync_tree_calls.append(+ (root_uri, target_uri, file_change_status, lease_ref, delete_temp_after)+ )+ self.mutation_leases.append(("sync_tree", root_uri, target_uri, lease_ref))+ return SyncDiff(updated_files=["viking://resources/root/a.md"])- async def stat(self, uri, ctx=None):- return {"size": len(self.contents.get(uri, ""))}+ async def delete_temp(self, uri, ctx=None):+ self.mutation_leases.append(("delete_temp", uri, None))+ self.deleted_temp.append(uri)async def read_file(self, uri, ctx=None):- return self.contents.get(uri, "")+ return "{}"- async def rm(self, uri, recursive=False, ctx=None, lease_ref=None):- self.mutation_leases.append(("rm", uri, lease_ref))- self.contents.pop(uri, None)+ async def stat(self, uri, ctx=None):+ raise FileNotFoundError(uri)- async def mv(self, src, dst, ctx=None, lease_ref=None):- self.mutation_leases.append(("mv", dst, lease_ref))- self.contents[dst] = self.contents.pop(src)+ async def write_file(self, uri, content, ctx=None, lease_ref=None):+ self.mutation_leases.append(("write_file", uri, lease_ref))- async def mkdir(self, uri, exist_ok=False, ctx=None, lease_ref=None):- self.mutation_leases.append(("mkdir", uri, lease_ref))- self.entries.setdefault(uri, [])-- async def delete_temp(self, uri, ctx=None, lease_ref=None):- self.mutation_leases.append(("delete_temp", uri, lease_ref))- self.deleted_temp.append(uri)+ async def glob(self, pattern, uri=None, ctx=None):+ return {"matches": []}class _FakeDagExecutor:@@ -105,7 +100,7 @@ async def test_target_source_syncs_before_semantic_dag(monkeypatch):processor = SemanticProcessor()processor._enqueue_parent_refresh = AsyncMock()processor._sync_topdown_recursive = AsyncMock(- return_value=DiffResult(+ return_value=SyncDiff(updated_files=["viking://resources/org/repo/a.md"],))@@ -129,8 +124,9 @@ async def test_target_source_syncs_before_semantic_dag(monkeypatch):@pytest.mark.asyncio-async def test_sync_diff_reports_target_uris_and_preserves_sidecars(monkeypatch):- fake_fs = _SyncVikingFS()+async def test_sync_wrapper_delegates_to_sync_tree_and_cleans_temp(monkeypatch):+ """The wrapper calls viking_fs.sync_tree and then deletes the temp tree."""+ fake_fs = _SyncWrapperVikingFS(target_exists=True)lease = {"lease_ref": "outer-tree-ref","owner_id": "outer-tree-owner",@@ -140,6 +136,10 @@ async def test_sync_diff_reports_target_uris_and_preserves_sidecars(monkeypatch)"openviking.storage.queuefs.semantic_processor.get_viking_fs",lambda: fake_fs,)+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_processor.rewrite_image_uris",+ AsyncMock(),+ )diff = await SemanticProcessor()._sync_topdown_recursive("viking://temp/import",@@ -147,24 +147,41 @@ async def test_sync_diff_reports_target_uris_and_preserves_sidecars(monkeypatch)lock=lease,)+ assert len(fake_fs.sync_tree_calls) == 1+ root_uri, target_uri, fcs, lr, dta = fake_fs.sync_tree_calls[0]+ assert root_uri == "viking://temp/import"+ assert target_uri == "viking://resources/root"+ assert lr is lease+ assert dta is Falseassert diff.to_changes() == {"added": [],"modified": ["viking://resources/root/a.md"],"deleted": [],}- assert fake_fs.contents["viking://resources/root/a.md"] == "new"- assert fake_fs.contents["viking://resources/root/.overview.md"] == (- "FILES:\n- b.md: old summary"- )- assert fake_fs.contents["viking://resources/root/.abstract.md"] == "old abstract"assert fake_fs.deleted_temp == ["viking://temp/import"]- assert fake_fs.mutation_leases- assert all(- mutation_lease is lease- for operation, _, mutation_lease in fake_fs.mutation_leases- if operation != "delete_temp"+++async def test_sync_wrapper_whole_tree_mv_for_new_target(monkeypatch):+ """When the target does not exist, sync_tree does a full mv and the wrapper does not delete (source is gone)."""+ fake_fs = _SyncWrapperVikingFS(target_exists=False)+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_processor.get_viking_fs",+ lambda: fake_fs,+ )+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_processor.rewrite_image_uris",+ AsyncMock(),)- assert fake_fs.mutation_leases[-1] == ("delete_temp", "viking://temp/import", None)++ diff = await SemanticProcessor()._sync_topdown_recursive(+ "viking://temp/import",+ "viking://resources/root",+ lock=None,+ )++ assert len(fake_fs.sync_tree_calls) == 1+ assert fake_fs.deleted_temp == [] # whole-tree mv already consumed the source@pytest.mark.asyncio@@ -183,5 +200,5 @@ async def test_sync_missing_source_never_touches_target(monkeypatch):lock=None,)- fake_fs.ls.assert_not_awaited()+ fake_fs.sync_tree.assert_not_awaited()fake_fs.rm.assert_not_awaited()