Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(fs): canonicalize URIs before move (#3219)
Qin Haojie4f0bd86f32c5
diff --git a/openviking/service/pack_service.py b/openviking/service/pack_service.pyindex 146b1cee3..be963f753 100644--- a/openviking/service/pack_service.py+++ b/openviking/service/pack_service.py@@ -8,6 +8,7 @@ Provides ovpack export/import and backup/restore operations.from typing import Optional+from openviking.core.namespace import canonicalize_urifrom openviking.core.uri_validation import validate_viking_urifrom openviking.server.identity import RequestContextfrom openviking.storage.ovpack.operations import backup_ovpack as local_backup_ovpack@@ -60,7 +61,7 @@ class PackService:Exported file path"""viking_fs = self._ensure_initialized()- uri = validate_viking_uri(uri)+ uri = canonicalize_uri(validate_viking_uri(uri), ctx)return await local_export_ovpack(viking_fs,uri,@@ -105,7 +106,7 @@ class PackService:Imported root resource URI"""viking_fs = self._ensure_initialized()- parent = validate_viking_uri(parent, field_name="parent")+ parent = canonicalize_uri(validate_viking_uri(parent, field_name="parent"), ctx)return await local_import_ovpack(viking_fs,file_path,diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex dd2b54fae..d3af8aa61 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -712,6 +712,10 @@ class VikingFS:details={"from_uri": old_uri, "to_uri": new_uri},)+ real_ctx = self._ctx_or_default(ctx)+ old_uri = canonicalize_uri(old_uri, real_ctx)+ new_uri = canonicalize_uri(new_uri, real_ctx)+lock_context = (LockContext(get_lock_manager(),@@ -913,6 +917,10 @@ class VikingFS:Returns:Dict with matches, count, match_count, files_scanned"""+ real_ctx = self._ctx_or_default(ctx)+ uri = canonicalize_uri(uri, real_ctx)+ if exclude_uri:+ exclude_uri = canonicalize_uri(exclude_uri, real_ctx)self._ensure_access(uri, ctx)# Skip vector_store.count() — the count field is not needed for grep,# and avoiding it saves one VikingDB API call.@@ -1088,7 +1096,7 @@ class VikingFS:filter_expr = PathScope("uri", uri, depth=level_limit)excluded_prefix = Noneif exclude_uri:- excluded_prefix = self._normalize_uri(exclude_uri).rstrip("/")+ excluded_prefix = exclude_uri.rstrip("/")self._ensure_access(excluded_prefix, ctx)filter_expr = And([@@ -1234,7 +1242,7 @@ class VikingFS:excluded_path = Noneif exclude_uri:- normalized_excluded_uri = self._normalize_uri(exclude_uri).rstrip("/")+ normalized_excluded_uri = exclude_uri.rstrip("/")self._ensure_access(normalized_excluded_uri, ctx)excluded_path = self._uri_to_path(normalized_excluded_uri, ctx=ctx)@@ -1331,7 +1339,7 @@ class VikingFS:compiled_pattern = re.compile(pattern, flags)excluded_prefix = Noneif exclude_uri:- excluded_prefix = self._normalize_uri(exclude_uri).rstrip("/")+ excluded_prefix = exclude_uri.rstrip("/")self._ensure_access(excluded_prefix, ctx)file_uris = await self._collect_grep_files(uri,@@ -1366,7 +1374,7 @@ class VikingFS:if current_depth > level_limit:return- normalized_current_uri = self._normalize_uri(current_uri)+ normalized_current_uri = current_uriif excluded_prefix and (normalized_current_uri == excluded_prefixor normalized_current_uri.startswith(excluded_prefix + "/")@@ -1392,7 +1400,7 @@ class VikingFS:else:file_uris.append(entry_uri)- normalized_uri = self._normalize_uri(uri)+ normalized_uri = uriif excluded_prefix and (normalized_uri == excluded_prefix or normalized_uri.startswith(excluded_prefix + "/")):@@ -2944,7 +2952,6 @@ class VikingFS:old_base: str,new_base: str,ctx: Optional[RequestContext] = None,- levels: Optional[List[int]] = None,) -> None:"""Update URIs in vector store (when moving files).@@ -2954,83 +2961,20 @@ class VikingFS:if not vector_store:return- old_base_uri = self._path_to_uri(old_base, ctx=ctx)- new_base_uri = self._path_to_uri(new_base, ctx=ctx)+ real_ctx = self._ctx_or_default(ctx)for uri in uris:try:- new_uri = uri.replace(old_base_uri, new_base_uri, 1)-- await vector_store.update_uri_mapping(- ctx=self._ctx_or_default(ctx),+ new_uri = new_base + uri[len(old_base) :]+ if await vector_store.update_uri_mapping(+ ctx=real_ctx,uri=uri,new_uri=new_uri,- levels=levels,- )- logger.debug(f"[VikingFS] Updated URI: {uri} -> {new_uri}")+ ):+ logger.debug(f"[VikingFS] Updated URI: {uri} -> {new_uri}")except Exception as e:logger.warning(f"[VikingFS] Failed to update {uri} in vector store: {e}")- async def _mv_vector_store_l0_l1(- self,- old_uri: str,- new_uri: str,- ctx: Optional[RequestContext] = None,- lock_handle: Optional["LockHandle"] = None,- ) -> None:- from openviking.storage.errors import LockAcquisitionError, ResourceBusyError- from openviking.storage.transaction import LockContext, get_lock_manager-- self._ensure_access(old_uri, ctx)- self._ensure_access(new_uri, ctx)-- real_ctx = self._ctx_or_default(ctx)- old_dir = VikingURI.normalize(old_uri).rstrip("/")- new_dir = VikingURI.normalize(new_uri).rstrip("/")- if old_dir == new_dir:- return-- for uri in (old_dir, new_dir):- if uri.endswith(("/.abstract.md", "/.overview.md")):- raise ValueError(f"mv_vector_store expects directory URIs, got: {uri}")-- try:- old_stat = await self.stat(old_dir, ctx=real_ctx)- except Exception as e:- raise FileNotFoundError(f"mv_vector_store old_uri not found: {old_dir}") from e- try:- new_stat = await self.stat(new_dir, ctx=real_ctx)- except Exception as e:- raise FileNotFoundError(f"mv_vector_store new_uri not found: {new_dir}") from e-- if not (isinstance(old_stat, dict) and old_stat.get("isDir", False)):- raise ValueError(f"mv_vector_store expects old_uri to be a directory: {old_dir}")- if not (isinstance(new_stat, dict) and new_stat.get("isDir", False)):- raise ValueError(f"mv_vector_store expects new_uri to be a directory: {new_dir}")-- old_path = self._uri_to_path(old_dir, ctx=real_ctx)- new_path = self._uri_to_path(new_dir, ctx=real_ctx)-- try:- async with LockContext(- get_lock_manager(),- [old_path],- lock_mode="mv",- mv_dst_path=new_path,- src_is_dir=True,- handle=lock_handle,- ):- await self._update_vector_store_uris(- uris=[old_dir],- old_base=old_dir,- new_base=new_dir,- ctx=real_ctx,- levels=[0, 1],- )-- except LockAcquisitionError:- raise ResourceBusyError(f"Resource is being processed: {old_dir}", uri=old_dir)-def _get_vector_store(self) -> Optional["VikingVectorIndexBackend"]:"""Get vector store instance."""return self.vector_storediff --git a/tests/storage/test_semantic_processor_mv_vector_store.py b/tests/storage/test_semantic_processor_mv_vector_store.pyindex 8ba7fad24..015861819 100644--- a/tests/storage/test_semantic_processor_mv_vector_store.py+++ b/tests/storage/test_semantic_processor_mv_vector_store.py@@ -1,235 +1,55 @@-from __future__ import annotations--import hashlib-from typing import Any, Dict, List, Optional+from contextlib import asynccontextmanager+from unittest.mock import AsyncMockimport pytestfrom openviking.server.identity import RequestContext, Role-from openviking.storage.expr import And, Eq, In+from openviking.storage.viking_fs import VikingFSfrom openviking_cli.session.user_id import UserIdentifier-class _FakeVectorStore:- def __init__(self, records: List[Dict[str, Any]]):- self.records = list(records)- self.deleted_ids: List[str] = []-- async def update_uri_mapping(- self,- *,- ctx: RequestContext,- uri: str,- new_uri: str,- levels: Optional[List[int]] = None,- ) -> bool:- def seed_uri_for_id(target_uri: str, level: int) -> str:- if level == 0:- return (- target_uri- if target_uri.endswith("/.abstract.md")- else f"{target_uri}/.abstract.md"- )- if level == 1:- return (- target_uri- if target_uri.endswith("/.overview.md")- else f"{target_uri}/.overview.md"- )- return target_uri-- touched = False- ids_to_delete: List[str] = []- for record in list(self.records):- if record.get("account_id") != ctx.account_id:- continue- if record.get("uri") != uri:- continue- try:- level = int(record.get("level", 2))- except (TypeError, ValueError):- level = 2- if levels is not None and level not in set(levels):- continue-- seed_uri = seed_uri_for_id(new_uri, level)- new_id = hashlib.md5(f"{ctx.account_id}:{seed_uri}".encode("utf-8")).hexdigest()- new_record = dict(record)- new_record["id"] = new_id- new_record["uri"] = new_uri- new_record.pop("parent_uri", None)- self.records.append(new_record)- touched = True-- old_id = record.get("id")- if old_id and old_id != new_id:- ids_to_delete.append(old_id)-- if ids_to_delete:- await self.delete(list(set(ids_to_delete)), ctx=ctx)-- return touched-- async def filter(self, *, filter=None, limit: int = 100, ctx: RequestContext):- conds = []- if filter is not None:- if isinstance(filter, And):- conds = list(filter.conds)- else:- conds = [filter]-- uri: Optional[str] = None- account_id: Optional[str] = None- owner_space: Optional[str] = None- levels: Optional[List[int]] = None-- for cond in conds:- if isinstance(cond, Eq) and cond.field == "uri":- uri = cond.value- elif isinstance(cond, Eq) and cond.field == "account_id":- account_id = cond.value- elif isinstance(cond, Eq) and cond.field == "owner_space":- owner_space = cond.value- elif isinstance(cond, In) and cond.field == "level":- levels = [int(v) for v in cond.values]-- matched = [- r- for r in self.records- if (uri is None or r.get("uri") == uri)- and (account_id is None or r.get("account_id") == account_id)- and (owner_space is None or r.get("owner_space") == owner_space)- and (levels is None or int(r.get("level", 2)) in levels)- ]- return matched[:limit]-- async def delete(self, ids: List[str], *, ctx: RequestContext) -> int:- id_set = set(ids)- self.deleted_ids.extend(ids)- self.records = [r for r in self.records if r.get("id") not in id_set]- return len(ids)---class _NoopLockContext:- def __init__(self, *_args, **_kwargs):- return None-- async def __aenter__(self):- return None-- async def __aexit__(self, exc_type, exc, tb):- return False--@pytest.mark.asyncio-async def test_mv_vector_store_moves_records(monkeypatch):- from openviking.storage.viking_fs import VikingFS+async def test_mv_canonicalizes_user_shorthand_before_vector_update(monkeypatch):+ ctx = RequestContext(user=UserIdentifier("acc", "default"), role=Role.ROOT)+ fs = VikingFS.__new__(VikingFS)+ fs._async_agfs = AsyncMock()+ fs._async_agfs.stat.return_value = {"isDir": False}+ fs._collect_uris = AsyncMock(return_value=[])+ fs._copy_for_mv = AsyncMock()+ fs._update_vector_store_uris = AsyncMock()++ @asynccontextmanager+ async def unlocked(*_args, **_kwargs):+ yield None- ctx = RequestContext(user=UserIdentifier("acc", "user"), role=Role.ROOT)- old_uri = "viking://resources/a"- new_uri = "viking://resources/b"+ monkeypatch.setattr("openviking.storage.transaction.get_lock_manager", lambda: None)+ monkeypatch.setattr("openviking.storage.transaction.LockContext", unlocked)- store = _FakeVectorStore(- [- {- "id": "l0",- "uri": old_uri,- "level": 0,- "account_id": ctx.account_id,- "owner_space": "",- },- {- "id": "l1",- "uri": old_uri,- "level": 1,- "account_id": ctx.account_id,- "owner_space": "",- },- {- "id": "l2",- "uri": old_uri,- "level": 2,- "account_id": ctx.account_id,- "owner_space": "",- },- {- "id": "child-l0",- "uri": f"{old_uri}/x",- "level": 0,- "account_id": ctx.account_id,- "owner_space": "",- },- ]+ await fs.mv(+ "viking://user/peers/vaka/memories/profile.md",+ "viking://user/memories/profile.md",+ ctx=ctx,)- class _FakeAGFS:- def rm(self, _path, recursive: bool = False):- return None-- class _FakeVikingFS(VikingFS):- def __init__(self):- super().__init__(agfs=_FakeAGFS(), vector_store=store)-- def _uri_to_path(self, uri, ctx=None):- return f"/mock/{uri.replace('viking://', '')}"-- async def stat(self, uri, ctx=None):- return {"isDir": True}-- def _ensure_access(self, uri, ctx):- return None-- monkeypatch.setattr(- "openviking.storage.viking_fs.get_viking_fs",- lambda: _FakeVikingFS(),+ fs._update_vector_store_uris.assert_awaited_once_with(+ ["viking://user/default/peers/vaka/memories/profile.md"],+ "viking://user/default/peers/vaka/memories/profile.md",+ "viking://user/default/memories/profile.md",+ ctx=ctx,)- monkeypatch.setattr("openviking.storage.transaction.get_lock_manager", lambda: None)- monkeypatch.setattr("openviking.storage.transaction.LockContext", _NoopLockContext)-- fs = _FakeVikingFS()- await fs._mv_vector_store_l0_l1(old_uri, new_uri, ctx=ctx)-- assert {r["id"] for r in store.records if r.get("uri") == old_uri} == {"l2"}- assert {r["id"] for r in store.records if r.get("uri") == f"{old_uri}/x"} == {"child-l0"}- assert {int(r["level"]) for r in store.records if r.get("uri") == new_uri} == {0, 1}- assert all("parent_uri" not in r for r in store.records if r.get("uri") == new_uri)- assert set(store.deleted_ids) == {"l0", "l1"}@pytest.mark.asyncio-async def test_mv_vector_store_requires_directories(monkeypatch):- from openviking.storage.viking_fs import VikingFS-- ctx = RequestContext(user=UserIdentifier("acc", "user"), role=Role.ROOT)- old_uri = "viking://resources/a"- new_uri = "viking://resources/b"-- store = _FakeVectorStore([])-- class _FakeAGFS:- def rm(self, _path, recursive: bool = False):- return None-- class _FakeVikingFS(VikingFS):- def __init__(self):- super().__init__(agfs=_FakeAGFS(), vector_store=store)-- def _uri_to_path(self, uri, ctx=None):- return f"/mock/{uri.replace('viking://', '')}"-- async def stat(self, uri, ctx=None):- return {"isDir": uri == old_uri}-- def _ensure_access(self, uri, ctx):- return None-- monkeypatch.setattr(- "openviking.storage.viking_fs.get_viking_fs",- lambda: _FakeVikingFS(),+async def test_update_vector_store_uris_swallows_update_failure():+ ctx = RequestContext(user=UserIdentifier("acc", "default"), role=Role.ROOT)+ fs = VikingFS.__new__(VikingFS)+ fs.vector_store = AsyncMock()+ fs.vector_store.update_uri_mapping.side_effect = RuntimeError("vector unavailable")++ await fs._update_vector_store_uris(+ ["viking://user/default/source.md"],+ "viking://user/default/source.md",+ "viking://user/default/target.md",+ ctx=ctx,)- monkeypatch.setattr("openviking.storage.transaction.get_lock_manager", lambda: None)- monkeypatch.setattr("openviking.storage.transaction.LockContext", _NoopLockContext)-- fs = _FakeVikingFS()- with pytest.raises(ValueError):- await fs._mv_vector_store_l0_l1(old_uri, new_uri, ctx=ctx)+ fs.vector_store.update_uri_mapping.assert_awaited_once()