Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(fs): canonicalize URIs before move (#3219)

Qin Haojie4f0bd86f32c5
diff --git a/openviking/service/pack_service.py b/openviking/service/pack_service.py
index 146b1cee3..be963f753 100644
--- a/openviking/service/pack_service.py
+++ b/openviking/service/pack_service.py
@@ -8,6 +8,7 @@ Provides ovpack export/import and backup/restore operations.
from typing import Optional
+from openviking.core.namespace import canonicalize_uri
from openviking.core.uri_validation import validate_viking_uri
from openviking.server.identity import RequestContext
from openviking.storage.ovpack.operations import backup_ovpack as local_backup_ovpack
@@ -60,7 +61,7 @@ class PackService:
Exported file path
"""
viking_fs = self._ensure_initialized()
- uri = validate_viking_uri(uri)
+ uri = canonicalize_uri(validate_viking_uri(uri), ctx)
return await local_export_ovpack(
viking_fs,
uri,
@@ -105,7 +106,7 @@ class PackService:
Imported root resource URI
"""
viking_fs = self._ensure_initialized()
- parent = validate_viking_uri(parent, field_name="parent")
+ parent = canonicalize_uri(validate_viking_uri(parent, field_name="parent"), ctx)
return await local_import_ovpack(
viking_fs,
file_path,
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index dd2b54fae..d3af8aa61 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -712,6 +712,10 @@ class VikingFS:
details={"from_uri": old_uri, "to_uri": new_uri},
)
+ real_ctx = self._ctx_or_default(ctx)
+ old_uri = canonicalize_uri(old_uri, real_ctx)
+ new_uri = canonicalize_uri(new_uri, real_ctx)
+
lock_context = (
LockContext(
get_lock_manager(),
@@ -913,6 +917,10 @@ class VikingFS:
Returns:
Dict with matches, count, match_count, files_scanned
"""
+ real_ctx = self._ctx_or_default(ctx)
+ uri = canonicalize_uri(uri, real_ctx)
+ if exclude_uri:
+ exclude_uri = canonicalize_uri(exclude_uri, real_ctx)
self._ensure_access(uri, ctx)
# Skip vector_store.count() — the count field is not needed for grep,
# and avoiding it saves one VikingDB API call.
@@ -1088,7 +1096,7 @@ class VikingFS:
filter_expr = PathScope("uri", uri, depth=level_limit)
excluded_prefix = None
if exclude_uri:
- excluded_prefix = self._normalize_uri(exclude_uri).rstrip("/")
+ excluded_prefix = exclude_uri.rstrip("/")
self._ensure_access(excluded_prefix, ctx)
filter_expr = And(
[
@@ -1234,7 +1242,7 @@ class VikingFS:
excluded_path = None
if exclude_uri:
- normalized_excluded_uri = self._normalize_uri(exclude_uri).rstrip("/")
+ normalized_excluded_uri = exclude_uri.rstrip("/")
self._ensure_access(normalized_excluded_uri, ctx)
excluded_path = self._uri_to_path(normalized_excluded_uri, ctx=ctx)
@@ -1331,7 +1339,7 @@ class VikingFS:
compiled_pattern = re.compile(pattern, flags)
excluded_prefix = None
if exclude_uri:
- excluded_prefix = self._normalize_uri(exclude_uri).rstrip("/")
+ excluded_prefix = exclude_uri.rstrip("/")
self._ensure_access(excluded_prefix, ctx)
file_uris = await self._collect_grep_files(
uri,
@@ -1366,7 +1374,7 @@ class VikingFS:
if current_depth > level_limit:
return
- normalized_current_uri = self._normalize_uri(current_uri)
+ normalized_current_uri = current_uri
if excluded_prefix and (
normalized_current_uri == excluded_prefix
or normalized_current_uri.startswith(excluded_prefix + "/")
@@ -1392,7 +1400,7 @@ class VikingFS:
else:
file_uris.append(entry_uri)
- normalized_uri = self._normalize_uri(uri)
+ normalized_uri = uri
if excluded_prefix and (
normalized_uri == excluded_prefix or normalized_uri.startswith(excluded_prefix + "/")
):
@@ -2944,7 +2952,6 @@ class VikingFS:
old_base: str,
new_base: str,
ctx: Optional[RequestContext] = None,
- levels: Optional[List[int]] = None,
) -> None:
"""Update URIs in vector store (when moving files).
@@ -2954,83 +2961,20 @@ class VikingFS:
if not vector_store:
return
- old_base_uri = self._path_to_uri(old_base, ctx=ctx)
- new_base_uri = self._path_to_uri(new_base, ctx=ctx)
+ real_ctx = self._ctx_or_default(ctx)
for uri in uris:
try:
- new_uri = uri.replace(old_base_uri, new_base_uri, 1)
-
- await vector_store.update_uri_mapping(
- ctx=self._ctx_or_default(ctx),
+ new_uri = new_base + uri[len(old_base) :]
+ if await vector_store.update_uri_mapping(
+ ctx=real_ctx,
uri=uri,
new_uri=new_uri,
- levels=levels,
- )
- logger.debug(f"[VikingFS] Updated URI: {uri} -> {new_uri}")
+ ):
+ logger.debug(f"[VikingFS] Updated URI: {uri} -> {new_uri}")
except Exception as e:
logger.warning(f"[VikingFS] Failed to update {uri} in vector store: {e}")
- async def _mv_vector_store_l0_l1(
- self,
- old_uri: str,
- new_uri: str,
- ctx: Optional[RequestContext] = None,
- lock_handle: Optional["LockHandle"] = None,
- ) -> None:
- from openviking.storage.errors import LockAcquisitionError, ResourceBusyError
- from openviking.storage.transaction import LockContext, get_lock_manager
-
- self._ensure_access(old_uri, ctx)
- self._ensure_access(new_uri, ctx)
-
- real_ctx = self._ctx_or_default(ctx)
- old_dir = VikingURI.normalize(old_uri).rstrip("/")
- new_dir = VikingURI.normalize(new_uri).rstrip("/")
- if old_dir == new_dir:
- return
-
- for uri in (old_dir, new_dir):
- if uri.endswith(("/.abstract.md", "/.overview.md")):
- raise ValueError(f"mv_vector_store expects directory URIs, got: {uri}")
-
- try:
- old_stat = await self.stat(old_dir, ctx=real_ctx)
- except Exception as e:
- raise FileNotFoundError(f"mv_vector_store old_uri not found: {old_dir}") from e
- try:
- new_stat = await self.stat(new_dir, ctx=real_ctx)
- except Exception as e:
- raise FileNotFoundError(f"mv_vector_store new_uri not found: {new_dir}") from e
-
- if not (isinstance(old_stat, dict) and old_stat.get("isDir", False)):
- raise ValueError(f"mv_vector_store expects old_uri to be a directory: {old_dir}")
- if not (isinstance(new_stat, dict) and new_stat.get("isDir", False)):
- raise ValueError(f"mv_vector_store expects new_uri to be a directory: {new_dir}")
-
- old_path = self._uri_to_path(old_dir, ctx=real_ctx)
- new_path = self._uri_to_path(new_dir, ctx=real_ctx)
-
- try:
- async with LockContext(
- get_lock_manager(),
- [old_path],
- lock_mode="mv",
- mv_dst_path=new_path,
- src_is_dir=True,
- handle=lock_handle,
- ):
- await self._update_vector_store_uris(
- uris=[old_dir],
- old_base=old_dir,
- new_base=new_dir,
- ctx=real_ctx,
- levels=[0, 1],
- )
-
- except LockAcquisitionError:
- raise ResourceBusyError(f"Resource is being processed: {old_dir}", uri=old_dir)
-
def _get_vector_store(self) -> Optional["VikingVectorIndexBackend"]:
"""Get vector store instance."""
return self.vector_store
diff --git a/tests/storage/test_semantic_processor_mv_vector_store.py b/tests/storage/test_semantic_processor_mv_vector_store.py
index 8ba7fad24..015861819 100644
--- a/tests/storage/test_semantic_processor_mv_vector_store.py
+++ b/tests/storage/test_semantic_processor_mv_vector_store.py
@@ -1,235 +1,55 @@
-from __future__ import annotations
-
-import hashlib
-from typing import Any, Dict, List, Optional
+from contextlib import asynccontextmanager
+from unittest.mock import AsyncMock
import pytest
from openviking.server.identity import RequestContext, Role
-from openviking.storage.expr import And, Eq, In
+from openviking.storage.viking_fs import VikingFS
from openviking_cli.session.user_id import UserIdentifier
-class _FakeVectorStore:
- def __init__(self, records: List[Dict[str, Any]]):
- self.records = list(records)
- self.deleted_ids: List[str] = []
-
- async def update_uri_mapping(
- self,
- *,
- ctx: RequestContext,
- uri: str,
- new_uri: str,
- levels: Optional[List[int]] = None,
- ) -> bool:
- def seed_uri_for_id(target_uri: str, level: int) -> str:
- if level == 0:
- return (
- target_uri
- if target_uri.endswith("/.abstract.md")
- else f"{target_uri}/.abstract.md"
- )
- if level == 1:
- return (
- target_uri
- if target_uri.endswith("/.overview.md")
- else f"{target_uri}/.overview.md"
- )
- return target_uri
-
- touched = False
- ids_to_delete: List[str] = []
- for record in list(self.records):
- if record.get("account_id") != ctx.account_id:
- continue
- if record.get("uri") != uri:
- continue
- try:
- level = int(record.get("level", 2))
- except (TypeError, ValueError):
- level = 2
- if levels is not None and level not in set(levels):
- continue
-
- seed_uri = seed_uri_for_id(new_uri, level)
- new_id = hashlib.md5(f"{ctx.account_id}:{seed_uri}".encode("utf-8")).hexdigest()
- new_record = dict(record)
- new_record["id"] = new_id
- new_record["uri"] = new_uri
- new_record.pop("parent_uri", None)
- self.records.append(new_record)
- touched = True
-
- old_id = record.get("id")
- if old_id and old_id != new_id:
- ids_to_delete.append(old_id)
-
- if ids_to_delete:
- await self.delete(list(set(ids_to_delete)), ctx=ctx)
-
- return touched
-
- async def filter(self, *, filter=None, limit: int = 100, ctx: RequestContext):
- conds = []
- if filter is not None:
- if isinstance(filter, And):
- conds = list(filter.conds)
- else:
- conds = [filter]
-
- uri: Optional[str] = None
- account_id: Optional[str] = None
- owner_space: Optional[str] = None
- levels: Optional[List[int]] = None
-
- for cond in conds:
- if isinstance(cond, Eq) and cond.field == "uri":
- uri = cond.value
- elif isinstance(cond, Eq) and cond.field == "account_id":
- account_id = cond.value
- elif isinstance(cond, Eq) and cond.field == "owner_space":
- owner_space = cond.value
- elif isinstance(cond, In) and cond.field == "level":
- levels = [int(v) for v in cond.values]
-
- matched = [
- r
- for r in self.records
- if (uri is None or r.get("uri") == uri)
- and (account_id is None or r.get("account_id") == account_id)
- and (owner_space is None or r.get("owner_space") == owner_space)
- and (levels is None or int(r.get("level", 2)) in levels)
- ]
- return matched[:limit]
-
- async def delete(self, ids: List[str], *, ctx: RequestContext) -> int:
- id_set = set(ids)
- self.deleted_ids.extend(ids)
- self.records = [r for r in self.records if r.get("id") not in id_set]
- return len(ids)
-
-
-class _NoopLockContext:
- def __init__(self, *_args, **_kwargs):
- return None
-
- async def __aenter__(self):
- return None
-
- async def __aexit__(self, exc_type, exc, tb):
- return False
-
-
@pytest.mark.asyncio
-async def test_mv_vector_store_moves_records(monkeypatch):
- from openviking.storage.viking_fs import VikingFS
+async def test_mv_canonicalizes_user_shorthand_before_vector_update(monkeypatch):
+ ctx = RequestContext(user=UserIdentifier("acc", "default"), role=Role.ROOT)
+ fs = VikingFS.__new__(VikingFS)
+ fs._async_agfs = AsyncMock()
+ fs._async_agfs.stat.return_value = {"isDir": False}
+ fs._collect_uris = AsyncMock(return_value=[])
+ fs._copy_for_mv = AsyncMock()
+ fs._update_vector_store_uris = AsyncMock()
+
+ @asynccontextmanager
+ async def unlocked(*_args, **_kwargs):
+ yield None
- ctx = RequestContext(user=UserIdentifier("acc", "user"), role=Role.ROOT)
- old_uri = "viking://resources/a"
- new_uri = "viking://resources/b"
+ monkeypatch.setattr("openviking.storage.transaction.get_lock_manager", lambda: None)
+ monkeypatch.setattr("openviking.storage.transaction.LockContext", unlocked)
- store = _FakeVectorStore(
- [
- {
- "id": "l0",
- "uri": old_uri,
- "level": 0,
- "account_id": ctx.account_id,
- "owner_space": "",
- },
- {
- "id": "l1",
- "uri": old_uri,
- "level": 1,
- "account_id": ctx.account_id,
- "owner_space": "",
- },
- {
- "id": "l2",
- "uri": old_uri,
- "level": 2,
- "account_id": ctx.account_id,
- "owner_space": "",
- },
- {
- "id": "child-l0",
- "uri": f"{old_uri}/x",
- "level": 0,
- "account_id": ctx.account_id,
- "owner_space": "",
- },
- ]
+ await fs.mv(
+ "viking://user/peers/vaka/memories/profile.md",
+ "viking://user/memories/profile.md",
+ ctx=ctx,
)
- class _FakeAGFS:
- def rm(self, _path, recursive: bool = False):
- return None
-
- class _FakeVikingFS(VikingFS):
- def __init__(self):
- super().__init__(agfs=_FakeAGFS(), vector_store=store)
-
- def _uri_to_path(self, uri, ctx=None):
- return f"/mock/{uri.replace('viking://', '')}"
-
- async def stat(self, uri, ctx=None):
- return {"isDir": True}
-
- def _ensure_access(self, uri, ctx):
- return None
-
- monkeypatch.setattr(
- "openviking.storage.viking_fs.get_viking_fs",
- lambda: _FakeVikingFS(),
+ fs._update_vector_store_uris.assert_awaited_once_with(
+ ["viking://user/default/peers/vaka/memories/profile.md"],
+ "viking://user/default/peers/vaka/memories/profile.md",
+ "viking://user/default/memories/profile.md",
+ ctx=ctx,
)
- monkeypatch.setattr("openviking.storage.transaction.get_lock_manager", lambda: None)
- monkeypatch.setattr("openviking.storage.transaction.LockContext", _NoopLockContext)
-
- fs = _FakeVikingFS()
- await fs._mv_vector_store_l0_l1(old_uri, new_uri, ctx=ctx)
-
- assert {r["id"] for r in store.records if r.get("uri") == old_uri} == {"l2"}
- assert {r["id"] for r in store.records if r.get("uri") == f"{old_uri}/x"} == {"child-l0"}
- assert {int(r["level"]) for r in store.records if r.get("uri") == new_uri} == {0, 1}
- assert all("parent_uri" not in r for r in store.records if r.get("uri") == new_uri)
- assert set(store.deleted_ids) == {"l0", "l1"}
@pytest.mark.asyncio
-async def test_mv_vector_store_requires_directories(monkeypatch):
- from openviking.storage.viking_fs import VikingFS
-
- ctx = RequestContext(user=UserIdentifier("acc", "user"), role=Role.ROOT)
- old_uri = "viking://resources/a"
- new_uri = "viking://resources/b"
-
- store = _FakeVectorStore([])
-
- class _FakeAGFS:
- def rm(self, _path, recursive: bool = False):
- return None
-
- class _FakeVikingFS(VikingFS):
- def __init__(self):
- super().__init__(agfs=_FakeAGFS(), vector_store=store)
-
- def _uri_to_path(self, uri, ctx=None):
- return f"/mock/{uri.replace('viking://', '')}"
-
- async def stat(self, uri, ctx=None):
- return {"isDir": uri == old_uri}
-
- def _ensure_access(self, uri, ctx):
- return None
-
- monkeypatch.setattr(
- "openviking.storage.viking_fs.get_viking_fs",
- lambda: _FakeVikingFS(),
+async def test_update_vector_store_uris_swallows_update_failure():
+ ctx = RequestContext(user=UserIdentifier("acc", "default"), role=Role.ROOT)
+ fs = VikingFS.__new__(VikingFS)
+ fs.vector_store = AsyncMock()
+ fs.vector_store.update_uri_mapping.side_effect = RuntimeError("vector unavailable")
+
+ await fs._update_vector_store_uris(
+ ["viking://user/default/source.md"],
+ "viking://user/default/source.md",
+ "viking://user/default/target.md",
+ ctx=ctx,
)
- monkeypatch.setattr("openviking.storage.transaction.get_lock_manager", lambda: None)
- monkeypatch.setattr("openviking.storage.transaction.LockContext", _NoopLockContext)
-
- fs = _FakeVikingFS()
- with pytest.raises(ValueError):
- await fs._mv_vector_store_l0_l1(old_uri, new_uri, ctx=ctx)
+ fs.vector_store.update_uri_mapping.assert_awaited_once()