Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Fix dir ops locks (#3054)

baojun-zhang22131233a9c2
diff --git a/openviking/session/compressor_v2.py b/openviking/session/compressor_v2.py
index ced3830a5..8322cd048 100644
--- a/openviking/session/compressor_v2.py
+++ b/openviking/session/compressor_v2.py
@@ -1029,13 +1029,13 @@ class SessionCompressorV2:
[lock_path],
lock_mode="exact",
handle=lock_handle,
- ):
+ ) as active_lock_handle:
await self._append_trajectory_metadata(
exp_uri,
normalized_traj_uris,
ctx,
viking_fs,
- lock_handle=lock_handle,
+ lock_handle=active_lock_handle,
)
except Exception as e:
logger.warning(f"Failed to append source trajectories to {exp_uri}: {e}")
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 701d11de2..cdaf2502c 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -729,7 +729,7 @@ class VikingFS:
)
)
- async with lock_context:
+ async with lock_context as active_lock_handle:
uris_to_move = await self._collect_uris(old_path, recursive=True, ctx=ctx)
uris_to_move.append(target_uri)
@@ -746,6 +746,7 @@ class VikingFS:
is_dir=is_dir,
is_temp=is_temp,
ctx=ctx,
+ lock_handle=active_lock_handle,
)
except Exception as e:
if "not found" in str(e).lower():
@@ -811,6 +812,7 @@ class VikingFS:
is_dir: bool,
is_temp: bool,
ctx: Optional[RequestContext] = None,
+ lock_handle: Optional["LockHandle"] = None,
) -> None:
"""Copy source to destination for mv without deleting source."""
if is_temp:
@@ -823,15 +825,23 @@ class VikingFS:
return
if is_dir:
- await self._copy_dir_through_vikingfs(old_uri, new_uri, ctx=ctx)
+ await self._copy_dir_through_vikingfs(
+ old_uri, new_uri, ctx=ctx, lock_handle=lock_handle
+ )
else:
- await self._copy_file_through_vikingfs(old_uri, new_uri, ctx=ctx)
+ await self._copy_file_through_vikingfs(
+ old_uri,
+ new_uri,
+ ctx=ctx,
+ lock_handle=lock_handle,
+ )
async def _copy_dir_through_vikingfs(
self,
old_uri: str,
new_uri: str,
ctx: Optional[RequestContext] = None,
+ lock_handle: Optional["LockHandle"] = None,
) -> None:
"""Recursively copy a directory through VikingFS read/write hooks."""
await self.mkdir(new_uri, exist_ok=True, ctx=ctx)
@@ -844,19 +854,30 @@ class VikingFS:
old_child_uri = f"{old_uri.rstrip('/')}/{name}"
new_child_uri = f"{new_uri.rstrip('/')}/{name}"
if entry.get("isDir"):
- await self._copy_dir_through_vikingfs(old_child_uri, new_child_uri, ctx=ctx)
+ await self._copy_dir_through_vikingfs(
+ old_child_uri,
+ new_child_uri,
+ ctx=ctx,
+ lock_handle=lock_handle,
+ )
else:
- await self._copy_file_through_vikingfs(old_child_uri, new_child_uri, ctx=ctx)
+ await self._copy_file_through_vikingfs(
+ old_child_uri,
+ new_child_uri,
+ ctx=ctx,
+ lock_handle=lock_handle,
+ )
async def _copy_file_through_vikingfs(
self,
from_uri: str,
to_uri: str,
ctx: Optional[RequestContext] = None,
+ lock_handle: Optional["LockHandle"] = None,
) -> None:
"""Copy one file through VikingFS read/write hooks without deleting source."""
content_bytes = await self.read_file_bytes(from_uri, ctx=ctx)
- await self.write_file_bytes(to_uri, content_bytes, ctx=ctx)
+ await self.write_file_bytes(to_uri, content_bytes, ctx=ctx, lock_handle=lock_handle)
async def grep(
self,
diff --git a/tests/agfs/test_viking_fs_git.py b/tests/agfs/test_viking_fs_git.py
index fd8aa07ff..63ebf8c0c 100644
--- a/tests/agfs/test_viking_fs_git.py
+++ b/tests/agfs/test_viking_fs_git.py
@@ -10,16 +10,13 @@ from __future__ import annotations
import asyncio
import os
import secrets
-import tempfile
import shutil
+import tempfile
from pathlib import Path
from typing import Tuple
import pytest
-
-ragfs_python = pytest.importorskip("ragfs_python")
-
from openviking.pyagfs.exceptions import (
AGFSInvalidOperationError,
AGFSNotFoundError,
@@ -27,8 +24,10 @@ from openviking.pyagfs.exceptions import (
)
from openviking.server.identity import RequestContext, Role
from openviking.storage.viking_fs import VikingFS
+from openviking_cli.exceptions import NotFoundError
from openviking_cli.session.user_id import UserIdentifier
+ragfs_python = pytest.importorskip("ragfs_python")
# ----------------------------- helpers -----------------------------
@@ -352,7 +351,10 @@ async def test_derived_files_versioned_with_source(vfs):
assert c1["changed"] == 2
# show finds both
- assert await vfs.show("main", path="viking://resources/x.md.abstract.md", ctx=ctx) == b"abstract-v1"
+ assert (
+ await vfs.show("main", path="viking://resources/x.md.abstract.md", ctx=ctx)
+ == b"abstract-v1"
+ )
# Update derived file
await vfs.write_file("viking://resources/x.md.abstract.md", b"abstract-v2", ctx=ctx)
@@ -428,19 +430,20 @@ async def test_double_encryption_restore_preserves_plaintext(vfs_encrypted):
await vfs_encrypted.write_file("viking://resources/secret.md", plaintext_v1, ctx=ctx)
c1 = await vfs_encrypted.commit(
- message="v1", paths=["viking://resources/secret.md"], ctx=ctx,
+ message="v1",
+ paths=["viking://resources/secret.md"],
+ ctx=ctx,
)
assert c1["result"] == "created"
# Modify
await vfs_encrypted.write_file("viking://resources/secret.md", plaintext_v2, ctx=ctx)
await vfs_encrypted.commit(
- message="v2", paths=["viking://resources/secret.md"], ctx=ctx,
- )
- assert (
- await vfs_encrypted.read("viking://resources/secret.md", ctx=ctx)
- == plaintext_v2
+ message="v2",
+ paths=["viking://resources/secret.md"],
+ ctx=ctx,
)
+ assert await vfs_encrypted.read("viking://resources/secret.md", ctx=ctx) == plaintext_v2
# Restore
result = await vfs_encrypted.restore(
@@ -457,6 +460,24 @@ async def test_double_encryption_restore_preserves_plaintext(vfs_encrypted):
assert restored == plaintext_v1
+async def test_encrypted_mv_file_reuses_outer_lock_handle(vfs_encrypted):
+ ctx = _make_ctx(account="acct_enc")
+ src_uri = "viking://resources/src.md"
+ dst_uri = "viking://resources/dst.md"
+ plaintext = b"top-secret-mv"
+
+ await vfs_encrypted.write_file(src_uri, plaintext, ctx=ctx)
+
+ await vfs_encrypted.mv(src_uri, dst_uri, ctx=ctx)
+
+ with pytest.raises(NotFoundError):
+ await vfs_encrypted.read(src_uri, ctx=ctx)
+
+ moved = await vfs_encrypted.read(dst_uri, ctx=ctx)
+ assert moved == plaintext
+
+
# =========================================================================
# 8. Feature disabled
# =========================================================================
@@ -486,42 +507,54 @@ def test_classify_restore_path(vfs):
from openviking.core.context import ContextLevel
# Directory-level markers -> (op, dir_uri, level)
- assert vfs._classify_restore_path(
- "resources/proj/.abstract.md", deleted=False
- ) == ("reindex_marker", "viking://resources/proj", ContextLevel.ABSTRACT)
- assert vfs._classify_restore_path(
- "resources/proj/.overview.md", deleted=False
- ) == ("reindex_marker", "viking://resources/proj", ContextLevel.OVERVIEW)
- assert vfs._classify_restore_path(
- "resources/proj/.abstract.md", deleted=True
- ) == ("delete", "viking://resources/proj", ContextLevel.ABSTRACT)
- assert vfs._classify_restore_path(
- "resources/proj/.overview.md", deleted=True
- ) == ("delete", "viking://resources/proj", ContextLevel.OVERVIEW)
+ assert vfs._classify_restore_path("resources/proj/.abstract.md", deleted=False) == (
+ "reindex_marker",
+ "viking://resources/proj",
+ ContextLevel.ABSTRACT,
+ )
+ assert vfs._classify_restore_path("resources/proj/.overview.md", deleted=False) == (
+ "reindex_marker",
+ "viking://resources/proj",
+ ContextLevel.OVERVIEW,
+ )
+ assert vfs._classify_restore_path("resources/proj/.abstract.md", deleted=True) == (
+ "delete",
+ "viking://resources/proj",
+ ContextLevel.ABSTRACT,
+ )
+ assert vfs._classify_restore_path("resources/proj/.overview.md", deleted=True) == (
+ "delete",
+ "viking://resources/proj",
+ ContextLevel.OVERVIEW,
+ )
# .relations.json has no vector side-effect
- assert vfs._classify_restore_path(
- "resources/proj/.relations.json", deleted=False
- ) is None
- assert vfs._classify_restore_path(
- "resources/proj/.relations.json", deleted=True
- ) is None
+ assert vfs._classify_restore_path("resources/proj/.relations.json", deleted=False) is None
+ assert vfs._classify_restore_path("resources/proj/.relations.json", deleted=True) is None
# Per-file sidecars do NOT exist in production -> treated as ordinary source files
- assert vfs._classify_restore_path(
- "resources/proj/x.md.abstract.md", deleted=False
- ) == ("reindex_file", "viking://resources/proj/x.md.abstract.md", ContextLevel.DETAIL)
- assert vfs._classify_restore_path(
- "resources/proj/x.md.overview.md", deleted=True
- ) == ("delete", "viking://resources/proj/x.md.overview.md", ContextLevel.DETAIL)
+ assert vfs._classify_restore_path("resources/proj/x.md.abstract.md", deleted=False) == (
+ "reindex_file",
+ "viking://resources/proj/x.md.abstract.md",
+ ContextLevel.DETAIL,
+ )
+ assert vfs._classify_restore_path("resources/proj/x.md.overview.md", deleted=True) == (
+ "delete",
+ "viking://resources/proj/x.md.overview.md",
+ ContextLevel.DETAIL,
+ )
# Source files -> DETAIL reindex/delete
- assert vfs._classify_restore_path(
- "resources/proj/x.md", deleted=False
- ) == ("reindex_file", "viking://resources/proj/x.md", ContextLevel.DETAIL)
- assert vfs._classify_restore_path(
- "resources/proj/x.md", deleted=True
- ) == ("delete", "viking://resources/proj/x.md", ContextLevel.DETAIL)
+ assert vfs._classify_restore_path("resources/proj/x.md", deleted=False) == (
+ "reindex_file",
+ "viking://resources/proj/x.md",
+ ContextLevel.DETAIL,
+ )
+ assert vfs._classify_restore_path("resources/proj/x.md", deleted=True) == (
+ "delete",
+ "viking://resources/proj/x.md",
+ ContextLevel.DETAIL,
+ )
# Directory marker at the account root -> None (no parent dir to scope)
assert vfs._classify_restore_path(".abstract.md", deleted=False) is None
@@ -558,20 +591,17 @@ async def test_restore_schedules_reindex_for_derived_only_change(vfs, monkeypatc
spy = _SpyExecutor()
import openviking.service.reindex_executor as reindex_mod
+
monkeypatch.setattr(reindex_mod, "get_reindex_executor", lambda: spy)
ctx = _make_ctx(account="acct_derived_only")
await vfs.write_file("viking://resources/proj/x.md", b"body", ctx=ctx)
- await vfs.write_file(
- "viking://resources/proj/.abstract.md", b"abs-v1", ctx=ctx
- )
+ await vfs.write_file("viking://resources/proj/.abstract.md", b"abs-v1", ctx=ctx)
c1 = await vfs.commit(message="v1", ctx=ctx)
assert c1["result"] == "created"
# Modify ONLY the directory marker; source file untouched
- await vfs.write_file(
- "viking://resources/proj/.abstract.md", b"abs-v2", ctx=ctx
- )
+ await vfs.write_file("viking://resources/proj/.abstract.md", b"abs-v2", ctx=ctx)
c2 = await vfs.commit(
message="v2",
paths=["viking://resources/proj/.abstract.md"],
@@ -604,21 +634,18 @@ async def test_restore_schedules_marker_and_files_independently(vfs, monkeypatch
spy = _SpyExecutor()
import openviking.service.reindex_executor as reindex_mod
+
monkeypatch.setattr(reindex_mod, "get_reindex_executor", lambda: spy)
ctx = _make_ctx(account="acct_dedup")
await vfs.write_file("viking://resources/proj/x.md", b"v1", ctx=ctx)
await vfs.write_file("viking://resources/proj/y.md", b"yv1", ctx=ctx)
- await vfs.write_file(
- "viking://resources/proj/.abstract.md", b"a-v1", ctx=ctx
- )
+ await vfs.write_file("viking://resources/proj/.abstract.md", b"a-v1", ctx=ctx)
c1 = await vfs.commit(message="v1", ctx=ctx)
await vfs.write_file("viking://resources/proj/x.md", b"v2", ctx=ctx)
await vfs.write_file("viking://resources/proj/y.md", b"yv2", ctx=ctx)
- await vfs.write_file(
- "viking://resources/proj/.abstract.md", b"a-v2", ctx=ctx
- )
+ await vfs.write_file("viking://resources/proj/.abstract.md", b"a-v2", ctx=ctx)
await vfs.commit(message="v2", ctx=ctx)
await vfs.restore(
@@ -630,11 +657,13 @@ async def test_restore_schedules_marker_and_files_independently(vfs, monkeypatch
await asyncio.sleep(0)
# Directory marker recompute + each source file's DETAIL, all independent.
- assert sorted(spy.calls) == sorted([
- ("reindex_marker", "viking://resources/proj", 0),
- ("reindex_file", "viking://resources/proj/x.md"),
- ("reindex_file", "viking://resources/proj/y.md"),
- ])
+ assert sorted(spy.calls) == sorted(
+ [
+ ("reindex_marker", "viking://resources/proj", 0),
+ ("reindex_file", "viking://resources/proj/x.md"),
+ ("reindex_file", "viking://resources/proj/y.md"),
+ ]
+ )
@pytest.mark.asyncio
@@ -645,22 +674,19 @@ async def test_restore_schedules_siblings_independently(vfs, monkeypatch):
spy = _SpyExecutor()
import openviking.service.reindex_executor as reindex_mod
+
monkeypatch.setattr(reindex_mod, "get_reindex_executor", lambda: spy)
ctx = _make_ctx(account="acct_subsume_sibling")
# proj_a: source file + directory marker
await vfs.write_file("viking://resources/proj_a/x.md", b"v1", ctx=ctx)
- await vfs.write_file(
- "viking://resources/proj_a/.abstract.md", b"a-v1", ctx=ctx
- )
+ await vfs.write_file("viking://resources/proj_a/.abstract.md", b"a-v1", ctx=ctx)
# proj_b: source file only — sibling directory
await vfs.write_file("viking://resources/proj_b/y.md", b"v1", ctx=ctx)
c1 = await vfs.commit(message="v1", ctx=ctx)
await vfs.write_file("viking://resources/proj_a/x.md", b"v2", ctx=ctx)
- await vfs.write_file(
- "viking://resources/proj_a/.abstract.md", b"a-v2", ctx=ctx
- )
+ await vfs.write_file("viking://resources/proj_a/.abstract.md", b"a-v2", ctx=ctx)
await vfs.write_file("viking://resources/proj_b/y.md", b"v2", ctx=ctx)
await vfs.commit(message="v2", ctx=ctx)
@@ -673,11 +699,13 @@ async def test_restore_schedules_siblings_independently(vfs, monkeypatch):
await asyncio.sleep(0)
await asyncio.sleep(0)
- assert sorted(spy.calls) == sorted([
- ("reindex_marker", "viking://resources/proj_a", 0),
- ("reindex_file", "viking://resources/proj_a/x.md"),
- ("reindex_file", "viking://resources/proj_b/y.md"),
- ])
+ assert sorted(spy.calls) == sorted(
+ [
+ ("reindex_marker", "viking://resources/proj_a", 0),
+ ("reindex_file", "viking://resources/proj_a/x.md"),
+ ("reindex_file", "viking://resources/proj_b/y.md"),
+ ]
+ )
@pytest.mark.asyncio
@@ -689,6 +717,7 @@ async def test_restore_deletes_marker_and_source_vectors(vfs, monkeypatch):
spy = _SpyExecutor()
import openviking.service.reindex_executor as reindex_mod
+
monkeypatch.setattr(reindex_mod, "get_reindex_executor", lambda: spy)
ctx = _make_ctx(account="acct_del_marker")
@@ -726,17 +755,14 @@ async def test_restore_relations_json_has_no_vector_side_effect(vfs, monkeypatch
spy = _SpyExecutor()
import openviking.service.reindex_executor as reindex_mod
+
monkeypatch.setattr(reindex_mod, "get_reindex_executor", lambda: spy)
ctx = _make_ctx(account="acct_relations")
- await vfs.write_file(
- "viking://resources/proj/.relations.json", b"{\"v\":1}", ctx=ctx
- )
+ await vfs.write_file("viking://resources/proj/.relations.json", b'{"v":1}', ctx=ctx)
c1 = await vfs.commit(message="v1", ctx=ctx)
- await vfs.write_file(
- "viking://resources/proj/.relations.json", b"{\"v\":2}", ctx=ctx
- )
+ await vfs.write_file("viking://resources/proj/.relations.json", b'{"v":2}', ctx=ctx)
c2 = await vfs.commit(
message="v2",
paths=["viking://resources/proj/.relations.json"],
@@ -766,6 +792,7 @@ async def test_restore_returns_pollable_task_id(vfs, monkeypatch):
spy = _SpyExecutor()
import openviking.service.reindex_executor as reindex_mod
+
monkeypatch.setattr(reindex_mod, "get_reindex_executor", lambda: spy)
from openviking.service.task_tracker import (
@@ -814,9 +841,7 @@ async def test_restore_returns_pollable_task_id(vfs, monkeypatch):
from openviking.service.task_tracker import get_task_tracker
tracker = get_task_tracker()
- task = await tracker.get(
- task_id, account_id=ctx.account_id, user_id=ctx.user.user_id
- )
+ task = await tracker.get(task_id, account_id=ctx.account_id, user_id=ctx.user.user_id)
assert task is not None
assert task.task_type == "snapshot_restore_reindex"
assert task.status.value == "completed"
@@ -840,6 +865,7 @@ async def test_restore_concurrent_same_dir_is_rejected(vfs, monkeypatch):
spy = _SpyExecutor()
import openviking.service.reindex_executor as reindex_mod
+
monkeypatch.setattr(reindex_mod, "get_reindex_executor", lambda: spy)
ctx = _make_ctx(account="acct_lock_same")
@@ -900,6 +926,7 @@ async def test_restore_concurrent_sibling_dirs_do_not_block(vfs, monkeypatch):
"""
spy = _SpyExecutor()
import openviking.service.reindex_executor as reindex_mod
+
monkeypatch.setattr(reindex_mod, "get_reindex_executor", lambda: spy)
ctx = _make_ctx(account="acct_lock_sibling")
@@ -927,7 +954,9 @@ async def test_restore_concurrent_sibling_dirs_do_not_block(vfs, monkeypatch):
if entered == 2:
both_in.set()
# proj_a is released first; proj_b waits on its own gate.
- await (release_a if kwargs.get("project_dir", "").endswith("proj_a") else release_b).wait()
+ await (
+ release_a if kwargs.get("project_dir", "").endswith("proj_a") else release_b
+ ).wait()
return await orig_run(method_name, *args, **kwargs)
monkeypatch.setattr(vfs._async_agfs, "run", gated_run)
@@ -1021,7 +1050,6 @@ async def test_vikingfs_set_gitignore_rejects_oversized_content(vfs):
assert result["ignored"] == 0
-
@pytest.mark.asyncio
async def test_vikingfs_get_gitignore_maps_non_utf8_to_invalid_operation(vfs):
ctx = _make_ctx(account="acct_gitignore_bad_utf8")
@@ -1035,4 +1063,3 @@ async def test_vikingfs_get_gitignore_maps_non_utf8_to_invalid_operation(vfs):
with pytest.raises(AGFSInvalidOperationError) as excinfo:
await vfs.get_gitignore(ctx=ctx)
assert "utf-8" in str(excinfo.value).lower()
-
diff --git a/tests/session/memory/test_compressor_v2.py b/tests/session/memory/test_compressor_v2.py
index e02f23321..68c6340e1 100644
--- a/tests/session/memory/test_compressor_v2.py
+++ b/tests/session/memory/test_compressor_v2.py
@@ -164,7 +164,11 @@ class MockVikingFS:
content = entry.get("content", "")
if query_lower in name.lower() or query_lower in content.lower():
memories.append(
- {"uri": uri, "name": name, "abstract": content[:200] if content else ""}
+ {
+ "uri": uri,
+ "name": name,
+ "abstract": content[:200] if content else "",
+ }
)
return {
@@ -739,7 +743,7 @@ class TestCompressorV2:
events.append("read")
return self.files.get(uri, "")
- async def write_file(self, uri: str, content: str, ctx=None):
+ async def write_file(self, uri: str, content: str, ctx=None, lock_handle=None):
events.append("write")
self.files[uri] = content
@@ -875,7 +879,9 @@ class TestExtractLoopPatchRepair:
]
self.messages = []
- async def get_completion_async(self, messages, tools=None, tool_choice=None, thinking=False):
+ async def get_completion_async(
+ self, messages, tools=None, tool_choice=None, thinking=False
+ ):
self.messages.append(list(messages))
return self.responses.pop(0)
@@ -975,7 +981,9 @@ class TestExtractLoopPatchRepair:
]
self.messages = []
- async def get_completion_async(self, messages, tools=None, tool_choice=None, thinking=False):
+ async def get_completion_async(
+ self, messages, tools=None, tool_choice=None, thinking=False
+ ):
self.messages.append(list(messages))
return self.responses.pop(0)
@@ -1073,7 +1081,9 @@ class TestExtractLoopPatchRepair:
]
self.messages = []
- async def get_completion_async(self, messages, tools=None, tool_choice=None, thinking=False):
+ async def get_completion_async(
+ self, messages, tools=None, tool_choice=None, thinking=False
+ ):
self.messages.append(list(messages))
return self.responses.pop(0)