Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Template lock contention must not starve storage I/O in the default executor."""
import asyncio
import threading
from concurrent.futures import ThreadPoolExecutor
from types import SimpleNamespace
import pytest
from openviking.pyagfs import AGFSNotFoundError
from openviking.session.memory.account_templates import (
account_memory_template_path,
account_memory_template_uri,
default_memory_template,
legacy_account_memory_template_path,
read_account_memory_template,
resolve_account_memory_registry,
update_account_memory_template,
)
from openviking.session.memory.memory_type_registry import MemoryTypeRegistry
from openviking.storage.errors import LockAcquisitionError, ResourceBusyError
from openviking_cli.exceptions import FailedPreconditionError
class LockingAGFS:
"""Blocking sync backend, including the legacy long-wait failure mode."""
def __init__(self):
self.files = {}
self.locks = {}
self.leases = {}
self.timeouts = []
async def pathlock_acquire_batch_async(self, ctx, requests, *args):
paths = [request["path"] for request in requests]
return await asyncio.to_thread(self.pathlock_acquire_exact_batch, ctx, paths, *args)
def pathlock_acquire_exact_batch(self, ctx, paths, timeout_secs=0, owner_lease_ref=None):
self.timeouts.append(timeout_secs)
held = []
for path in sorted(paths):
lock = self.locks.setdefault(path, threading.Lock())
# Bound failures on the old implementation so the regression is fast.
if not lock.acquire(timeout=min(timeout_secs, 0.15)):
for previous in held:
self.locks[previous].release()
raise LockAcquisitionError("test lock conflict")
held.append(path)
lease = {"lease_ref": paths[0], "paths": held}
self.leases[paths[0]] = lease
return lease
def pathlock_release(self, ctx, lease):
path = lease["lease_ref"]
assert ctx["account_id"] == "locking-test"
assert self.leases.pop(path) == lease
for locked_path in lease["paths"]:
self.locks[locked_path].release()
def read(self, path, **kwargs):
if path not in self.files:
raise AGFSNotFoundError(path)
return self.files[path]
def write(self, path, data, *, ctx):
if not path.endswith(".backup"):
assert path.endswith(".tmp"), "the active file must never be overwritten in place"
assert path in self.leases[ctx["lease_ref"]]["paths"]
self.files[path] = data
def mv(self, old_path, new_path, *, ctx):
held = self.leases[ctx["lease_ref"]]["paths"]
assert old_path in held and new_path in held
self.files[new_path] = self.files.pop(old_path)
def ensure_parent_dirs(self, path, **kwargs):
pass
def rm(self, path, *, ctx):
assert path in self.leases[ctx["lease_ref"]]["paths"]
self.files.pop(path, None)
@pytest.mark.parametrize("workers", [1, 2])
def test_template_readers_finish_with_a_small_default_executor(workers):
agfs = LockingAGFS()
fs = SimpleNamespace(agfs=agfs)
async def run():
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=workers))
results = await asyncio.wait_for(
asyncio.gather(
*(read_account_memory_template(fs, "locking-test", "profile") for _ in range(8)),
return_exceptions=True,
),
timeout=4,
)
assert results == [None] * 8
assert not agfs.leases
assert not agfs.timeouts, "readers must not acquire locks at all"
asyncio.run(run())
@pytest.mark.parametrize("workers", [1, 2])
def test_concurrent_template_writers_do_not_starve_the_lock_holder(workers):
agfs = LockingAGFS()
fs = SimpleNamespace(agfs=agfs)
registry = MemoryTypeRegistry()
async def run():
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=workers))
results = await asyncio.wait_for(
asyncio.gather(
*(
update_account_memory_template(
fs, "locking-test", "profile", {"description": f"Writer {index}"}, registry
)
for index in range(8)
)
),
timeout=3,
)
assert [item["description"] for item in results] == [
f"Writer {index}" for index in range(8)
]
assert not agfs.leases
assert set(agfs.timeouts) == {0.0}
assert not any(path.endswith(".tmp") for path in agfs.files)
asyncio.run(run())
@pytest.mark.parametrize("workers", [1, 2])
def test_template_resolve_and_publish_share_a_small_executor(workers):
agfs = LockingAGFS()
fs = SimpleNamespace(agfs=agfs)
registry = MemoryTypeRegistry()
async def run():
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=workers))
await update_account_memory_template(
fs, "locking-test", "profile", {"description": "Before"}, registry
)
results = await asyncio.wait_for(
asyncio.gather(
*(resolve_account_memory_registry(fs, "locking-test", registry) for _ in range(8)),
update_account_memory_template(
fs, "locking-test", "profile", {"description": "After"}, registry
),
),
timeout=4,
)
assert all(
result.get("profile").description in {"Before", "After"} for result in results[:-1]
)
assert results[-1]["description"] == "After"
assert not agfs.leases
asyncio.run(run())
async def _wait_for_flag(flag):
async def wait():
while not flag.is_set():
await asyncio.sleep(0.001)
await asyncio.wait_for(wait(), timeout=2)
@pytest.mark.parametrize("existing", [False, True], ids=["first-publication", "replacement"])
def test_readers_never_see_a_partial_staging_file(monkeypatch, existing):
agfs = LockingAGFS()
fs = SimpleNamespace(agfs=agfs)
registry = MemoryTypeRegistry()
entered = threading.Event()
proceed = threading.Event()
async def run():
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=2))
if existing:
await update_account_memory_template(
fs, "locking-test", "profile", {"description": "Before"}, registry
)
write = agfs.write
def slow_write(path, data, **kwargs):
if path.endswith(".tmp"):
agfs.files[path] = b"partial YAML"
entered.set()
assert proceed.wait(3)
return write(path, data, **kwargs)
monkeypatch.setattr(agfs, "write", slow_write)
writer = asyncio.create_task(
update_account_memory_template(
fs, "locking-test", "profile", {"description": "After"}, registry
)
)
try:
await _wait_for_flag(entered)
readers = await asyncio.wait_for(
asyncio.gather(
*(
read_account_memory_template(fs, "locking-test", "profile")
for _ in range(16)
)
),
timeout=1,
)
assert all(
result["description"] == "Before" if existing else result is None
for result in readers
)
finally:
proceed.set()
await writer
assert (await read_account_memory_template(fs, "locking-test", "profile"))[
"description"
] == "After"
assert not agfs.leases
assert not any(path.endswith(".tmp") for path in agfs.files)
asyncio.run(run())
@pytest.mark.parametrize("phase", ["acquire", "write", "move", "release"])
def test_cancelled_publication_drains_io_and_releases_leases(monkeypatch, phase):
agfs = LockingAGFS()
fs = SimpleNamespace(agfs=agfs)
registry = MemoryTypeRegistry()
entered = threading.Event()
proceed = threading.Event()
method = {
"acquire": "pathlock_acquire_exact_batch",
"write": "write",
"move": "mv",
"release": "pathlock_release",
}[phase]
original = getattr(agfs, method)
def blocked(*args, **kwargs):
result = original(*args, **kwargs) if phase != "release" else None
entered.set()
assert proceed.wait(3)
return original(*args, **kwargs) if phase == "release" else result
monkeypatch.setattr(agfs, method, blocked)
async def run():
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1))
writer = asyncio.create_task(
update_account_memory_template(
fs, "locking-test", "profile", {"description": "Complete"}, registry
)
)
try:
await _wait_for_flag(entered)
writer.cancel()
await asyncio.sleep(0.01)
writer.cancel()
await asyncio.sleep(0.01)
assert not writer.done(), "cancellation must drain the in-flight native call"
assert agfs.leases, "do not release the lease before the worker finishes"
finally:
proceed.set()
with pytest.raises(asyncio.CancelledError):
await writer
assert not agfs.leases
assert all(not lock.locked() for lock in agfs.locks.values())
assert not any(path.endswith(".tmp") for path in agfs.files)
result = await read_account_memory_template(fs, "locking-test", "profile")
assert result is None if phase == "acquire" else result["description"] == "Complete"
asyncio.run(run())
@pytest.mark.parametrize("cancel", [False, True], ids=["timeout", "cancel"])
def test_waiting_writer_does_not_block_readers_or_leak_locks(monkeypatch, cancel):
import openviking.session.memory.account_templates as templates
monkeypatch.setattr(templates, "_TEMPLATE_LOCK_TIMEOUT_SECONDS", 0.05)
agfs = LockingAGFS()
fs = SimpleNamespace(agfs=agfs)
registry = MemoryTypeRegistry()
path = account_memory_template_path("locking-test", "profile")
lock = threading.Lock()
agfs.locks[path] = lock
async def run():
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1))
with lock:
writer = asyncio.create_task(
update_account_memory_template(
fs, "locking-test", "profile", {"description": "Waiting"}, registry
)
)
while not agfs.timeouts:
await asyncio.sleep(0.001)
assert await read_account_memory_template(fs, "locking-test", "profile") is None
if cancel:
writer.cancel()
with pytest.raises(asyncio.CancelledError):
await writer
else:
with pytest.raises(ResourceBusyError) as exc:
await writer
assert exc.value.uri == account_memory_template_uri("profile")
assert exc.value.conflict_type == "memory_templates_busy"
assert agfs.timeouts and set(agfs.timeouts) == {0.0}
assert not agfs.leases
assert not agfs.files
assert all(not held.locked() for held in agfs.locks.values())
asyncio.run(run())
@pytest.mark.parametrize("existing", [False, True])
@pytest.mark.parametrize("failure", ["staging", "before-move", "after-move", "cleanup"])
def test_publication_failures_never_overwrite_the_active_file(monkeypatch, existing, failure):
agfs = LockingAGFS()
fs = SimpleNamespace(agfs=agfs)
registry = MemoryTypeRegistry()
path = account_memory_template_path("locking-test", "profile")
async def run():
if existing:
await update_account_memory_template(
fs, "locking-test", "profile", {"description": "Before"}, registry
)
old = agfs.files.get(path)
write, move, remove = agfs.write, agfs.mv, agfs.rm
def faulty_write(target, data, **kwargs):
if failure == "staging" and target.endswith(".tmp"):
agfs.files[target] = b"partial"
raise OSError("staging failed")
return write(target, data, **kwargs)
def faulty_move(source, target, **kwargs):
if failure == "before-move":
raise OSError("publish failed")
if failure == "after-move":
# S3 CopyObject succeeded but deleting its source failed.
agfs.files[target] = agfs.files[source]
raise OSError("source cleanup failed")
return move(source, target, **kwargs)
def faulty_cleanup(target, **kwargs):
if failure in {"cleanup", "after-move"} and target.endswith(".tmp"):
raise OSError("cleanup failed")
return remove(target, **kwargs)
monkeypatch.setattr(agfs, "write", faulty_write)
monkeypatch.setattr(agfs, "mv", faulty_move)
monkeypatch.setattr(agfs, "rm", faulty_cleanup)
update = update_account_memory_template(
fs, "locking-test", "profile", {"description": "After"}, registry
)
if failure in {"staging", "before-move"}:
with pytest.raises(OSError):
await update
assert agfs.files.get(path) == old
else:
assert (await update)["description"] == "After"
assert (await read_account_memory_template(fs, "locking-test", "profile"))[
"description"
] == "After"
assert not agfs.leases
assert all(not held.locked() for held in agfs.locks.values())
assert path + ".backup" not in agfs.files
asyncio.run(run())
@pytest.mark.parametrize("failure", [None, "remove"])
def test_saving_defaults_uses_locked_reset_and_preserves_failed_writes(monkeypatch, failure):
agfs = LockingAGFS()
fs = SimpleNamespace(agfs=agfs)
registry = MemoryTypeRegistry()
path = account_memory_template_path("locking-test", "profile")
legacy_path = legacy_account_memory_template_path("locking-test", "profile")
async def run():
await update_account_memory_template(
fs, "locking-test", "profile", {"description": "Custom"}, registry
)
previous = agfs.files[path]
backup_path = path + ".backup"
agfs.files[backup_path] = b"obsolete"
remove = agfs.rm
def checked_remove(target, **kwargs):
if target == legacy_path:
return remove(target, **kwargs)
if target == backup_path:
assert path not in agfs.files
return remove(target, **kwargs)
assert target == path
if failure == "remove":
raise OSError("remove failed")
return remove(target, **kwargs)
monkeypatch.setattr(agfs, "rm", checked_remove)
update = update_account_memory_template(
fs, "locking-test", "profile", default_memory_template(registry, "profile"), registry
)
if failure == "remove":
with pytest.raises(FailedPreconditionError, match="current memory template"):
await update
assert agfs.files[path] == previous
assert agfs.files[backup_path] == b"obsolete"
else:
assert await update is None
assert path not in agfs.files
assert backup_path not in agfs.files
assert not agfs.leases
assert all(not lock.locked() for lock in agfs.locks.values())
assert not any(name.endswith(".tmp") for name in agfs.files)
asyncio.run(run())
@pytest.mark.parametrize("workers", [None, 1], ids=["default-executor", "one-worker"])
@pytest.mark.parametrize("encrypted", [False, True], ids=["plaintext", "encrypted"])
def test_native_template_read_publish_and_reset(tmp_path, workers, encrypted):
from openviking.pyagfs import get_binding_client
from openviking.utils.agfs_utils import RagfsBindingConfig, mount_agfs_backend
from openviking_cli.utils.config.agfs_config import AGFSConfig
try:
client_type, _ = get_binding_client()
except ImportError:
pytest.skip("RAGFS native binding is not installed")
if not hasattr(client_type, "pathlock_acquire_exact_batch"):
pytest.skip("RAGFS native binding predates the current PathLock API")
config = RagfsBindingConfig(
agfs=AGFSConfig(path=str(tmp_path), backend="local"),
root_key=b"k" * 32 if encrypted else None,
provider_type=1 if encrypted else None,
)
native = client_type(None, config=config.to_binding_dict())
fs = SimpleNamespace(agfs=native)
registry = MemoryTypeRegistry()
async def run():
if workers is not None:
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=workers))
# Original B2 reproduction: no override, no writer, concurrent registry reads.
empty = await asyncio.wait_for(
asyncio.gather(
*(resolve_account_memory_registry(fs, "locking-test", registry) for _ in range(8))
),
timeout=5,
)
assert all(
result.get("profile").description == registry.get("profile").description
for result in empty
)
await update_account_memory_template(
fs, "locking-test", "profile", {"description": "Before"}, registry
)
for index in range(3):
results = await asyncio.wait_for(
asyncio.gather(
*(
read_account_memory_template(fs, "locking-test", "profile")
for _ in range(24)
),
update_account_memory_template(
fs, "locking-test", "profile", {"description": f"Version {index}"}, registry
),
),
timeout=5,
)
expected_old = "Before" if index == 0 else f"Version {index - 1}"
assert all(
result["description"] in {expected_old, f"Version {index}"} for result in results
)
results = await asyncio.wait_for(
asyncio.gather(
*(
update_account_memory_template(
fs,
"locking-test",
"profile",
{"description": f"Concurrent {index}"},
registry,
)
for index in range(4)
)
),
timeout=5,
)
assert [result["description"] for result in results] == [
f"Concurrent {index}" for index in range(4)
]
final = await read_account_memory_template(fs, "locking-test", "profile")
results = await asyncio.wait_for(
asyncio.gather(
*(read_account_memory_template(fs, "locking-test", "profile") for _ in range(24)),
update_account_memory_template(fs, "locking-test", "profile", None, registry),
),
timeout=5,
)
assert all(
result is None or result["description"] == final["description"] for result in results
)
assert await read_account_memory_template(fs, "locking-test", "profile") is None
try:
mount_agfs_backend(native, config)
asyncio.run(run())
finally:
native.close()