Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(storage): avoid blocking executor on PathLock waits (#4548)
Qin Haojieb0d1d633fb53
diff --git a/Cargo.lock b/Cargo.lockindex 626c9324f..405b573e1 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3901,6 +3901,19 @@ dependencies = ["unindent",]+[[package]]+name = "pyo3-async-runtimes"+version = "0.27.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "57ddb5b570751e93cc6777e81fee8087e59cd53b5043292f2a6d59d5bd80fdfd"+dependencies = [+ "futures",+ "once_cell",+ "pin-project-lite",+ "pyo3",+ "tokio",+]+[[package]]name = "pyo3-build-config"version = "0.27.2"@@ -4108,6 +4121,7 @@ dependencies = ["bytes","gix-hash","pyo3",+ "pyo3-async-runtimes","ragfs","serde","serde_json",diff --git a/crates/ragfs-python/Cargo.toml b/crates/ragfs-python/Cargo.tomlindex 340130ae4..2fcc8f6d0 100644--- a/crates/ragfs-python/Cargo.toml+++ b/crates/ragfs-python/Cargo.toml@@ -20,6 +20,7 @@ s3 = ["ragfs/s3"][dependencies]ragfs = { path = "../ragfs", features = ["cache"] }pyo3 = { version = "0.27", features = ["abi3", "abi3-py310"] }+pyo3-async-runtimes = { version = "0.27", features = ["tokio-runtime"] }tokio = { version = "1", features = ["full"] }tracing = "0.1"tracing-appender = "0.2"diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rsindex de9219683..19b10ce59 100644--- a/crates/ragfs-python/src/lib.rs+++ b/crates/ragfs-python/src/lib.rs@@ -2659,6 +2659,36 @@ impl RAGFSBindingClient {Python::attach(|py| owned_lease_to_py_dict(py, &lease))}+ /// Acquire a batch on Tokio; the Python adapter settles cancellation before releasing it.+ #[pyo3(signature = (ctx, requests, timeout_secs=0.0, owner_lease_ref=None))]+ fn pathlock_acquire_batch_async<'py>(+ &self,+ py: Python<'py>,+ ctx: Option<HashMap<String, String>>,+ requests: Vec<HashMap<String, String>>,+ timeout_secs: f64,+ owner_lease_ref: Option<Py<PyAny>>,+ ) -> PyResult<Bound<'py, PyAny>> {+ let fs_ctx = build_fs_context(ctx);+ let timeout = validate_timeout_secs(timeout_secs)?;+ let owner = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?;+ let requests = parse_pathlock_request_batch(&requests)?;+ let manager = self.clone_pathlock_manager();+ let task = self.rt.spawn(FS_CTX.scope(fs_ctx, async move {+ let capability = owner+ .as_ref()+ .map(|(lease_ref, ownership_ref)| (lease_ref.as_str(), ownership_ref.as_str()));+ manager.acquire_batch(&requests, timeout, capability).await+ }));+ pyo3_async_runtimes::tokio::future_into_py(py, async move {+ let lease = task+ .await+ .map_err(|error| PyRuntimeError::new_err(error.to_string()))?+ .map_err(pathlock_err_to_py)?;+ Python::attach(|py| owned_lease_to_py_dict(py, &lease))+ })+ }+/// Create a borrowed view of an owned lease.#[pyo3(signature = (ctx, owned_lease_ref))]fn pathlock_as_borrowed(diff --git a/openviking/pyagfs/async_client.py b/openviking/pyagfs/async_client.pyindex 4c6657e8c..481d30d61 100644--- a/openviking/pyagfs/async_client.py+++ b/openviking/pyagfs/async_client.py@@ -83,10 +83,9 @@ def ensure_same_encryption_account(src_path: str, dst_path: str) -> None:class AsyncAGFSClient:"""Run blocking AGFS binding client operations off the event loop.- This is intentionally a thin adapter over the synchronous RAGFS binding- client (``RAGFSBindingClient``). If the binding later provides native async- methods, they can be swapped in here without changing storage and- transaction call sites.+ This is a thin adapter over ``RAGFSBindingClient``. Blocking methods use a+ worker thread; native async methods, currently PathLock acquisition, run+ without occupying that shared executor."""def __init__(self, client: AGFSSyncClientProtocol):@@ -360,12 +359,11 @@ class AsyncAGFSClient:fs_ctx: Dict[str, str] | None = None,) -> Dict[str, Any]:"""Acquire an exact lock on a single path."""- return await self.run(- "pathlock_acquire_exact",- _fs_ctx_or_default(path, fs_ctx),- path,+ return await self.pathlock_acquire_batch(+ [{"path": path, "kind": "exact"}],timeout_secs,owner_lease_ref,+ fs_ctx=fs_ctx,)async def pathlock_acquire_exact_batch(@@ -377,12 +375,11 @@ class AsyncAGFSClient:fs_ctx: Dict[str, str] | None = None,) -> Dict[str, Any]:"""Acquire exact locks on multiple paths."""- return await self.run(- "pathlock_acquire_exact_batch",- _fs_ctx_or_default(paths[0] if paths else "/", fs_ctx),- paths,+ return await self.pathlock_acquire_batch(+ [{"path": path, "kind": "exact"} for path in paths],timeout_secs,owner_lease_ref,+ fs_ctx=fs_ctx,)async def pathlock_acquire_tree(@@ -394,12 +391,11 @@ class AsyncAGFSClient:fs_ctx: Dict[str, str] | None = None,) -> Dict[str, Any]:"""Acquire a tree lock on a single path."""- return await self.run(- "pathlock_acquire_tree",- _fs_ctx_or_default(path, fs_ctx),- path,+ return await self.pathlock_acquire_batch(+ [{"path": path, "kind": "tree"}],timeout_secs,owner_lease_ref,+ fs_ctx=fs_ctx,)async def pathlock_acquire_tree_batch(@@ -411,12 +407,11 @@ class AsyncAGFSClient:fs_ctx: Dict[str, str] | None = None,) -> Dict[str, Any]:"""Acquire tree locks on multiple paths."""- return await self.run(- "pathlock_acquire_tree_batch",- _fs_ctx_or_default(paths[0] if paths else "/", fs_ctx),- paths,+ return await self.pathlock_acquire_batch(+ [{"path": path, "kind": "tree"} for path in paths],timeout_secs,owner_lease_ref,+ fs_ctx=fs_ctx,)async def pathlock_acquire_exact_tree_batch(@@ -429,14 +424,13 @@ class AsyncAGFSClient:fs_ctx: Dict[str, str] | None = None,) -> Dict[str, Any]:"""Acquire a mixed batch of exact and tree locks."""- first = exact_paths[0] if exact_paths else (tree_paths[0] if tree_paths else "/")- return await self.run(- "pathlock_acquire_exact_tree_batch",- _fs_ctx_or_default(first, fs_ctx),- exact_paths,- tree_paths,+ requests = [{"path": path, "kind": "exact"} for path in exact_paths]+ requests.extend({"path": path, "kind": "tree"} for path in tree_paths)+ return await self.pathlock_acquire_batch(+ requests,timeout_secs,owner_lease_ref,+ fs_ctx=fs_ctx,)async def pathlock_acquire_batch(@@ -456,14 +450,23 @@ class AsyncAGFSClient:raise ValueError("pathlock request.path must be an absolute path")if request.get("kind") not in {"exact", "tree"}:raise ValueError("pathlock request.kind must be 'exact' or 'tree'")- first = requests[0]["path"]- return await self.run(- "pathlock_acquire_batch",- _fs_ctx_or_default(first, fs_ctx),- requests,- timeout_secs,- owner_lease_ref,+ fs_ctx = _fs_ctx_or_default(requests[0]["path"], fs_ctx)+ future = asyncio.ensure_future(+ self._client.pathlock_acquire_batch_async(+ fs_ctx, requests, timeout_secs, owner_lease_ref+ ))+ try:+ return await asyncio.shield(future)+ except asyncio.CancelledError as cancelled:++ async def release_acquired() -> None:+ await self.pathlock_release(await future, fs_ctx=fs_ctx)++ try:+ await run_to_completion(release_acquired)+ finally:+ raise cancelledasync def pathlock_as_borrowed(self,diff --git a/openviking/pyagfs/protocols.py b/openviking/pyagfs/protocols.pyindex 5c2a484b2..eee0471fa 100644--- a/openviking/pyagfs/protocols.py+++ b/openviking/pyagfs/protocols.py@@ -3,14 +3,14 @@from __future__ import annotationsfrom collections.abc import Iterator-from typing import Any, BinaryIO, Dict, Literal, Protocol, overload, runtime_checkable+from typing import Any, Awaitable, BinaryIO, Dict, Literal, Protocol, overload, runtime_checkableAGFSByteStream = Iterator[bytes]@runtime_checkableclass AGFSSyncClientProtocol(Protocol):- """Minimal synchronous AGFS client contract used by OpenViking."""+ """AGFS binding contract used by OpenViking."""def ls(self,@@ -118,3 +118,12 @@ class AGFSSyncClientProtocol(Protocol):continuation_token: str | None = None,) -> Dict[str, Any]:"""Return one page of flat glob matches for the given AGFS directory."""++ def pathlock_acquire_batch_async(+ self,+ ctx: Dict[str, str],+ requests: list[Dict[str, str]],+ timeout_secs: float = 0.0,+ owner_lease_ref: Dict[str, Any] | None = None,+ ) -> Awaitable[Dict[str, Any]]:+ """Acquire a PathLock batch without blocking a Python worker thread."""diff --git a/tests/agfs/test_async_client.py b/tests/agfs/test_async_client.pyindex cb0897771..9c8b025d4 100644--- a/tests/agfs/test_async_client.py+++ b/tests/agfs/test_async_client.py@@ -6,17 +6,10 @@ import threadingimport pytest-from openviking.pyagfs import AsyncAGFSClient, RAGFSBindingClient+from openviking.pyagfs import AsyncAGFSClientfrom openviking.storage.viking_vector_index_backend import _AsyncVectorAdapter-def test_removed_sync_methods_are_not_exposed():- """Removed sync methods must stay absent from Python and native clients."""- for client_type in (AsyncAGFSClient, RAGFSBindingClient):- assert not hasattr(client_type, "system_sync_status")- assert not hasattr(client_type, "system_sync_retry")--@pytest.mark.asyncio@pytest.mark.parametrize("backend", ["agfs", "vector"])async def test_async_agfs_client_hides_threadpool(backend):diff --git a/tests/agfs/test_fs_binding.py b/tests/agfs/test_fs_binding.pyindex 1599b5c00..6ef1e5256 100644--- a/tests/agfs/test_fs_binding.py+++ b/tests/agfs/test_fs_binding.py@@ -7,9 +7,11 @@ Tests the python binding mode of VikingFS which directly uses AGFS implementatiowithout HTTP server."""+import asyncioimport osimport shutilimport uuid+from concurrent.futures import ThreadPoolExecutorimport pytest@@ -279,22 +281,23 @@ class TestVikingFSBindingLocal:await vfs._async_agfs.pathlock_release(adopted)await vfs._async_agfs.pathlock_release(owned)- @pytest.mark.parametrize(- "requests",- [- [],- [{"kind": "exact"}],- [{"path": "", "kind": "exact"}],- [{"path": "relative", "kind": "tree"}],- [{"path": "/local/default/temp/file", "kind": "other"}],- ],- )- async def test_pathlock_batch_rejects_invalid_requests(- self, viking_fs_binding_instance, requests- ):- """Reject malformed pathlock requests without applying defaults."""- with pytest.raises(ValueError):- await viking_fs_binding_instance._async_agfs.pathlock_acquire_batch(requests)+ async def test_pathlock_wait_and_cancel_release_resources(self, viking_fs_binding_instance):+ """Lock waits leave storage responsive; cancelled acquisitions release their lease."""+ client = viking_fs_binding_instance._async_agfs+ asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1))+ path = f"/local/default/temp/wait_{uuid.uuid4().hex}"+ owner = await client.pathlock_acquire_exact(path)+ waiter = asyncio.create_task(client.pathlock_acquire_exact(path, timeout_secs=2))+ try:+ await asyncio.sleep(0.01)+ await asyncio.wait_for(client.stat("/local"), timeout=0.5)+ finally:+ waiter.cancel()+ await client.pathlock_release(owner)+ with pytest.raises(asyncio.CancelledError):+ await waiter+ lease = await client.pathlock_acquire_exact(path)+ await client.pathlock_release(lease)async def test_tree_operations(self, viking_fs_binding_instance):"""Test VikingFS tree operations."""diff --git a/tests/server/account_store_fakes.py b/tests/server/account_store_fakes.pyindex 880dfe55f..be9727d4e 100644--- a/tests/server/account_store_fakes.py+++ b/tests/server/account_store_fakes.py@@ -2,6 +2,7 @@# SPDX-License-Identifier: AGPL-3.0"""Test doubles for file-backed account-store tests."""+import asyncioimport threadingimport timeimport uuid@@ -47,17 +48,19 @@ class InMemoryAGFS:def ensure_parent_dirs(self, path: str, ctx=None) -> None:del path, ctx- def pathlock_acquire_exact(+ async def pathlock_acquire_batch_async(self,ctx,- path: str,+ requests,timeout_secs: float = 0.0,owner_lease_ref=None,) -> dict:del ctx, owner_lease_ref+ (request,) = requests+ path = request["path"]with self._guard:lock = self._locks.setdefault(path, threading.Lock())- if not lock.acquire(timeout=timeout_secs):+ if not await asyncio.to_thread(lock.acquire, timeout=timeout_secs):raise TimeoutError(f"timed out acquiring account store lock: {path}")lease_ref = uuid.uuid4().hexwith self._guard:diff --git a/tests/server/test_admin_api.py b/tests/server/test_admin_api.pyindex ac43dd73c..d540a0403 100644--- a/tests/server/test_admin_api.py+++ b/tests/server/test_admin_api.py@@ -101,8 +101,8 @@ class _FakeAGFS:def pathlock_acquire_exact(self, ctx, path, timeout_secs=0.0, owner_lease_ref=None):return {"lease_ref": f"test:{path}"}- def pathlock_acquire_exact_batch(self, ctx, paths, timeout_secs=0.0, owner_lease_ref=None):- return self.pathlock_acquire_exact(ctx, paths[0], timeout_secs, owner_lease_ref)+ async def pathlock_acquire_batch_async(self, ctx, requests, *args):+ return self.pathlock_acquire_exact(ctx, requests[0]["path"], *args)def pathlock_release(self, ctx, lease):return Nonediff --git a/tests/server/test_agent_evolution_global_setting.py b/tests/server/test_agent_evolution_global_setting.pyindex ebde7ff68..8c0b487e0 100644--- a/tests/server/test_agent_evolution_global_setting.py+++ b/tests/server/test_agent_evolution_global_setting.py@@ -52,11 +52,7 @@ class _FakeAGFS:self.files.pop(path, None)return {}- def pathlock_acquire_exact(self, ctx, path, timeout_secs, owner_lease_ref):- del ctx- del path- del timeout_secs- del owner_lease_ref+ async def pathlock_acquire_batch_async(self, *args):return {"lease_ref": "test-lease"}def pathlock_release(self, ctx, lease):diff --git a/tests/session/memory/test_account_template_locking.py b/tests/session/memory/test_account_template_locking.pyindex 723bd72b2..16762b329 100644--- a/tests/session/memory/test_account_template_locking.py+++ b/tests/session/memory/test_account_template_locking.py@@ -30,8 +30,9 @@ class LockingAGFS:self.leases = {}self.timeouts = []- def pathlock_acquire_exact(self, ctx, path, timeout_secs=0, owner_lease_ref=None):- return self.pathlock_acquire_exact_batch(ctx, [path], timeout_secs, owner_lease_ref)+ async def pathlock_acquire_batch_async(self, ctx, requests, *args):+ paths = [request["path"] for request in requests]+ return await asyncio.to_thread(self.pathlock_acquire_exact_batch, ctx, paths, *args)def pathlock_acquire_exact_batch(self, ctx, paths, timeout_secs=0, owner_lease_ref=None):self.timeouts.append(timeout_secs)diff --git a/tests/utils/mock_agfs.py b/tests/utils/mock_agfs.pyindex c259a4c61..44307e903 100644--- a/tests/utils/mock_agfs.py+++ b/tests/utils/mock_agfs.py@@ -1,3 +1,4 @@+import asyncioimport shutilimport threadingimport uuid@@ -172,18 +173,20 @@ class MockLocalAGFS:def bind_request_context(self, ctx):return MagicMock(__enter__=lambda x: None, __exit__=lambda x, y, z: None)- def pathlock_acquire_tree(+ async def pathlock_acquire_batch_async(self,ctx,- path,+ requests,timeout_secs=0.0,owner_lease_ref=None,):del ctx, owner_lease_ref+ (request,) = requests+ path = request["path"]with self._pathlocks_guard:lock = self._pathlocks.setdefault(path, threading.Lock())- acquired = lock.acquire(timeout=timeout_secs)+ acquired = await asyncio.to_thread(lock.acquire, timeout=timeout_secs)if not acquired:raise TimeoutError(f"timed out acquiring test path lock: {path}")@@ -198,8 +201,6 @@ class MockLocalAGFS:self._pathlock_leases[lease_ref] = lockreturn lease- pathlock_acquire_exact = pathlock_acquire_tree-def pathlock_release(self, ctx, owned_lease_ref):del ctxlease_ref = owned_lease_ref["lease_ref"]