Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(storage): avoid blocking executor on PathLock waits (#4548)

Qin Haojieb0d1d633fb53
diff --git a/Cargo.lock b/Cargo.lock
index 626c9324f..405b573e1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3901,6 +3901,19 @@ dependencies = [
"unindent",
]
+[[package]]
+name = "pyo3-async-runtimes"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57ddb5b570751e93cc6777e81fee8087e59cd53b5043292f2a6d59d5bd80fdfd"
+dependencies = [
+ "futures",
+ "once_cell",
+ "pin-project-lite",
+ "pyo3",
+ "tokio",
+]
+
[[package]]
name = "pyo3-build-config"
version = "0.27.2"
@@ -4108,6 +4121,7 @@ dependencies = [
"bytes",
"gix-hash",
"pyo3",
+ "pyo3-async-runtimes",
"ragfs",
"serde",
"serde_json",
diff --git a/crates/ragfs-python/Cargo.toml b/crates/ragfs-python/Cargo.toml
index 340130ae4..2fcc8f6d0 100644
--- a/crates/ragfs-python/Cargo.toml
+++ b/crates/ragfs-python/Cargo.toml
@@ -20,6 +20,7 @@ s3 = ["ragfs/s3"]
[dependencies]
ragfs = { path = "../ragfs", features = ["cache"] }
pyo3 = { version = "0.27", features = ["abi3", "abi3-py310"] }
+pyo3-async-runtimes = { version = "0.27", features = ["tokio-runtime"] }
tokio = { version = "1", features = ["full"] }
tracing = "0.1"
tracing-appender = "0.2"
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs
index de9219683..19b10ce59 100644
--- a/crates/ragfs-python/src/lib.rs
+++ b/crates/ragfs-python/src/lib.rs
@@ -2659,6 +2659,36 @@ impl RAGFSBindingClient {
Python::attach(|py| owned_lease_to_py_dict(py, &lease))
}
+ /// Acquire a batch on Tokio; the Python adapter settles cancellation before releasing it.
+ #[pyo3(signature = (ctx, requests, timeout_secs=0.0, owner_lease_ref=None))]
+ fn pathlock_acquire_batch_async<'py>(
+ &self,
+ py: Python<'py>,
+ ctx: Option<HashMap<String, String>>,
+ requests: Vec<HashMap<String, String>>,
+ timeout_secs: f64,
+ owner_lease_ref: Option<Py<PyAny>>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ let fs_ctx = build_fs_context(ctx);
+ let timeout = validate_timeout_secs(timeout_secs)?;
+ let owner = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?;
+ let requests = parse_pathlock_request_batch(&requests)?;
+ let manager = self.clone_pathlock_manager();
+ let task = self.rt.spawn(FS_CTX.scope(fs_ctx, async move {
+ let capability = owner
+ .as_ref()
+ .map(|(lease_ref, ownership_ref)| (lease_ref.as_str(), ownership_ref.as_str()));
+ manager.acquire_batch(&requests, timeout, capability).await
+ }));
+ pyo3_async_runtimes::tokio::future_into_py(py, async move {
+ let lease = task
+ .await
+ .map_err(|error| PyRuntimeError::new_err(error.to_string()))?
+ .map_err(pathlock_err_to_py)?;
+ Python::attach(|py| owned_lease_to_py_dict(py, &lease))
+ })
+ }
+
/// Create a borrowed view of an owned lease.
#[pyo3(signature = (ctx, owned_lease_ref))]
fn pathlock_as_borrowed(
diff --git a/openviking/pyagfs/async_client.py b/openviking/pyagfs/async_client.py
index 4c6657e8c..481d30d61 100644
--- a/openviking/pyagfs/async_client.py
+++ b/openviking/pyagfs/async_client.py
@@ -83,10 +83,9 @@ def ensure_same_encryption_account(src_path: str, dst_path: str) -> None:
class AsyncAGFSClient:
"""Run blocking AGFS binding client operations off the event loop.
- This is intentionally a thin adapter over the synchronous RAGFS binding
- client (``RAGFSBindingClient``). If the binding later provides native async
- methods, they can be swapped in here without changing storage and
- transaction call sites.
+ This is a thin adapter over ``RAGFSBindingClient``. Blocking methods use a
+ worker thread; native async methods, currently PathLock acquisition, run
+ without occupying that shared executor.
"""
def __init__(self, client: AGFSSyncClientProtocol):
@@ -360,12 +359,11 @@ class AsyncAGFSClient:
fs_ctx: Dict[str, str] | None = None,
) -> Dict[str, Any]:
"""Acquire an exact lock on a single path."""
- return await self.run(
- "pathlock_acquire_exact",
- _fs_ctx_or_default(path, fs_ctx),
- path,
+ return await self.pathlock_acquire_batch(
+ [{"path": path, "kind": "exact"}],
timeout_secs,
owner_lease_ref,
+ fs_ctx=fs_ctx,
)
async def pathlock_acquire_exact_batch(
@@ -377,12 +375,11 @@ class AsyncAGFSClient:
fs_ctx: Dict[str, str] | None = None,
) -> Dict[str, Any]:
"""Acquire exact locks on multiple paths."""
- return await self.run(
- "pathlock_acquire_exact_batch",
- _fs_ctx_or_default(paths[0] if paths else "/", fs_ctx),
- paths,
+ return await self.pathlock_acquire_batch(
+ [{"path": path, "kind": "exact"} for path in paths],
timeout_secs,
owner_lease_ref,
+ fs_ctx=fs_ctx,
)
async def pathlock_acquire_tree(
@@ -394,12 +391,11 @@ class AsyncAGFSClient:
fs_ctx: Dict[str, str] | None = None,
) -> Dict[str, Any]:
"""Acquire a tree lock on a single path."""
- return await self.run(
- "pathlock_acquire_tree",
- _fs_ctx_or_default(path, fs_ctx),
- path,
+ return await self.pathlock_acquire_batch(
+ [{"path": path, "kind": "tree"}],
timeout_secs,
owner_lease_ref,
+ fs_ctx=fs_ctx,
)
async def pathlock_acquire_tree_batch(
@@ -411,12 +407,11 @@ class AsyncAGFSClient:
fs_ctx: Dict[str, str] | None = None,
) -> Dict[str, Any]:
"""Acquire tree locks on multiple paths."""
- return await self.run(
- "pathlock_acquire_tree_batch",
- _fs_ctx_or_default(paths[0] if paths else "/", fs_ctx),
- paths,
+ return await self.pathlock_acquire_batch(
+ [{"path": path, "kind": "tree"} for path in paths],
timeout_secs,
owner_lease_ref,
+ fs_ctx=fs_ctx,
)
async def pathlock_acquire_exact_tree_batch(
@@ -429,14 +424,13 @@ class AsyncAGFSClient:
fs_ctx: Dict[str, str] | None = None,
) -> Dict[str, Any]:
"""Acquire a mixed batch of exact and tree locks."""
- first = exact_paths[0] if exact_paths else (tree_paths[0] if tree_paths else "/")
- return await self.run(
- "pathlock_acquire_exact_tree_batch",
- _fs_ctx_or_default(first, fs_ctx),
- exact_paths,
- tree_paths,
+ requests = [{"path": path, "kind": "exact"} for path in exact_paths]
+ requests.extend({"path": path, "kind": "tree"} for path in tree_paths)
+ return await self.pathlock_acquire_batch(
+ requests,
timeout_secs,
owner_lease_ref,
+ fs_ctx=fs_ctx,
)
async def pathlock_acquire_batch(
@@ -456,14 +450,23 @@ class AsyncAGFSClient:
raise ValueError("pathlock request.path must be an absolute path")
if request.get("kind") not in {"exact", "tree"}:
raise ValueError("pathlock request.kind must be 'exact' or 'tree'")
- first = requests[0]["path"]
- return await self.run(
- "pathlock_acquire_batch",
- _fs_ctx_or_default(first, fs_ctx),
- requests,
- timeout_secs,
- owner_lease_ref,
+ fs_ctx = _fs_ctx_or_default(requests[0]["path"], fs_ctx)
+ future = asyncio.ensure_future(
+ self._client.pathlock_acquire_batch_async(
+ fs_ctx, requests, timeout_secs, owner_lease_ref
+ )
)
+ try:
+ return await asyncio.shield(future)
+ except asyncio.CancelledError as cancelled:
+
+ async def release_acquired() -> None:
+ await self.pathlock_release(await future, fs_ctx=fs_ctx)
+
+ try:
+ await run_to_completion(release_acquired)
+ finally:
+ raise cancelled
async def pathlock_as_borrowed(
self,
diff --git a/openviking/pyagfs/protocols.py b/openviking/pyagfs/protocols.py
index 5c2a484b2..eee0471fa 100644
--- a/openviking/pyagfs/protocols.py
+++ b/openviking/pyagfs/protocols.py
@@ -3,14 +3,14 @@
from __future__ import annotations
from collections.abc import Iterator
-from typing import Any, BinaryIO, Dict, Literal, Protocol, overload, runtime_checkable
+from typing import Any, Awaitable, BinaryIO, Dict, Literal, Protocol, overload, runtime_checkable
AGFSByteStream = Iterator[bytes]
@runtime_checkable
class AGFSSyncClientProtocol(Protocol):
- """Minimal synchronous AGFS client contract used by OpenViking."""
+ """AGFS binding contract used by OpenViking."""
def ls(
self,
@@ -118,3 +118,12 @@ class AGFSSyncClientProtocol(Protocol):
continuation_token: str | None = None,
) -> Dict[str, Any]:
"""Return one page of flat glob matches for the given AGFS directory."""
+
+ def pathlock_acquire_batch_async(
+ self,
+ ctx: Dict[str, str],
+ requests: list[Dict[str, str]],
+ timeout_secs: float = 0.0,
+ owner_lease_ref: Dict[str, Any] | None = None,
+ ) -> Awaitable[Dict[str, Any]]:
+ """Acquire a PathLock batch without blocking a Python worker thread."""
diff --git a/tests/agfs/test_async_client.py b/tests/agfs/test_async_client.py
index cb0897771..9c8b025d4 100644
--- a/tests/agfs/test_async_client.py
+++ b/tests/agfs/test_async_client.py
@@ -6,17 +6,10 @@ import threading
import pytest
-from openviking.pyagfs import AsyncAGFSClient, RAGFSBindingClient
+from openviking.pyagfs import AsyncAGFSClient
from openviking.storage.viking_vector_index_backend import _AsyncVectorAdapter
-def test_removed_sync_methods_are_not_exposed():
- """Removed sync methods must stay absent from Python and native clients."""
- for client_type in (AsyncAGFSClient, RAGFSBindingClient):
- assert not hasattr(client_type, "system_sync_status")
- assert not hasattr(client_type, "system_sync_retry")
-
-
@pytest.mark.asyncio
@pytest.mark.parametrize("backend", ["agfs", "vector"])
async def test_async_agfs_client_hides_threadpool(backend):
diff --git a/tests/agfs/test_fs_binding.py b/tests/agfs/test_fs_binding.py
index 1599b5c00..6ef1e5256 100644
--- a/tests/agfs/test_fs_binding.py
+++ b/tests/agfs/test_fs_binding.py
@@ -7,9 +7,11 @@ Tests the python binding mode of VikingFS which directly uses AGFS implementatio
without HTTP server.
"""
+import asyncio
import os
import shutil
import uuid
+from concurrent.futures import ThreadPoolExecutor
import pytest
@@ -279,22 +281,23 @@ class TestVikingFSBindingLocal:
await vfs._async_agfs.pathlock_release(adopted)
await vfs._async_agfs.pathlock_release(owned)
- @pytest.mark.parametrize(
- "requests",
- [
- [],
- [{"kind": "exact"}],
- [{"path": "", "kind": "exact"}],
- [{"path": "relative", "kind": "tree"}],
- [{"path": "/local/default/temp/file", "kind": "other"}],
- ],
- )
- async def test_pathlock_batch_rejects_invalid_requests(
- self, viking_fs_binding_instance, requests
- ):
- """Reject malformed pathlock requests without applying defaults."""
- with pytest.raises(ValueError):
- await viking_fs_binding_instance._async_agfs.pathlock_acquire_batch(requests)
+ async def test_pathlock_wait_and_cancel_release_resources(self, viking_fs_binding_instance):
+ """Lock waits leave storage responsive; cancelled acquisitions release their lease."""
+ client = viking_fs_binding_instance._async_agfs
+ asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1))
+ path = f"/local/default/temp/wait_{uuid.uuid4().hex}"
+ owner = await client.pathlock_acquire_exact(path)
+ waiter = asyncio.create_task(client.pathlock_acquire_exact(path, timeout_secs=2))
+ try:
+ await asyncio.sleep(0.01)
+ await asyncio.wait_for(client.stat("/local"), timeout=0.5)
+ finally:
+ waiter.cancel()
+ await client.pathlock_release(owner)
+ with pytest.raises(asyncio.CancelledError):
+ await waiter
+ lease = await client.pathlock_acquire_exact(path)
+ await client.pathlock_release(lease)
async def test_tree_operations(self, viking_fs_binding_instance):
"""Test VikingFS tree operations."""
diff --git a/tests/server/account_store_fakes.py b/tests/server/account_store_fakes.py
index 880dfe55f..be9727d4e 100644
--- a/tests/server/account_store_fakes.py
+++ b/tests/server/account_store_fakes.py
@@ -2,6 +2,7 @@
# SPDX-License-Identifier: AGPL-3.0
"""Test doubles for file-backed account-store tests."""
+import asyncio
import threading
import time
import uuid
@@ -47,17 +48,19 @@ class InMemoryAGFS:
def ensure_parent_dirs(self, path: str, ctx=None) -> None:
del path, ctx
- def pathlock_acquire_exact(
+ async def pathlock_acquire_batch_async(
self,
ctx,
- path: str,
+ requests,
timeout_secs: float = 0.0,
owner_lease_ref=None,
) -> dict:
del ctx, owner_lease_ref
+ (request,) = requests
+ path = request["path"]
with self._guard:
lock = self._locks.setdefault(path, threading.Lock())
- if not lock.acquire(timeout=timeout_secs):
+ if not await asyncio.to_thread(lock.acquire, timeout=timeout_secs):
raise TimeoutError(f"timed out acquiring account store lock: {path}")
lease_ref = uuid.uuid4().hex
with self._guard:
diff --git a/tests/server/test_admin_api.py b/tests/server/test_admin_api.py
index ac43dd73c..d540a0403 100644
--- a/tests/server/test_admin_api.py
+++ b/tests/server/test_admin_api.py
@@ -101,8 +101,8 @@ class _FakeAGFS:
def pathlock_acquire_exact(self, ctx, path, timeout_secs=0.0, owner_lease_ref=None):
return {"lease_ref": f"test:{path}"}
- def pathlock_acquire_exact_batch(self, ctx, paths, timeout_secs=0.0, owner_lease_ref=None):
- return self.pathlock_acquire_exact(ctx, paths[0], timeout_secs, owner_lease_ref)
+ async def pathlock_acquire_batch_async(self, ctx, requests, *args):
+ return self.pathlock_acquire_exact(ctx, requests[0]["path"], *args)
def pathlock_release(self, ctx, lease):
return None
diff --git a/tests/server/test_agent_evolution_global_setting.py b/tests/server/test_agent_evolution_global_setting.py
index ebde7ff68..8c0b487e0 100644
--- a/tests/server/test_agent_evolution_global_setting.py
+++ b/tests/server/test_agent_evolution_global_setting.py
@@ -52,11 +52,7 @@ class _FakeAGFS:
self.files.pop(path, None)
return {}
- def pathlock_acquire_exact(self, ctx, path, timeout_secs, owner_lease_ref):
- del ctx
- del path
- del timeout_secs
- del owner_lease_ref
+ async def pathlock_acquire_batch_async(self, *args):
return {"lease_ref": "test-lease"}
def pathlock_release(self, ctx, lease):
diff --git a/tests/session/memory/test_account_template_locking.py b/tests/session/memory/test_account_template_locking.py
index 723bd72b2..16762b329 100644
--- a/tests/session/memory/test_account_template_locking.py
+++ b/tests/session/memory/test_account_template_locking.py
@@ -30,8 +30,9 @@ class LockingAGFS:
self.leases = {}
self.timeouts = []
- def pathlock_acquire_exact(self, ctx, path, timeout_secs=0, owner_lease_ref=None):
- return self.pathlock_acquire_exact_batch(ctx, [path], timeout_secs, owner_lease_ref)
+ async def pathlock_acquire_batch_async(self, ctx, requests, *args):
+ paths = [request["path"] for request in requests]
+ return await asyncio.to_thread(self.pathlock_acquire_exact_batch, ctx, paths, *args)
def pathlock_acquire_exact_batch(self, ctx, paths, timeout_secs=0, owner_lease_ref=None):
self.timeouts.append(timeout_secs)
diff --git a/tests/utils/mock_agfs.py b/tests/utils/mock_agfs.py
index c259a4c61..44307e903 100644
--- a/tests/utils/mock_agfs.py
+++ b/tests/utils/mock_agfs.py
@@ -1,3 +1,4 @@
+import asyncio
import shutil
import threading
import uuid
@@ -172,18 +173,20 @@ class MockLocalAGFS:
def bind_request_context(self, ctx):
return MagicMock(__enter__=lambda x: None, __exit__=lambda x, y, z: None)
- def pathlock_acquire_tree(
+ async def pathlock_acquire_batch_async(
self,
ctx,
- path,
+ requests,
timeout_secs=0.0,
owner_lease_ref=None,
):
del ctx, owner_lease_ref
+ (request,) = requests
+ path = request["path"]
with self._pathlocks_guard:
lock = self._pathlocks.setdefault(path, threading.Lock())
- acquired = lock.acquire(timeout=timeout_secs)
+ acquired = await asyncio.to_thread(lock.acquire, timeout=timeout_secs)
if not acquired:
raise TimeoutError(f"timed out acquiring test path lock: {path}")
@@ -198,8 +201,6 @@ class MockLocalAGFS:
self._pathlock_leases[lease_ref] = lock
return lease
- pathlock_acquire_exact = pathlock_acquire_tree
-
def pathlock_release(self, ctx, owned_lease_ref):
del ctx
lease_ref = owned_lease_ref["lease_ref"]