Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(pathlock): split lock busy from io for retryable stale cleanup (#3659)

baojun-zhang89c2705192b0
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs
index c5e4f4273..00b0265d1 100644
--- a/crates/ragfs-python/src/lib.rs
+++ b/crates/ragfs-python/src/lib.rs
@@ -20,7 +20,7 @@ fn pathlock_err_to_py(err: PathLockError) -> PyErr {
PathLockError::Conflict { .. }
| PathLockError::Timeout { .. }
| PathLockError::HandoffFailed(_)
- | PathLockError::Io(_) => {
+ | PathLockError::Busy { .. } => {
#[allow(deprecated)]
Python::with_gil(|py| {
let ty = LOCK_ACQUISITION_ERROR_TYPE
@@ -30,7 +30,9 @@ fn pathlock_err_to_py(err: PathLockError) -> PyErr {
})
}
PathLockError::InvalidRequest(_) => PyValueError::new_err(err.to_string()),
- _ => PyRuntimeError::new_err(err.to_string()),
+ PathLockError::Io(_) | PathLockError::InvalidToken(_) | PathLockError::Internal(_) => {
+ PyRuntimeError::new_err(err.to_string())
+ }
}
}
use std::fs;
@@ -642,6 +644,7 @@ fn to_py_err(e: ragfs::core::Error) -> PyErr {
ragfs::core::Error::Serialization(_) => new_py_err("AGFSSerializationError", msg),
ragfs::core::Error::Network(_) => new_py_err("AGFSNetworkError", msg),
ragfs::core::Error::Timeout(_) => new_py_err("AGFSTimeoutError", msg),
+ ragfs::core::Error::WouldBlock(_) => new_py_err("AGFSTimeoutError", msg),
ragfs::core::Error::SyncWriteQuorum { .. } => new_py_err("AGFSInternalError", msg),
ragfs::core::Error::ContextMissing(_) => new_py_err("AGFSInternalError", msg),
ragfs::core::Error::Internal(_) => new_py_err("AGFSInternalError", msg),
@@ -2605,7 +2608,7 @@ mod tests {
}
#[test]
- fn pathlock_io_error_maps_to_lock_acquisition_error() {
+ fn pathlock_io_error_maps_to_runtime_error() {
Python::initialize();
Python::attach(|py| {
let errors_mod = py.import("openviking.storage.errors").unwrap();
@@ -2619,6 +2622,27 @@ mod tests {
let error =
pathlock_err_to_py(PathLockError::Io("failed to create lock dir".to_string()));
+ assert!(error.is_instance_of::<PyRuntimeError>(py));
+ });
+ }
+
+ #[test]
+ fn pathlock_busy_error_maps_to_lock_acquisition_error() {
+ Python::initialize();
+ Python::attach(|py| {
+ let errors_mod = py.import("openviking.storage.errors").unwrap();
+ let lock_error_type: Py<PyType> = errors_mod
+ .getattr("LockAcquisitionError")
+ .unwrap()
+ .extract()
+ .unwrap();
+ let _ = LOCK_ACQUISITION_ERROR_TYPE.set(lock_error_type.clone_ref(py));
+
+ let error = pathlock_err_to_py(PathLockError::Busy {
+ lock_path: "/data/.path.ovlock".to_string(),
+ operation: "remove".to_string(),
+ });
+
assert!(error.is_instance(py, lock_error_type.bind(py)));
});
}
diff --git a/crates/ragfs/src/core/errors.rs b/crates/ragfs/src/core/errors.rs
index f6ff185f6..f7cd32bfa 100644
--- a/crates/ragfs/src/core/errors.rs
+++ b/crates/ragfs/src/core/errors.rs
@@ -88,6 +88,10 @@ pub enum Error {
#[error("operation timed out: {0}")]
Timeout(String),
+ /// Non-blocking operation would block and should be retried by the caller
+ #[error("operation would block: {0}")]
+ WouldBlock(String),
+
/// Multi-write synchronous fanout failed to reach the required acknowledgement quorum
#[error(
"sync write quorum failed: {succeeded}/{attempted} backups succeeded (required {required}); failures: {failures:?}"
@@ -164,6 +168,11 @@ impl Error {
Self::Timeout(msg.into())
}
+ /// Create a WouldBlock error
+ pub fn would_block(msg: impl Into<String>) -> Self {
+ Self::WouldBlock(msg.into())
+ }
+
/// Return a stable error kind name for structured reporting.
pub fn kind_name(&self) -> &'static str {
match self {
@@ -183,6 +192,7 @@ impl Error {
Self::Serialization(_) => "serialization",
Self::Network(_) => "network",
Self::Timeout(_) => "timeout",
+ Self::WouldBlock(_) => "would_block",
Self::SyncWriteQuorum { .. } => "sync_write_quorum",
Self::ContextMissing(_) => "context_missing",
Self::Internal(_) => "internal",
diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rs
index 10707cd00..ce3aa711a 100644
--- a/crates/ragfs/src/lock/manager.rs
+++ b/crates/ragfs/src/lock/manager.rs
@@ -432,6 +432,14 @@ impl PathLockManager {
now_ns.saturating_sub(token.time_ns) > expire_ns
}
+ /// Return whether an acquire-loop error should be retried within the wait budget.
+ fn is_retryable_error(error: &PathLockError) -> bool {
+ matches!(
+ error,
+ PathLockError::Conflict { .. } | PathLockError::Busy { .. }
+ )
+ }
+
/// Resolve a trusted reentrant owner from an active local lease capability.
async fn resolve_owner_id(
&self,
@@ -658,7 +666,7 @@ impl PathLockManager {
.await?;
acquired_lock_paths.clear();
- if !matches!(err, PathLockError::Conflict { .. }) {
+ if !Self::is_retryable_error(&err) {
if is_waiting {
let mut metrics = self.metrics.write().await;
metrics.waiting_lock_count = metrics.waiting_lock_count.saturating_sub(1);
@@ -778,7 +786,7 @@ impl PathLockManager {
.await?;
acquired_lock_paths.clear();
- if !matches!(err, PathLockError::Conflict { .. }) {
+ if !Self::is_retryable_error(&err) {
if is_waiting {
let mut metrics = self.metrics.write().await;
metrics.waiting_lock_count = metrics.waiting_lock_count.saturating_sub(1);
@@ -925,8 +933,8 @@ impl PathLockManager {
kind: existing.lock_type,
});
}
- // Stale — remove it.
- let _ = self.provider.remove_token(lock_path, &existing.owner_id).await;
+ // Stale — remove it before attempting to create our own token.
+ self.provider.remove_token(lock_path, &existing.owner_id).await?;
}
self.provider.try_create_token(lock_path, &token).await?;
@@ -1622,6 +1630,8 @@ mod tests {
fail_next_read: AtomicBool,
fail_next_remove: AtomicBool,
return_false_next_remove: AtomicBool,
+ busy_next_remove: AtomicBool,
+ busy_remove_count: AtomicUsize,
}
impl FailNextRemoveProvider {
@@ -1632,9 +1642,22 @@ mod tests {
fail_next_read: AtomicBool::new(true),
fail_next_remove: AtomicBool::new(false),
return_false_next_remove: AtomicBool::new(false),
+ busy_next_remove: AtomicBool::new(false),
+ busy_remove_count: AtomicUsize::new(0),
}
}
+ /// Build a memory provider whose next remove reports a retryable busy error.
+ fn with_busy_remove() -> Self {
+ Self {
+ inner: crate::lock::provider::MemoryPathLockProvider::new(),
+ fail_next_read: AtomicBool::new(false),
+ fail_next_remove: AtomicBool::new(false),
+ return_false_next_remove: AtomicBool::new(false),
+ busy_next_remove: AtomicBool::new(true),
+ busy_remove_count: AtomicUsize::new(0),
+ }
+ }
}
#[async_trait]
@@ -1685,6 +1708,13 @@ mod tests {
lock_path: &str,
owner_id: &str,
) -> PathLockResult<bool> {
+ if self.busy_next_remove.swap(false, Ordering::SeqCst) {
+ self.busy_remove_count.fetch_add(1, Ordering::SeqCst);
+ return Err(PathLockError::Busy {
+ lock_path: lock_path.to_string(),
+ operation: "remove".to_string(),
+ });
+ }
if self.return_false_next_remove.swap(false, Ordering::SeqCst) {
return Ok(false);
}
@@ -2033,4 +2063,39 @@ mod tests {
Err(PathLockError::Io(message)) if message == "injected read failure"
));
}
+
+ #[tokio::test]
+ async fn busy_cleanup_is_retried() {
+ let fs = Arc::new(MemFileSystem::new());
+ fs.mkdir("/data", 0o755).await.unwrap();
+ let provider = Arc::new(FailNextRemoveProvider::with_busy_remove());
+ provider
+ .inner
+ .try_create_token(
+ "/data/file.txt/.path.ovlock",
+ &LockToken {
+ owner_id: "stale-owner".to_string(),
+ time_ns: 1,
+ lock_type: PathLockKind::Tree,
+ },
+ )
+ .await
+ .unwrap();
+ let mgr = PathLockManager::new(
+ fs,
+ provider.clone(),
+ PathLockConfig {
+ lock_expire_secs: 1.0,
+ ..PathLockConfig::default()
+ },
+ );
+
+ let lease = mgr
+ .acquire_tree("/data/file.txt", Duration::from_millis(200), None)
+ .await
+ .unwrap();
+
+ assert_eq!(provider.busy_remove_count.load(Ordering::SeqCst), 1);
+ mgr.release(&lease).await.unwrap();
+ }
}
diff --git a/crates/ragfs/src/lock/provider.rs b/crates/ragfs/src/lock/provider.rs
index 731516f2b..1459d3885 100644
--- a/crates/ragfs/src/lock/provider.rs
+++ b/crates/ragfs/src/lock/provider.rs
@@ -190,6 +190,17 @@ impl FilesystemPathLockProvider {
}
}
}
+
+ /// Map filesystem CAS failures into pathlock busy vs. fatal I/O outcomes.
+ fn map_cas_error(operation: &str, lock_path: &str, error: crate::core::Error) -> PathLockError {
+ match error {
+ crate::core::Error::WouldBlock(message) => PathLockError::Busy {
+ lock_path: lock_path.to_string(),
+ operation: format!("{operation} CAS ({message})"),
+ },
+ other => PathLockError::Io(format!("{operation} CAS failed: {other}")),
+ }
+ }
}
#[cfg(test)]
@@ -255,6 +266,16 @@ mod tests {
assert_eq!(token.time_ns, 123);
assert_eq!(token.lock_type, crate::lock::PathLockKind::Exact);
}
+
+ #[test]
+ fn compare_and_remove_would_block_maps_to_busy() {
+ let err = FilesystemPathLockProvider::map_cas_error(
+ "remove",
+ "/data/.path.ovlock",
+ Error::WouldBlock("lock would block".to_string()),
+ );
+ assert!(matches!(err, PathLockError::Busy { .. }));
+ }
}
#[async_trait]
@@ -279,11 +300,7 @@ impl PathLockProvider for FilesystemPathLockProvider {
.fs
.compare_and_remove(lock_path, &data)
.await
- .map_err(|e| {
- PathLockError::Io(format!(
- "legacy encrypted lock cleanup failed: {e}"
- ))
- })?
+ .map_err(|e| Self::map_cas_error("legacy encrypted lock cleanup", lock_path, e))?
{
return Ok(None);
}
@@ -335,7 +352,7 @@ impl PathLockProvider for FilesystemPathLockProvider {
self.fs
.compare_and_write(lock_path, expected.as_bytes(), replacement.as_bytes())
.await
- .map_err(|e| PathLockError::Io(format!("token CAS failed: {e}")))
+ .map_err(|e| Self::map_cas_error("compare_and_write", lock_path, e))
}
async fn refresh_token(
@@ -360,7 +377,7 @@ impl PathLockProvider for FilesystemPathLockProvider {
self.fs
.compare_and_write(lock_path, &raw, encoded.as_bytes())
.await
- .map_err(|e| PathLockError::Io(format!("refresh CAS failed: {e}")))
+ .map_err(|e| Self::map_cas_error("refresh", lock_path, e))
}
async fn remove_token(&self, lock_path: &str, owner_id: &str) -> PathLockResult<bool> {
@@ -374,7 +391,7 @@ impl PathLockProvider for FilesystemPathLockProvider {
self.fs
.compare_and_remove(lock_path, &raw)
.await
- .map_err(|e| PathLockError::Io(format!("remove CAS failed: {e}")))
+ .map_err(|e| Self::map_cas_error("remove", lock_path, e))
}
async fn scan_descendant_locks(&self, root: &str) -> PathLockResult<Vec<String>> {
diff --git a/crates/ragfs/src/lock/types.rs b/crates/ragfs/src/lock/types.rs
index 1bb1c34da..7b7104411 100644
--- a/crates/ragfs/src/lock/types.rs
+++ b/crates/ragfs/src/lock/types.rs
@@ -175,6 +175,15 @@ pub enum PathLockError {
elapsed_ms: u64,
},
+ /// Lock token mutation is temporarily blocked and should be retried.
+ #[error("lock busy at '{lock_path}' during {operation}")]
+ Busy {
+ /// Lock path whose token mutation is temporarily blocked.
+ lock_path: String,
+ /// Mutation operation being attempted.
+ operation: String,
+ },
+
/// I/O error during lock operation.
#[error("lock I/O error: {0}")]
Io(String),
diff --git a/crates/ragfs/src/plugins/localfs/mod.rs b/crates/ragfs/src/plugins/localfs/mod.rs
index 162b5efd2..426cc6a57 100644
--- a/crates/ragfs/src/plugins/localfs/mod.rs
+++ b/crates/ragfs/src/plugins/localfs/mod.rs
@@ -193,7 +193,7 @@ impl LocalFileSystem {
filelocks::LockKind::Exclusive,
filelocks::LockMode::NonBlocking,
)
- .map_err(|e| Error::plugin(format!("failed to lock for compare_and_write: {}", e)))?;
+ .map_err(|e| Self::map_lock_error("compare_and_write", &e.to_string()))?;
let file = lock.file_mut();
if !Self::open_file_matches_path(&file, &local_path)? {
return Ok(false);
@@ -246,7 +246,7 @@ impl LocalFileSystem {
filelocks::LockKind::Exclusive,
filelocks::LockMode::NonBlocking,
)
- .map_err(|e| Error::plugin(format!("failed to lock for compare_and_remove: {}", e)))?;
+ .map_err(|e| Self::map_lock_error("compare_and_remove", &e.to_string()))?;
let file = lock.file_mut();
if !Self::open_file_matches_path(&file, &local_path)? {
return Ok(false);
@@ -266,6 +266,14 @@ impl LocalFileSystem {
Ok(true)
}
+ /// Map a non-blocking file-lock acquisition error into a retryable filesystem error.
+ fn map_lock_error(operation: &str, error: &str) -> Error {
+ if error.to_ascii_lowercase().contains("would block") {
+ return Error::would_block(format!("failed to lock for {operation}: {error}"));
+ }
+ Error::plugin(format!("failed to lock for {operation}: {error}"))
+ }
+
/// Run blocking local filesystem work on a dedicated thread.
async fn run_blocking_fs<T, F>(job: F) -> Result<T>
where
@@ -1503,6 +1511,15 @@ mod tests {
}
}
+ #[test]
+ fn test_localfs_maps_would_block_lock_errors() {
+ let err = LocalFileSystem::map_lock_error(
+ "compare_and_remove",
+ "failed to lock for compare_and_remove: lock would block",
+ );
+ assert!(matches!(err, Error::WouldBlock(_)));
+ }
+
/// Install a fake rg executable and prepend it to PATH for the current test.
#[cfg(unix)]
fn install_fake_rg(dir: &TempDir, script: &str) -> EnvVarGuard {