Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(pathlock): split lock busy from io for retryable stale cleanup (#3659)
baojun-zhang89c2705192b0
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rsindex c5e4f4273..00b0265d1 100644--- a/crates/ragfs-python/src/lib.rs+++ b/crates/ragfs-python/src/lib.rs@@ -20,7 +20,7 @@ fn pathlock_err_to_py(err: PathLockError) -> PyErr {PathLockError::Conflict { .. }| PathLockError::Timeout { .. }| PathLockError::HandoffFailed(_)- | PathLockError::Io(_) => {+ | PathLockError::Busy { .. } => {#[allow(deprecated)]Python::with_gil(|py| {let ty = LOCK_ACQUISITION_ERROR_TYPE@@ -30,7 +30,9 @@ fn pathlock_err_to_py(err: PathLockError) -> PyErr {})}PathLockError::InvalidRequest(_) => PyValueError::new_err(err.to_string()),- _ => PyRuntimeError::new_err(err.to_string()),+ PathLockError::Io(_) | PathLockError::InvalidToken(_) | PathLockError::Internal(_) => {+ PyRuntimeError::new_err(err.to_string())+ }}}use std::fs;@@ -642,6 +644,7 @@ fn to_py_err(e: ragfs::core::Error) -> PyErr {ragfs::core::Error::Serialization(_) => new_py_err("AGFSSerializationError", msg),ragfs::core::Error::Network(_) => new_py_err("AGFSNetworkError", msg),ragfs::core::Error::Timeout(_) => new_py_err("AGFSTimeoutError", msg),+ ragfs::core::Error::WouldBlock(_) => new_py_err("AGFSTimeoutError", msg),ragfs::core::Error::SyncWriteQuorum { .. } => new_py_err("AGFSInternalError", msg),ragfs::core::Error::ContextMissing(_) => new_py_err("AGFSInternalError", msg),ragfs::core::Error::Internal(_) => new_py_err("AGFSInternalError", msg),@@ -2605,7 +2608,7 @@ mod tests {}#[test]- fn pathlock_io_error_maps_to_lock_acquisition_error() {+ fn pathlock_io_error_maps_to_runtime_error() {Python::initialize();Python::attach(|py| {let errors_mod = py.import("openviking.storage.errors").unwrap();@@ -2619,6 +2622,27 @@ mod tests {let error =pathlock_err_to_py(PathLockError::Io("failed to create lock dir".to_string()));+ assert!(error.is_instance_of::<PyRuntimeError>(py));+ });+ }++ #[test]+ fn pathlock_busy_error_maps_to_lock_acquisition_error() {+ Python::initialize();+ Python::attach(|py| {+ let errors_mod = py.import("openviking.storage.errors").unwrap();+ let lock_error_type: Py<PyType> = errors_mod+ .getattr("LockAcquisitionError")+ .unwrap()+ .extract()+ .unwrap();+ let _ = LOCK_ACQUISITION_ERROR_TYPE.set(lock_error_type.clone_ref(py));++ let error = pathlock_err_to_py(PathLockError::Busy {+ lock_path: "/data/.path.ovlock".to_string(),+ operation: "remove".to_string(),+ });+assert!(error.is_instance(py, lock_error_type.bind(py)));});}diff --git a/crates/ragfs/src/core/errors.rs b/crates/ragfs/src/core/errors.rsindex f6ff185f6..f7cd32bfa 100644--- a/crates/ragfs/src/core/errors.rs+++ b/crates/ragfs/src/core/errors.rs@@ -88,6 +88,10 @@ pub enum Error {#[error("operation timed out: {0}")]Timeout(String),+ /// Non-blocking operation would block and should be retried by the caller+ #[error("operation would block: {0}")]+ WouldBlock(String),+/// Multi-write synchronous fanout failed to reach the required acknowledgement quorum#[error("sync write quorum failed: {succeeded}/{attempted} backups succeeded (required {required}); failures: {failures:?}"@@ -164,6 +168,11 @@ impl Error {Self::Timeout(msg.into())}+ /// Create a WouldBlock error+ pub fn would_block(msg: impl Into<String>) -> Self {+ Self::WouldBlock(msg.into())+ }+/// Return a stable error kind name for structured reporting.pub fn kind_name(&self) -> &'static str {match self {@@ -183,6 +192,7 @@ impl Error {Self::Serialization(_) => "serialization",Self::Network(_) => "network",Self::Timeout(_) => "timeout",+ Self::WouldBlock(_) => "would_block",Self::SyncWriteQuorum { .. } => "sync_write_quorum",Self::ContextMissing(_) => "context_missing",Self::Internal(_) => "internal",diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rsindex 10707cd00..ce3aa711a 100644--- a/crates/ragfs/src/lock/manager.rs+++ b/crates/ragfs/src/lock/manager.rs@@ -432,6 +432,14 @@ impl PathLockManager {now_ns.saturating_sub(token.time_ns) > expire_ns}+ /// Return whether an acquire-loop error should be retried within the wait budget.+ fn is_retryable_error(error: &PathLockError) -> bool {+ matches!(+ error,+ PathLockError::Conflict { .. } | PathLockError::Busy { .. }+ )+ }+/// Resolve a trusted reentrant owner from an active local lease capability.async fn resolve_owner_id(&self,@@ -658,7 +666,7 @@ impl PathLockManager {.await?;acquired_lock_paths.clear();- if !matches!(err, PathLockError::Conflict { .. }) {+ if !Self::is_retryable_error(&err) {if is_waiting {let mut metrics = self.metrics.write().await;metrics.waiting_lock_count = metrics.waiting_lock_count.saturating_sub(1);@@ -778,7 +786,7 @@ impl PathLockManager {.await?;acquired_lock_paths.clear();- if !matches!(err, PathLockError::Conflict { .. }) {+ if !Self::is_retryable_error(&err) {if is_waiting {let mut metrics = self.metrics.write().await;metrics.waiting_lock_count = metrics.waiting_lock_count.saturating_sub(1);@@ -925,8 +933,8 @@ impl PathLockManager {kind: existing.lock_type,});}- // Stale — remove it.- let _ = self.provider.remove_token(lock_path, &existing.owner_id).await;+ // Stale — remove it before attempting to create our own token.+ self.provider.remove_token(lock_path, &existing.owner_id).await?;}self.provider.try_create_token(lock_path, &token).await?;@@ -1622,6 +1630,8 @@ mod tests {fail_next_read: AtomicBool,fail_next_remove: AtomicBool,return_false_next_remove: AtomicBool,+ busy_next_remove: AtomicBool,+ busy_remove_count: AtomicUsize,}impl FailNextRemoveProvider {@@ -1632,9 +1642,22 @@ mod tests {fail_next_read: AtomicBool::new(true),fail_next_remove: AtomicBool::new(false),return_false_next_remove: AtomicBool::new(false),+ busy_next_remove: AtomicBool::new(false),+ busy_remove_count: AtomicUsize::new(0),}}+ /// Build a memory provider whose next remove reports a retryable busy error.+ fn with_busy_remove() -> Self {+ Self {+ inner: crate::lock::provider::MemoryPathLockProvider::new(),+ fail_next_read: AtomicBool::new(false),+ fail_next_remove: AtomicBool::new(false),+ return_false_next_remove: AtomicBool::new(false),+ busy_next_remove: AtomicBool::new(true),+ busy_remove_count: AtomicUsize::new(0),+ }+ }}#[async_trait]@@ -1685,6 +1708,13 @@ mod tests {lock_path: &str,owner_id: &str,) -> PathLockResult<bool> {+ if self.busy_next_remove.swap(false, Ordering::SeqCst) {+ self.busy_remove_count.fetch_add(1, Ordering::SeqCst);+ return Err(PathLockError::Busy {+ lock_path: lock_path.to_string(),+ operation: "remove".to_string(),+ });+ }if self.return_false_next_remove.swap(false, Ordering::SeqCst) {return Ok(false);}@@ -2033,4 +2063,39 @@ mod tests {Err(PathLockError::Io(message)) if message == "injected read failure"));}++ #[tokio::test]+ async fn busy_cleanup_is_retried() {+ let fs = Arc::new(MemFileSystem::new());+ fs.mkdir("/data", 0o755).await.unwrap();+ let provider = Arc::new(FailNextRemoveProvider::with_busy_remove());+ provider+ .inner+ .try_create_token(+ "/data/file.txt/.path.ovlock",+ &LockToken {+ owner_id: "stale-owner".to_string(),+ time_ns: 1,+ lock_type: PathLockKind::Tree,+ },+ )+ .await+ .unwrap();+ let mgr = PathLockManager::new(+ fs,+ provider.clone(),+ PathLockConfig {+ lock_expire_secs: 1.0,+ ..PathLockConfig::default()+ },+ );++ let lease = mgr+ .acquire_tree("/data/file.txt", Duration::from_millis(200), None)+ .await+ .unwrap();++ assert_eq!(provider.busy_remove_count.load(Ordering::SeqCst), 1);+ mgr.release(&lease).await.unwrap();+ }}diff --git a/crates/ragfs/src/lock/provider.rs b/crates/ragfs/src/lock/provider.rsindex 731516f2b..1459d3885 100644--- a/crates/ragfs/src/lock/provider.rs+++ b/crates/ragfs/src/lock/provider.rs@@ -190,6 +190,17 @@ impl FilesystemPathLockProvider {}}}++ /// Map filesystem CAS failures into pathlock busy vs. fatal I/O outcomes.+ fn map_cas_error(operation: &str, lock_path: &str, error: crate::core::Error) -> PathLockError {+ match error {+ crate::core::Error::WouldBlock(message) => PathLockError::Busy {+ lock_path: lock_path.to_string(),+ operation: format!("{operation} CAS ({message})"),+ },+ other => PathLockError::Io(format!("{operation} CAS failed: {other}")),+ }+ }}#[cfg(test)]@@ -255,6 +266,16 @@ mod tests {assert_eq!(token.time_ns, 123);assert_eq!(token.lock_type, crate::lock::PathLockKind::Exact);}++ #[test]+ fn compare_and_remove_would_block_maps_to_busy() {+ let err = FilesystemPathLockProvider::map_cas_error(+ "remove",+ "/data/.path.ovlock",+ Error::WouldBlock("lock would block".to_string()),+ );+ assert!(matches!(err, PathLockError::Busy { .. }));+ }}#[async_trait]@@ -279,11 +300,7 @@ impl PathLockProvider for FilesystemPathLockProvider {.fs.compare_and_remove(lock_path, &data).await- .map_err(|e| {- PathLockError::Io(format!(- "legacy encrypted lock cleanup failed: {e}"- ))- })?+ .map_err(|e| Self::map_cas_error("legacy encrypted lock cleanup", lock_path, e))?{return Ok(None);}@@ -335,7 +352,7 @@ impl PathLockProvider for FilesystemPathLockProvider {self.fs.compare_and_write(lock_path, expected.as_bytes(), replacement.as_bytes()).await- .map_err(|e| PathLockError::Io(format!("token CAS failed: {e}")))+ .map_err(|e| Self::map_cas_error("compare_and_write", lock_path, e))}async fn refresh_token(@@ -360,7 +377,7 @@ impl PathLockProvider for FilesystemPathLockProvider {self.fs.compare_and_write(lock_path, &raw, encoded.as_bytes()).await- .map_err(|e| PathLockError::Io(format!("refresh CAS failed: {e}")))+ .map_err(|e| Self::map_cas_error("refresh", lock_path, e))}async fn remove_token(&self, lock_path: &str, owner_id: &str) -> PathLockResult<bool> {@@ -374,7 +391,7 @@ impl PathLockProvider for FilesystemPathLockProvider {self.fs.compare_and_remove(lock_path, &raw).await- .map_err(|e| PathLockError::Io(format!("remove CAS failed: {e}")))+ .map_err(|e| Self::map_cas_error("remove", lock_path, e))}async fn scan_descendant_locks(&self, root: &str) -> PathLockResult<Vec<String>> {diff --git a/crates/ragfs/src/lock/types.rs b/crates/ragfs/src/lock/types.rsindex 1bb1c34da..7b7104411 100644--- a/crates/ragfs/src/lock/types.rs+++ b/crates/ragfs/src/lock/types.rs@@ -175,6 +175,15 @@ pub enum PathLockError {elapsed_ms: u64,},+ /// Lock token mutation is temporarily blocked and should be retried.+ #[error("lock busy at '{lock_path}' during {operation}")]+ Busy {+ /// Lock path whose token mutation is temporarily blocked.+ lock_path: String,+ /// Mutation operation being attempted.+ operation: String,+ },+/// I/O error during lock operation.#[error("lock I/O error: {0}")]Io(String),diff --git a/crates/ragfs/src/plugins/localfs/mod.rs b/crates/ragfs/src/plugins/localfs/mod.rsindex 162b5efd2..426cc6a57 100644--- a/crates/ragfs/src/plugins/localfs/mod.rs+++ b/crates/ragfs/src/plugins/localfs/mod.rs@@ -193,7 +193,7 @@ impl LocalFileSystem {filelocks::LockKind::Exclusive,filelocks::LockMode::NonBlocking,)- .map_err(|e| Error::plugin(format!("failed to lock for compare_and_write: {}", e)))?;+ .map_err(|e| Self::map_lock_error("compare_and_write", &e.to_string()))?;let file = lock.file_mut();if !Self::open_file_matches_path(&file, &local_path)? {return Ok(false);@@ -246,7 +246,7 @@ impl LocalFileSystem {filelocks::LockKind::Exclusive,filelocks::LockMode::NonBlocking,)- .map_err(|e| Error::plugin(format!("failed to lock for compare_and_remove: {}", e)))?;+ .map_err(|e| Self::map_lock_error("compare_and_remove", &e.to_string()))?;let file = lock.file_mut();if !Self::open_file_matches_path(&file, &local_path)? {return Ok(false);@@ -266,6 +266,14 @@ impl LocalFileSystem {Ok(true)}+ /// Map a non-blocking file-lock acquisition error into a retryable filesystem error.+ fn map_lock_error(operation: &str, error: &str) -> Error {+ if error.to_ascii_lowercase().contains("would block") {+ return Error::would_block(format!("failed to lock for {operation}: {error}"));+ }+ Error::plugin(format!("failed to lock for {operation}: {error}"))+ }+/// Run blocking local filesystem work on a dedicated thread.async fn run_blocking_fs<T, F>(job: F) -> Result<T>where@@ -1503,6 +1511,15 @@ mod tests {}}+ #[test]+ fn test_localfs_maps_would_block_lock_errors() {+ let err = LocalFileSystem::map_lock_error(+ "compare_and_remove",+ "failed to lock for compare_and_remove: lock would block",+ );+ assert!(matches!(err, Error::WouldBlock(_)));+ }+/// Install a fake rg executable and prepend it to PATH for the current test.#[cfg(unix)]fn install_fake_rg(dir: &TempDir, script: &str) -> EnvVarGuard {