Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(pathlock): revoke lease authorization after token loss (#6704)

zhanglistar4adcab749848
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs
index 777a1e9c2..25a53b686 100644
--- a/crates/ragfs-python/src/lib.rs
+++ b/crates/ragfs-python/src/lib.rs
@@ -2834,7 +2834,7 @@ impl RAGFSBindingClient {
let ownership = ownership_ref.clone();
async move {
let lease = mgr
- .get_owned_lease_by_capability(&lr, &ownership)
+ .get_releasable_lease_by_capability(&lr, &ownership)
.await
.ok_or_else(|| {
PathLockError::InvalidRequest(format!(
@@ -2867,7 +2867,7 @@ impl RAGFSBindingClient {
let paths = lock_paths.clone();
async move {
let lease = mgr
- .get_owned_lease_by_capability(&lr, &ownership)
+ .get_releasable_lease_by_capability(&lr, &ownership)
.await
.ok_or_else(|| {
PathLockError::InvalidRequest(format!(
diff --git a/crates/ragfs/src/lock/lost_lease_tests.rs b/crates/ragfs/src/lock/lost_lease_tests.rs
new file mode 100644
index 000000000..3eefdc880
--- /dev/null
+++ b/crates/ragfs/src/lock/lost_lease_tests.rs
@@ -0,0 +1,1031 @@
+//! Lost lease authorization, renewal deadlines, and bounded cleanup regressions.
+use super::*;
+use crate::core::{FsContextInner, PathLockContext, WriteFlag, FS_CTX};
+use crate::lock::provider::{FilesystemPathLockProvider, MemoryPathLockProvider};
+use crate::lock::wrapper::PathLockWrappedFS;
+use crate::plugins::memfs::MemFileSystem;
+use async_trait::async_trait;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+struct FaultProvider {
+ inner: Arc<dyn PathLockProvider>,
+ /// false: definite loss; true: uncertain transport failure.
+ failures: StdMutex<HashMap<String, bool>>,
+ fail_remove: AtomicBool,
+ refresh_calls: AtomicUsize,
+ remove_calls: AtomicUsize,
+ delay: Duration,
+}
+
+impl FaultProvider {
+ fn new(inner: Arc<dyn PathLockProvider>) -> Self {
+ Self {
+ inner,
+ failures: StdMutex::new(HashMap::new()),
+ fail_remove: AtomicBool::new(false),
+ refresh_calls: AtomicUsize::new(0),
+ remove_calls: AtomicUsize::new(0),
+ delay: Duration::ZERO,
+ }
+ }
+}
+
+#[async_trait]
+impl PathLockProvider for FaultProvider {
+ fn name(&self) -> &'static str {
+ "fault-injection"
+ }
+ fn handle_mode(&self) -> PathLockHandleMode {
+ self.inner.handle_mode()
+ }
+ async fn try_acquire_batch_atomic(
+ &self,
+ requests: &[PathLockRequest],
+ owner: &str,
+ now: u128,
+ stale: u128,
+ ) -> PathLockResult<Option<Vec<AtomicAcquisition>>> {
+ self.inner
+ .try_acquire_batch_atomic(requests, owner, now, stale)
+ .await
+ }
+ async fn read_token(&self, path: &str) -> PathLockResult<Option<LockToken>> {
+ self.inner.read_token(path).await
+ }
+ async fn try_create_token(&self, path: &str, token: &LockToken) -> PathLockResult<()> {
+ self.inner.try_create_token(path, token).await
+ }
+ async fn compare_and_write_token(
+ &self,
+ path: &str,
+ expected: &LockToken,
+ replacement: &LockToken,
+ ) -> PathLockResult<bool> {
+ self.inner
+ .compare_and_write_token(path, expected, replacement)
+ .await
+ }
+ async fn refresh_token(&self, path: &str, owner: &str, now: u128) -> PathLockResult<bool> {
+ self.refresh_calls.fetch_add(1, Ordering::SeqCst);
+ if !self.delay.is_zero() {
+ tokio::time::sleep(self.delay).await;
+ }
+ let fault = self.failures.lock().unwrap().get(path).copied();
+ match fault {
+ Some(true) => Err(PathLockError::Io("injected refresh timeout".into())),
+ Some(false) => Ok(false),
+ None => self.inner.refresh_token(path, owner, now).await,
+ }
+ }
+ async fn remove_token(&self, path: &str, owner: &str, force: bool) -> PathLockResult<bool> {
+ assert!(!force, "cleanup must use ownership CAS");
+ self.remove_calls.fetch_add(1, Ordering::SeqCst);
+ if self.fail_remove.load(Ordering::SeqCst) {
+ Err(PathLockError::Io("injected cleanup timeout".into()))
+ } else {
+ self.inner.remove_token(path, owner, force).await
+ }
+ }
+ async fn scan_descendant_locks(&self, root: &str) -> PathLockResult<Vec<String>> {
+ self.inner.scan_descendant_locks(root).await
+ }
+}
+
+async fn fixture() -> (PathLockManager, Arc<FaultProvider>) {
+ let fs = Arc::new(MemFileSystem::new());
+ fs.mkdir("/data", 0o755).await.unwrap();
+ let provider = Arc::new(FaultProvider::new(Arc::new(MemoryPathLockProvider::new())));
+ let manager = PathLockManager::new(fs, provider.clone(), PathLockConfig::default());
+ (manager, provider)
+}
+
+fn capability(lease: &OwnedPathLockLease) -> Option<(&str, &str)> {
+ Some((&lease.lease.lease_ref, &lease.ownership_ref))
+}
+
+async fn age_token(
+ manager: &PathLockManager,
+ lease: &OwnedPathLockLease,
+ path: &str,
+ age: Duration,
+) {
+ let owner = manager
+ .lease_registry
+ .resolve(&lease.lease.lease_ref)
+ .unwrap();
+ owner
+ .registry
+ .lock()
+ .await
+ .token_health
+ .get_mut(path)
+ .unwrap()
+ .last_confirmed_at = Instant::now() - age;
+}
+
+async fn lease_invalidation(
+ manager: &PathLockManager,
+ lease: &OwnedPathLockLease,
+) -> LeaseInvalidation {
+ let owner = manager
+ .lease_registry
+ .resolve(&lease.lease.lease_ref)
+ .unwrap();
+ let registry = owner.registry.lock().await;
+ registry.entries[&lease.lease.lease_ref]
+ .invalidation
+ .unwrap()
+}
+
+#[tokio::test]
+async fn loss_invalidates_whole_leases_sharing_tokens_but_not_exact_descendants() {
+ let (manager, provider) = fixture().await;
+ let tree = manager
+ .acquire_tree("/data", Duration::ZERO, None)
+ .await
+ .unwrap();
+ let shared = manager
+ .acquire_tree_batch(
+ &["/data".into(), "/other".into()],
+ Duration::ZERO,
+ capability(&tree),
+ )
+ .await
+ .unwrap();
+ let child = manager
+ .acquire_exact("/data/child", Duration::ZERO, capability(&tree))
+ .await
+ .unwrap();
+ assert_eq!(shared.lease.lock_paths[0], tree.lease.lock_paths[0]);
+ assert_ne!(child.lease.lock_paths[0], tree.lease.lock_paths[0]);
+ provider
+ .failures
+ .lock()
+ .unwrap()
+ .insert(tree.lease.lock_paths[0].clone(), false);
+ assert_eq!(manager.refresh(&tree).await.unwrap(), "lost");
+ for lease in [&tree, &shared] {
+ assert_eq!(
+ lease_invalidation(&manager, lease).await.reason,
+ InvalidationReason::TokenLost
+ );
+ assert!(manager
+ .get_owned_lease_by_ref(&lease.lease.lease_ref)
+ .await
+ .is_none());
+ assert!(manager
+ .require_covered_lease_ref(&lease.lease.lease_ref, &lease.lease.covered_paths)
+ .await
+ .is_err());
+ assert!(manager
+ .acquire_exact("/new", Duration::ZERO, capability(lease))
+ .await
+ .is_err());
+ assert!(manager.handoff(lease).await.is_err());
+ assert!(manager.refresh(lease).await.is_err());
+ assert!(manager
+ .get_releasable_lease_by_capability(&lease.lease.lease_ref, &lease.ownership_ref)
+ .await
+ .is_some());
+ }
+ assert!(manager
+ .get_owned_lease_by_ref(&child.lease.lease_ref)
+ .await
+ .is_some());
+ manager.release(&shared).await.unwrap();
+ assert!(provider
+ .read_token(&child.lease.lock_paths[0])
+ .await
+ .unwrap()
+ .is_some());
+ manager.release(&tree).await.unwrap();
+ manager.release(&child).await.unwrap();
+}
+
+#[tokio::test]
+async fn uncertain_partial_refresh_preserves_short_jitter_but_not_failed_token_deadline() {
+ let (manager, provider) = fixture().await;
+ let lease = manager
+ .acquire_exact_batch(&["/data/a".into(), "/data/b".into()], Duration::ZERO, None)
+ .await
+ .unwrap();
+ let failed = lease.lease.lock_paths[0].clone();
+ provider
+ .failures
+ .lock()
+ .unwrap()
+ .insert(failed.clone(), true);
+ age_token(&manager, &lease, &failed, Duration::from_secs(23)).await;
+ let owner = manager
+ .lease_registry
+ .resolve(&lease.lease.lease_ref)
+ .unwrap();
+ let before = owner.registry.lock().await.token_health[&failed].last_confirmed_at;
+ assert_eq!(manager.refresh(&lease).await.unwrap(), "failed");
+ assert!(manager
+ .get_owned_lease_by_ref(&lease.lease.lease_ref)
+ .await
+ .is_some());
+ assert_eq!(
+ owner.registry.lock().await.token_health[&failed].last_confirmed_at,
+ before
+ );
+ assert_eq!(
+ owner.registry.lock().await.token_health[&failed].consecutive_failures,
+ 1
+ );
+ provider.failures.lock().unwrap().clear();
+ assert_eq!(manager.refresh(&lease).await.unwrap(), "refreshed");
+ assert_eq!(
+ owner.registry.lock().await.token_health[&failed].consecutive_failures,
+ 0
+ );
+ provider
+ .failures
+ .lock()
+ .unwrap()
+ .insert(failed.clone(), true);
+ age_token(&manager, &lease, &failed, Duration::from_secs(25)).await;
+ assert!(manager
+ .require_covered_lease_ref(&lease.lease.lease_ref, &lease.lease.covered_paths)
+ .await
+ .is_err());
+ let invalidation = lease_invalidation(&manager, &lease).await;
+ assert_eq!(
+ invalidation.reason,
+ InvalidationReason::ConfirmationDeadlineExceeded
+ );
+ // A later definite-loss result must retain the initial deadline cause.
+ {
+ let mut registry = owner.registry.lock().await;
+ assert_eq!(
+ registry.apply_refresh_results(
+ &lease.lease.lease_ref,
+ &lease.lease.lock_paths,
+ vec![Ok(false), Ok(true)],
+ Instant::now(),
+ PathLockManager::lease_validity(&manager.config),
+ ),
+ "lost"
+ );
+ assert_eq!(
+ registry.entries[&lease.lease.lease_ref].invalidation,
+ Some(invalidation)
+ );
+ }
+ provider.failures.lock().unwrap().clear();
+ assert!(
+ manager.refresh(&lease).await.is_err(),
+ "late recovery cannot resurrect"
+ );
+ manager.release(&lease).await.unwrap();
+}
+
+#[tokio::test]
+async fn reentry_and_handoff_do_not_restart_token_deadlines() {
+ let (manager, _) = fixture().await;
+ let lease = manager
+ .acquire_tree("/data", Duration::ZERO, None)
+ .await
+ .unwrap();
+ let path = &lease.lease.lock_paths[0];
+ age_token(&manager, &lease, path, Duration::from_secs(23)).await;
+ let owner = manager
+ .lease_registry
+ .resolve(&lease.lease.lease_ref)
+ .unwrap();
+ let before = owner.registry.lock().await.token_health[path].last_confirmed_at;
+ let reentrant = manager
+ .acquire_tree("/data", Duration::ZERO, capability(&lease))
+ .await
+ .unwrap();
+ assert_eq!(
+ owner.registry.lock().await.token_health[path].last_confirmed_at,
+ before
+ );
+ let handoff = manager.to_handoff(&lease);
+ manager.handoff(&lease).await.unwrap();
+ let adopted = manager.adopt(&handoff).await.unwrap();
+ assert_eq!(
+ owner.registry.lock().await.token_health[path].last_confirmed_at,
+ before
+ );
+ age_token(&manager, &adopted, path, Duration::from_secs(25)).await;
+ assert!(manager
+ .get_owned_lease_by_ref(&adopted.lease.lease_ref)
+ .await
+ .is_none());
+ assert!(manager
+ .get_owned_lease_by_ref(&reentrant.lease.lease_ref)
+ .await
+ .is_none());
+ manager.release(&reentrant).await.unwrap();
+ manager.release(&adopted).await.unwrap();
+}
+
+#[tokio::test]
+async fn invalidated_pending_handoff_cannot_be_adopted() {
+ let (manager, provider) = fixture().await;
+ let lease = manager
+ .acquire_tree("/data", Duration::ZERO, None)
+ .await
+ .unwrap();
+ let handoff = manager.to_handoff(&lease);
+ manager.handoff(&lease).await.unwrap();
+ let owner = manager
+ .lease_registry
+ .resolve(&lease.lease.lease_ref)
+ .unwrap();
+ let mut registry = owner.registry.lock().await;
+ provider
+ .failures
+ .lock()
+ .unwrap()
+ .insert(lease.lease.lock_paths[0].clone(), false);
+ assert_eq!(
+ PathLockManager::refresh_entry_with(
+ &manager.provider,
+ &mut registry,
+ &owner.owner_id,
+ &lease.lease.lease_ref,
+ PathLockManager::lease_validity(&manager.config),
+ )
+ .await,
+ "lost"
+ );
+ drop(registry);
+ assert!(manager.adopt(&handoff).await.is_err());
+ assert!(PathLockManager::release_lease_paths_with(
+ &manager.provider,
+ &manager.lease_registry,
+ owner,
+ &lease.lease.lease_ref,
+ None,
+ None,
+ Some(PathLockManager::cleanup_window(&manager.config)),
+ )
+ .await
+ .unwrap());
+}
+
+#[tokio::test]
+async fn failed_cleanup_keeps_capability_and_refs_until_recovery() {
+ let (manager, provider) = fixture().await;
+ let lease = manager
+ .acquire_tree("/data", Duration::ZERO, None)
+ .await
+ .unwrap();
+ provider.fail_remove.store(true, Ordering::SeqCst);
+ assert!(manager.release(&lease).await.is_err());
+ let owner = manager
+ .lease_registry
+ .resolve(&lease.lease.lease_ref)
+ .unwrap();
+ let invalidation = lease_invalidation(&manager, &lease).await;
+ assert_eq!(invalidation.reason, InvalidationReason::ReleaseFailed);
+ assert!(manager
+ .get_owned_lease_by_ref(&lease.lease.lease_ref)
+ .await
+ .is_none());
+ assert!(manager
+ .get_releasable_lease_by_capability(&lease.lease.lease_ref, "wrong")
+ .await
+ .is_none());
+ assert!(manager.release(&lease).await.is_err());
+ assert_eq!(
+ owner.registry.lock().await.entries[&lease.lease.lease_ref].invalidation,
+ Some(invalidation)
+ );
+ assert_eq!(
+ owner.registry.lock().await.lock_refs[&lease.lease.lock_paths[0]],
+ 1
+ );
+ provider.fail_remove.store(false, Ordering::SeqCst);
+ let cleanup = manager
+ .get_releasable_lease_by_capability(&lease.lease.lease_ref, &lease.ownership_ref)
+ .await
+ .unwrap();
+ manager.release(&cleanup).await.unwrap();
+ assert!(owner.registry.lock().await.lock_refs.is_empty());
+ assert!(owner.registry.lock().await.token_health.is_empty());
+ assert!(manager
+ .lease_registry
+ .resolve(&lease.lease.lease_ref)
+ .is_none());
+}
+
+#[tokio::test]
+async fn bounded_cleanup_forgets_only_local_refs_and_preserves_valid_shared_token() {
+ let (manager, provider) = fixture().await;
+ let lease = manager
+ .acquire_tree_batch(&["/data".into(), "/other".into()], Duration::ZERO, None)
+ .await
+ .unwrap();
+ let valid = manager
+ .acquire_tree("/other", Duration::ZERO, capability(&lease))
+ .await
+ .unwrap();
+ provider
+ .failures
+ .lock()
+ .unwrap()
+ .insert(lease.lease.lock_paths[0].clone(), false);
+ assert_eq!(manager.refresh(&lease).await.unwrap(), "lost");
+ let invalidation = lease_invalidation(&manager, &lease).await;
+ assert_eq!(invalidation.reason, InvalidationReason::TokenLost);
+ provider.fail_remove.store(true, Ordering::SeqCst);
+ assert!(manager.release(&lease).await.is_err());
+ assert_eq!(lease_invalidation(&manager, &lease).await, invalidation);
+ let owner = manager
+ .lease_registry
+ .resolve(&lease.lease.lease_ref)
+ .unwrap();
+ let calls = provider.remove_calls.load(Ordering::SeqCst);
+ assert!(PathLockManager::release_lease_paths_with(
+ &manager.provider,
+ &manager.lease_registry,
+ owner.clone(),
+ &lease.lease.lease_ref,
+ None,
+ None,
+ Some(Duration::ZERO),
+ )
+ .await
+ .unwrap());
+ assert_eq!(provider.remove_calls.load(Ordering::SeqCst), calls);
+ assert_eq!(
+ owner.registry.lock().await.lock_refs[&valid.lease.lock_paths[0]],
+ 1
+ );
+ assert!(manager
+ .get_owned_lease_by_ref(&valid.lease.lease_ref)
+ .await
+ .is_some());
+ assert!(!PathLockManager::release_lease_paths_with(
+ &manager.provider,
+ &manager.lease_registry,
+ owner,
+ &lease.lease.lease_ref,
+ None,
+ None,
+ Some(Duration::ZERO),
+ )
+ .await
+ .unwrap());
+ provider.fail_remove.store(false, Ordering::SeqCst);
+ manager.release(&valid).await.unwrap();
+}
+
+#[tokio::test]
+async fn delayed_result_for_old_handoff_ref_cannot_update_the_adopted_lease() {
+ let (manager, _) = fixture().await;
+ let lease = manager
+ .acquire_tree("/data", Duration::ZERO, None)
+ .await
+ .unwrap();
+ let handoff = manager.to_handoff(&lease);
+ manager.handoff(&lease).await.unwrap();
+ let adopted = manager.adopt(&handoff).await.unwrap();
+ let owner = manager
+ .lease_registry
+ .resolve(&adopted.lease.lease_ref)
+ .unwrap();
+ let mut registry = owner.registry.lock().await;
+ let before = registry.token_health[&lease.lease.lock_paths[0]].last_confirmed_at;
+ assert_eq!(
+ registry.apply_refresh_results(
+ &lease.lease.lease_ref,
+ &lease.lease.lock_paths,
+ vec![Ok(false)],
+ Instant::now(),
+ PathLockManager::lease_validity(&manager.config),
+ ),
+ "lost"
+ );
+ assert_eq!(
+ registry.token_health[&lease.lease.lock_paths[0]].last_confirmed_at,
+ before
+ );
+ assert!(registry.entries[&adopted.lease.lease_ref]
+ .invalidation
+ .is_none());
+ drop(registry);
+ assert!(manager.release(&lease).await.is_err());
+ manager.release(&adopted).await.unwrap();
+}
+
+#[tokio::test]
+async fn cross_manager_adoption_cannot_revive_an_expired_token() {
+ let (producer, provider) = fixture().await;
+ let lease = producer
+ .acquire_tree("/data", Duration::ZERO, None)
+ .await
+ .unwrap();
+ let handoff = producer.to_handoff(&lease);
+ producer.handoff(&lease).await.unwrap();
+ let path = &lease.lease.lock_paths[0];
+ let token = provider.read_token(path).await.unwrap().unwrap();
+ let expired = LockToken {
+ time_ns: 1,
+ ..token.clone()
+ };
+ assert!(provider
+ .compare_and_write_token(path, &token, &expired)
+ .await
+ .unwrap());
+ let consumer = PathLockManager::new(
+ producer.resolver.fs.clone(),
+ provider.clone(),
+ PathLockConfig::default(),
+ );
+ assert!(consumer.adopt(&handoff).await.is_err());
+ assert_eq!(provider.read_token(path).await.unwrap(), Some(expired));
+}
+
+#[tokio::test]
+async fn late_refresh_success_does_not_resurrect_an_expired_lease() {
+ let fs = Arc::new(MemFileSystem::new());
+ fs.mkdir("/data", 0o755).await.unwrap();
+ let mut fault = FaultProvider::new(Arc::new(MemoryPathLockProvider::new()));
+ fault.delay = Duration::from_millis(100);
+ // Retain the invalidated entry while inspecting it; heartbeat cleanup races
+ // the explicit caller after the delayed refresh releases the owner mutex.
+ fault.fail_remove.store(true, Ordering::SeqCst);
+ let manager = PathLockManager::new(
+ fs,
+ Arc::new(fault),
+ PathLockConfig {
+ lock_expire_secs: 0.2,
+ ..PathLockConfig::default()
+ },
+ );
+ let lease = manager
+ .acquire_tree("/data", Duration::ZERO, None)
+ .await
+ .unwrap();
+ age_token(
+ &manager,
+ &lease,
+ &lease.lease.lock_paths[0],
+ Duration::from_millis(100),
+ )
+ .await;
+ assert_eq!(manager.refresh(&lease).await.unwrap(), "lost");
+ assert!(manager
+ .get_owned_lease_by_ref(&lease.lease.lease_ref)
+ .await
+ .is_none());
+ assert!(manager.release(&lease).await.is_err());
+}
+
+#[tokio::test]
+async fn background_refresh_invalidates_and_bounds_cleanup_during_outage() {
+ for definite_loss in [true, false] {
+ let fs = Arc::new(MemFileSystem::new());
+ fs.mkdir("/data", 0o755).await.unwrap();
+ let provider = Arc::new(FaultProvider::new(Arc::new(MemoryPathLockProvider::new())));
+ let manager = PathLockManager::new(
+ fs,
+ provider.clone(),
+ PathLockConfig {
+ lock_expire_secs: 0.3,
+ ..PathLockConfig::default()
+ },
+ );
+ let lease = manager
+ .acquire_tree("/data", Duration::ZERO, None)
+ .await
+ .unwrap();
+ provider
+ .failures
+ .lock()
+ .unwrap()
+ .insert(lease.lease.lock_paths[0].clone(), !definite_loss);
+ provider.fail_remove.store(true, Ordering::SeqCst);
+ let owner = manager
+ .lease_registry
+ .resolve(&lease.lease.lease_ref)
+ .unwrap();
+ tokio::time::timeout(Duration::from_secs(2), async {
+ loop {
+ if owner
+ .registry
+ .lock()
+ .await
+ .entries
+ .get(&lease.lease.lease_ref)
+ .is_some_and(|e| e.invalidation.is_some())
+ {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(5)).await;
+ }
+ })
+ .await
+ .unwrap();
+ assert!(manager
+ .get_owned_lease_by_ref(&lease.lease.lease_ref)
+ .await
+ .is_none());
+ let calls = provider.refresh_calls.load(Ordering::SeqCst);
+ tokio::time::timeout(Duration::from_secs(2), async {
+ while manager
+ .lease_registry
+ .resolve(&lease.lease.lease_ref)
+ .is_some()
+ {
+ tokio::time::sleep(Duration::from_millis(5)).await;
+ }
+ })
+ .await
+ .unwrap();
+ assert_eq!(provider.refresh_calls.load(Ordering::SeqCst), calls);
+ assert!(provider.remove_calls.load(Ordering::SeqCst) >= 2);
+ assert!(owner.registry.lock().await.entries.is_empty());
+ assert!(owner.registry.lock().await.lock_refs.is_empty());
+ }
+}
+
+async fn takeover_rejects_protected_write(
+ provider: Arc<dyn PathLockProvider>,
+ fs: Arc<MemFileSystem>,
+) {
+ let manager = Arc::new(PathLockManager::new(
+ fs.clone(),
+ provider.clone(),
+ PathLockConfig::default(),
+ ));
+ let lease = manager
+ .acquire_exact("/data/file", Duration::ZERO, None)
+ .await
+ .unwrap();
+ let lock_path = &lease.lease.lock_paths[0];
+ let old = provider.read_token(lock_path).await.unwrap().unwrap();
+ let replacement = LockToken {
+ owner_id: "new-owner".into(),
+ time_ns: PathLockManager::now_ns(),
+ ..old.clone()
+ };
+ assert!(provider
+ .compare_and_write_token(lock_path, &old, &replacement)
+ .await
+ .unwrap());
+ assert_eq!(manager.refresh(&lease).await.unwrap(), "lost");
+ let wrapped = PathLockWrappedFS::new(manager.clone(), fs.clone());
+ let context = Arc::new(FsContextInner::with_pathlock(
+ "",
+ PathLockContext {
+ lease_ref: Some(lease.lease.lease_ref.clone()),
+ ..PathLockContext::default()
+ },
+ ));
+ let result = FS_CTX
+ .scope(context, async {
+ wrapped
+ .write("/data/file", b"corruption", 0, WriteFlag::Create)
+ .await
+ })
+ .await;
+ assert!(result.is_err(), "old lease must be refused before writing");
+ assert!(fs.stat("/data/file").await.is_err());
+ manager.release(&lease).await.unwrap();
+ assert_eq!(
+ provider.read_token(lock_path).await.unwrap(),
+ Some(replacement)
+ );
+}
+
+#[tokio::test]
+async fn takeover_denies_writes_and_cleanup_preserves_new_owner_for_builtin_providers() {
+ for filesystem in [false, true] {
+ let fs = Arc::new(MemFileSystem::new());
+ fs.mkdir("/data", 0o755).await.unwrap();
+ let provider: Arc<dyn PathLockProvider> = if filesystem {
+ Arc::new(FilesystemPathLockProvider::new(fs.clone(), 30.0))
+ } else {
+ Arc::new(MemoryPathLockProvider::new())
+ };
+ takeover_rejects_protected_write(provider, fs).await;
+ }
+ let dir = tempfile::TempDir::new().unwrap();
+ let provider = Arc::new(
+ crate::lock::SQLitePathLockProvider::new(
+ dir.path().join("locks.db").to_str().unwrap(),
+ "lost-lease",
+ )
+ .unwrap(),
+ );
+ let fs = Arc::new(MemFileSystem::new());
+ fs.mkdir("/data", 0o755).await.unwrap();
+ takeover_rejects_protected_write(provider, fs).await;
+}
+
+#[cfg(feature = "cache")]
+struct RedisResponseDelayProxy {
+ endpoint: String,
+ delay_next_response: Arc<AtomicBool>,
+ response_forwarded: Arc<tokio::sync::Notify>,
+ task: tokio::task::JoinHandle<()>,
+}
+
+#[cfg(feature = "cache")]
+impl RedisResponseDelayProxy {
+ async fn start(endpoint: &str) -> Self {
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
+ use tokio::net::{TcpListener, TcpStream};
+
+ let mut url = url::Url::parse(endpoint).unwrap();
+ assert_eq!(
+ url.scheme(),
+ "redis",
+ "delay proxy requires a plain TCP Redis endpoint"
+ );
+ let host = url.host_str().unwrap().to_string();
+ let port = url.port().unwrap_or(6379);
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ url.set_host(Some("127.0.0.1")).unwrap();
+ url.set_port(Some(listener.local_addr().unwrap().port()))
+ .unwrap();
+ let delay_next_response = Arc::new(AtomicBool::new(false));
+ let response_forwarded = Arc::new(tokio::sync::Notify::new());
+ let armed = delay_next_response.clone();
+ let forwarded = response_forwarded.clone();
+ let task = tokio::spawn(async move {
+ // JoinSet aborts every connection pump when this accept task is dropped.
+ let mut connections = tokio::task::JoinSet::new();
+ loop {
+ tokio::select! {
+ accepted = listener.accept() => {
+ let (client, _) = accepted.unwrap();
+ let host = host.clone();
+ let armed = armed.clone();
+ let forwarded = forwarded.clone();
+ connections.spawn(async move {
+ let upstream = TcpStream::connect((host.as_str(), port)).await.unwrap();
+ client.set_nodelay(true).unwrap();
+ upstream.set_nodelay(true).unwrap();
+ let (mut client_read, mut client_write) = client.into_split();
+ let (mut server_read, mut server_write) = upstream.into_split();
+ let requests = async {
+ tokio::io::copy(&mut client_read, &mut server_write).await?;
+ server_write.shutdown().await
+ };
+ let responses = async {
+ let mut buffer = [0u8; 16 * 1024];
+ loop {
+ let count = server_read.read(&mut buffer).await?;
+ if count == 0 { return client_write.shutdown().await; }
+ let delayed = armed.swap(false, Ordering::SeqCst);
+ if delayed {
+ // Redis has executed the command, but its reply arrives
+ // after the real client's 20 ms deadline.
+ tokio::time::sleep(Duration::from_millis(80)).await;
+ }
+ client_write.write_all(&buffer[..count]).await?;
+ if delayed { forwarded.notify_one(); }
+ }
+ };
+ tokio::try_join!(requests, responses)
+ });
+ }
+ completed = connections.join_next(), if !connections.is_empty() => {
+ completed.unwrap().unwrap().unwrap();
+ }
+ }
+ }
+ });
+ Self {
+ endpoint: url.to_string(),
+ delay_next_response,
+ response_forwarded,
+ task,
+ }
+ }
+
+ fn delay_once(&self) {
+ assert!(!self.delay_next_response.swap(true, Ordering::SeqCst));
+ }
+
+ async fn wait_for_recovery(&self) {
+ tokio::time::timeout(Duration::from_secs(2), self.response_forwarded.notified())
+ .await
+ .expect("delayed Redis response must be forwarded");
+ assert!(!self.delay_next_response.load(Ordering::SeqCst));
+ }
+}
+
+#[cfg(feature = "cache")]
+impl Drop for RedisResponseDelayProxy {
+ fn drop(&mut self) {
+ self.task.abort();
+ }
+}
+
+#[cfg(feature = "cache")]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn redis_20ms_renew_timeout_preserves_lease_and_recovers_after_network_delay() {
+ let Ok(endpoint) = std::env::var("REDIS_URL") else {
+ return;
+ };
+ let proxy = RedisResponseDelayProxy::start(&endpoint).await;
+ let runtime =
+ crate::cache_runtime::CacheRuntime::redis(crate::cache_runtime::RedisProviderConfig {
+ endpoints: vec![proxy.endpoint.clone()],
+ command_timeout_ms: 20,
+ ..crate::cache_runtime::RedisProviderConfig::default()
+ })
+ .await
+ .unwrap();
+ let provider = Arc::new(
+ crate::lock::RedisPathLockProvider::new(
+ runtime.clone(),
+ &format!("renew-network-delay-{}", Uuid::new_v4()),
+ 30.0,
+ )
+ .unwrap(),
+ );
+ let dir = tempfile::TempDir::new().unwrap();
+ let fs = Arc::new(
+ crate::plugins::localfs::LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap(),
+ );
+ fs.mkdir("/local", 0o755).await.unwrap();
+ fs.mkdir("/local/account", 0o755).await.unwrap();
+ let file = "/local/account/file";
+ let manager = Arc::new(PathLockManager::new(
+ fs.clone(),
+ provider.clone(),
+ PathLockConfig::default(),
+ ));
+ let lease = manager
+ .acquire_exact(file, Duration::ZERO, None)
+ .await
+ .unwrap();
+ let path = &lease.lease.lock_paths[0];
+ // Load the real renewal script before arming the proxy so the delayed
+ // response belongs to renewal rather than SCRIPT LOAD.
+ assert_eq!(manager.refresh(&lease).await.unwrap(), "refreshed");
+
+ proxy.delay_once();
+ let result = provider
+ .refresh_tokens(
+ &lease.lease.lock_paths,
+ &lease.lease.owner_id,
+ PathLockManager::now_ns(),
+ )
+ .await;
+ assert_eq!(result.len(), 1);
+ assert!(
+ matches!(&result[0], Err(PathLockError::Io(message))
+ if message.to_ascii_lowercase().contains("timeout")
+ || message.contains("exceeded 20 ms")),
+ "expected a real Redis command timeout: {result:?}"
+ );
+ proxy.wait_for_recovery().await;
+ assert_eq!(manager.refresh(&lease).await.unwrap(), "refreshed");
+
+ let owner = manager
+ .lease_registry
+ .resolve(&lease.lease.lease_ref)
+ .unwrap();
+ let confirmed_before = owner.registry.lock().await.token_health[path].last_confirmed_at;
+ proxy.delay_once();
+ assert_eq!(manager.refresh(&lease).await.unwrap(), "failed");
+ {
+ let registry = owner.registry.lock().await;
+ assert!(registry.entries[&lease.lease.lease_ref]
+ .invalidation
+ .is_none());
+ assert_eq!(
+ registry.token_health[path].last_confirmed_at,
+ confirmed_before
+ );
+ assert_eq!(registry.token_health[path].consecutive_failures, 1);
+ }
+ assert!(manager
+ .get_owned_lease_by_capability(&lease.lease.lease_ref, &lease.ownership_ref)
+ .await
+ .is_some());
+ manager
+ .require_covered_lease_ref(&lease.lease.lease_ref, &lease.lease.covered_paths)
+ .await
+ .unwrap();
+ let wrapped = PathLockWrappedFS::new(manager.clone(), fs.clone());
+ let context = Arc::new(FsContextInner::with_pathlock(
+ "account",
+ PathLockContext {
+ lease_ref: Some(lease.lease.lease_ref.clone()),
+ ..PathLockContext::default()
+ },
+ ));
+ FS_CTX
+ .scope(context.clone(), async {
+ wrapped.write(file, b"before", 0, WriteFlag::Create).await
+ })
+ .await
+ .unwrap();
+ assert_eq!(fs.read(file, 0, 100).await.unwrap(), b"before");
+
+ proxy.wait_for_recovery().await;
+ assert_eq!(manager.refresh(&lease).await.unwrap(), "refreshed");
+ {
+ let registry = owner.registry.lock().await;
+ assert!(registry.entries[&lease.lease.lease_ref]
+ .invalidation
+ .is_none());
+ assert!(registry.token_health[path].last_confirmed_at > confirmed_before);
+ assert_eq!(registry.token_health[path].consecutive_failures, 0);
+ }
+ let token = provider.read_token(path).await.unwrap().unwrap();
+ assert_eq!(token.owner_id, lease.lease.owner_id);
+ FS_CTX
+ .scope(context, async {
+ wrapped.write(file, b"after!", 0, WriteFlag::Create).await
+ })
+ .await
+ .unwrap();
+ assert_eq!(fs.read(file, 0, 100).await.unwrap(), b"after!");
+ manager.release(&lease).await.unwrap();
+ assert!(provider.read_token(path).await.unwrap().is_none());
+ runtime.close().await.unwrap();
+}
+
+#[cfg(feature = "cache")]
+#[tokio::test]
+async fn redis_takeover_denies_writes_and_cleanup_preserves_new_owner() {
+ let Ok(endpoint) = std::env::var("REDIS_URL") else {
+ return;
+ };
+ let runtime =
+ crate::cache_runtime::CacheRuntime::redis(crate::cache_runtime::RedisProviderConfig {
+ endpoints: vec![endpoint],
+ command_timeout_ms: 1_000,
+ ..crate::cache_runtime::RedisProviderConfig::default()
+ })
+ .await
+ .unwrap();
+ let provider = Arc::new(
+ crate::lock::cache_provder::RedisPathLockProvider::new(
+ runtime.clone(),
+ &format!("lost-lease-{}", Uuid::new_v4()),
+ 30.0,
+ )
+ .unwrap(),
+ );
+ let dir = tempfile::TempDir::new().unwrap();
+ let fs = Arc::new(
+ crate::plugins::localfs::LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap(),
+ );
+ fs.mkdir("/local", 0o755).await.unwrap();
+ fs.mkdir("/local/account", 0o755).await.unwrap();
+ fs.write("/local/account/file", b"original", 0, WriteFlag::Create)
+ .await
+ .unwrap();
+ // Redis handles are scoped logical paths.
+ let manager = Arc::new(PathLockManager::new(
+ fs.clone(),
+ provider.clone(),
+ PathLockConfig::default(),
+ ));
+ let lease = manager
+ .acquire_exact("/local/account/file", Duration::ZERO, None)
+ .await
+ .unwrap();
+ let path = &lease.lease.lock_paths[0];
+ let old = provider.read_token(path).await.unwrap().unwrap();
+ let replacement = LockToken {
+ owner_id: "new-owner".into(),
+ ..old.clone()
+ };
+ assert!(provider
+ .compare_and_write_token(path, &old, &replacement)
+ .await
+ .unwrap());
+ assert_eq!(manager.refresh(&lease).await.unwrap(), "lost");
+ let wrapped = PathLockWrappedFS::new(manager.clone(), fs.clone());
+ let context = Arc::new(FsContextInner::with_pathlock(
+ "account",
+ PathLockContext {
+ lease_ref: Some(lease.lease.lease_ref.clone()),
+ ..PathLockContext::default()
+ },
+ ));
+ assert!(FS_CTX
+ .scope(context, async {
+ wrapped
+ .write("/local/account/file", b"corruption", 0, WriteFlag::Create)
+ .await
+ })
+ .await
+ .is_err());
+ assert_eq!(
+ fs.read("/local/account/file", 0, 100).await.unwrap(),
+ b"original"
+ );
+ manager.release(&lease).await.unwrap();
+ assert_eq!(provider.read_token(path).await.unwrap(), Some(replacement));
+ provider
+ .remove_token(path, "new-owner", false)
+ .await
+ .unwrap();
+ runtime.close().await.unwrap();
+}
diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rs
index 3372e2539..a45faea89 100644
--- a/crates/ragfs/src/lock/manager.rs
+++ b/crates/ragfs/src/lock/manager.rs
@@ -62,6 +62,10 @@ const MAX_POLL_INTERVAL_MS: u64 = 500;
const POLL_BACKOFF_NUMERATOR: u64 = 3;
const POLL_BACKOFF_DENOMINATOR: u64 = 2;
const POLL_JITTER_PERCENT: u64 = 20;
+/// Revoke authorization before the provider's expiry, leaving a safety margin.
+const LEASE_VALIDITY_RATIO: f64 = 0.8;
+/// Retry invalidated cleanup for six heartbeat intervals, then forget locally.
+const CLEANUP_RETRY_INTERVALS: f64 = 6.0;
impl PathLockManager {
fn poll_interval_bounds(attempt: u32) -> (u64, u64) {
@@ -117,6 +121,45 @@ struct LeaseEntry {
pending_handoff: bool,
lock_kinds: HashMap<String, PathLockKind>,
last_active_at: Instant,
+ /// Irreversible; retain the capability and references until cleanup finishes.
+ invalidation: Option<LeaseInvalidation>,
+}
+
+/// The first irreversible authorization revocation, retained through cleanup.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct LeaseInvalidation {
+ at: Instant,
+ reason: InvalidationReason,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum InvalidationReason {
+ TokenLost,
+ ConfirmationDeadlineExceeded,
+ ReleaseFailed,
+}
+
+impl LeaseEntry {
+ /// Keep the first cause and timestamp, including during cleanup retries.
+ fn invalidate(&mut self, at: Instant, reason: InvalidationReason, paths: &[&str]) {
+ if self.invalidation.is_some() {
+ return;
+ }
+ self.invalidation = Some(LeaseInvalidation { at, reason });
+ warn!(
+ lease_ref = %self.lease.lease_ref,
+ owner_id = %self.lease.owner_id,
+ ?reason,
+ affected_paths = ?paths,
+ "pathlock lease invalidated"
+ );
+ }
+}
+
+#[derive(Debug)]
+struct TokenHealth {
+ last_confirmed_at: Instant,
+ consecutive_failures: u64,
}
#[derive(Debug)]
@@ -132,6 +175,7 @@ enum DowngradeTokenResult {
struct OwnerRegistry {
entries: HashMap<String, LeaseEntry>,
lock_refs: HashMap<String, usize>,
+ token_health: HashMap<String, TokenHealth>,
// ponytail: grows with handoffs while this owner lives; encode generations if measured.
consumed_handoff_refs: HashSet<String>,
}
@@ -144,6 +188,7 @@ impl OwnerRegistry {
lease: PathLockLease,
ownership_ref: String,
lock_kinds: &[PathLockKind],
+ confirmed_at: Instant,
) -> PathLockResult<()> {
let lease_ref = lease.lease_ref.clone();
if lease.owner_id != expected_owner_id {
@@ -172,6 +217,12 @@ impl OwnerRegistry {
for lock_path in &lease.lock_paths {
if unique_paths.insert(lock_path.clone()) {
*self.lock_refs.entry(lock_path.clone()).or_default() += 1;
+ self.token_health
+ .entry(lock_path.clone())
+ .or_insert(TokenHealth {
+ last_confirmed_at: confirmed_at,
+ consecutive_failures: 0,
+ });
}
}
self.entries.insert(
@@ -182,6 +233,7 @@ impl OwnerRegistry {
pending_handoff: false,
lock_kinds,
last_active_at: Instant::now(),
+ invalidation: None,
},
);
Ok(())
@@ -200,12 +252,14 @@ impl OwnerRegistry {
}
/// True when the entry exists, is owned (not parked for handoff) and its
- /// ownership_ref matches. Used to gate release/refresh against a stale
+ /// ownership_ref matches. Used to gate acquisition/refresh against a stale
/// capability under the same write guard that performs the mutation.
fn capability_matches(&self, lease_ref: &str, ownership_ref: &str) -> bool {
- self.entries
- .get(lease_ref)
- .is_some_and(|entry| !entry.pending_handoff && entry.ownership_ref == ownership_ref)
+ self.entries.get(lease_ref).is_some_and(|entry| {
+ !entry.pending_handoff
+ && entry.invalidation.is_none()
+ && entry.ownership_ref == ownership_ref
+ })
}
/// Return the strongest remaining kind after one lease drops a lock path.
@@ -213,6 +267,7 @@ impl OwnerRegistry {
self.entries
.iter()
.filter(|(candidate_ref, _)| candidate_ref.as_str() != lease_ref)
+ .filter(|(_, entry)| entry.invalidation.is_none())
.filter_map(|(_, entry)| entry.lock_kinds.get(lock_path).copied())
.max_by_key(|kind| match kind {
PathLockKind::Exact => 0,
@@ -253,7 +308,11 @@ impl OwnerRegistry {
/// but reject the original ownership_ref. Requires a matching, not-yet-parked entry.
fn mark_pending_handoff(&mut self, lease_ref: &str, ownership_ref: &str) -> PathLockResult<()> {
match self.entries.get_mut(lease_ref) {
- Some(entry) if entry.ownership_ref == ownership_ref && !entry.pending_handoff => {
+ Some(entry)
+ if entry.ownership_ref == ownership_ref
+ && !entry.pending_handoff
+ && entry.invalidation.is_none() =>
+ {
entry.pending_handoff = true;
Ok(())
}
@@ -282,7 +341,9 @@ impl OwnerRegistry {
new_lease_ref: String,
new_ownership_ref: String,
) -> Option<OwnedPathLockLease> {
- if !self.entries.get(lease_ref)?.pending_handoff {
+ if !self.entries.get(lease_ref)?.pending_handoff
+ || self.entries.get(lease_ref)?.invalidation.is_some()
+ {
return None;
}
let mut entry = self.entries.remove(lease_ref)?;
@@ -309,6 +370,7 @@ impl OwnerRegistry {
Some(count) if *count > 1 => *count -= 1,
Some(_) => {
self.lock_refs.remove(lock_path);
+ self.token_health.remove(lock_path);
}
None => {}
}
@@ -321,6 +383,135 @@ impl OwnerRegistry {
entry.last_active_at = Instant::now();
}
}
+
+ /// Invalidate entire leases sharing an actual token, never logical ancestors.
+ fn invalidate_paths(
+ &mut self,
+ paths: &HashSet<String>,
+ now: Instant,
+ reason: InvalidationReason,
+ ) {
+ for entry in self.entries.values_mut() {
+ if entry.invalidation.is_some() {
+ continue;
+ }
+ let affected_paths: Vec<String> = entry
+ .lease
+ .lock_paths
+ .iter()
+ .filter(|path| paths.contains(*path))
+ .cloned()
+ .collect();
+ if !affected_paths.is_empty() {
+ if reason == InvalidationReason::ConfirmationDeadlineExceeded {
+ for path in &affected_paths {
+ if let Some(health) = self.token_health.get(path) {
+ warn!(
+ lease_ref = %entry.lease.lease_ref,
+ owner_id = %entry.lease.owner_id,
+ ?reason,
+ lock_path = %path,
+ since_last_confirmation_ms = now
+ .saturating_duration_since(health.last_confirmed_at)
+ .as_millis() as u64,
+ failures = health.consecutive_failures,
+ "pathlock token confirmation deadline exceeded"
+ );
+ }
+ }
+ }
+ entry.invalidate(
+ now,
+ reason,
+ &affected_paths
+ .iter()
+ .map(String::as_str)
+ .collect::<Vec<_>>(),
+ );
+ }
+ }
+ }
+
+ fn invalidate_expired(&mut self, now: Instant, validity: Duration) {
+ let expired = self
+ .token_health
+ .iter()
+ .filter(|(_, health)| {
+ now.saturating_duration_since(health.last_confirmed_at) >= validity
+ })
+ .map(|(path, _)| path.clone())
+ .collect();
+ self.invalidate_paths(
+ &expired,
+ now,
+ InvalidationReason::ConfirmationDeadlineExceeded,
+ );
+ }
+
+ fn has_only_invalid_refs(&self, path: &str) -> bool {
+ self.lock_refs.contains_key(path)
+ && !self.entries.values().any(|entry| {
+ entry.invalidation.is_none()
+ && entry
+ .lease
+ .lock_paths
+ .iter()
+ .any(|lock_path| lock_path == path)
+ })
+ }
+
+ /// Both heartbeat and explicit refresh use this state transition.
+ fn apply_refresh_results(
+ &mut self,
+ lease_ref: &str,
+ paths: &[String],
+ results: Vec<PathLockResult<bool>>,
+ started_at: Instant,
+ validity: Duration,
+ ) -> &'static str {
+ // A late success must never restore a lease that crossed its deadline.
+ let now = Instant::now();
+ self.invalidate_expired(now, validity);
+ if !self.entries.contains_key(lease_ref) {
+ return "lost";
+ }
+ let mut lost_paths = HashSet::new();
+ let mut all_ok = results.len() == paths.len();
+ let mut results = results.into_iter();
+ for path in paths {
+ let Some(health) = self.token_health.get_mut(path) else {
+ continue;
+ };
+ match results.next() {
+ Some(Ok(true)) => {
+ health.last_confirmed_at = started_at;
+ health.consecutive_failures = 0;
+ }
+ Some(Ok(false)) => {
+ lost_paths.insert(path.clone());
+ all_ok = false;
+ }
+ result => {
+ health.consecutive_failures = health.consecutive_failures.saturating_add(1);
+ warn!(%lease_ref, owner_id = %self.entries[lease_ref].lease.owner_id,
+ lock_path = %path, failures = health.consecutive_failures,
+ since_last_confirmation_ms = now.saturating_duration_since(health.last_confirmed_at).as_millis() as u64,
+ ?result, "pathlock refresh uncertain");
+ all_ok = false;
+ }
+ }
+ }
+ self.invalidate_paths(&lost_paths, now, InvalidationReason::TokenLost);
+ self.invalidate_expired(now, validity);
+ if self.entries[lease_ref].invalidation.is_some() {
+ "lost"
+ } else if all_ok {
+ self.touch(lease_ref);
+ "refreshed"
+ } else {
+ "failed"
+ }
+ }
}
#[derive(Debug)]
@@ -517,73 +708,25 @@ impl PathLockManager {
tokio::spawn(async move {
loop {
tokio::time::sleep(refresh_interval).await;
- let now_ns = Self::now_ns();
let leases = refresh_registry.snapshot();
for (lease_ref, owner) in leases {
let mut owner_registry = owner.registry.lock().await;
+ owner_registry
+ .invalidate_expired(Instant::now(), Self::lease_validity(&refresh_config));
let Some(entry) = owner_registry.entries.get(&lease_ref) else {
continue;
};
- let lock_paths = entry.lease.lock_paths.clone();
- let pending_handoff = entry.pending_handoff;
- let mut any_ok = false;
- let mut released_handoff_paths = Vec::new();
- let refresh_results = refresh_provider
- .refresh_tokens(&lock_paths, &owner.owner_id, now_ns)
+ if entry.invalidation.is_none() {
+ Self::refresh_entry_with(
+ &refresh_provider,
+ &mut owner_registry,
+ &owner.owner_id,
+ &lease_ref,
+ Self::lease_validity(&refresh_config),
+ )
.await;
- for (lp, result) in lock_paths.iter().zip(refresh_results) {
- match result {
- Ok(true) => {
- any_ok = true;
- debug!(lease_ref = %lease_ref, owner_id = %owner.owner_id, lock_path = %lp, "pathlock lease automatically refreshed");
- }
- Ok(false) => {
- if pending_handoff {
- // A different process may have adopted and released this
- // handoff. Its completed release makes the producer's
- // local reference obsolete. Do not remove the provider
- // token: another process may own it or have released it.
- released_handoff_paths.push(lp.clone());
- } else {
- warn!(lease_ref = %lease_ref, owner_id = %owner.owner_id, lock_path = %lp, "pathlock lease automatic refresh failed");
- }
- }
- Err(error) => {
- warn!(lease_ref = %lease_ref, owner_id = %owner.owner_id, lock_path = %lp, error = %error, "pathlock lease automatic refresh failed");
- }
- }
- }
- if any_ok {
- owner_registry.touch(&lease_ref);
- }
- for lock_path in released_handoff_paths {
- let is_last_path =
- owner_registry.entries.get(&lease_ref).is_some_and(|entry| {
- entry.lease.lock_paths.iter().all(|path| path == &lock_path)
- });
- if is_last_path {
- if let Err(error) = refresh_registry.unpublish(&lease_ref, &owner) {
- error!(
- lease_ref = %lease_ref,
- owner_id = %owner.owner_id,
- error = %error,
- "failed to forget released pathlock handoff"
- );
- continue;
- }
- }
- owner_registry.remove_path(&lease_ref, &lock_path);
- if is_last_path {
- debug!(
- lease_ref = %lease_ref,
- owner_id = %owner.owner_id,
- "forgot locally released pathlock handoff"
- );
- }
}
}
- let stale_cutoff =
- Instant::now() - Duration::from_secs_f64(refresh_config.lock_expire_secs * 2.0);
let stale_candidates = refresh_registry.snapshot();
let mut removed = 0;
for (lease_ref, owner) in stale_candidates {
@@ -594,7 +737,7 @@ impl PathLockManager {
&lease_ref,
None,
None,
- Some(stale_cutoff),
+ Some(Self::cleanup_window(&refresh_config)),
)
.await
{
@@ -702,6 +845,29 @@ impl PathLockManager {
(self.config.lock_expire_secs * 1_000_000_000.0) as u128
}
+ fn lease_validity(config: &PathLockConfig) -> Duration {
+ Duration::from_secs_f64(config.lock_expire_secs * LEASE_VALIDITY_RATIO)
+ }
+
+ fn cleanup_window(config: &PathLockConfig) -> Duration {
+ Duration::from_secs_f64(config.lock_expire_secs / 3.0 * CLEANUP_RETRY_INTERVALS)
+ }
+
+ async fn refresh_entry_with(
+ provider: &Arc<dyn PathLockProvider>,
+ registry: &mut OwnerRegistry,
+ owner_id: &str,
+ lease_ref: &str,
+ validity: Duration,
+ ) -> &'static str {
+ let paths = registry.entries[lease_ref].lease.lock_paths.clone();
+ let started_at = Instant::now();
+ let results = provider
+ .refresh_tokens(&paths, owner_id, Self::now_ns())
+ .await;
+ registry.apply_refresh_results(lease_ref, &paths, results, started_at, validity)
+ }
+
/// Check if a token is stale based on config.
fn is_stale(&self, token: &LockToken, now_ns: u128) -> bool {
now_ns.saturating_sub(token.time_ns) > self.lock_expire_ns()
@@ -886,6 +1052,7 @@ impl PathLockManager {
elapsed_ms: start.elapsed().as_millis() as u64,
});
}
+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));
if let Some((lease_ref, ownership_ref)) = owner_capability {
if !owner_registry.capability_matches(lease_ref, ownership_ref) {
break Err(PathLockError::InvalidRequest(format!(
@@ -894,6 +1061,7 @@ impl PathLockManager {
}
}
first_attempt = false;
+ let confirmed_at = Instant::now();
let acquired_lock_paths = match self.try_acquire_batch_once(&sorted, &owner_id).await {
Ok(acquired) => acquired,
Err(err) => {
@@ -937,6 +1105,28 @@ impl PathLockManager {
}
};
+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));
+ let source_invalid = owner_capability.is_some_and(|(lease_ref, ownership_ref)| {
+ !owner_registry.capability_matches(lease_ref, ownership_ref)
+ });
+ let reuses_invalid_token = acquired_lock_paths
+ .iter()
+ .any(|acquisition| owner_registry.has_only_invalid_refs(&acquisition.handle));
+ if source_invalid
+ || reuses_invalid_token
+ || confirmed_at.elapsed() >= Self::lease_validity(&self.config)
+ {
+ let error = PathLockError::InvalidRequest(
+ "pathlock lease expired during acquisition".to_string(),
+ );
+ let rollback = self
+ .rollback_acquisitions(&acquired_lock_paths, &owner_id)
+ .await;
+ break match rollback {
+ Ok(()) => Err(error),
+ Err(rollback_error) => Err(Self::rollback_error(error, rollback_error)),
+ };
+ }
let lease = PathLockLease {
lease_ref: Self::new_owner_id(),
owner_id: owner_id.clone(),
@@ -948,9 +1138,13 @@ impl PathLockManager {
};
let ownership_ref = Self::new_owner_id();
let lock_kinds: Vec<_> = sorted.iter().map(|request| request.kind).collect();
- if let Err(error) =
- owner_registry.insert(&owner_id, lease.clone(), ownership_ref.clone(), &lock_kinds)
- {
+ if let Err(error) = owner_registry.insert(
+ &owner_id,
+ lease.clone(),
+ ownership_ref.clone(),
+ &lock_kinds,
+ confirmed_at,
+ ) {
let rollback = self
.rollback_acquisitions(&acquired_lock_paths, &owner_id)
.await;
@@ -1352,49 +1546,21 @@ impl PathLockManager {
))
})?;
let mut owner_registry = owner.registry.lock().await;
+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));
if !owner_registry.capability_matches(&lease.lease.lease_ref, &lease.ownership_ref) {
return Err(PathLockError::InvalidRequest(format!(
"owned lease capability does not match ref '{}'",
lease.lease.lease_ref
)));
}
- let lock_paths = owner_registry
- .entries
- .get(&lease.lease.lease_ref)
- .map(|entry| entry.lease.lock_paths.clone())
- .ok_or_else(|| {
- PathLockError::Internal(format!(
- "published pathlock lease '{}' is missing from owner state",
- lease.lease.lease_ref
- ))
- })?;
- let now_ns = Self::now_ns();
- let mut all_ok = true;
- let mut any_ok = false;
-
- for result in self
- .provider
- .refresh_tokens(&lock_paths, &owner.owner_id, now_ns)
- .await
- {
- match result {
- Ok(true) => any_ok = true,
- Ok(false) => all_ok = false,
- Err(_) => all_ok = false,
- }
- }
-
- if any_ok {
- owner_registry.touch(&lease.lease.lease_ref);
- }
-
- let result = if all_ok && any_ok {
- "refreshed"
- } else if any_ok {
- "lost"
- } else {
- "failed"
- };
+ let result = Self::refresh_entry_with(
+ &self.provider,
+ &mut owner_registry,
+ &owner.owner_id,
+ &lease.lease.lease_ref,
+ Self::lease_validity(&self.config),
+ )
+ .await;
if result == "refreshed" {
debug!(lease_ref = %lease.lease.lease_ref, owner_id = %lease.lease.owner_id, lock_paths = ?lease.lease.lock_paths, result = %result, "pathlock refresh completed");
} else {
@@ -1478,7 +1644,7 @@ impl PathLockManager {
lease_ref: &str,
ownership_ref: Option<&str>,
selected_paths: Option<&[String]>,
- stale_cutoff: Option<Instant>,
+ cleanup_window: Option<Duration>,
) -> PathLockResult<bool> {
let mut owner_registry = owner.registry.lock().await;
let Some(entry) = owner_registry.entries.get(lease_ref) else {
@@ -1496,8 +1662,19 @@ impl PathLockManager {
)));
}
}
- if stale_cutoff.is_some_and(|cutoff| entry.last_active_at > cutoff) {
- return Ok(false);
+ if let Some(window) = cleanup_window {
+ let Some(invalidation) = entry.invalidation else {
+ return Ok(false);
+ };
+ if invalidation.at.elapsed() >= window {
+ error!(%lease_ref, owner_id = %owner.owner_id, reason = ?invalidation.reason,
+ invalidated_for_ms = invalidation.at.elapsed().as_millis() as u64,
+ remaining_paths = ?entry.lease.lock_paths,
+ "pathlock cleanup retry window exhausted; forgetting local lease");
+ lease_registry.unpublish(lease_ref, &owner)?;
+ owner_registry.rollback_insert(lease_ref);
+ return Ok(true);
+ }
}
let selected: Option<HashSet<&str>> =
selected_paths.map(|paths| paths.iter().map(String::as_str).collect());
@@ -1529,24 +1706,47 @@ impl PathLockManager {
})?;
let released_kind = entry.lock_kinds.get(&lock_path).copied();
let remaining_kind = owner_registry.strongest_kind_excluding(lease_ref, &lock_path);
+ let invalidated = entry.invalidation.is_some();
if ref_count == 1 {
- if owner_registry
- .entries
- .get(lease_ref)
- .is_some_and(|entry| entry.lease.lock_paths.iter().all(|p| p == &lock_path))
- {
- lease_registry.unpublish(lease_ref, &owner)?;
- }
- owner_registry.remove_path(lease_ref, &lock_path);
- match Self::remove_owned_token_with(provider, &lock_path, &owner.owner_id).await {
- Ok(true) => {}
- Ok(false) => {
- if first_error.is_none() {
- first_error = Some(Self::release_changed_error(&lock_path));
+ let result = if invalidated {
+ provider
+ .remove_token(&lock_path, &owner.owner_id, false)
+ .await
+ } else {
+ Self::remove_owned_token_with(provider, &lock_path, &owner.owner_id).await
+ };
+ match result {
+ Ok(removed) => {
+ if !removed && !invalidated {
+ owner_registry.invalidate_paths(
+ &HashSet::from([lock_path.clone()]),
+ Instant::now(),
+ InvalidationReason::TokenLost,
+ );
+ }
+ if owner_registry.entries[lease_ref]
+ .lease
+ .lock_paths
+ .iter()
+ .all(|p| p == &lock_path)
+ {
+ lease_registry.unpublish(lease_ref, &owner)?;
}
+ owner_registry.remove_path(lease_ref, &lock_path);
}
Err(release_error) => {
+ // Stop authorizing writes while failed release is retried.
+ if let Some(entry) = owner_registry.entries.get_mut(lease_ref) {
+ entry.invalidate(
+ Instant::now(),
+ InvalidationReason::ReleaseFailed,
+ &[&lock_path],
+ );
+ warn!(%lease_ref, owner_id = %owner.owner_id, lock_path = %lock_path,
+ reason = ?entry.invalidation.unwrap().reason, error = %release_error,
+ "pathlock release failed; retaining lease for cleanup");
+ }
if first_error.is_none() {
first_error = Some(release_error);
}
@@ -1587,15 +1787,20 @@ impl PathLockManager {
owner_registry.remove_path(lease_ref, &lock_path);
}
Ok(false) => {
- Self::discard_lost_path_refs(
- lease_registry,
- &owner,
- &mut owner_registry,
- &lock_path,
- )?;
- if first_error.is_none() {
- first_error = Some(Self::release_changed_error(&lock_path));
+ owner_registry.invalidate_paths(
+ &HashSet::from([lock_path.clone()]),
+ Instant::now(),
+ InvalidationReason::TokenLost,
+ );
+ if owner_registry.entries[lease_ref]
+ .lease
+ .lock_paths
+ .iter()
+ .all(|p| p == &lock_path)
+ {
+ lease_registry.unpublish(lease_ref, &owner)?;
}
+ owner_registry.remove_path(lease_ref, &lock_path);
}
Err(release_error) => {
let owner_mismatch = matches!(
@@ -1604,12 +1809,21 @@ impl PathLockManager {
if current_owner != &owner.owner_id
);
if owner_mismatch {
- Self::discard_lost_path_refs(
- lease_registry,
- &owner,
- &mut owner_registry,
- &lock_path,
- )?;
+ owner_registry.invalidate_paths(
+ &HashSet::from([lock_path.clone()]),
+ Instant::now(),
+ InvalidationReason::TokenLost,
+ );
+ }
+ if let Some(entry) = owner_registry.entries.get_mut(lease_ref) {
+ entry.invalidate(
+ Instant::now(),
+ InvalidationReason::ReleaseFailed,
+ &[&lock_path],
+ );
+ warn!(%lease_ref, owner_id = %owner.owner_id, lock_path = %lock_path,
+ reason = ?entry.invalidation.unwrap().reason, error = %release_error,
+ "pathlock release failed; retaining lease for cleanup");
}
if first_error.is_none() {
first_error = Some(release_error);
@@ -1627,35 +1841,6 @@ impl PathLockManager {
Ok(removed)
}
- /// Drop every local reference to a path whose token belongs to another owner.
- fn discard_lost_path_refs(
- lease_registry: &LeaseRegistry,
- owner: &Arc<OwnerState>,
- owner_registry: &mut OwnerRegistry,
- lock_path: &str,
- ) -> PathLockResult<()> {
- let affected: Vec<String> = owner_registry
- .entries
- .iter()
- .filter(|(_, entry)| entry.lease.lock_paths.iter().any(|path| path == lock_path))
- .map(|(lease_ref, _)| lease_ref.clone())
- .collect();
- for lease_ref in &affected {
- if owner_registry.entries[lease_ref]
- .lease
- .lock_paths
- .iter()
- .all(|path| path == lock_path)
- {
- lease_registry.unpublish(lease_ref, owner)?;
- }
- }
- for lease_ref in affected {
- owner_registry.remove_path(&lease_ref, lock_path);
- }
- Ok(())
- }
-
/// Atomically reduce one same-owner Tree token to Exact.
async fn downgrade_token_to_exact_with(
provider: &Arc<dyn PathLockProvider>,
@@ -1725,11 +1910,9 @@ impl PathLockManager {
lease.lease.lease_ref
))
})?;
- owner
- .registry
- .lock()
- .await
- .mark_pending_handoff(&lease.lease.lease_ref, &lease.ownership_ref)?;
+ let mut owner_registry = owner.registry.lock().await;
+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));
+ owner_registry.mark_pending_handoff(&lease.lease.lease_ref, &lease.ownership_ref)?;
let active_count = self.lease_registry.active_count();
debug!(lease_ref = %lease.lease.lease_ref, owner_id = %lease.lease.owner_id, lock_paths = ?lease.lease.lock_paths, active_count = active_count, "parked pathlock lease for handoff");
Ok(())
@@ -1752,6 +1935,16 @@ impl PathLockManager {
if let Some(lease_ref) = handoff.lease_ref.as_deref() {
if let Some(owner) = self.lease_registry.resolve(lease_ref) {
let mut owner_registry = owner.registry.lock().await;
+ owner_registry
+ .invalidate_expired(Instant::now(), Self::lease_validity(&self.config));
+ if owner_registry
+ .get_by_ref(lease_ref)
+ .is_some_and(|entry| entry.invalidation.is_some())
+ {
+ return Err(PathLockError::HandoffFailed(format!(
+ "pathlock lease '{lease_ref}' is invalidated"
+ )));
+ }
let local = owner_registry.get_by_ref(lease_ref).map(|entry| {
(
entry.lease.owner_id == handoff.owner_id
@@ -1818,7 +2011,6 @@ impl PathLockManager {
"handoff lock_paths and covered_paths must have equal lengths".to_string(),
));
}
- let now_ns = Self::now_ns();
let mut expected_kinds = Vec::with_capacity(handoff.lock_paths.len());
// ponytail: legacy durable handoffs only persisted owner_id/handle_id + lock_paths.
@@ -1887,6 +2079,16 @@ impl PathLockManager {
.lease_registry
.resolve_or_create_owner(&handoff.owner_id);
let mut owner_registry = owner.registry.lock().await;
+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));
+ if handoff
+ .lock_paths
+ .iter()
+ .any(|path| owner_registry.has_only_invalid_refs(path))
+ {
+ return Err(PathLockError::HandoffFailed(
+ "handoff shares an invalidated token".to_string(),
+ ));
+ }
let already_consumed = match handoff.lease_ref.as_deref() {
Some(source_lease_ref) => owner_registry
.consumed_handoff_refs
@@ -1904,10 +2106,15 @@ impl PathLockManager {
handoff.owner_id
)));
}
+ let confirmed_at = Instant::now();
+ let now_ns = Self::now_ns();
for (lock_path, expected_kind) in handoff.lock_paths.iter().zip(&expected_kinds) {
match self.provider.read_token(lock_path).await? {
Some(token)
- if token.owner_id == handoff.owner_id && token.lock_type == *expected_kind => {}
+ if token.owner_id == handoff.owner_id
+ && token.lock_type == *expected_kind
+ && now_ns.saturating_sub(token.time_ns)
+ < (self.lock_expire_ns() as f64 * LEASE_VALIDITY_RATIO) as u128 => {}
_ => {
return Err(PathLockError::HandoffFailed(format!(
"lock path '{lock_path}' is no longer owned by '{}'",
@@ -1932,6 +2139,17 @@ impl PathLockManager {
debug!(lock_path = %lock_path, owner_id = %handoff.owner_id, kind = ?expected_kind, "refreshed pathlock token during adopt");
}
+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));
+ if confirmed_at.elapsed() >= Self::lease_validity(&self.config)
+ || handoff
+ .lock_paths
+ .iter()
+ .any(|path| owner_registry.has_only_invalid_refs(path))
+ {
+ return Err(PathLockError::HandoffFailed(
+ "handoff expired during adoption".to_string(),
+ ));
+ }
let lease = PathLockLease {
lease_ref: Self::new_owner_id(),
owner_id: handoff.owner_id.clone(),
@@ -1944,6 +2162,7 @@ impl PathLockManager {
lease.clone(),
ownership_ref.clone(),
&expected_kinds,
+ confirmed_at,
)?;
if let Err(error) = self
.lease_registry
@@ -2028,11 +2247,12 @@ impl PathLockManager {
/// Look up an owned lease by owner_id for cross-FFI lease operations.
pub async fn get_owned_lease(&self, owner_id: &str) -> Option<OwnedPathLockLease> {
let owner = self.lease_registry.resolve_owner(owner_id)?;
- let owner_registry = owner.registry.lock().await;
+ let mut owner_registry = owner.registry.lock().await;
+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));
owner_registry
.entries
.values()
- .find(|entry| !entry.pending_handoff)
+ .find(|entry| !entry.pending_handoff && entry.invalidation.is_none())
.map(|entry| OwnedPathLockLease {
lease: entry.lease.clone(),
ownership_ref: entry.ownership_ref.clone(),
@@ -2042,10 +2262,11 @@ impl PathLockManager {
/// Look up an owned lease by opaque lease_ref for cross-FFI lease operations.
pub async fn get_owned_lease_by_ref(&self, lease_ref: &str) -> Option<OwnedPathLockLease> {
let owner = self.lease_registry.resolve(lease_ref)?;
- let owner_registry = owner.registry.lock().await;
+ let mut owner_registry = owner.registry.lock().await;
+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));
owner_registry
.get_by_ref(lease_ref)
- .filter(|entry| !entry.pending_handoff)
+ .filter(|entry| !entry.pending_handoff && entry.invalidation.is_none())
.map(|entry| OwnedPathLockLease {
lease: entry.lease.clone(),
ownership_ref: entry.ownership_ref.clone(),
@@ -2059,8 +2280,30 @@ impl PathLockManager {
ownership_ref: &str,
) -> Option<OwnedPathLockLease> {
let owner = self.lease_registry.resolve(lease_ref)?;
- let owner_registry = owner.registry.lock().await;
+ let mut owner_registry = owner.registry.lock().await;
+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));
owner_registry
+ .get_by_ref(lease_ref)
+ .filter(|entry| {
+ !entry.pending_handoff
+ && entry.invalidation.is_none()
+ && entry.ownership_ref == ownership_ref
+ })
+ .map(|entry| OwnedPathLockLease {
+ lease: entry.lease.clone(),
+ ownership_ref: entry.ownership_ref.clone(),
+ })
+ }
+
+ /// Resolve a release capability even after authorization has been revoked.
+ pub async fn get_releasable_lease_by_capability(
+ &self,
+ lease_ref: &str,
+ ownership_ref: &str,
+ ) -> Option<OwnedPathLockLease> {
+ let owner = self.lease_registry.resolve(lease_ref)?;
+ let registry = owner.registry.lock().await;
+ registry
.get_by_ref(lease_ref)
.filter(|entry| !entry.pending_handoff && entry.ownership_ref == ownership_ref)
.map(|entry| OwnedPathLockLease {
@@ -2081,10 +2324,11 @@ impl PathLockManager {
let owner = self.lease_registry.resolve(lease_ref).ok_or_else(|| {
PathLockError::InvalidRequest(format!("unknown pathlock lease ref '{lease_ref}'"))
})?;
- let owner_registry = owner.registry.lock().await;
+ let mut owner_registry = owner.registry.lock().await;
+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));
let entry = owner_registry
.get_by_ref(lease_ref)
- .filter(|entry| !entry.pending_handoff)
+ .filter(|entry| !entry.pending_handoff && entry.invalidation.is_none())
.ok_or_else(|| {
PathLockError::InvalidRequest(format!("unknown pathlock lease ref '{lease_ref}'"))
})?;
@@ -2140,6 +2384,10 @@ impl PathLockManager {
}
}
+#[cfg(test)]
+#[path = "lost_lease_tests.rs"]
+mod lost_lease_tests;
+
#[cfg(test)]
mod tests {
use super::*;
@@ -2962,7 +3210,7 @@ mod tests {
}
#[tokio::test]
- async fn pending_handoff_keeps_unreleased_paths_refreshing() {
+ async fn pending_handoff_token_loss_invalidates_the_entire_lease() {
let fs = Arc::new(MemFileSystem::new());
fs.mkdir("/data", 0o755).await.unwrap();
let provider = Arc::new(crate::lock::provider::MemoryPathLockProvider::new());
@@ -2991,18 +3239,20 @@ mod tests {
.await
.unwrap();
- // Wait beyond the producer stale-GC cutoff. The remaining path must
- // retain the producer's local renewal state while the consumer holds it.
+ // Losing one required token invalidates the entire parked lease.
tokio::time::sleep(Duration::from_millis(700)).await;
- assert_eq!(producer.observe().await.active_locks, 1);
+ assert_eq!(producer.observe().await.active_locks, 0);
assert_eq!(provider.read_token(&first_lock_path).await.unwrap(), None);
assert!(provider
.read_token(&second_lock_path)
.await
.unwrap()
- .is_some());
+ .is_none());
- consumer.release(&adopted).await.unwrap();
+ assert!(consumer
+ .get_owned_lease_by_ref(&adopted.lease.lease_ref)
+ .await
+ .is_none());
tokio::time::timeout(Duration::from_millis(400), async {
loop {
if producer.observe().await.active_locks == 0 {
@@ -3352,6 +3602,10 @@ mod tests {
assert!(mgr
.get_owned_lease_by_capability(&tree.lease.lease_ref, &tree.ownership_ref)
.await
+ .is_none());
+ assert!(mgr
+ .get_releasable_lease_by_capability(&tree.lease.lease_ref, &tree.ownership_ref)
+ .await
.is_some());
mgr.release(&exact).await.unwrap();
diff --git a/docs/en/concepts/09-transaction.md b/docs/en/concepts/09-transaction.md
index 77504802b..cbeec1375 100644
--- a/docs/en/concepts/09-transaction.md
+++ b/docs/en/concepts/09-transaction.md
@@ -375,7 +375,15 @@ Acquisition rollback and normal release clean up tokens without guaranteeing rem
**Stale lock detection**: PathLockEngine checks the ownership token timestamp. Locks older than `lock_expire` (default 30s) are considered stale and are removed automatically during acquisition.
-**In-process cleanup**: During refresh, Rust PathLockManager checks leases that have not refreshed successfully for `2 × lock_expire` and attempts cleanup with ownership validation.
+**Lost lease authorization**: Explicit and background refresh use the same rules. A missing token or owner mismatch immediately invalidates every local lease under that owner sharing the actual token path (`lock_paths`). Losing any required token invalidates the entire lease, including a pending handoff. A Tree ancestor and an Exact descendant have different tokens and do not invalidate each other.
+
+A transport error preserves authorization only until `0.8 × lock_expire` since that token's last successful confirmation (24 seconds by default). Each token has its own monotonic deadline; partial refresh, reentry, and local handoff/adoption do not extend failed tokens. Authorization checks enforce the deadline even before the next heartbeat. Invalidated leases cannot authorize subsequent protected writes, refresh, owner reuse, or handoff/adoption, and cannot become valid again.
+
+**In-process cleanup**: Invalidated leases retain their release capability and local references while ownership-checked cleanup is retried. A missing or foreign-owned token completes cleanup without deleting a new owner's token. Tokens shared with another valid local lease remain protected. After six heartbeat intervals from the first invalidation (`2 × lock_expire`, 60 seconds by default), the manager logs remaining paths and forgets the local entry even if the provider is unavailable; remaining provider tokens rely on expiry. Cleanup retries never extend this window.
+
+**Invalidation diagnostics**: The manager retains the first invalidation time and reason through cleanup. The `pathlock lease invalidated` log includes `reason`, `lease_ref`, `owner_id`, and `affected_paths`. `TokenLost` means a required token is missing or ownership no longer matches; inspect token expiry, removal, and takeover. `ConfirmationDeadlineExceeded` means ownership could not be confirmed before the safety deadline; inspect provider health, network latency, and renewal scheduling. Its token log includes `since_last_confirmation_ms` and `failures`. `ReleaseFailed` means a release or downgrade error revoked authorization while cleanup is retried; the release log includes the error. Later results and cleanup failures do not replace the first reason or reset its time; the cleanup-window exhaustion log retains that reason.
+
+These checks revoke authorization for future operations after loss is detected or the deadline passes. They do not cancel writes that already passed authorization or provide storage-side fencing.
**Orphan locks**: Provider tokens left behind after a process crash are automatically removed via stale lock detection when a later acquisition checks the same path or scope.
diff --git a/docs/zh/concepts/09-transaction.md b/docs/zh/concepts/09-transaction.md
index ed5771919..3d3feabfe 100644
--- a/docs/zh/concepts/09-transaction.md
+++ b/docs/zh/concepts/09-transaction.md
@@ -383,7 +383,15 @@ fencing token 校验通过的一方成功持有 `TreeLock(java-guide)`;失败
**陈旧锁检测**:PathLockEngine 检查归属 token 中的时间戳。超过 `lock_expire`(默认 30s)的锁被视为陈旧锁,在加锁过程中自动移除。
-**进程内清理**:Rust PathLockManager 在续期循环中检查长期未成功续期的 lease,以 `2 × lock_expire` 为阈值尝试清理,并校验归属后释放 token。
+**失锁后的授权失效**:手动和后台续租使用相同规则。token 不存在或 owner 不匹配时,同 owner 下共享实际 token 路径(`lock_paths`)的所有本地 lease 立即整份失效,包括待交接的 lease。Tree 祖先和 Exact 后代使用不同 token,不会因逻辑覆盖关系连带失效。
+
+网络等不确定错误允许短暂重试,但每个 token 距离最近一次成功确认达到 `0.8 × lock_expire`(默认 24 秒)时撤销授权。使用单调时钟分别记录各 token 的确认时间;部分续租成功、重入和本地交接/接管不会延长失败 token 的期限。写入授权检查也会检查期限,无需等待下次心跳。失效 lease 不能再授权后续受保护写入、续租、复用 owner 或交接/接管,且不会恢复有效。
+
+**进程内清理**:失效 lease 保留释放能力和本地引用,后台重试带 owner 校验的清理。token 已不存在或已被其他 owner 接管均视为清理完成,不会删除新 owner 的 token;仍被其他有效本地 lease 引用的 token 也不会被删除。从首次失效起超过六个心跳间隔(`2 × lock_expire`,默认 60 秒),即使 Provider 持续不可用也记录剩余路径并移除本地 entry,遗留的远端 token 依赖过期清理。重试不会延长该窗口。
+
+**失效诊断**:manager 在清理期间保留首次失效的时间与原因。`pathlock lease invalidated` 日志包含 `reason`、`lease_ref`、`owner_id` 和 `affected_paths`。`TokenLost` 表示必要 token 缺失或归属不符,应检查 token 过期、删除和接管情况;`ConfirmationDeadlineExceeded` 表示安全期限内未能确认归属,应检查 Provider 健康、网络延迟和续租调度,对应 token 日志还包含 `since_last_confirmation_ms` 和 `failures`;`ReleaseFailed` 表示释放或降级错误导致撤销授权并进入清理重试,释放日志会记录具体错误。后续响应和清理失败不会覆盖首次原因或重置时间,清理窗口耗尽的日志也会保留该原因。
+
+上述检查在发现失锁或达到期限后拒绝后续操作;不会中断已经通过授权检查的写入,也不提供存储侧 fencing。
**孤儿锁**:进程崩溃后遗留的 Provider token,在后续 acquire 检查同一路径或 scope 时通过 stale lock 检测自动移除。