Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(pathlock): revoke lease authorization after token loss (#6704)
zhanglistar4adcab749848
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rsindex 777a1e9c2..25a53b686 100644--- a/crates/ragfs-python/src/lib.rs+++ b/crates/ragfs-python/src/lib.rs@@ -2834,7 +2834,7 @@ impl RAGFSBindingClient {let ownership = ownership_ref.clone();async move {let lease = mgr- .get_owned_lease_by_capability(&lr, &ownership)+ .get_releasable_lease_by_capability(&lr, &ownership).await.ok_or_else(|| {PathLockError::InvalidRequest(format!(@@ -2867,7 +2867,7 @@ impl RAGFSBindingClient {let paths = lock_paths.clone();async move {let lease = mgr- .get_owned_lease_by_capability(&lr, &ownership)+ .get_releasable_lease_by_capability(&lr, &ownership).await.ok_or_else(|| {PathLockError::InvalidRequest(format!(diff --git a/crates/ragfs/src/lock/lost_lease_tests.rs b/crates/ragfs/src/lock/lost_lease_tests.rsnew file mode 100644index 000000000..3eefdc880--- /dev/null+++ b/crates/ragfs/src/lock/lost_lease_tests.rs@@ -0,0 +1,1031 @@+//! Lost lease authorization, renewal deadlines, and bounded cleanup regressions.+use super::*;+use crate::core::{FsContextInner, PathLockContext, WriteFlag, FS_CTX};+use crate::lock::provider::{FilesystemPathLockProvider, MemoryPathLockProvider};+use crate::lock::wrapper::PathLockWrappedFS;+use crate::plugins::memfs::MemFileSystem;+use async_trait::async_trait;+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};++struct FaultProvider {+ inner: Arc<dyn PathLockProvider>,+ /// false: definite loss; true: uncertain transport failure.+ failures: StdMutex<HashMap<String, bool>>,+ fail_remove: AtomicBool,+ refresh_calls: AtomicUsize,+ remove_calls: AtomicUsize,+ delay: Duration,+}++impl FaultProvider {+ fn new(inner: Arc<dyn PathLockProvider>) -> Self {+ Self {+ inner,+ failures: StdMutex::new(HashMap::new()),+ fail_remove: AtomicBool::new(false),+ refresh_calls: AtomicUsize::new(0),+ remove_calls: AtomicUsize::new(0),+ delay: Duration::ZERO,+ }+ }+}++#[async_trait]+impl PathLockProvider for FaultProvider {+ fn name(&self) -> &'static str {+ "fault-injection"+ }+ fn handle_mode(&self) -> PathLockHandleMode {+ self.inner.handle_mode()+ }+ async fn try_acquire_batch_atomic(+ &self,+ requests: &[PathLockRequest],+ owner: &str,+ now: u128,+ stale: u128,+ ) -> PathLockResult<Option<Vec<AtomicAcquisition>>> {+ self.inner+ .try_acquire_batch_atomic(requests, owner, now, stale)+ .await+ }+ async fn read_token(&self, path: &str) -> PathLockResult<Option<LockToken>> {+ self.inner.read_token(path).await+ }+ async fn try_create_token(&self, path: &str, token: &LockToken) -> PathLockResult<()> {+ self.inner.try_create_token(path, token).await+ }+ async fn compare_and_write_token(+ &self,+ path: &str,+ expected: &LockToken,+ replacement: &LockToken,+ ) -> PathLockResult<bool> {+ self.inner+ .compare_and_write_token(path, expected, replacement)+ .await+ }+ async fn refresh_token(&self, path: &str, owner: &str, now: u128) -> PathLockResult<bool> {+ self.refresh_calls.fetch_add(1, Ordering::SeqCst);+ if !self.delay.is_zero() {+ tokio::time::sleep(self.delay).await;+ }+ let fault = self.failures.lock().unwrap().get(path).copied();+ match fault {+ Some(true) => Err(PathLockError::Io("injected refresh timeout".into())),+ Some(false) => Ok(false),+ None => self.inner.refresh_token(path, owner, now).await,+ }+ }+ async fn remove_token(&self, path: &str, owner: &str, force: bool) -> PathLockResult<bool> {+ assert!(!force, "cleanup must use ownership CAS");+ self.remove_calls.fetch_add(1, Ordering::SeqCst);+ if self.fail_remove.load(Ordering::SeqCst) {+ Err(PathLockError::Io("injected cleanup timeout".into()))+ } else {+ self.inner.remove_token(path, owner, force).await+ }+ }+ async fn scan_descendant_locks(&self, root: &str) -> PathLockResult<Vec<String>> {+ self.inner.scan_descendant_locks(root).await+ }+}++async fn fixture() -> (PathLockManager, Arc<FaultProvider>) {+ let fs = Arc::new(MemFileSystem::new());+ fs.mkdir("/data", 0o755).await.unwrap();+ let provider = Arc::new(FaultProvider::new(Arc::new(MemoryPathLockProvider::new())));+ let manager = PathLockManager::new(fs, provider.clone(), PathLockConfig::default());+ (manager, provider)+}++fn capability(lease: &OwnedPathLockLease) -> Option<(&str, &str)> {+ Some((&lease.lease.lease_ref, &lease.ownership_ref))+}++async fn age_token(+ manager: &PathLockManager,+ lease: &OwnedPathLockLease,+ path: &str,+ age: Duration,+) {+ let owner = manager+ .lease_registry+ .resolve(&lease.lease.lease_ref)+ .unwrap();+ owner+ .registry+ .lock()+ .await+ .token_health+ .get_mut(path)+ .unwrap()+ .last_confirmed_at = Instant::now() - age;+}++async fn lease_invalidation(+ manager: &PathLockManager,+ lease: &OwnedPathLockLease,+) -> LeaseInvalidation {+ let owner = manager+ .lease_registry+ .resolve(&lease.lease.lease_ref)+ .unwrap();+ let registry = owner.registry.lock().await;+ registry.entries[&lease.lease.lease_ref]+ .invalidation+ .unwrap()+}++#[tokio::test]+async fn loss_invalidates_whole_leases_sharing_tokens_but_not_exact_descendants() {+ let (manager, provider) = fixture().await;+ let tree = manager+ .acquire_tree("/data", Duration::ZERO, None)+ .await+ .unwrap();+ let shared = manager+ .acquire_tree_batch(+ &["/data".into(), "/other".into()],+ Duration::ZERO,+ capability(&tree),+ )+ .await+ .unwrap();+ let child = manager+ .acquire_exact("/data/child", Duration::ZERO, capability(&tree))+ .await+ .unwrap();+ assert_eq!(shared.lease.lock_paths[0], tree.lease.lock_paths[0]);+ assert_ne!(child.lease.lock_paths[0], tree.lease.lock_paths[0]);+ provider+ .failures+ .lock()+ .unwrap()+ .insert(tree.lease.lock_paths[0].clone(), false);+ assert_eq!(manager.refresh(&tree).await.unwrap(), "lost");+ for lease in [&tree, &shared] {+ assert_eq!(+ lease_invalidation(&manager, lease).await.reason,+ InvalidationReason::TokenLost+ );+ assert!(manager+ .get_owned_lease_by_ref(&lease.lease.lease_ref)+ .await+ .is_none());+ assert!(manager+ .require_covered_lease_ref(&lease.lease.lease_ref, &lease.lease.covered_paths)+ .await+ .is_err());+ assert!(manager+ .acquire_exact("/new", Duration::ZERO, capability(lease))+ .await+ .is_err());+ assert!(manager.handoff(lease).await.is_err());+ assert!(manager.refresh(lease).await.is_err());+ assert!(manager+ .get_releasable_lease_by_capability(&lease.lease.lease_ref, &lease.ownership_ref)+ .await+ .is_some());+ }+ assert!(manager+ .get_owned_lease_by_ref(&child.lease.lease_ref)+ .await+ .is_some());+ manager.release(&shared).await.unwrap();+ assert!(provider+ .read_token(&child.lease.lock_paths[0])+ .await+ .unwrap()+ .is_some());+ manager.release(&tree).await.unwrap();+ manager.release(&child).await.unwrap();+}++#[tokio::test]+async fn uncertain_partial_refresh_preserves_short_jitter_but_not_failed_token_deadline() {+ let (manager, provider) = fixture().await;+ let lease = manager+ .acquire_exact_batch(&["/data/a".into(), "/data/b".into()], Duration::ZERO, None)+ .await+ .unwrap();+ let failed = lease.lease.lock_paths[0].clone();+ provider+ .failures+ .lock()+ .unwrap()+ .insert(failed.clone(), true);+ age_token(&manager, &lease, &failed, Duration::from_secs(23)).await;+ let owner = manager+ .lease_registry+ .resolve(&lease.lease.lease_ref)+ .unwrap();+ let before = owner.registry.lock().await.token_health[&failed].last_confirmed_at;+ assert_eq!(manager.refresh(&lease).await.unwrap(), "failed");+ assert!(manager+ .get_owned_lease_by_ref(&lease.lease.lease_ref)+ .await+ .is_some());+ assert_eq!(+ owner.registry.lock().await.token_health[&failed].last_confirmed_at,+ before+ );+ assert_eq!(+ owner.registry.lock().await.token_health[&failed].consecutive_failures,+ 1+ );+ provider.failures.lock().unwrap().clear();+ assert_eq!(manager.refresh(&lease).await.unwrap(), "refreshed");+ assert_eq!(+ owner.registry.lock().await.token_health[&failed].consecutive_failures,+ 0+ );+ provider+ .failures+ .lock()+ .unwrap()+ .insert(failed.clone(), true);+ age_token(&manager, &lease, &failed, Duration::from_secs(25)).await;+ assert!(manager+ .require_covered_lease_ref(&lease.lease.lease_ref, &lease.lease.covered_paths)+ .await+ .is_err());+ let invalidation = lease_invalidation(&manager, &lease).await;+ assert_eq!(+ invalidation.reason,+ InvalidationReason::ConfirmationDeadlineExceeded+ );+ // A later definite-loss result must retain the initial deadline cause.+ {+ let mut registry = owner.registry.lock().await;+ assert_eq!(+ registry.apply_refresh_results(+ &lease.lease.lease_ref,+ &lease.lease.lock_paths,+ vec![Ok(false), Ok(true)],+ Instant::now(),+ PathLockManager::lease_validity(&manager.config),+ ),+ "lost"+ );+ assert_eq!(+ registry.entries[&lease.lease.lease_ref].invalidation,+ Some(invalidation)+ );+ }+ provider.failures.lock().unwrap().clear();+ assert!(+ manager.refresh(&lease).await.is_err(),+ "late recovery cannot resurrect"+ );+ manager.release(&lease).await.unwrap();+}++#[tokio::test]+async fn reentry_and_handoff_do_not_restart_token_deadlines() {+ let (manager, _) = fixture().await;+ let lease = manager+ .acquire_tree("/data", Duration::ZERO, None)+ .await+ .unwrap();+ let path = &lease.lease.lock_paths[0];+ age_token(&manager, &lease, path, Duration::from_secs(23)).await;+ let owner = manager+ .lease_registry+ .resolve(&lease.lease.lease_ref)+ .unwrap();+ let before = owner.registry.lock().await.token_health[path].last_confirmed_at;+ let reentrant = manager+ .acquire_tree("/data", Duration::ZERO, capability(&lease))+ .await+ .unwrap();+ assert_eq!(+ owner.registry.lock().await.token_health[path].last_confirmed_at,+ before+ );+ let handoff = manager.to_handoff(&lease);+ manager.handoff(&lease).await.unwrap();+ let adopted = manager.adopt(&handoff).await.unwrap();+ assert_eq!(+ owner.registry.lock().await.token_health[path].last_confirmed_at,+ before+ );+ age_token(&manager, &adopted, path, Duration::from_secs(25)).await;+ assert!(manager+ .get_owned_lease_by_ref(&adopted.lease.lease_ref)+ .await+ .is_none());+ assert!(manager+ .get_owned_lease_by_ref(&reentrant.lease.lease_ref)+ .await+ .is_none());+ manager.release(&reentrant).await.unwrap();+ manager.release(&adopted).await.unwrap();+}++#[tokio::test]+async fn invalidated_pending_handoff_cannot_be_adopted() {+ let (manager, provider) = fixture().await;+ let lease = manager+ .acquire_tree("/data", Duration::ZERO, None)+ .await+ .unwrap();+ let handoff = manager.to_handoff(&lease);+ manager.handoff(&lease).await.unwrap();+ let owner = manager+ .lease_registry+ .resolve(&lease.lease.lease_ref)+ .unwrap();+ let mut registry = owner.registry.lock().await;+ provider+ .failures+ .lock()+ .unwrap()+ .insert(lease.lease.lock_paths[0].clone(), false);+ assert_eq!(+ PathLockManager::refresh_entry_with(+ &manager.provider,+ &mut registry,+ &owner.owner_id,+ &lease.lease.lease_ref,+ PathLockManager::lease_validity(&manager.config),+ )+ .await,+ "lost"+ );+ drop(registry);+ assert!(manager.adopt(&handoff).await.is_err());+ assert!(PathLockManager::release_lease_paths_with(+ &manager.provider,+ &manager.lease_registry,+ owner,+ &lease.lease.lease_ref,+ None,+ None,+ Some(PathLockManager::cleanup_window(&manager.config)),+ )+ .await+ .unwrap());+}++#[tokio::test]+async fn failed_cleanup_keeps_capability_and_refs_until_recovery() {+ let (manager, provider) = fixture().await;+ let lease = manager+ .acquire_tree("/data", Duration::ZERO, None)+ .await+ .unwrap();+ provider.fail_remove.store(true, Ordering::SeqCst);+ assert!(manager.release(&lease).await.is_err());+ let owner = manager+ .lease_registry+ .resolve(&lease.lease.lease_ref)+ .unwrap();+ let invalidation = lease_invalidation(&manager, &lease).await;+ assert_eq!(invalidation.reason, InvalidationReason::ReleaseFailed);+ assert!(manager+ .get_owned_lease_by_ref(&lease.lease.lease_ref)+ .await+ .is_none());+ assert!(manager+ .get_releasable_lease_by_capability(&lease.lease.lease_ref, "wrong")+ .await+ .is_none());+ assert!(manager.release(&lease).await.is_err());+ assert_eq!(+ owner.registry.lock().await.entries[&lease.lease.lease_ref].invalidation,+ Some(invalidation)+ );+ assert_eq!(+ owner.registry.lock().await.lock_refs[&lease.lease.lock_paths[0]],+ 1+ );+ provider.fail_remove.store(false, Ordering::SeqCst);+ let cleanup = manager+ .get_releasable_lease_by_capability(&lease.lease.lease_ref, &lease.ownership_ref)+ .await+ .unwrap();+ manager.release(&cleanup).await.unwrap();+ assert!(owner.registry.lock().await.lock_refs.is_empty());+ assert!(owner.registry.lock().await.token_health.is_empty());+ assert!(manager+ .lease_registry+ .resolve(&lease.lease.lease_ref)+ .is_none());+}++#[tokio::test]+async fn bounded_cleanup_forgets_only_local_refs_and_preserves_valid_shared_token() {+ let (manager, provider) = fixture().await;+ let lease = manager+ .acquire_tree_batch(&["/data".into(), "/other".into()], Duration::ZERO, None)+ .await+ .unwrap();+ let valid = manager+ .acquire_tree("/other", Duration::ZERO, capability(&lease))+ .await+ .unwrap();+ provider+ .failures+ .lock()+ .unwrap()+ .insert(lease.lease.lock_paths[0].clone(), false);+ assert_eq!(manager.refresh(&lease).await.unwrap(), "lost");+ let invalidation = lease_invalidation(&manager, &lease).await;+ assert_eq!(invalidation.reason, InvalidationReason::TokenLost);+ provider.fail_remove.store(true, Ordering::SeqCst);+ assert!(manager.release(&lease).await.is_err());+ assert_eq!(lease_invalidation(&manager, &lease).await, invalidation);+ let owner = manager+ .lease_registry+ .resolve(&lease.lease.lease_ref)+ .unwrap();+ let calls = provider.remove_calls.load(Ordering::SeqCst);+ assert!(PathLockManager::release_lease_paths_with(+ &manager.provider,+ &manager.lease_registry,+ owner.clone(),+ &lease.lease.lease_ref,+ None,+ None,+ Some(Duration::ZERO),+ )+ .await+ .unwrap());+ assert_eq!(provider.remove_calls.load(Ordering::SeqCst), calls);+ assert_eq!(+ owner.registry.lock().await.lock_refs[&valid.lease.lock_paths[0]],+ 1+ );+ assert!(manager+ .get_owned_lease_by_ref(&valid.lease.lease_ref)+ .await+ .is_some());+ assert!(!PathLockManager::release_lease_paths_with(+ &manager.provider,+ &manager.lease_registry,+ owner,+ &lease.lease.lease_ref,+ None,+ None,+ Some(Duration::ZERO),+ )+ .await+ .unwrap());+ provider.fail_remove.store(false, Ordering::SeqCst);+ manager.release(&valid).await.unwrap();+}++#[tokio::test]+async fn delayed_result_for_old_handoff_ref_cannot_update_the_adopted_lease() {+ let (manager, _) = fixture().await;+ let lease = manager+ .acquire_tree("/data", Duration::ZERO, None)+ .await+ .unwrap();+ let handoff = manager.to_handoff(&lease);+ manager.handoff(&lease).await.unwrap();+ let adopted = manager.adopt(&handoff).await.unwrap();+ let owner = manager+ .lease_registry+ .resolve(&adopted.lease.lease_ref)+ .unwrap();+ let mut registry = owner.registry.lock().await;+ let before = registry.token_health[&lease.lease.lock_paths[0]].last_confirmed_at;+ assert_eq!(+ registry.apply_refresh_results(+ &lease.lease.lease_ref,+ &lease.lease.lock_paths,+ vec![Ok(false)],+ Instant::now(),+ PathLockManager::lease_validity(&manager.config),+ ),+ "lost"+ );+ assert_eq!(+ registry.token_health[&lease.lease.lock_paths[0]].last_confirmed_at,+ before+ );+ assert!(registry.entries[&adopted.lease.lease_ref]+ .invalidation+ .is_none());+ drop(registry);+ assert!(manager.release(&lease).await.is_err());+ manager.release(&adopted).await.unwrap();+}++#[tokio::test]+async fn cross_manager_adoption_cannot_revive_an_expired_token() {+ let (producer, provider) = fixture().await;+ let lease = producer+ .acquire_tree("/data", Duration::ZERO, None)+ .await+ .unwrap();+ let handoff = producer.to_handoff(&lease);+ producer.handoff(&lease).await.unwrap();+ let path = &lease.lease.lock_paths[0];+ let token = provider.read_token(path).await.unwrap().unwrap();+ let expired = LockToken {+ time_ns: 1,+ ..token.clone()+ };+ assert!(provider+ .compare_and_write_token(path, &token, &expired)+ .await+ .unwrap());+ let consumer = PathLockManager::new(+ producer.resolver.fs.clone(),+ provider.clone(),+ PathLockConfig::default(),+ );+ assert!(consumer.adopt(&handoff).await.is_err());+ assert_eq!(provider.read_token(path).await.unwrap(), Some(expired));+}++#[tokio::test]+async fn late_refresh_success_does_not_resurrect_an_expired_lease() {+ let fs = Arc::new(MemFileSystem::new());+ fs.mkdir("/data", 0o755).await.unwrap();+ let mut fault = FaultProvider::new(Arc::new(MemoryPathLockProvider::new()));+ fault.delay = Duration::from_millis(100);+ // Retain the invalidated entry while inspecting it; heartbeat cleanup races+ // the explicit caller after the delayed refresh releases the owner mutex.+ fault.fail_remove.store(true, Ordering::SeqCst);+ let manager = PathLockManager::new(+ fs,+ Arc::new(fault),+ PathLockConfig {+ lock_expire_secs: 0.2,+ ..PathLockConfig::default()+ },+ );+ let lease = manager+ .acquire_tree("/data", Duration::ZERO, None)+ .await+ .unwrap();+ age_token(+ &manager,+ &lease,+ &lease.lease.lock_paths[0],+ Duration::from_millis(100),+ )+ .await;+ assert_eq!(manager.refresh(&lease).await.unwrap(), "lost");+ assert!(manager+ .get_owned_lease_by_ref(&lease.lease.lease_ref)+ .await+ .is_none());+ assert!(manager.release(&lease).await.is_err());+}++#[tokio::test]+async fn background_refresh_invalidates_and_bounds_cleanup_during_outage() {+ for definite_loss in [true, false] {+ let fs = Arc::new(MemFileSystem::new());+ fs.mkdir("/data", 0o755).await.unwrap();+ let provider = Arc::new(FaultProvider::new(Arc::new(MemoryPathLockProvider::new())));+ let manager = PathLockManager::new(+ fs,+ provider.clone(),+ PathLockConfig {+ lock_expire_secs: 0.3,+ ..PathLockConfig::default()+ },+ );+ let lease = manager+ .acquire_tree("/data", Duration::ZERO, None)+ .await+ .unwrap();+ provider+ .failures+ .lock()+ .unwrap()+ .insert(lease.lease.lock_paths[0].clone(), !definite_loss);+ provider.fail_remove.store(true, Ordering::SeqCst);+ let owner = manager+ .lease_registry+ .resolve(&lease.lease.lease_ref)+ .unwrap();+ tokio::time::timeout(Duration::from_secs(2), async {+ loop {+ if owner+ .registry+ .lock()+ .await+ .entries+ .get(&lease.lease.lease_ref)+ .is_some_and(|e| e.invalidation.is_some())+ {+ break;+ }+ tokio::time::sleep(Duration::from_millis(5)).await;+ }+ })+ .await+ .unwrap();+ assert!(manager+ .get_owned_lease_by_ref(&lease.lease.lease_ref)+ .await+ .is_none());+ let calls = provider.refresh_calls.load(Ordering::SeqCst);+ tokio::time::timeout(Duration::from_secs(2), async {+ while manager+ .lease_registry+ .resolve(&lease.lease.lease_ref)+ .is_some()+ {+ tokio::time::sleep(Duration::from_millis(5)).await;+ }+ })+ .await+ .unwrap();+ assert_eq!(provider.refresh_calls.load(Ordering::SeqCst), calls);+ assert!(provider.remove_calls.load(Ordering::SeqCst) >= 2);+ assert!(owner.registry.lock().await.entries.is_empty());+ assert!(owner.registry.lock().await.lock_refs.is_empty());+ }+}++async fn takeover_rejects_protected_write(+ provider: Arc<dyn PathLockProvider>,+ fs: Arc<MemFileSystem>,+) {+ let manager = Arc::new(PathLockManager::new(+ fs.clone(),+ provider.clone(),+ PathLockConfig::default(),+ ));+ let lease = manager+ .acquire_exact("/data/file", Duration::ZERO, None)+ .await+ .unwrap();+ let lock_path = &lease.lease.lock_paths[0];+ let old = provider.read_token(lock_path).await.unwrap().unwrap();+ let replacement = LockToken {+ owner_id: "new-owner".into(),+ time_ns: PathLockManager::now_ns(),+ ..old.clone()+ };+ assert!(provider+ .compare_and_write_token(lock_path, &old, &replacement)+ .await+ .unwrap());+ assert_eq!(manager.refresh(&lease).await.unwrap(), "lost");+ let wrapped = PathLockWrappedFS::new(manager.clone(), fs.clone());+ let context = Arc::new(FsContextInner::with_pathlock(+ "",+ PathLockContext {+ lease_ref: Some(lease.lease.lease_ref.clone()),+ ..PathLockContext::default()+ },+ ));+ let result = FS_CTX+ .scope(context, async {+ wrapped+ .write("/data/file", b"corruption", 0, WriteFlag::Create)+ .await+ })+ .await;+ assert!(result.is_err(), "old lease must be refused before writing");+ assert!(fs.stat("/data/file").await.is_err());+ manager.release(&lease).await.unwrap();+ assert_eq!(+ provider.read_token(lock_path).await.unwrap(),+ Some(replacement)+ );+}++#[tokio::test]+async fn takeover_denies_writes_and_cleanup_preserves_new_owner_for_builtin_providers() {+ for filesystem in [false, true] {+ let fs = Arc::new(MemFileSystem::new());+ fs.mkdir("/data", 0o755).await.unwrap();+ let provider: Arc<dyn PathLockProvider> = if filesystem {+ Arc::new(FilesystemPathLockProvider::new(fs.clone(), 30.0))+ } else {+ Arc::new(MemoryPathLockProvider::new())+ };+ takeover_rejects_protected_write(provider, fs).await;+ }+ let dir = tempfile::TempDir::new().unwrap();+ let provider = Arc::new(+ crate::lock::SQLitePathLockProvider::new(+ dir.path().join("locks.db").to_str().unwrap(),+ "lost-lease",+ )+ .unwrap(),+ );+ let fs = Arc::new(MemFileSystem::new());+ fs.mkdir("/data", 0o755).await.unwrap();+ takeover_rejects_protected_write(provider, fs).await;+}++#[cfg(feature = "cache")]+struct RedisResponseDelayProxy {+ endpoint: String,+ delay_next_response: Arc<AtomicBool>,+ response_forwarded: Arc<tokio::sync::Notify>,+ task: tokio::task::JoinHandle<()>,+}++#[cfg(feature = "cache")]+impl RedisResponseDelayProxy {+ async fn start(endpoint: &str) -> Self {+ use tokio::io::{AsyncReadExt, AsyncWriteExt};+ use tokio::net::{TcpListener, TcpStream};++ let mut url = url::Url::parse(endpoint).unwrap();+ assert_eq!(+ url.scheme(),+ "redis",+ "delay proxy requires a plain TCP Redis endpoint"+ );+ let host = url.host_str().unwrap().to_string();+ let port = url.port().unwrap_or(6379);+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ url.set_host(Some("127.0.0.1")).unwrap();+ url.set_port(Some(listener.local_addr().unwrap().port()))+ .unwrap();+ let delay_next_response = Arc::new(AtomicBool::new(false));+ let response_forwarded = Arc::new(tokio::sync::Notify::new());+ let armed = delay_next_response.clone();+ let forwarded = response_forwarded.clone();+ let task = tokio::spawn(async move {+ // JoinSet aborts every connection pump when this accept task is dropped.+ let mut connections = tokio::task::JoinSet::new();+ loop {+ tokio::select! {+ accepted = listener.accept() => {+ let (client, _) = accepted.unwrap();+ let host = host.clone();+ let armed = armed.clone();+ let forwarded = forwarded.clone();+ connections.spawn(async move {+ let upstream = TcpStream::connect((host.as_str(), port)).await.unwrap();+ client.set_nodelay(true).unwrap();+ upstream.set_nodelay(true).unwrap();+ let (mut client_read, mut client_write) = client.into_split();+ let (mut server_read, mut server_write) = upstream.into_split();+ let requests = async {+ tokio::io::copy(&mut client_read, &mut server_write).await?;+ server_write.shutdown().await+ };+ let responses = async {+ let mut buffer = [0u8; 16 * 1024];+ loop {+ let count = server_read.read(&mut buffer).await?;+ if count == 0 { return client_write.shutdown().await; }+ let delayed = armed.swap(false, Ordering::SeqCst);+ if delayed {+ // Redis has executed the command, but its reply arrives+ // after the real client's 20 ms deadline.+ tokio::time::sleep(Duration::from_millis(80)).await;+ }+ client_write.write_all(&buffer[..count]).await?;+ if delayed { forwarded.notify_one(); }+ }+ };+ tokio::try_join!(requests, responses)+ });+ }+ completed = connections.join_next(), if !connections.is_empty() => {+ completed.unwrap().unwrap().unwrap();+ }+ }+ }+ });+ Self {+ endpoint: url.to_string(),+ delay_next_response,+ response_forwarded,+ task,+ }+ }++ fn delay_once(&self) {+ assert!(!self.delay_next_response.swap(true, Ordering::SeqCst));+ }++ async fn wait_for_recovery(&self) {+ tokio::time::timeout(Duration::from_secs(2), self.response_forwarded.notified())+ .await+ .expect("delayed Redis response must be forwarded");+ assert!(!self.delay_next_response.load(Ordering::SeqCst));+ }+}++#[cfg(feature = "cache")]+impl Drop for RedisResponseDelayProxy {+ fn drop(&mut self) {+ self.task.abort();+ }+}++#[cfg(feature = "cache")]+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn redis_20ms_renew_timeout_preserves_lease_and_recovers_after_network_delay() {+ let Ok(endpoint) = std::env::var("REDIS_URL") else {+ return;+ };+ let proxy = RedisResponseDelayProxy::start(&endpoint).await;+ let runtime =+ crate::cache_runtime::CacheRuntime::redis(crate::cache_runtime::RedisProviderConfig {+ endpoints: vec![proxy.endpoint.clone()],+ command_timeout_ms: 20,+ ..crate::cache_runtime::RedisProviderConfig::default()+ })+ .await+ .unwrap();+ let provider = Arc::new(+ crate::lock::RedisPathLockProvider::new(+ runtime.clone(),+ &format!("renew-network-delay-{}", Uuid::new_v4()),+ 30.0,+ )+ .unwrap(),+ );+ let dir = tempfile::TempDir::new().unwrap();+ let fs = Arc::new(+ crate::plugins::localfs::LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap(),+ );+ fs.mkdir("/local", 0o755).await.unwrap();+ fs.mkdir("/local/account", 0o755).await.unwrap();+ let file = "/local/account/file";+ let manager = Arc::new(PathLockManager::new(+ fs.clone(),+ provider.clone(),+ PathLockConfig::default(),+ ));+ let lease = manager+ .acquire_exact(file, Duration::ZERO, None)+ .await+ .unwrap();+ let path = &lease.lease.lock_paths[0];+ // Load the real renewal script before arming the proxy so the delayed+ // response belongs to renewal rather than SCRIPT LOAD.+ assert_eq!(manager.refresh(&lease).await.unwrap(), "refreshed");++ proxy.delay_once();+ let result = provider+ .refresh_tokens(+ &lease.lease.lock_paths,+ &lease.lease.owner_id,+ PathLockManager::now_ns(),+ )+ .await;+ assert_eq!(result.len(), 1);+ assert!(+ matches!(&result[0], Err(PathLockError::Io(message))+ if message.to_ascii_lowercase().contains("timeout")+ || message.contains("exceeded 20 ms")),+ "expected a real Redis command timeout: {result:?}"+ );+ proxy.wait_for_recovery().await;+ assert_eq!(manager.refresh(&lease).await.unwrap(), "refreshed");++ let owner = manager+ .lease_registry+ .resolve(&lease.lease.lease_ref)+ .unwrap();+ let confirmed_before = owner.registry.lock().await.token_health[path].last_confirmed_at;+ proxy.delay_once();+ assert_eq!(manager.refresh(&lease).await.unwrap(), "failed");+ {+ let registry = owner.registry.lock().await;+ assert!(registry.entries[&lease.lease.lease_ref]+ .invalidation+ .is_none());+ assert_eq!(+ registry.token_health[path].last_confirmed_at,+ confirmed_before+ );+ assert_eq!(registry.token_health[path].consecutive_failures, 1);+ }+ assert!(manager+ .get_owned_lease_by_capability(&lease.lease.lease_ref, &lease.ownership_ref)+ .await+ .is_some());+ manager+ .require_covered_lease_ref(&lease.lease.lease_ref, &lease.lease.covered_paths)+ .await+ .unwrap();+ let wrapped = PathLockWrappedFS::new(manager.clone(), fs.clone());+ let context = Arc::new(FsContextInner::with_pathlock(+ "account",+ PathLockContext {+ lease_ref: Some(lease.lease.lease_ref.clone()),+ ..PathLockContext::default()+ },+ ));+ FS_CTX+ .scope(context.clone(), async {+ wrapped.write(file, b"before", 0, WriteFlag::Create).await+ })+ .await+ .unwrap();+ assert_eq!(fs.read(file, 0, 100).await.unwrap(), b"before");++ proxy.wait_for_recovery().await;+ assert_eq!(manager.refresh(&lease).await.unwrap(), "refreshed");+ {+ let registry = owner.registry.lock().await;+ assert!(registry.entries[&lease.lease.lease_ref]+ .invalidation+ .is_none());+ assert!(registry.token_health[path].last_confirmed_at > confirmed_before);+ assert_eq!(registry.token_health[path].consecutive_failures, 0);+ }+ let token = provider.read_token(path).await.unwrap().unwrap();+ assert_eq!(token.owner_id, lease.lease.owner_id);+ FS_CTX+ .scope(context, async {+ wrapped.write(file, b"after!", 0, WriteFlag::Create).await+ })+ .await+ .unwrap();+ assert_eq!(fs.read(file, 0, 100).await.unwrap(), b"after!");+ manager.release(&lease).await.unwrap();+ assert!(provider.read_token(path).await.unwrap().is_none());+ runtime.close().await.unwrap();+}++#[cfg(feature = "cache")]+#[tokio::test]+async fn redis_takeover_denies_writes_and_cleanup_preserves_new_owner() {+ let Ok(endpoint) = std::env::var("REDIS_URL") else {+ return;+ };+ let runtime =+ crate::cache_runtime::CacheRuntime::redis(crate::cache_runtime::RedisProviderConfig {+ endpoints: vec![endpoint],+ command_timeout_ms: 1_000,+ ..crate::cache_runtime::RedisProviderConfig::default()+ })+ .await+ .unwrap();+ let provider = Arc::new(+ crate::lock::cache_provder::RedisPathLockProvider::new(+ runtime.clone(),+ &format!("lost-lease-{}", Uuid::new_v4()),+ 30.0,+ )+ .unwrap(),+ );+ let dir = tempfile::TempDir::new().unwrap();+ let fs = Arc::new(+ crate::plugins::localfs::LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap(),+ );+ fs.mkdir("/local", 0o755).await.unwrap();+ fs.mkdir("/local/account", 0o755).await.unwrap();+ fs.write("/local/account/file", b"original", 0, WriteFlag::Create)+ .await+ .unwrap();+ // Redis handles are scoped logical paths.+ let manager = Arc::new(PathLockManager::new(+ fs.clone(),+ provider.clone(),+ PathLockConfig::default(),+ ));+ let lease = manager+ .acquire_exact("/local/account/file", Duration::ZERO, None)+ .await+ .unwrap();+ let path = &lease.lease.lock_paths[0];+ let old = provider.read_token(path).await.unwrap().unwrap();+ let replacement = LockToken {+ owner_id: "new-owner".into(),+ ..old.clone()+ };+ assert!(provider+ .compare_and_write_token(path, &old, &replacement)+ .await+ .unwrap());+ assert_eq!(manager.refresh(&lease).await.unwrap(), "lost");+ let wrapped = PathLockWrappedFS::new(manager.clone(), fs.clone());+ let context = Arc::new(FsContextInner::with_pathlock(+ "account",+ PathLockContext {+ lease_ref: Some(lease.lease.lease_ref.clone()),+ ..PathLockContext::default()+ },+ ));+ assert!(FS_CTX+ .scope(context, async {+ wrapped+ .write("/local/account/file", b"corruption", 0, WriteFlag::Create)+ .await+ })+ .await+ .is_err());+ assert_eq!(+ fs.read("/local/account/file", 0, 100).await.unwrap(),+ b"original"+ );+ manager.release(&lease).await.unwrap();+ assert_eq!(provider.read_token(path).await.unwrap(), Some(replacement));+ provider+ .remove_token(path, "new-owner", false)+ .await+ .unwrap();+ runtime.close().await.unwrap();+}diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rsindex 3372e2539..a45faea89 100644--- a/crates/ragfs/src/lock/manager.rs+++ b/crates/ragfs/src/lock/manager.rs@@ -62,6 +62,10 @@ const MAX_POLL_INTERVAL_MS: u64 = 500;const POLL_BACKOFF_NUMERATOR: u64 = 3;const POLL_BACKOFF_DENOMINATOR: u64 = 2;const POLL_JITTER_PERCENT: u64 = 20;+/// Revoke authorization before the provider's expiry, leaving a safety margin.+const LEASE_VALIDITY_RATIO: f64 = 0.8;+/// Retry invalidated cleanup for six heartbeat intervals, then forget locally.+const CLEANUP_RETRY_INTERVALS: f64 = 6.0;impl PathLockManager {fn poll_interval_bounds(attempt: u32) -> (u64, u64) {@@ -117,6 +121,45 @@ struct LeaseEntry {pending_handoff: bool,lock_kinds: HashMap<String, PathLockKind>,last_active_at: Instant,+ /// Irreversible; retain the capability and references until cleanup finishes.+ invalidation: Option<LeaseInvalidation>,+}++/// The first irreversible authorization revocation, retained through cleanup.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+struct LeaseInvalidation {+ at: Instant,+ reason: InvalidationReason,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum InvalidationReason {+ TokenLost,+ ConfirmationDeadlineExceeded,+ ReleaseFailed,+}++impl LeaseEntry {+ /// Keep the first cause and timestamp, including during cleanup retries.+ fn invalidate(&mut self, at: Instant, reason: InvalidationReason, paths: &[&str]) {+ if self.invalidation.is_some() {+ return;+ }+ self.invalidation = Some(LeaseInvalidation { at, reason });+ warn!(+ lease_ref = %self.lease.lease_ref,+ owner_id = %self.lease.owner_id,+ ?reason,+ affected_paths = ?paths,+ "pathlock lease invalidated"+ );+ }+}++#[derive(Debug)]+struct TokenHealth {+ last_confirmed_at: Instant,+ consecutive_failures: u64,}#[derive(Debug)]@@ -132,6 +175,7 @@ enum DowngradeTokenResult {struct OwnerRegistry {entries: HashMap<String, LeaseEntry>,lock_refs: HashMap<String, usize>,+ token_health: HashMap<String, TokenHealth>,// ponytail: grows with handoffs while this owner lives; encode generations if measured.consumed_handoff_refs: HashSet<String>,}@@ -144,6 +188,7 @@ impl OwnerRegistry {lease: PathLockLease,ownership_ref: String,lock_kinds: &[PathLockKind],+ confirmed_at: Instant,) -> PathLockResult<()> {let lease_ref = lease.lease_ref.clone();if lease.owner_id != expected_owner_id {@@ -172,6 +217,12 @@ impl OwnerRegistry {for lock_path in &lease.lock_paths {if unique_paths.insert(lock_path.clone()) {*self.lock_refs.entry(lock_path.clone()).or_default() += 1;+ self.token_health+ .entry(lock_path.clone())+ .or_insert(TokenHealth {+ last_confirmed_at: confirmed_at,+ consecutive_failures: 0,+ });}}self.entries.insert(@@ -182,6 +233,7 @@ impl OwnerRegistry {pending_handoff: false,lock_kinds,last_active_at: Instant::now(),+ invalidation: None,},);Ok(())@@ -200,12 +252,14 @@ impl OwnerRegistry {}/// True when the entry exists, is owned (not parked for handoff) and its- /// ownership_ref matches. Used to gate release/refresh against a stale+ /// ownership_ref matches. Used to gate acquisition/refresh against a stale/// capability under the same write guard that performs the mutation.fn capability_matches(&self, lease_ref: &str, ownership_ref: &str) -> bool {- self.entries- .get(lease_ref)- .is_some_and(|entry| !entry.pending_handoff && entry.ownership_ref == ownership_ref)+ self.entries.get(lease_ref).is_some_and(|entry| {+ !entry.pending_handoff+ && entry.invalidation.is_none()+ && entry.ownership_ref == ownership_ref+ })}/// Return the strongest remaining kind after one lease drops a lock path.@@ -213,6 +267,7 @@ impl OwnerRegistry {self.entries.iter().filter(|(candidate_ref, _)| candidate_ref.as_str() != lease_ref)+ .filter(|(_, entry)| entry.invalidation.is_none()).filter_map(|(_, entry)| entry.lock_kinds.get(lock_path).copied()).max_by_key(|kind| match kind {PathLockKind::Exact => 0,@@ -253,7 +308,11 @@ impl OwnerRegistry {/// but reject the original ownership_ref. Requires a matching, not-yet-parked entry.fn mark_pending_handoff(&mut self, lease_ref: &str, ownership_ref: &str) -> PathLockResult<()> {match self.entries.get_mut(lease_ref) {- Some(entry) if entry.ownership_ref == ownership_ref && !entry.pending_handoff => {+ Some(entry)+ if entry.ownership_ref == ownership_ref+ && !entry.pending_handoff+ && entry.invalidation.is_none() =>+ {entry.pending_handoff = true;Ok(())}@@ -282,7 +341,9 @@ impl OwnerRegistry {new_lease_ref: String,new_ownership_ref: String,) -> Option<OwnedPathLockLease> {- if !self.entries.get(lease_ref)?.pending_handoff {+ if !self.entries.get(lease_ref)?.pending_handoff+ || self.entries.get(lease_ref)?.invalidation.is_some()+ {return None;}let mut entry = self.entries.remove(lease_ref)?;@@ -309,6 +370,7 @@ impl OwnerRegistry {Some(count) if *count > 1 => *count -= 1,Some(_) => {self.lock_refs.remove(lock_path);+ self.token_health.remove(lock_path);}None => {}}@@ -321,6 +383,135 @@ impl OwnerRegistry {entry.last_active_at = Instant::now();}}++ /// Invalidate entire leases sharing an actual token, never logical ancestors.+ fn invalidate_paths(+ &mut self,+ paths: &HashSet<String>,+ now: Instant,+ reason: InvalidationReason,+ ) {+ for entry in self.entries.values_mut() {+ if entry.invalidation.is_some() {+ continue;+ }+ let affected_paths: Vec<String> = entry+ .lease+ .lock_paths+ .iter()+ .filter(|path| paths.contains(*path))+ .cloned()+ .collect();+ if !affected_paths.is_empty() {+ if reason == InvalidationReason::ConfirmationDeadlineExceeded {+ for path in &affected_paths {+ if let Some(health) = self.token_health.get(path) {+ warn!(+ lease_ref = %entry.lease.lease_ref,+ owner_id = %entry.lease.owner_id,+ ?reason,+ lock_path = %path,+ since_last_confirmation_ms = now+ .saturating_duration_since(health.last_confirmed_at)+ .as_millis() as u64,+ failures = health.consecutive_failures,+ "pathlock token confirmation deadline exceeded"+ );+ }+ }+ }+ entry.invalidate(+ now,+ reason,+ &affected_paths+ .iter()+ .map(String::as_str)+ .collect::<Vec<_>>(),+ );+ }+ }+ }++ fn invalidate_expired(&mut self, now: Instant, validity: Duration) {+ let expired = self+ .token_health+ .iter()+ .filter(|(_, health)| {+ now.saturating_duration_since(health.last_confirmed_at) >= validity+ })+ .map(|(path, _)| path.clone())+ .collect();+ self.invalidate_paths(+ &expired,+ now,+ InvalidationReason::ConfirmationDeadlineExceeded,+ );+ }++ fn has_only_invalid_refs(&self, path: &str) -> bool {+ self.lock_refs.contains_key(path)+ && !self.entries.values().any(|entry| {+ entry.invalidation.is_none()+ && entry+ .lease+ .lock_paths+ .iter()+ .any(|lock_path| lock_path == path)+ })+ }++ /// Both heartbeat and explicit refresh use this state transition.+ fn apply_refresh_results(+ &mut self,+ lease_ref: &str,+ paths: &[String],+ results: Vec<PathLockResult<bool>>,+ started_at: Instant,+ validity: Duration,+ ) -> &'static str {+ // A late success must never restore a lease that crossed its deadline.+ let now = Instant::now();+ self.invalidate_expired(now, validity);+ if !self.entries.contains_key(lease_ref) {+ return "lost";+ }+ let mut lost_paths = HashSet::new();+ let mut all_ok = results.len() == paths.len();+ let mut results = results.into_iter();+ for path in paths {+ let Some(health) = self.token_health.get_mut(path) else {+ continue;+ };+ match results.next() {+ Some(Ok(true)) => {+ health.last_confirmed_at = started_at;+ health.consecutive_failures = 0;+ }+ Some(Ok(false)) => {+ lost_paths.insert(path.clone());+ all_ok = false;+ }+ result => {+ health.consecutive_failures = health.consecutive_failures.saturating_add(1);+ warn!(%lease_ref, owner_id = %self.entries[lease_ref].lease.owner_id,+ lock_path = %path, failures = health.consecutive_failures,+ since_last_confirmation_ms = now.saturating_duration_since(health.last_confirmed_at).as_millis() as u64,+ ?result, "pathlock refresh uncertain");+ all_ok = false;+ }+ }+ }+ self.invalidate_paths(&lost_paths, now, InvalidationReason::TokenLost);+ self.invalidate_expired(now, validity);+ if self.entries[lease_ref].invalidation.is_some() {+ "lost"+ } else if all_ok {+ self.touch(lease_ref);+ "refreshed"+ } else {+ "failed"+ }+ }}#[derive(Debug)]@@ -517,73 +708,25 @@ impl PathLockManager {tokio::spawn(async move {loop {tokio::time::sleep(refresh_interval).await;- let now_ns = Self::now_ns();let leases = refresh_registry.snapshot();for (lease_ref, owner) in leases {let mut owner_registry = owner.registry.lock().await;+ owner_registry+ .invalidate_expired(Instant::now(), Self::lease_validity(&refresh_config));let Some(entry) = owner_registry.entries.get(&lease_ref) else {continue;};- let lock_paths = entry.lease.lock_paths.clone();- let pending_handoff = entry.pending_handoff;- let mut any_ok = false;- let mut released_handoff_paths = Vec::new();- let refresh_results = refresh_provider- .refresh_tokens(&lock_paths, &owner.owner_id, now_ns)+ if entry.invalidation.is_none() {+ Self::refresh_entry_with(+ &refresh_provider,+ &mut owner_registry,+ &owner.owner_id,+ &lease_ref,+ Self::lease_validity(&refresh_config),+ ).await;- for (lp, result) in lock_paths.iter().zip(refresh_results) {- match result {- Ok(true) => {- any_ok = true;- debug!(lease_ref = %lease_ref, owner_id = %owner.owner_id, lock_path = %lp, "pathlock lease automatically refreshed");- }- Ok(false) => {- if pending_handoff {- // A different process may have adopted and released this- // handoff. Its completed release makes the producer's- // local reference obsolete. Do not remove the provider- // token: another process may own it or have released it.- released_handoff_paths.push(lp.clone());- } else {- warn!(lease_ref = %lease_ref, owner_id = %owner.owner_id, lock_path = %lp, "pathlock lease automatic refresh failed");- }- }- Err(error) => {- warn!(lease_ref = %lease_ref, owner_id = %owner.owner_id, lock_path = %lp, error = %error, "pathlock lease automatic refresh failed");- }- }- }- if any_ok {- owner_registry.touch(&lease_ref);- }- for lock_path in released_handoff_paths {- let is_last_path =- owner_registry.entries.get(&lease_ref).is_some_and(|entry| {- entry.lease.lock_paths.iter().all(|path| path == &lock_path)- });- if is_last_path {- if let Err(error) = refresh_registry.unpublish(&lease_ref, &owner) {- error!(- lease_ref = %lease_ref,- owner_id = %owner.owner_id,- error = %error,- "failed to forget released pathlock handoff"- );- continue;- }- }- owner_registry.remove_path(&lease_ref, &lock_path);- if is_last_path {- debug!(- lease_ref = %lease_ref,- owner_id = %owner.owner_id,- "forgot locally released pathlock handoff"- );- }}}- let stale_cutoff =- Instant::now() - Duration::from_secs_f64(refresh_config.lock_expire_secs * 2.0);let stale_candidates = refresh_registry.snapshot();let mut removed = 0;for (lease_ref, owner) in stale_candidates {@@ -594,7 +737,7 @@ impl PathLockManager {&lease_ref,None,None,- Some(stale_cutoff),+ Some(Self::cleanup_window(&refresh_config)),).await{@@ -702,6 +845,29 @@ impl PathLockManager {(self.config.lock_expire_secs * 1_000_000_000.0) as u128}+ fn lease_validity(config: &PathLockConfig) -> Duration {+ Duration::from_secs_f64(config.lock_expire_secs * LEASE_VALIDITY_RATIO)+ }++ fn cleanup_window(config: &PathLockConfig) -> Duration {+ Duration::from_secs_f64(config.lock_expire_secs / 3.0 * CLEANUP_RETRY_INTERVALS)+ }++ async fn refresh_entry_with(+ provider: &Arc<dyn PathLockProvider>,+ registry: &mut OwnerRegistry,+ owner_id: &str,+ lease_ref: &str,+ validity: Duration,+ ) -> &'static str {+ let paths = registry.entries[lease_ref].lease.lock_paths.clone();+ let started_at = Instant::now();+ let results = provider+ .refresh_tokens(&paths, owner_id, Self::now_ns())+ .await;+ registry.apply_refresh_results(lease_ref, &paths, results, started_at, validity)+ }+/// Check if a token is stale based on config.fn is_stale(&self, token: &LockToken, now_ns: u128) -> bool {now_ns.saturating_sub(token.time_ns) > self.lock_expire_ns()@@ -886,6 +1052,7 @@ impl PathLockManager {elapsed_ms: start.elapsed().as_millis() as u64,});}+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));if let Some((lease_ref, ownership_ref)) = owner_capability {if !owner_registry.capability_matches(lease_ref, ownership_ref) {break Err(PathLockError::InvalidRequest(format!(@@ -894,6 +1061,7 @@ impl PathLockManager {}}first_attempt = false;+ let confirmed_at = Instant::now();let acquired_lock_paths = match self.try_acquire_batch_once(&sorted, &owner_id).await {Ok(acquired) => acquired,Err(err) => {@@ -937,6 +1105,28 @@ impl PathLockManager {}};+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));+ let source_invalid = owner_capability.is_some_and(|(lease_ref, ownership_ref)| {+ !owner_registry.capability_matches(lease_ref, ownership_ref)+ });+ let reuses_invalid_token = acquired_lock_paths+ .iter()+ .any(|acquisition| owner_registry.has_only_invalid_refs(&acquisition.handle));+ if source_invalid+ || reuses_invalid_token+ || confirmed_at.elapsed() >= Self::lease_validity(&self.config)+ {+ let error = PathLockError::InvalidRequest(+ "pathlock lease expired during acquisition".to_string(),+ );+ let rollback = self+ .rollback_acquisitions(&acquired_lock_paths, &owner_id)+ .await;+ break match rollback {+ Ok(()) => Err(error),+ Err(rollback_error) => Err(Self::rollback_error(error, rollback_error)),+ };+ }let lease = PathLockLease {lease_ref: Self::new_owner_id(),owner_id: owner_id.clone(),@@ -948,9 +1138,13 @@ impl PathLockManager {};let ownership_ref = Self::new_owner_id();let lock_kinds: Vec<_> = sorted.iter().map(|request| request.kind).collect();- if let Err(error) =- owner_registry.insert(&owner_id, lease.clone(), ownership_ref.clone(), &lock_kinds)- {+ if let Err(error) = owner_registry.insert(+ &owner_id,+ lease.clone(),+ ownership_ref.clone(),+ &lock_kinds,+ confirmed_at,+ ) {let rollback = self.rollback_acquisitions(&acquired_lock_paths, &owner_id).await;@@ -1352,49 +1546,21 @@ impl PathLockManager {))})?;let mut owner_registry = owner.registry.lock().await;+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));if !owner_registry.capability_matches(&lease.lease.lease_ref, &lease.ownership_ref) {return Err(PathLockError::InvalidRequest(format!("owned lease capability does not match ref '{}'",lease.lease.lease_ref)));}- let lock_paths = owner_registry- .entries- .get(&lease.lease.lease_ref)- .map(|entry| entry.lease.lock_paths.clone())- .ok_or_else(|| {- PathLockError::Internal(format!(- "published pathlock lease '{}' is missing from owner state",- lease.lease.lease_ref- ))- })?;- let now_ns = Self::now_ns();- let mut all_ok = true;- let mut any_ok = false;-- for result in self- .provider- .refresh_tokens(&lock_paths, &owner.owner_id, now_ns)- .await- {- match result {- Ok(true) => any_ok = true,- Ok(false) => all_ok = false,- Err(_) => all_ok = false,- }- }-- if any_ok {- owner_registry.touch(&lease.lease.lease_ref);- }-- let result = if all_ok && any_ok {- "refreshed"- } else if any_ok {- "lost"- } else {- "failed"- };+ let result = Self::refresh_entry_with(+ &self.provider,+ &mut owner_registry,+ &owner.owner_id,+ &lease.lease.lease_ref,+ Self::lease_validity(&self.config),+ )+ .await;if result == "refreshed" {debug!(lease_ref = %lease.lease.lease_ref, owner_id = %lease.lease.owner_id, lock_paths = ?lease.lease.lock_paths, result = %result, "pathlock refresh completed");} else {@@ -1478,7 +1644,7 @@ impl PathLockManager {lease_ref: &str,ownership_ref: Option<&str>,selected_paths: Option<&[String]>,- stale_cutoff: Option<Instant>,+ cleanup_window: Option<Duration>,) -> PathLockResult<bool> {let mut owner_registry = owner.registry.lock().await;let Some(entry) = owner_registry.entries.get(lease_ref) else {@@ -1496,8 +1662,19 @@ impl PathLockManager {)));}}- if stale_cutoff.is_some_and(|cutoff| entry.last_active_at > cutoff) {- return Ok(false);+ if let Some(window) = cleanup_window {+ let Some(invalidation) = entry.invalidation else {+ return Ok(false);+ };+ if invalidation.at.elapsed() >= window {+ error!(%lease_ref, owner_id = %owner.owner_id, reason = ?invalidation.reason,+ invalidated_for_ms = invalidation.at.elapsed().as_millis() as u64,+ remaining_paths = ?entry.lease.lock_paths,+ "pathlock cleanup retry window exhausted; forgetting local lease");+ lease_registry.unpublish(lease_ref, &owner)?;+ owner_registry.rollback_insert(lease_ref);+ return Ok(true);+ }}let selected: Option<HashSet<&str>> =selected_paths.map(|paths| paths.iter().map(String::as_str).collect());@@ -1529,24 +1706,47 @@ impl PathLockManager {})?;let released_kind = entry.lock_kinds.get(&lock_path).copied();let remaining_kind = owner_registry.strongest_kind_excluding(lease_ref, &lock_path);+ let invalidated = entry.invalidation.is_some();if ref_count == 1 {- if owner_registry- .entries- .get(lease_ref)- .is_some_and(|entry| entry.lease.lock_paths.iter().all(|p| p == &lock_path))- {- lease_registry.unpublish(lease_ref, &owner)?;- }- owner_registry.remove_path(lease_ref, &lock_path);- match Self::remove_owned_token_with(provider, &lock_path, &owner.owner_id).await {- Ok(true) => {}- Ok(false) => {- if first_error.is_none() {- first_error = Some(Self::release_changed_error(&lock_path));+ let result = if invalidated {+ provider+ .remove_token(&lock_path, &owner.owner_id, false)+ .await+ } else {+ Self::remove_owned_token_with(provider, &lock_path, &owner.owner_id).await+ };+ match result {+ Ok(removed) => {+ if !removed && !invalidated {+ owner_registry.invalidate_paths(+ &HashSet::from([lock_path.clone()]),+ Instant::now(),+ InvalidationReason::TokenLost,+ );+ }+ if owner_registry.entries[lease_ref]+ .lease+ .lock_paths+ .iter()+ .all(|p| p == &lock_path)+ {+ lease_registry.unpublish(lease_ref, &owner)?;}+ owner_registry.remove_path(lease_ref, &lock_path);}Err(release_error) => {+ // Stop authorizing writes while failed release is retried.+ if let Some(entry) = owner_registry.entries.get_mut(lease_ref) {+ entry.invalidate(+ Instant::now(),+ InvalidationReason::ReleaseFailed,+ &[&lock_path],+ );+ warn!(%lease_ref, owner_id = %owner.owner_id, lock_path = %lock_path,+ reason = ?entry.invalidation.unwrap().reason, error = %release_error,+ "pathlock release failed; retaining lease for cleanup");+ }if first_error.is_none() {first_error = Some(release_error);}@@ -1587,15 +1787,20 @@ impl PathLockManager {owner_registry.remove_path(lease_ref, &lock_path);}Ok(false) => {- Self::discard_lost_path_refs(- lease_registry,- &owner,- &mut owner_registry,- &lock_path,- )?;- if first_error.is_none() {- first_error = Some(Self::release_changed_error(&lock_path));+ owner_registry.invalidate_paths(+ &HashSet::from([lock_path.clone()]),+ Instant::now(),+ InvalidationReason::TokenLost,+ );+ if owner_registry.entries[lease_ref]+ .lease+ .lock_paths+ .iter()+ .all(|p| p == &lock_path)+ {+ lease_registry.unpublish(lease_ref, &owner)?;}+ owner_registry.remove_path(lease_ref, &lock_path);}Err(release_error) => {let owner_mismatch = matches!(@@ -1604,12 +1809,21 @@ impl PathLockManager {if current_owner != &owner.owner_id);if owner_mismatch {- Self::discard_lost_path_refs(- lease_registry,- &owner,- &mut owner_registry,- &lock_path,- )?;+ owner_registry.invalidate_paths(+ &HashSet::from([lock_path.clone()]),+ Instant::now(),+ InvalidationReason::TokenLost,+ );+ }+ if let Some(entry) = owner_registry.entries.get_mut(lease_ref) {+ entry.invalidate(+ Instant::now(),+ InvalidationReason::ReleaseFailed,+ &[&lock_path],+ );+ warn!(%lease_ref, owner_id = %owner.owner_id, lock_path = %lock_path,+ reason = ?entry.invalidation.unwrap().reason, error = %release_error,+ "pathlock release failed; retaining lease for cleanup");}if first_error.is_none() {first_error = Some(release_error);@@ -1627,35 +1841,6 @@ impl PathLockManager {Ok(removed)}- /// Drop every local reference to a path whose token belongs to another owner.- fn discard_lost_path_refs(- lease_registry: &LeaseRegistry,- owner: &Arc<OwnerState>,- owner_registry: &mut OwnerRegistry,- lock_path: &str,- ) -> PathLockResult<()> {- let affected: Vec<String> = owner_registry- .entries- .iter()- .filter(|(_, entry)| entry.lease.lock_paths.iter().any(|path| path == lock_path))- .map(|(lease_ref, _)| lease_ref.clone())- .collect();- for lease_ref in &affected {- if owner_registry.entries[lease_ref]- .lease- .lock_paths- .iter()- .all(|path| path == lock_path)- {- lease_registry.unpublish(lease_ref, owner)?;- }- }- for lease_ref in affected {- owner_registry.remove_path(&lease_ref, lock_path);- }- Ok(())- }-/// Atomically reduce one same-owner Tree token to Exact.async fn downgrade_token_to_exact_with(provider: &Arc<dyn PathLockProvider>,@@ -1725,11 +1910,9 @@ impl PathLockManager {lease.lease.lease_ref))})?;- owner- .registry- .lock()- .await- .mark_pending_handoff(&lease.lease.lease_ref, &lease.ownership_ref)?;+ let mut owner_registry = owner.registry.lock().await;+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));+ owner_registry.mark_pending_handoff(&lease.lease.lease_ref, &lease.ownership_ref)?;let active_count = self.lease_registry.active_count();debug!(lease_ref = %lease.lease.lease_ref, owner_id = %lease.lease.owner_id, lock_paths = ?lease.lease.lock_paths, active_count = active_count, "parked pathlock lease for handoff");Ok(())@@ -1752,6 +1935,16 @@ impl PathLockManager {if let Some(lease_ref) = handoff.lease_ref.as_deref() {if let Some(owner) = self.lease_registry.resolve(lease_ref) {let mut owner_registry = owner.registry.lock().await;+ owner_registry+ .invalidate_expired(Instant::now(), Self::lease_validity(&self.config));+ if owner_registry+ .get_by_ref(lease_ref)+ .is_some_and(|entry| entry.invalidation.is_some())+ {+ return Err(PathLockError::HandoffFailed(format!(+ "pathlock lease '{lease_ref}' is invalidated"+ )));+ }let local = owner_registry.get_by_ref(lease_ref).map(|entry| {(entry.lease.owner_id == handoff.owner_id@@ -1818,7 +2011,6 @@ impl PathLockManager {"handoff lock_paths and covered_paths must have equal lengths".to_string(),));}- let now_ns = Self::now_ns();let mut expected_kinds = Vec::with_capacity(handoff.lock_paths.len());// ponytail: legacy durable handoffs only persisted owner_id/handle_id + lock_paths.@@ -1887,6 +2079,16 @@ impl PathLockManager {.lease_registry.resolve_or_create_owner(&handoff.owner_id);let mut owner_registry = owner.registry.lock().await;+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));+ if handoff+ .lock_paths+ .iter()+ .any(|path| owner_registry.has_only_invalid_refs(path))+ {+ return Err(PathLockError::HandoffFailed(+ "handoff shares an invalidated token".to_string(),+ ));+ }let already_consumed = match handoff.lease_ref.as_deref() {Some(source_lease_ref) => owner_registry.consumed_handoff_refs@@ -1904,10 +2106,15 @@ impl PathLockManager {handoff.owner_id)));}+ let confirmed_at = Instant::now();+ let now_ns = Self::now_ns();for (lock_path, expected_kind) in handoff.lock_paths.iter().zip(&expected_kinds) {match self.provider.read_token(lock_path).await? {Some(token)- if token.owner_id == handoff.owner_id && token.lock_type == *expected_kind => {}+ if token.owner_id == handoff.owner_id+ && token.lock_type == *expected_kind+ && now_ns.saturating_sub(token.time_ns)+ < (self.lock_expire_ns() as f64 * LEASE_VALIDITY_RATIO) as u128 => {}_ => {return Err(PathLockError::HandoffFailed(format!("lock path '{lock_path}' is no longer owned by '{}'",@@ -1932,6 +2139,17 @@ impl PathLockManager {debug!(lock_path = %lock_path, owner_id = %handoff.owner_id, kind = ?expected_kind, "refreshed pathlock token during adopt");}+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));+ if confirmed_at.elapsed() >= Self::lease_validity(&self.config)+ || handoff+ .lock_paths+ .iter()+ .any(|path| owner_registry.has_only_invalid_refs(path))+ {+ return Err(PathLockError::HandoffFailed(+ "handoff expired during adoption".to_string(),+ ));+ }let lease = PathLockLease {lease_ref: Self::new_owner_id(),owner_id: handoff.owner_id.clone(),@@ -1944,6 +2162,7 @@ impl PathLockManager {lease.clone(),ownership_ref.clone(),&expected_kinds,+ confirmed_at,)?;if let Err(error) = self.lease_registry@@ -2028,11 +2247,12 @@ impl PathLockManager {/// Look up an owned lease by owner_id for cross-FFI lease operations.pub async fn get_owned_lease(&self, owner_id: &str) -> Option<OwnedPathLockLease> {let owner = self.lease_registry.resolve_owner(owner_id)?;- let owner_registry = owner.registry.lock().await;+ let mut owner_registry = owner.registry.lock().await;+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));owner_registry.entries.values()- .find(|entry| !entry.pending_handoff)+ .find(|entry| !entry.pending_handoff && entry.invalidation.is_none()).map(|entry| OwnedPathLockLease {lease: entry.lease.clone(),ownership_ref: entry.ownership_ref.clone(),@@ -2042,10 +2262,11 @@ impl PathLockManager {/// Look up an owned lease by opaque lease_ref for cross-FFI lease operations.pub async fn get_owned_lease_by_ref(&self, lease_ref: &str) -> Option<OwnedPathLockLease> {let owner = self.lease_registry.resolve(lease_ref)?;- let owner_registry = owner.registry.lock().await;+ let mut owner_registry = owner.registry.lock().await;+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));owner_registry.get_by_ref(lease_ref)- .filter(|entry| !entry.pending_handoff)+ .filter(|entry| !entry.pending_handoff && entry.invalidation.is_none()).map(|entry| OwnedPathLockLease {lease: entry.lease.clone(),ownership_ref: entry.ownership_ref.clone(),@@ -2059,8 +2280,30 @@ impl PathLockManager {ownership_ref: &str,) -> Option<OwnedPathLockLease> {let owner = self.lease_registry.resolve(lease_ref)?;- let owner_registry = owner.registry.lock().await;+ let mut owner_registry = owner.registry.lock().await;+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));owner_registry+ .get_by_ref(lease_ref)+ .filter(|entry| {+ !entry.pending_handoff+ && entry.invalidation.is_none()+ && entry.ownership_ref == ownership_ref+ })+ .map(|entry| OwnedPathLockLease {+ lease: entry.lease.clone(),+ ownership_ref: entry.ownership_ref.clone(),+ })+ }++ /// Resolve a release capability even after authorization has been revoked.+ pub async fn get_releasable_lease_by_capability(+ &self,+ lease_ref: &str,+ ownership_ref: &str,+ ) -> Option<OwnedPathLockLease> {+ let owner = self.lease_registry.resolve(lease_ref)?;+ let registry = owner.registry.lock().await;+ registry.get_by_ref(lease_ref).filter(|entry| !entry.pending_handoff && entry.ownership_ref == ownership_ref).map(|entry| OwnedPathLockLease {@@ -2081,10 +2324,11 @@ impl PathLockManager {let owner = self.lease_registry.resolve(lease_ref).ok_or_else(|| {PathLockError::InvalidRequest(format!("unknown pathlock lease ref '{lease_ref}'"))})?;- let owner_registry = owner.registry.lock().await;+ let mut owner_registry = owner.registry.lock().await;+ owner_registry.invalidate_expired(Instant::now(), Self::lease_validity(&self.config));let entry = owner_registry.get_by_ref(lease_ref)- .filter(|entry| !entry.pending_handoff)+ .filter(|entry| !entry.pending_handoff && entry.invalidation.is_none()).ok_or_else(|| {PathLockError::InvalidRequest(format!("unknown pathlock lease ref '{lease_ref}'"))})?;@@ -2140,6 +2384,10 @@ impl PathLockManager {}}+#[cfg(test)]+#[path = "lost_lease_tests.rs"]+mod lost_lease_tests;+#[cfg(test)]mod tests {use super::*;@@ -2962,7 +3210,7 @@ mod tests {}#[tokio::test]- async fn pending_handoff_keeps_unreleased_paths_refreshing() {+ async fn pending_handoff_token_loss_invalidates_the_entire_lease() {let fs = Arc::new(MemFileSystem::new());fs.mkdir("/data", 0o755).await.unwrap();let provider = Arc::new(crate::lock::provider::MemoryPathLockProvider::new());@@ -2991,18 +3239,20 @@ mod tests {.await.unwrap();- // Wait beyond the producer stale-GC cutoff. The remaining path must- // retain the producer's local renewal state while the consumer holds it.+ // Losing one required token invalidates the entire parked lease.tokio::time::sleep(Duration::from_millis(700)).await;- assert_eq!(producer.observe().await.active_locks, 1);+ assert_eq!(producer.observe().await.active_locks, 0);assert_eq!(provider.read_token(&first_lock_path).await.unwrap(), None);assert!(provider.read_token(&second_lock_path).await.unwrap()- .is_some());+ .is_none());- consumer.release(&adopted).await.unwrap();+ assert!(consumer+ .get_owned_lease_by_ref(&adopted.lease.lease_ref)+ .await+ .is_none());tokio::time::timeout(Duration::from_millis(400), async {loop {if producer.observe().await.active_locks == 0 {@@ -3352,6 +3602,10 @@ mod tests {assert!(mgr.get_owned_lease_by_capability(&tree.lease.lease_ref, &tree.ownership_ref).await+ .is_none());+ assert!(mgr+ .get_releasable_lease_by_capability(&tree.lease.lease_ref, &tree.ownership_ref)+ .await.is_some());mgr.release(&exact).await.unwrap();diff --git a/docs/en/concepts/09-transaction.md b/docs/en/concepts/09-transaction.mdindex 77504802b..cbeec1375 100644--- a/docs/en/concepts/09-transaction.md+++ b/docs/en/concepts/09-transaction.md@@ -375,7 +375,15 @@ Acquisition rollback and normal release clean up tokens without guaranteeing rem**Stale lock detection**: PathLockEngine checks the ownership token timestamp. Locks older than `lock_expire` (default 30s) are considered stale and are removed automatically during acquisition.-**In-process cleanup**: During refresh, Rust PathLockManager checks leases that have not refreshed successfully for `2 × lock_expire` and attempts cleanup with ownership validation.+**Lost lease authorization**: Explicit and background refresh use the same rules. A missing token or owner mismatch immediately invalidates every local lease under that owner sharing the actual token path (`lock_paths`). Losing any required token invalidates the entire lease, including a pending handoff. A Tree ancestor and an Exact descendant have different tokens and do not invalidate each other.++A transport error preserves authorization only until `0.8 × lock_expire` since that token's last successful confirmation (24 seconds by default). Each token has its own monotonic deadline; partial refresh, reentry, and local handoff/adoption do not extend failed tokens. Authorization checks enforce the deadline even before the next heartbeat. Invalidated leases cannot authorize subsequent protected writes, refresh, owner reuse, or handoff/adoption, and cannot become valid again.++**In-process cleanup**: Invalidated leases retain their release capability and local references while ownership-checked cleanup is retried. A missing or foreign-owned token completes cleanup without deleting a new owner's token. Tokens shared with another valid local lease remain protected. After six heartbeat intervals from the first invalidation (`2 × lock_expire`, 60 seconds by default), the manager logs remaining paths and forgets the local entry even if the provider is unavailable; remaining provider tokens rely on expiry. Cleanup retries never extend this window.++**Invalidation diagnostics**: The manager retains the first invalidation time and reason through cleanup. The `pathlock lease invalidated` log includes `reason`, `lease_ref`, `owner_id`, and `affected_paths`. `TokenLost` means a required token is missing or ownership no longer matches; inspect token expiry, removal, and takeover. `ConfirmationDeadlineExceeded` means ownership could not be confirmed before the safety deadline; inspect provider health, network latency, and renewal scheduling. Its token log includes `since_last_confirmation_ms` and `failures`. `ReleaseFailed` means a release or downgrade error revoked authorization while cleanup is retried; the release log includes the error. Later results and cleanup failures do not replace the first reason or reset its time; the cleanup-window exhaustion log retains that reason.++These checks revoke authorization for future operations after loss is detected or the deadline passes. They do not cancel writes that already passed authorization or provide storage-side fencing.**Orphan locks**: Provider tokens left behind after a process crash are automatically removed via stale lock detection when a later acquisition checks the same path or scope.diff --git a/docs/zh/concepts/09-transaction.md b/docs/zh/concepts/09-transaction.mdindex ed5771919..3d3feabfe 100644--- a/docs/zh/concepts/09-transaction.md+++ b/docs/zh/concepts/09-transaction.md@@ -383,7 +383,15 @@ fencing token 校验通过的一方成功持有 `TreeLock(java-guide)`;失败**陈旧锁检测**:PathLockEngine 检查归属 token 中的时间戳。超过 `lock_expire`(默认 30s)的锁被视为陈旧锁,在加锁过程中自动移除。-**进程内清理**:Rust PathLockManager 在续期循环中检查长期未成功续期的 lease,以 `2 × lock_expire` 为阈值尝试清理,并校验归属后释放 token。+**失锁后的授权失效**:手动和后台续租使用相同规则。token 不存在或 owner 不匹配时,同 owner 下共享实际 token 路径(`lock_paths`)的所有本地 lease 立即整份失效,包括待交接的 lease。Tree 祖先和 Exact 后代使用不同 token,不会因逻辑覆盖关系连带失效。++网络等不确定错误允许短暂重试,但每个 token 距离最近一次成功确认达到 `0.8 × lock_expire`(默认 24 秒)时撤销授权。使用单调时钟分别记录各 token 的确认时间;部分续租成功、重入和本地交接/接管不会延长失败 token 的期限。写入授权检查也会检查期限,无需等待下次心跳。失效 lease 不能再授权后续受保护写入、续租、复用 owner 或交接/接管,且不会恢复有效。++**进程内清理**:失效 lease 保留释放能力和本地引用,后台重试带 owner 校验的清理。token 已不存在或已被其他 owner 接管均视为清理完成,不会删除新 owner 的 token;仍被其他有效本地 lease 引用的 token 也不会被删除。从首次失效起超过六个心跳间隔(`2 × lock_expire`,默认 60 秒),即使 Provider 持续不可用也记录剩余路径并移除本地 entry,遗留的远端 token 依赖过期清理。重试不会延长该窗口。++**失效诊断**:manager 在清理期间保留首次失效的时间与原因。`pathlock lease invalidated` 日志包含 `reason`、`lease_ref`、`owner_id` 和 `affected_paths`。`TokenLost` 表示必要 token 缺失或归属不符,应检查 token 过期、删除和接管情况;`ConfirmationDeadlineExceeded` 表示安全期限内未能确认归属,应检查 Provider 健康、网络延迟和续租调度,对应 token 日志还包含 `since_last_confirmation_ms` 和 `failures`;`ReleaseFailed` 表示释放或降级错误导致撤销授权并进入清理重试,释放日志会记录具体错误。后续响应和清理失败不会覆盖首次原因或重置时间,清理窗口耗尽的日志也会保留该原因。++上述检查在发现失锁或达到期限后拒绝后续操作;不会中断已经通过授权检查的写入,也不提供存储侧 fencing。**孤儿锁**:进程崩溃后遗留的 Provider token,在后续 acquire 检查同一路径或 scope 时通过 stale lock 检测自动移除。