A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
drop the per-append session lease guard (#3378)
Sebastian Müller92dc7e504867
diff --git a/crates/pa-daemon/src/lease.rs b/crates/pa-daemon/src/lease.rsindex 18951fa446..68f7e4569b 100644--- a/crates/pa-daemon/src/lease.rs+++ b/crates/pa-daemon/src/lease.rs@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};use sha2::{Digest, Sha256};use std::fs;use std::path::{Path, PathBuf};-use std::time::Duration;+use std::time::{Duration, Instant};pub const SESSION_LEASES_ENABLED_ENV: &str = "PRIME_AGENT_INTERNAL_SESSION_LEASES";pub const SESSION_LEASE_OWNER_ID_ENV: &str = "PRIME_AGENT_INTERNAL_SESSION_LEASE_OWNER_ID";@@ -225,11 +225,16 @@ const FAST_GUARD_ATTEMPTS: u32 = 100;/// stale reclaim and the next acquisition attempt.const THROUGH_STALE_SLACK: Duration = Duration::from_millis(500);+/// How long `release` drains an append already inside its write before the+/// lease directory is removed anyway: appends are file writes at most, so+/// this bounds a wedged device, not a normal turn.+const APPEND_DRAIN_AFTER: Duration = Duration::from_secs(10);+/// How long a guard acquisition waits for a foreign holder.#[derive(Clone, Copy)]enum GuardWait {- /// The fast budget: the release and append paths hold the guard only- /// for sub-millisecond bookkeeping; a longer-busy guard fails fast.+ /// The fast budget: the release path holds the guard only for+ /// sub-millisecond bookkeeping; a longer-busy guard fails fast.Fast,/// Outlast the stale window: a holder killed mid-mutation can never/// release its guard, so the acquisition survives until the reclaim.@@ -275,7 +280,7 @@ fn with_lease_guard<T>(// Stale guard: a holder that died without cleanup. The// bare guard carries no holder identity, so the lease's// owner is the stand-in: steal only when the owner is- // provably dead (a live owner may hold the guard mid-append).+ // provably dead (a live owner may hold the guard mid-release).if crate::paths::mtime_age(&guard).is_some_and(|age| age > STALE_GUARD_AFTER)&& match read_owner(directory) {Ok(Some(owner)) => !owner_alive(&owner),@@ -314,6 +319,10 @@ pub struct SessionLease {directory: PathBuf,token: String,released: std::sync::atomic::AtomicBool,+ /// Appends between `append`'s released-check and its write: `release`+ /// drains this before removing the lease directory, so no row lands+ /// behind a successor's acquire.+ append_in_flight: std::sync::atomic::AtomicUsize,}impl SessionLease {@@ -324,6 +333,21 @@ impl SessionLease {{return;}+ // Fence against a concurrent append: one already past its+ // released-check must land before the lease directory is removed,+ // or its row reaches the file after a successor acquires and the+ // window cache certifies a generation the new owner never saw.+ // New appends are refused from the swap above; this only drains+ // writes already in flight.+ let drain_deadline = Instant::now() + APPEND_DRAIN_AFTER;+ while self+ .append_in_flight+ .load(std::sync::atomic::Ordering::SeqCst)+ > 0+ && Instant::now() < drain_deadline+ {+ std::thread::yield_now();+ }let _ = pa_core::session::window::flush_cache(&self.opened_path);// The usage-scan sidecar persists beside the window snapshot, in// the same lease-keyed, best-effort shape: only the lease holder@@ -358,19 +382,13 @@ impl SessionLease {}pub(crate) fn append(&self, path: &Path, bytes: &[u8]) -> Result<()> {- anyhow::ensure!(- !self.released.load(std::sync::atomic::Ordering::SeqCst)- && canonical_session_path(path) == self.session_path,- "session lease does not own append target"- );- with_lease_guard(&self.directory, GuardWait::Fast, || {- let owner = read_owner(&self.directory)?;+ self.append_in_flight+ .fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ let result = (|| {anyhow::ensure!(- owner- .as_ref()- .is_some_and(|owner| owner.token == self.token)- && !self.released.load(std::sync::atomic::Ordering::SeqCst),- "session lease lost ownership before append"+ !self.released.load(std::sync::atomic::Ordering::SeqCst)+ && canonical_session_path(path) == self.session_path,+ "session lease does not own append target");pa_core::session::window::append_cached(path,@@ -378,7 +396,10 @@ impl SessionLease {pa_core::session::window::AppendOwnership::SessionLeaseHeld,)?;Ok(())- })+ })();+ self.append_in_flight+ .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);+ result}}@@ -476,6 +497,7 @@ pub fn acquire_runtime_session_lease(directory: directory.clone(),token,released: std::sync::atomic::AtomicBool::new(false),+ append_in_flight: std::sync::atomic::AtomicUsize::new(0),});}Err(error) => {@@ -546,61 +568,6 @@ mod tests {assert_eq!(next.session_path, canonical_session_path(&path));}- #[test]- fn final_owner_token_check_allows_exactly_one_racing_writer() {- let dir = tempfile::tempdir().unwrap();- let path = dir.path().join("session.jsonl");- std::fs::write(&path, "{}\n").unwrap();- let stale = acquire_runtime_session_lease(&path, dir.path()).unwrap();- let directory = stale.directory.clone();- let replacement = LeaseOwner {- version: 1,- token: "replacement".to_owned(),- pid: std::process::id(),- process_start_id: get_process_start_id(std::process::id()),- active_session_id: None,- session_path: canonical_session_path(&path).to_string_lossy().to_string(),- created_at: crate::util::now_iso(),- };- std::fs::write(- directory.join("owner.json"),- serde_json::to_string_pretty(&replacement).unwrap() + "\n",- )- .unwrap();- let winner = SessionLease {- session_path: canonical_session_path(&path),- opened_path: path.clone(),- directory,- token: replacement.token,- released: std::sync::atomic::AtomicBool::new(false),- };- let barrier = std::sync::Arc::new(std::sync::Barrier::new(3));- let path_a = path.clone();- let path_b = path.clone();- let barrier_a = barrier.clone();- let barrier_b = barrier.clone();- let stale = std::sync::Arc::new(stale);- let winner = std::sync::Arc::new(winner);- let stale_task = {- let stale = stale;- std::thread::spawn(move || {- barrier_a.wait();- stale.append(&path_a, b"stale\n")- })- };- let winner_task = {- let winner = winner;- std::thread::spawn(move || {- barrier_b.wait();- winner.append(&path_b, b"winner\n")- })- };- barrier.wait();- let results = [stale_task.join().unwrap(), winner_task.join().unwrap()];- assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1);- assert_eq!(std::fs::read_to_string(path).unwrap(), "{}\nwinner\n");- }-#[test]fn lease_conflicts_and_releases() {let dir = std::env::temp_dir().join(format!("pa-lease-{}", uuid::Uuid::new_v4()));@@ -757,31 +724,6 @@ mod tests {let _ = fs::remove_dir_all(&directory);}- #[test]- fn append_fails_fast_behind_a_fresh_guard() {- let dir = tempfile::tempdir().unwrap();- let path = dir.path().join("session.jsonl");- std::fs::write(&path, "{}\n").unwrap();- let lease = acquire_runtime_session_lease(&path, dir.path()).unwrap();- let guard = format!("{}.guard", lease.directory.display());- std::fs::create_dir(&guard).unwrap();- let started = std::time::Instant::now();- let error = lease.append(&path, b"blocked\n").unwrap_err();- assert!(- error- .to_string()- .contains("Could not coordinate session lease"),- "unexpected append error: {error}"- );- assert!(- started.elapsed() < STALE_GUARD_AFTER,- "append waited out the stale window: {:?}",- started.elapsed()- );- std::fs::remove_dir(&guard).unwrap();- lease.release();- }-#[test]fn rename_target_contention_covers_exist_and_not_empty() {// TS: EEXIST and ENOTEMPTY are contention on every platform.@@ -855,4 +797,34 @@ mod tests {let unrelated = std::io::Error::from(std::io::ErrorKind::NotFound);assert_eq!(reclaim_retry_delay_ms(true, &unrelated, 1), None);}++ #[test]+ fn release_drains_an_append_still_in_flight() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("session.jsonl");+ std::fs::write(&path, "{}").unwrap();+ let lease = std::sync::Arc::new(acquire_runtime_session_lease(&path, dir.path()).unwrap());+ // An append past its released-check, still inside its write.+ lease+ .append_in_flight+ .fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ let directory = lease.directory.clone();+ let releaser = {+ let lease = std::sync::Arc::clone(&lease);+ std::thread::spawn(move || lease.release())+ };+ std::thread::sleep(Duration::from_millis(50));+ assert!(+ directory.exists(),+ "release must wait for the append still in flight"+ );+ lease+ .append_in_flight+ .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);+ releaser.join().unwrap();+ assert!(+ !directory.exists(),+ "release reclaims the lease once the append lands"+ );+ }}