A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
reach the give-up cap on a persistently failing relaunch (#3179)
Sebastian Müller332fe80e98ee
diff --git a/crates/pa-daemon/src/supervisor.rs b/crates/pa-daemon/src/supervisor.rsindex a2d99d1dc4..ab12c59e30 100644--- a/crates/pa-daemon/src/supervisor.rs+++ b/crates/pa-daemon/src/supervisor.rs@@ -39,7 +39,7 @@ mod tests;// STABLE_LIFETIME_MS is read only by this facade's in-file test modules (via the module's// pub(super) const); the lib-target import is flagged unused since only tests use it.#[allow(unused_imports)]-use supervision::STABLE_LIFETIME_MS;+use supervision::{MAX_CONSECUTIVE_FAILURES, STABLE_LIFETIME_MS};// The saved-session row builders are read only by this facade's in-file test// modules (via the module's pub(super) fns); the lib-target import is flaggeddiff --git a/crates/pa-daemon/src/supervisor/supervision.rs b/crates/pa-daemon/src/supervisor/supervision.rsindex 6bf1d3738f..4b2c2e1cf7 100644--- a/crates/pa-daemon/src/supervisor/supervision.rs+++ b/crates/pa-daemon/src/supervisor/supervision.rs@@ -12,7 +12,7 @@ use super::{use crate::lease::is_process_alive;use crate::registry::WorkerRelay;-const MAX_CONSECUTIVE_FAILURES: u32 = 5;+pub(super) const MAX_CONSECUTIVE_FAILURES: u32 = 5;/// A crash-path child that lived at least this long proved health: its death/// resets the failure count (a fresh count) instead of accumulating toward/// the give-up cap. Below it, a spawn-dies-fast child counts as another@@ -51,7 +51,7 @@ impl Supervisor {});}- async fn watch_worker(+ pub(super) async fn watch_worker(self: Arc<Self>,resident: Arc<ResidentWorker>,mut child: Option<Child>,@@ -105,6 +105,14 @@ impl Supervisor {.map_or(0, |elapsed| elapsed.as_millis() as u64);let failures = Self::next_failure_count(&resident, now_ms);if failures > MAX_CONSECUTIVE_FAILURES {+ // A stop that lands during the relaunch storm owns the+ // terminal state: the give-up must not persist `Failed`+ // over the stop's tombstone or recreate a descriptor the+ // stop already deleted (the next boot would adopt a+ // cleanly stopped worker as `GaveUp`).+ if self.is_stopping(&resident) {+ return;+ }let mut descriptor = resident.descriptor.lock().await;descriptor.lifecycle = DaemonWorkerLifecycle::Failed;descriptor.last_failure_at = Some(util::now_iso());@@ -182,6 +190,12 @@ impl Supervisor {));child = None;adopted_pid = 0;+ if self.is_stopping(&resident) {+ return;+ }+ // A failed relaunch has no lifetime of its own: clear+ // the spawn time so the count accumulates to the give-up cap.+ resident.spawned_at_ms.store(0, Ordering::SeqCst);}}}diff --git a/crates/pa-daemon/src/supervisor/tests.rs b/crates/pa-daemon/src/supervisor/tests.rsindex d9c52cbec8..e66a786126 100644--- a/crates/pa-daemon/src/supervisor/tests.rs+++ b/crates/pa-daemon/src/supervisor/tests.rs@@ -43,6 +43,132 @@ fn churn_accumulates_and_a_stable_lifetime_resets() {assert_eq!(Supervisor::next_failure_count(&resident, now), 2);}+/// A relaunch that FAILS produced no worker, so the count must+/// accumulate to the give-up cap instead of resetting against the old spawn.+#[tokio::test(start_paused = true)]+async fn a_failed_relaunch_accumulates_to_the_give_up_cap() {+ let dir = tempfile::TempDir::new().unwrap();+ let agent_dir = dir.path().join("agent");+ std::fs::create_dir_all(&agent_dir).unwrap();+ // The relaunch fails deterministically: the logs dir cannot be created+ // (a file stands where it would go), so every spawn dies at the worker+ // stderr log's open.+ std::fs::write(agent_dir.join("logs"), "not a directory").unwrap();+ let supervisor = Arc::new(+ Supervisor::new(SupervisorOptions {+ socket_path: dir.path().join("daemon.sock"),+ agent_dir: agent_dir.clone(),+ })+ .expect("supervisor"),+ );+ let descriptor: DaemonWorkerDescriptor = serde_json::from_value(serde_json::json!({+ "version": 2,+ "workerId": "w-relaunch",+ "pid": 0,+ "socketPath": "/tmp/none.sock",+ "recoveryJournalPath": dir.path().join("journal.jsonl").to_string_lossy(),+ "supervisorSocketPath": "/tmp/none.sock",+ "authenticationToken": "token",+ "rootActiveSessionId": "w-relaunch",+ "createdAt": "t",+ "updatedAt": "t",+ "lifecycle": "ready",+ "createCommand": {},+ "consecutiveFailures": 0,+ }))+ .expect("descriptor");+ let resident = Arc::new(ResidentWorker::new(+ "w-relaunch".to_string(),+ descriptor,+ dir.path().join("w-relaunch.descriptor.json"),+ ));+ let now_ms = std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .expect("the clock is after the epoch")+ .as_millis() as u64;+ resident+ .spawned_at_ms+ .store(now_ms - STABLE_LIFETIME_MS - 1, Ordering::SeqCst);+ supervisor.registry.insert(Arc::clone(&resident)).await;+ tokio::time::timeout(+ Duration::from_secs(60),+ Arc::clone(&supervisor).watch_worker(Arc::clone(&resident), None, 0),+ )+ .await+ .expect("the watch loop gives up instead of spinning");+ assert!(+ supervisor.registry.get("w-relaunch").await.is_none(),+ "the give-up removes the worker from the registry"+ );+ assert_eq!(+ resident.descriptor.lock().await.lifecycle,+ DaemonWorkerLifecycle::Failed+ );+}++/// A stop that lands while the watch loop is on the give-up cap leaves+/// the terminal state to the stop: the give-up arm returns without+/// persisting `Failed` (the stop's tombstone already owns the next boot's+/// verdict) and without removing the resident (the stop path owns the+/// removal), so a cleanly stopped worker is not adopted as `GaveUp`.+#[tokio::test(start_paused = true)]+async fn a_stop_during_the_storm_leaves_the_terminal_state_to_the_stop() {+ let dir = tempfile::TempDir::new().unwrap();+ let agent_dir = dir.path().join("agent");+ std::fs::create_dir_all(&agent_dir).unwrap();+ std::fs::write(agent_dir.join("logs"), "not a directory").unwrap();+ let supervisor = Arc::new(+ Supervisor::new(SupervisorOptions {+ socket_path: dir.path().join("daemon.sock"),+ agent_dir: agent_dir.clone(),+ })+ .expect("supervisor"),+ );+ let descriptor: DaemonWorkerDescriptor = serde_json::from_value(serde_json::json!({+ "version": 2,+ "workerId": "w-storm-stop",+ "pid": 0,+ "socketPath": "/tmp/none.sock",+ "recoveryJournalPath": dir.path().join("journal.jsonl").to_string_lossy(),+ "supervisorSocketPath": "/tmp/none.sock",+ "authenticationToken": "token",+ "rootActiveSessionId": "w-storm-stop",+ "createdAt": "t",+ "updatedAt": "t",+ "lifecycle": "ready",+ "createCommand": {},+ "consecutiveFailures": 0,+ }))+ .expect("descriptor");+ let resident = Arc::new(ResidentWorker::new(+ "w-storm-stop".to_string(),+ descriptor,+ dir.path().join("w-storm-stop.descriptor.json"),+ ));+ // The storm is at the cap and the stop already landed (the stop+ // path sets the flag before it finalizes the worker).+ resident+ .consecutive_failures+ .store(MAX_CONSECUTIVE_FAILURES, Ordering::SeqCst);+ resident.intentional_stop.store(true, Ordering::SeqCst);+ supervisor.registry.insert(Arc::clone(&resident)).await;+ tokio::time::timeout(+ Duration::from_secs(60),+ Arc::clone(&supervisor).watch_worker(Arc::clone(&resident), None, 0),+ )+ .await+ .expect("the give-up arm returns instead of spinning");+ assert_eq!(+ resident.descriptor.lock().await.lifecycle,+ DaemonWorkerLifecycle::Ready,+ "the give-up must not persist Failed over a concurrent stop"+ );+ assert!(+ supervisor.registry.get("w-storm-stop").await.is_some(),+ "the stop path owns the worker's removal"+ );+}+/// The saved-session surfaces (the `list --all` summary row and the/// `list_saved_sessions` catalog row) carry the persisted thinking/// level: the agents-view Model column renders "model:level" for