Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

reach the give-up cap on a persistently failing relaunch (#3179)

Sebastian Müller332fe80e98ee
diff --git a/crates/pa-daemon/src/supervisor.rs b/crates/pa-daemon/src/supervisor.rs
index a2d99d1dc4..ab12c59e30 100644
--- a/crates/pa-daemon/src/supervisor.rs
+++ b/crates/pa-daemon/src/supervisor.rs
@@ -39,7 +39,7 @@ mod tests;
// STABLE_LIFETIME_MS is read only by this facade's in-file test modules (via the module's
// pub(super) const); the lib-target import is flagged unused since only tests use it.
#[allow(unused_imports)]
-use supervision::STABLE_LIFETIME_MS;
+use supervision::{MAX_CONSECUTIVE_FAILURES, STABLE_LIFETIME_MS};
// The saved-session row builders are read only by this facade's in-file test
// modules (via the module's pub(super) fns); the lib-target import is flagged
diff --git a/crates/pa-daemon/src/supervisor/supervision.rs b/crates/pa-daemon/src/supervisor/supervision.rs
index 6bf1d3738f..4b2c2e1cf7 100644
--- a/crates/pa-daemon/src/supervisor/supervision.rs
+++ b/crates/pa-daemon/src/supervisor/supervision.rs
@@ -12,7 +12,7 @@ use super::{
use crate::lease::is_process_alive;
use crate::registry::WorkerRelay;
-const MAX_CONSECUTIVE_FAILURES: u32 = 5;
+pub(super) const MAX_CONSECUTIVE_FAILURES: u32 = 5;
/// A crash-path child that lived at least this long proved health: its death
/// resets the failure count (a fresh count) instead of accumulating toward
/// the give-up cap. Below it, a spawn-dies-fast child counts as another
@@ -51,7 +51,7 @@ impl Supervisor {
});
}
- async fn watch_worker(
+ pub(super) async fn watch_worker(
self: Arc<Self>,
resident: Arc<ResidentWorker>,
mut child: Option<Child>,
@@ -105,6 +105,14 @@ impl Supervisor {
.map_or(0, |elapsed| elapsed.as_millis() as u64);
let failures = Self::next_failure_count(&resident, now_ms);
if failures > MAX_CONSECUTIVE_FAILURES {
+ // A stop that lands during the relaunch storm owns the
+ // terminal state: the give-up must not persist `Failed`
+ // over the stop's tombstone or recreate a descriptor the
+ // stop already deleted (the next boot would adopt a
+ // cleanly stopped worker as `GaveUp`).
+ if self.is_stopping(&resident) {
+ return;
+ }
let mut descriptor = resident.descriptor.lock().await;
descriptor.lifecycle = DaemonWorkerLifecycle::Failed;
descriptor.last_failure_at = Some(util::now_iso());
@@ -182,6 +190,12 @@ impl Supervisor {
));
child = None;
adopted_pid = 0;
+ if self.is_stopping(&resident) {
+ return;
+ }
+ // A failed relaunch has no lifetime of its own: clear
+ // the spawn time so the count accumulates to the give-up cap.
+ resident.spawned_at_ms.store(0, Ordering::SeqCst);
}
}
}
diff --git a/crates/pa-daemon/src/supervisor/tests.rs b/crates/pa-daemon/src/supervisor/tests.rs
index d9c52cbec8..e66a786126 100644
--- a/crates/pa-daemon/src/supervisor/tests.rs
+++ b/crates/pa-daemon/src/supervisor/tests.rs
@@ -43,6 +43,132 @@ fn churn_accumulates_and_a_stable_lifetime_resets() {
assert_eq!(Supervisor::next_failure_count(&resident, now), 2);
}
+/// A relaunch that FAILS produced no worker, so the count must
+/// accumulate to the give-up cap instead of resetting against the old spawn.
+#[tokio::test(start_paused = true)]
+async fn a_failed_relaunch_accumulates_to_the_give_up_cap() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let agent_dir = dir.path().join("agent");
+ std::fs::create_dir_all(&agent_dir).unwrap();
+ // The relaunch fails deterministically: the logs dir cannot be created
+ // (a file stands where it would go), so every spawn dies at the worker
+ // stderr log's open.
+ std::fs::write(agent_dir.join("logs"), "not a directory").unwrap();
+ let supervisor = Arc::new(
+ Supervisor::new(SupervisorOptions {
+ socket_path: dir.path().join("daemon.sock"),
+ agent_dir: agent_dir.clone(),
+ })
+ .expect("supervisor"),
+ );
+ let descriptor: DaemonWorkerDescriptor = serde_json::from_value(serde_json::json!({
+ "version": 2,
+ "workerId": "w-relaunch",
+ "pid": 0,
+ "socketPath": "/tmp/none.sock",
+ "recoveryJournalPath": dir.path().join("journal.jsonl").to_string_lossy(),
+ "supervisorSocketPath": "/tmp/none.sock",
+ "authenticationToken": "token",
+ "rootActiveSessionId": "w-relaunch",
+ "createdAt": "t",
+ "updatedAt": "t",
+ "lifecycle": "ready",
+ "createCommand": {},
+ "consecutiveFailures": 0,
+ }))
+ .expect("descriptor");
+ let resident = Arc::new(ResidentWorker::new(
+ "w-relaunch".to_string(),
+ descriptor,
+ dir.path().join("w-relaunch.descriptor.json"),
+ ));
+ let now_ms = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .expect("the clock is after the epoch")
+ .as_millis() as u64;
+ resident
+ .spawned_at_ms
+ .store(now_ms - STABLE_LIFETIME_MS - 1, Ordering::SeqCst);
+ supervisor.registry.insert(Arc::clone(&resident)).await;
+ tokio::time::timeout(
+ Duration::from_secs(60),
+ Arc::clone(&supervisor).watch_worker(Arc::clone(&resident), None, 0),
+ )
+ .await
+ .expect("the watch loop gives up instead of spinning");
+ assert!(
+ supervisor.registry.get("w-relaunch").await.is_none(),
+ "the give-up removes the worker from the registry"
+ );
+ assert_eq!(
+ resident.descriptor.lock().await.lifecycle,
+ DaemonWorkerLifecycle::Failed
+ );
+}
+
+/// A stop that lands while the watch loop is on the give-up cap leaves
+/// the terminal state to the stop: the give-up arm returns without
+/// persisting `Failed` (the stop's tombstone already owns the next boot's
+/// verdict) and without removing the resident (the stop path owns the
+/// removal), so a cleanly stopped worker is not adopted as `GaveUp`.
+#[tokio::test(start_paused = true)]
+async fn a_stop_during_the_storm_leaves_the_terminal_state_to_the_stop() {
+ let dir = tempfile::TempDir::new().unwrap();
+ let agent_dir = dir.path().join("agent");
+ std::fs::create_dir_all(&agent_dir).unwrap();
+ std::fs::write(agent_dir.join("logs"), "not a directory").unwrap();
+ let supervisor = Arc::new(
+ Supervisor::new(SupervisorOptions {
+ socket_path: dir.path().join("daemon.sock"),
+ agent_dir: agent_dir.clone(),
+ })
+ .expect("supervisor"),
+ );
+ let descriptor: DaemonWorkerDescriptor = serde_json::from_value(serde_json::json!({
+ "version": 2,
+ "workerId": "w-storm-stop",
+ "pid": 0,
+ "socketPath": "/tmp/none.sock",
+ "recoveryJournalPath": dir.path().join("journal.jsonl").to_string_lossy(),
+ "supervisorSocketPath": "/tmp/none.sock",
+ "authenticationToken": "token",
+ "rootActiveSessionId": "w-storm-stop",
+ "createdAt": "t",
+ "updatedAt": "t",
+ "lifecycle": "ready",
+ "createCommand": {},
+ "consecutiveFailures": 0,
+ }))
+ .expect("descriptor");
+ let resident = Arc::new(ResidentWorker::new(
+ "w-storm-stop".to_string(),
+ descriptor,
+ dir.path().join("w-storm-stop.descriptor.json"),
+ ));
+ // The storm is at the cap and the stop already landed (the stop
+ // path sets the flag before it finalizes the worker).
+ resident
+ .consecutive_failures
+ .store(MAX_CONSECUTIVE_FAILURES, Ordering::SeqCst);
+ resident.intentional_stop.store(true, Ordering::SeqCst);
+ supervisor.registry.insert(Arc::clone(&resident)).await;
+ tokio::time::timeout(
+ Duration::from_secs(60),
+ Arc::clone(&supervisor).watch_worker(Arc::clone(&resident), None, 0),
+ )
+ .await
+ .expect("the give-up arm returns instead of spinning");
+ assert_eq!(
+ resident.descriptor.lock().await.lifecycle,
+ DaemonWorkerLifecycle::Ready,
+ "the give-up must not persist Failed over a concurrent stop"
+ );
+ assert!(
+ supervisor.registry.get("w-storm-stop").await.is_some(),
+ "the stop path owns the worker's removal"
+ );
+}
+
/// The saved-session surfaces (the `list --all` summary row and the
/// `list_saved_sessions` catalog row) carry the persisted thinking
/// level: the agents-view Model column renders "model:level" for