A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
make the stop escalation signal where there is no pidfd (#3404)
Sebastian Müller03ee3111eaca
diff --git a/.changes/fix-darwin-stop-escalation.md b/.changes/fix-darwin-stop-escalation.mdnew file mode 100644index 0000000000..7b69296796--- /dev/null+++ b/.changes/fix-darwin-stop-escalation.md@@ -0,0 +1 @@+- Terminal worker stops on macOS (and Linux kernels without pidfd) now send SIGTERM and SIGKILL via kill(2), gated by the start-id identity check before each signal, instead of silently reporting the worker as survived.diff --git a/crates/pa-core/src/platform/process.rs b/crates/pa-core/src/platform/process.rsindex 29ef61b189..e632fe5a6a 100644--- a/crates/pa-core/src/platform/process.rs+++ b/crates/pa-core/src/platform/process.rs@@ -239,9 +239,7 @@ pub fn pid_exists(pid: u32) -> bool {/// The kernel-held process handle (`pidfd_open`): pins the exact process/// behind the pid, so a signal through it ([`pidfd_signal`]) reaches that-/// process even if the numeric pid is recycled afterwards. The caller-/// treats an unobtainable handle as never-signal: a missed stop is-/// recoverable, a wrong one is not.+/// process even if the numeric pid is recycled afterwards.////// # Errors///diff --git a/crates/pa-daemon/src/boot_reap.rs b/crates/pa-daemon/src/boot_reap.rsindex 43e17ef33e..789b15fb54 100644--- a/crates/pa-daemon/src/boot_reap.rs+++ b/crates/pa-daemon/src/boot_reap.rs@@ -169,9 +169,9 @@ pub(crate) async fn reap_abandoned_workers(supervisor: &Arc<Supervisor>, worker_}}-/// Whether the pid still names the discovered process (the identity gate: a-/// recycled pid is never signaled). An UNVERIFIABLE identity never signals --/// safe for lease retention, not for termination.+/// Whether the current identity snapshot agrees with the discovered process.+/// This preflight check cannot itself bind a later numeric signal to an instance.+/// An UNVERIFIABLE identity never signals - safe for lease retention, not for termination.fn identity_current(target: &ReapTarget) -> bool {match &target.start_id {Some(expected) => {@@ -186,49 +186,87 @@ async fn stop_target(target: &ReapTarget) -> ReapOutcome {stop_target_within(target, TERM_GRACE, KILL_VERIFY).await}+struct PidfdGuard(i32);++impl Drop for PidfdGuard {+ fn drop(&mut self) {+ pa_core::platform::process::close_pidfd(self.0);+ }+}+/// Stop one target with explicit escalation budgets: gone check, SIGTERM,-/// grace, SIGKILL, verify. The signals ride the kernel-held pidfd: a pid-/// recycled in the check-then-signal window never receives the signal.+/// grace, SIGKILL, verify. The signals ride the kernel-held pidfd, which+/// pins the process, where the platform has one; on unix without one+/// they ride kill(2), with the identity check re-run immediately before+/// each signal, narrowing pid reuse to the check-to-kill gap.async fn stop_target_within(target: &ReapTarget,term_grace: Duration,kill_verify: Duration,) -> ReapOutcome {- // The handle opens BEFORE the identity check and the check runs WHILE- // it is held: a target that dies and has its pid recycled in between- // would otherwise leave the handle pinning the REPLACEMENT - open- // first, then verify the pid still names our process, and only then- // does any signal ride the held fd (a signal through this fd can- // reach the pinned process and nothing else, ever).- let Ok(pidfd) = pa_core::platform::process::open_pidfd(target.pid) else {- // The kernel-held handle is unavailable (an unsupported platform,- // an old kernel, or a process that just exited): the conservative- // default never signals - a missed reap is recoverable, a wrong- // one is not. A LIVE process behind an unobtainable handle is NOT- // gone: the terminal stop keeps its tombstoned descriptor (the- // next boot retries), never deletes it behind a false AlreadyGone.- if identity_current(target) && crate::lease::is_process_alive(target.pid).unwrap_or(false) {- return ReapOutcome::Survived;+ match pa_core::platform::process::open_pidfd(target.pid) {+ // The fd opens before the identity check and is held across it and+ // every signal, so only the pinned process can receive one.+ Ok(pidfd) => {+ let pidfd = PidfdGuard(pidfd);+ escalate(+ target,+ |signal| pa_core::platform::process::pidfd_signal(pidfd.0, signal),+ term_grace,+ kill_verify,+ )+ .await}- return ReapOutcome::AlreadyGone;- };+ Err(error) if cfg!(unix) && error.kind() == std::io::ErrorKind::Unsupported => {+ escalate(+ target,+ |signal| {+ if identity_current(target) {+ let _ = pa_core::platform::process::kill_pid(target.pid as i32, signal);+ }+ true+ },+ term_grace,+ kill_verify,+ )+ .await+ }+ // Every other unobtainable handle keeps the conservative default:+ // it never signals. A LIVE process behind it is NOT gone: the+ // terminal stop keeps its tombstoned descriptor (the next boot+ // retries), never deletes it behind a false AlreadyGone.+ Err(_) => {+ if identity_current(target)+ && crate::lease::is_process_alive(target.pid).unwrap_or(false)+ {+ ReapOutcome::Survived+ } else {+ ReapOutcome::AlreadyGone+ }+ }+ }+}++/// The escalation ladder both signal carriers ride: gone check, SIGTERM,+/// grace, SIGKILL, verify.+async fn escalate(+ target: &ReapTarget,+ signal: impl Fn(pa_core::platform::process::Signal) -> bool,+ term_grace: Duration,+ kill_verify: Duration,+) -> ReapOutcome {if !identity_current(target) || !crate::lease::is_process_alive(target.pid).unwrap_or(false) {- pa_core::platform::process::close_pidfd(pidfd);return ReapOutcome::AlreadyGone;}- if pa_core::platform::process::pidfd_signal(pidfd, pa_core::platform::process::Signal::Term) {+ if signal(pa_core::platform::process::Signal::Term) {if await_gone(target, term_grace).await {- pa_core::platform::process::close_pidfd(pidfd);return ReapOutcome::Term;}- if pa_core::platform::process::pidfd_signal(pidfd, pa_core::platform::process::Signal::Kill)- && await_gone(target, kill_verify).await+ if signal(pa_core::platform::process::Signal::Kill) && await_gone(target, kill_verify).await{- pa_core::platform::process::close_pidfd(pidfd);return ReapOutcome::Kill;}}- pa_core::platform::process::close_pidfd(pidfd);ReapOutcome::Survived}@@ -666,12 +704,9 @@ mod tests {/// Kills and reaps the child on any exit path (a failed assertion in/// between would otherwise leak the `sleep` into the test machine: the- /// std child kills nothing on drop). Linux only: the guard exists for- /// the /proc census tests, which never compile elsewhere.- #[cfg(target_os = "linux")]+ /// std child kills nothing on drop).struct ReapOnDrop(Option<std::process::Child>);- #[cfg(target_os = "linux")]impl Drop for ReapOnDrop {fn drop(&mut self) {if let Some(mut child) = self.0.take() {@@ -762,21 +797,23 @@ mod tests {assert_eq!(outcome, ReapOutcome::Term, "sleep must exit on SIGTERM");}- /// The `bash` ignores SIGTERM without spawning a child (a leaked grandchild would outlive- /// the guard's kill); its marker file is the readiness barrier.- #[cfg(target_os = "linux")]+ /// The `bash` records SIGTERM without exiting or spawning a child (a leaked+ /// grandchild would outlive the guard's kill); the trap marker is the readiness+ /// barrier and the TERM record the escalation must leave before its KILL.#[tokio::test]async fn a_term_ignoring_process_dies_to_the_stop_escalation() {let dir = tempfile::TempDir::new().expect("temp dir");let trap_armed = dir.path().join("trap-armed");+ let term_delivered = dir.path().join("term-delivered");let child = std::process::Command::new("bash").arg("-c")- .arg("trap '' TERM; : > \"$1\"; while :; do :; done")+ .arg("trap ': > \"$2\"' TERM; : > \"$1\"; while :; do :; done").arg("bash").arg(&trap_armed)+ .arg(&term_delivered).spawn().expect("spawn term-ignoring bash");- let guard = ReapOnDrop(Some(child));+ let mut guard = ReapOnDrop(Some(child));let deadline = tokio::time::Instant::now() + Duration::from_secs(10);while !trap_armed.exists() && tokio::time::Instant::now() < deadline {tokio::time::sleep(Duration::from_millis(10)).await;@@ -784,12 +821,22 @@ mod tests {assert!(trap_armed.exists(), "the trap never armed");let pid = guard.0.as_ref().expect("guard holds the child").id();let outcome = stop_process(pid, crate::lease::get_process_start_id(pid)).await;- drop(guard);assert_eq!(outcome,ReapOutcome::Kill,"the stop escalation must SIGKILL a term-ignoring worker");+ assert!(+ term_delivered.exists(),+ "the escalation must deliver its TERM before the KILL"+ );+ let mut child = guard.0.take().expect("guard holds the child");+ let status = child.wait().expect("reap the escalated child");+ assert_eq!(+ pa_core::platform::process::termination_signal(&status),+ Some(9),+ "the child's death must be the escalation's SIGKILL"+ );}#[tokio::test]diff --git a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs b/crates/pa-daemon/src/supervisor/worker_lifecycle.rsindex b35286d9a3..0a0ffce29e 100644--- a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs+++ b/crates/pa-daemon/src/supervisor/worker_lifecycle.rs@@ -67,11 +67,12 @@ impl Supervisor {// only with a provably-gone process — the settle is not a death certificate.let settled = self.finalize_worker_stop(resident, None).await;if settled {- self.retire_worker_after_stop(resident).await;- self.log_line(&format!(- "finished the tombstoned stop of session worker {}",- resident.worker_id- ));+ if self.retire_worker_after_stop(resident).await {+ self.log_line(&format!(+ "finished the tombstoned stop of session worker {}",+ resident.worker_id+ ));+ }} else {self.log_line(&format!("tombstoned stop of session worker {} not settled; descriptor kept",@@ -86,11 +87,12 @@ impl Supervisor {if resident.descriptor.lock().await.owner_client_id.is_some() {self.finalize_owned_stop(resident).await;}- self.retire_worker_after_stop(resident).await;- self.log_line(&format!(- "finished the tombstoned per-session stop of session worker {}",- resident.worker_id- ));+ if self.retire_worker_after_stop(resident).await {+ self.log_line(&format!(+ "finished the tombstoned per-session stop of session worker {}",+ resident.worker_id+ ));+ }}}@@ -794,8 +796,11 @@ impl Supervisor {/// Delete one stopped worker's descriptor only after its process is provably gone (TS/// `stopWorkerUntracked`'s contract): deleting the descriptor of a live worker orphans- /// it behind its lease.- pub(super) async fn retire_worker_after_stop(self: &Arc<Self>, resident: &Arc<ResidentWorker>) {+ /// it behind its lease. Returns whether the descriptor was deleted or already absent.+ pub(super) async fn retire_worker_after_stop(+ self: &Arc<Self>,+ resident: &Arc<ResidentWorker>,+ ) -> bool {let (pid, start_id, recovery_journal_path) = {let descriptor = resident.descriptor.lock().await;(@@ -816,12 +821,14 @@ impl Supervisor {"session worker {} survived the shutdown escalation; descriptor tombstoned for the next boot",resident.worker_id));+ false}_ if alive_unverified => {self.log_line(&format!("session worker {} cannot be identity-verified; descriptor tombstoned for the next boot",resident.worker_id));+ false}_ => {// The worker is provably gone, so its kernels can no longer@@ -838,12 +845,22 @@ impl Supervisor {) {let _ = std::fs::remove_file(&recovery_journal_path);}- let _ = std::fs::remove_file(&resident.descriptor_path);+ match std::fs::remove_file(&resident.descriptor_path) {+ Err(error) if error.kind() != std::io::ErrorKind::NotFound => {+ self.log_line(&format!(+ "failed to retire the descriptor of session worker {}: {error}; descriptor kept",+ resident.worker_id+ ));+ return false;+ }+ Ok(()) | Err(_) => {}+ }// The identity-pending side record dies with the descriptor it shadows// (an orphaned pending would shadow the next identity).let _ = crate::descriptor::clear_identity_pending(&resident.descriptor_path);// The reaper alone owns orphan-journal removal. A read or// identity-verified kill failure must preserve its retry records.+ true}}}@@ -1145,7 +1162,7 @@ mod tests {let resident =ResidentWorker::new("w-gone".to_string(), descriptor, descriptor_path.clone());- supervisor.retire_worker_after_stop(&resident).await;+ assert!(supervisor.retire_worker_after_stop(&resident).await);assert!(!journal_path.exists(),@@ -1156,6 +1173,53 @@ mod tests {assert_eq!(std::fs::read(unowned_journal).unwrap(), retained);assert_eq!(std::fs::read(unknown_journal).unwrap(), retained);}+ #[tokio::test]+ async fn retirement_reports_unlink_failure_and_keeps_pending_identity_for_retry() {+ let dir = tempfile::tempdir().expect("temp dir");+ let supervisor = supervisor_in(dir.path());+ let descriptor_path = supervisor.descriptor_dir.join("w-retire.json");+ // No recorded identity means this fixture never reaches a signal+ // carrier. A directory forces an unlink error even when run as root.+ let descriptor: DaemonWorkerDescriptor = serde_json::from_value(json!({+ "version": 2,+ "workerId": "w-retire",+ "pid": 0,+ "socketPath": dir.path().join("worker.sock"),+ "recoveryJournalPath": dir.path().join("recovery.jsonl"),+ "supervisorSocketPath": supervisor.options.socket_path,+ "authenticationToken": "token",+ "rootActiveSessionId": "w-retire",+ "createdAt": "2026-10-07T00:00:00Z",+ "updatedAt": "2026-10-07T00:00:00Z",+ "lifecycle": "stopping",+ "createCommand": {},+ "consecutiveFailures": 0,+ }))+ .expect("descriptor");+ std::fs::create_dir(&descriptor_path).expect("blocked descriptor path");+ crate::descriptor::write_identity_pending(+ &descriptor_path,+ "moved-session",+ "moved.jsonl",+ "2026-10-07T00:00:01Z",+ )+ .expect("pending identity");+ let resident =+ ResidentWorker::new("w-retire".to_string(), descriptor, descriptor_path.clone());++ assert!(!supervisor.retire_worker_after_stop(&resident).await);+ assert!(descriptor_path.is_dir());+ assert!(crate::descriptor::read_identity_pending(&descriptor_path).is_some());++ std::fs::remove_dir(&descriptor_path).expect("unblock descriptor path");+ persist_worker(&descriptor_path, &*resident.descriptor.lock().await)+ .expect("descriptor persisted for retry");+ assert!(supervisor.retire_worker_after_stop(&resident).await);+ assert!(!descriptor_path.exists());+ assert!(crate::descriptor::read_identity_pending(&descriptor_path).is_none());+ assert!(supervisor.retire_worker_after_stop(&resident).await);+ }+#[tokio::test]async fn a_live_worker_keeps_its_orphan_journal() {let dir = tempfile::tempdir().expect("temp dir");