Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

make the stop escalation signal where there is no pidfd (#3404)

Sebastian Müller03ee3111eaca
diff --git a/.changes/fix-darwin-stop-escalation.md b/.changes/fix-darwin-stop-escalation.md
new file mode 100644
index 0000000000..7b69296796
--- /dev/null
+++ b/.changes/fix-darwin-stop-escalation.md
@@ -0,0 +1 @@
+- Terminal worker stops on macOS (and Linux kernels without pidfd) now send SIGTERM and SIGKILL via kill(2), gated by the start-id identity check before each signal, instead of silently reporting the worker as survived.
diff --git a/crates/pa-core/src/platform/process.rs b/crates/pa-core/src/platform/process.rs
index 29ef61b189..e632fe5a6a 100644
--- a/crates/pa-core/src/platform/process.rs
+++ b/crates/pa-core/src/platform/process.rs
@@ -239,9 +239,7 @@ pub fn pid_exists(pid: u32) -> bool {
/// The kernel-held process handle (`pidfd_open`): pins the exact process
/// behind the pid, so a signal through it ([`pidfd_signal`]) reaches that
-/// process even if the numeric pid is recycled afterwards. The caller
-/// treats an unobtainable handle as never-signal: a missed stop is
-/// recoverable, a wrong one is not.
+/// process even if the numeric pid is recycled afterwards.
///
/// # Errors
///
diff --git a/crates/pa-daemon/src/boot_reap.rs b/crates/pa-daemon/src/boot_reap.rs
index 43e17ef33e..789b15fb54 100644
--- a/crates/pa-daemon/src/boot_reap.rs
+++ b/crates/pa-daemon/src/boot_reap.rs
@@ -169,9 +169,9 @@ pub(crate) async fn reap_abandoned_workers(supervisor: &Arc<Supervisor>, worker_
}
}
-/// Whether the pid still names the discovered process (the identity gate: a
-/// recycled pid is never signaled). An UNVERIFIABLE identity never signals -
-/// safe for lease retention, not for termination.
+/// Whether the current identity snapshot agrees with the discovered process.
+/// This preflight check cannot itself bind a later numeric signal to an instance.
+/// An UNVERIFIABLE identity never signals - safe for lease retention, not for termination.
fn identity_current(target: &ReapTarget) -> bool {
match &target.start_id {
Some(expected) => {
@@ -186,49 +186,87 @@ async fn stop_target(target: &ReapTarget) -> ReapOutcome {
stop_target_within(target, TERM_GRACE, KILL_VERIFY).await
}
+struct PidfdGuard(i32);
+
+impl Drop for PidfdGuard {
+ fn drop(&mut self) {
+ pa_core::platform::process::close_pidfd(self.0);
+ }
+}
+
/// Stop one target with explicit escalation budgets: gone check, SIGTERM,
-/// grace, SIGKILL, verify. The signals ride the kernel-held pidfd: a pid
-/// recycled in the check-then-signal window never receives the signal.
+/// grace, SIGKILL, verify. The signals ride the kernel-held pidfd, which
+/// pins the process, where the platform has one; on unix without one
+/// they ride kill(2), with the identity check re-run immediately before
+/// each signal, narrowing pid reuse to the check-to-kill gap.
async fn stop_target_within(
target: &ReapTarget,
term_grace: Duration,
kill_verify: Duration,
) -> ReapOutcome {
- // The handle opens BEFORE the identity check and the check runs WHILE
- // it is held: a target that dies and has its pid recycled in between
- // would otherwise leave the handle pinning the REPLACEMENT - open
- // first, then verify the pid still names our process, and only then
- // does any signal ride the held fd (a signal through this fd can
- // reach the pinned process and nothing else, ever).
- let Ok(pidfd) = pa_core::platform::process::open_pidfd(target.pid) else {
- // The kernel-held handle is unavailable (an unsupported platform,
- // an old kernel, or a process that just exited): the conservative
- // default never signals - a missed reap is recoverable, a wrong
- // one is not. A LIVE process behind an unobtainable handle is NOT
- // gone: the terminal stop keeps its tombstoned descriptor (the
- // next boot retries), never deletes it behind a false AlreadyGone.
- if identity_current(target) && crate::lease::is_process_alive(target.pid).unwrap_or(false) {
- return ReapOutcome::Survived;
+ match pa_core::platform::process::open_pidfd(target.pid) {
+ // The fd opens before the identity check and is held across it and
+ // every signal, so only the pinned process can receive one.
+ Ok(pidfd) => {
+ let pidfd = PidfdGuard(pidfd);
+ escalate(
+ target,
+ |signal| pa_core::platform::process::pidfd_signal(pidfd.0, signal),
+ term_grace,
+ kill_verify,
+ )
+ .await
}
- return ReapOutcome::AlreadyGone;
- };
+ Err(error) if cfg!(unix) && error.kind() == std::io::ErrorKind::Unsupported => {
+ escalate(
+ target,
+ |signal| {
+ if identity_current(target) {
+ let _ = pa_core::platform::process::kill_pid(target.pid as i32, signal);
+ }
+ true
+ },
+ term_grace,
+ kill_verify,
+ )
+ .await
+ }
+ // Every other unobtainable handle keeps the conservative default:
+ // it never signals. A LIVE process behind it is NOT gone: the
+ // terminal stop keeps its tombstoned descriptor (the next boot
+ // retries), never deletes it behind a false AlreadyGone.
+ Err(_) => {
+ if identity_current(target)
+ && crate::lease::is_process_alive(target.pid).unwrap_or(false)
+ {
+ ReapOutcome::Survived
+ } else {
+ ReapOutcome::AlreadyGone
+ }
+ }
+ }
+}
+
+/// The escalation ladder both signal carriers ride: gone check, SIGTERM,
+/// grace, SIGKILL, verify.
+async fn escalate(
+ target: &ReapTarget,
+ signal: impl Fn(pa_core::platform::process::Signal) -> bool,
+ term_grace: Duration,
+ kill_verify: Duration,
+) -> ReapOutcome {
if !identity_current(target) || !crate::lease::is_process_alive(target.pid).unwrap_or(false) {
- pa_core::platform::process::close_pidfd(pidfd);
return ReapOutcome::AlreadyGone;
}
- if pa_core::platform::process::pidfd_signal(pidfd, pa_core::platform::process::Signal::Term) {
+ if signal(pa_core::platform::process::Signal::Term) {
if await_gone(target, term_grace).await {
- pa_core::platform::process::close_pidfd(pidfd);
return ReapOutcome::Term;
}
- if pa_core::platform::process::pidfd_signal(pidfd, pa_core::platform::process::Signal::Kill)
- && await_gone(target, kill_verify).await
+ if signal(pa_core::platform::process::Signal::Kill) && await_gone(target, kill_verify).await
{
- pa_core::platform::process::close_pidfd(pidfd);
return ReapOutcome::Kill;
}
}
- pa_core::platform::process::close_pidfd(pidfd);
ReapOutcome::Survived
}
@@ -666,12 +704,9 @@ mod tests {
/// Kills and reaps the child on any exit path (a failed assertion in
/// between would otherwise leak the `sleep` into the test machine: the
- /// std child kills nothing on drop). Linux only: the guard exists for
- /// the /proc census tests, which never compile elsewhere.
- #[cfg(target_os = "linux")]
+ /// std child kills nothing on drop).
struct ReapOnDrop(Option<std::process::Child>);
- #[cfg(target_os = "linux")]
impl Drop for ReapOnDrop {
fn drop(&mut self) {
if let Some(mut child) = self.0.take() {
@@ -762,21 +797,23 @@ mod tests {
assert_eq!(outcome, ReapOutcome::Term, "sleep must exit on SIGTERM");
}
- /// The `bash` ignores SIGTERM without spawning a child (a leaked grandchild would outlive
- /// the guard's kill); its marker file is the readiness barrier.
- #[cfg(target_os = "linux")]
+ /// The `bash` records SIGTERM without exiting or spawning a child (a leaked
+ /// grandchild would outlive the guard's kill); the trap marker is the readiness
+ /// barrier and the TERM record the escalation must leave before its KILL.
#[tokio::test]
async fn a_term_ignoring_process_dies_to_the_stop_escalation() {
let dir = tempfile::TempDir::new().expect("temp dir");
let trap_armed = dir.path().join("trap-armed");
+ let term_delivered = dir.path().join("term-delivered");
let child = std::process::Command::new("bash")
.arg("-c")
- .arg("trap '' TERM; : > \"$1\"; while :; do :; done")
+ .arg("trap ': > \"$2\"' TERM; : > \"$1\"; while :; do :; done")
.arg("bash")
.arg(&trap_armed)
+ .arg(&term_delivered)
.spawn()
.expect("spawn term-ignoring bash");
- let guard = ReapOnDrop(Some(child));
+ let mut guard = ReapOnDrop(Some(child));
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
while !trap_armed.exists() && tokio::time::Instant::now() < deadline {
tokio::time::sleep(Duration::from_millis(10)).await;
@@ -784,12 +821,22 @@ mod tests {
assert!(trap_armed.exists(), "the trap never armed");
let pid = guard.0.as_ref().expect("guard holds the child").id();
let outcome = stop_process(pid, crate::lease::get_process_start_id(pid)).await;
- drop(guard);
assert_eq!(
outcome,
ReapOutcome::Kill,
"the stop escalation must SIGKILL a term-ignoring worker"
);
+ assert!(
+ term_delivered.exists(),
+ "the escalation must deliver its TERM before the KILL"
+ );
+ let mut child = guard.0.take().expect("guard holds the child");
+ let status = child.wait().expect("reap the escalated child");
+ assert_eq!(
+ pa_core::platform::process::termination_signal(&status),
+ Some(9),
+ "the child's death must be the escalation's SIGKILL"
+ );
}
#[tokio::test]
diff --git a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs b/crates/pa-daemon/src/supervisor/worker_lifecycle.rs
index b35286d9a3..0a0ffce29e 100644
--- a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs
+++ b/crates/pa-daemon/src/supervisor/worker_lifecycle.rs
@@ -67,11 +67,12 @@ impl Supervisor {
// only with a provably-gone process — the settle is not a death certificate.
let settled = self.finalize_worker_stop(resident, None).await;
if settled {
- self.retire_worker_after_stop(resident).await;
- self.log_line(&format!(
- "finished the tombstoned stop of session worker {}",
- resident.worker_id
- ));
+ if self.retire_worker_after_stop(resident).await {
+ self.log_line(&format!(
+ "finished the tombstoned stop of session worker {}",
+ resident.worker_id
+ ));
+ }
} else {
self.log_line(&format!(
"tombstoned stop of session worker {} not settled; descriptor kept",
@@ -86,11 +87,12 @@ impl Supervisor {
if resident.descriptor.lock().await.owner_client_id.is_some() {
self.finalize_owned_stop(resident).await;
}
- self.retire_worker_after_stop(resident).await;
- self.log_line(&format!(
- "finished the tombstoned per-session stop of session worker {}",
- resident.worker_id
- ));
+ if self.retire_worker_after_stop(resident).await {
+ self.log_line(&format!(
+ "finished the tombstoned per-session stop of session worker {}",
+ resident.worker_id
+ ));
+ }
}
}
@@ -794,8 +796,11 @@ impl Supervisor {
/// Delete one stopped worker's descriptor only after its process is provably gone (TS
/// `stopWorkerUntracked`'s contract): deleting the descriptor of a live worker orphans
- /// it behind its lease.
- pub(super) async fn retire_worker_after_stop(self: &Arc<Self>, resident: &Arc<ResidentWorker>) {
+ /// it behind its lease. Returns whether the descriptor was deleted or already absent.
+ pub(super) async fn retire_worker_after_stop(
+ self: &Arc<Self>,
+ resident: &Arc<ResidentWorker>,
+ ) -> bool {
let (pid, start_id, recovery_journal_path) = {
let descriptor = resident.descriptor.lock().await;
(
@@ -816,12 +821,14 @@ impl Supervisor {
"session worker {} survived the shutdown escalation; descriptor tombstoned for the next boot",
resident.worker_id
));
+ false
}
_ if alive_unverified => {
self.log_line(&format!(
"session worker {} cannot be identity-verified; descriptor tombstoned for the next boot",
resident.worker_id
));
+ false
}
_ => {
// The worker is provably gone, so its kernels can no longer
@@ -838,12 +845,22 @@ impl Supervisor {
) {
let _ = std::fs::remove_file(&recovery_journal_path);
}
- let _ = std::fs::remove_file(&resident.descriptor_path);
+ match std::fs::remove_file(&resident.descriptor_path) {
+ Err(error) if error.kind() != std::io::ErrorKind::NotFound => {
+ self.log_line(&format!(
+ "failed to retire the descriptor of session worker {}: {error}; descriptor kept",
+ resident.worker_id
+ ));
+ return false;
+ }
+ Ok(()) | Err(_) => {}
+ }
// The identity-pending side record dies with the descriptor it shadows
// (an orphaned pending would shadow the next identity).
let _ = crate::descriptor::clear_identity_pending(&resident.descriptor_path);
// The reaper alone owns orphan-journal removal. A read or
// identity-verified kill failure must preserve its retry records.
+ true
}
}
}
@@ -1145,7 +1162,7 @@ mod tests {
let resident =
ResidentWorker::new("w-gone".to_string(), descriptor, descriptor_path.clone());
- supervisor.retire_worker_after_stop(&resident).await;
+ assert!(supervisor.retire_worker_after_stop(&resident).await);
assert!(
!journal_path.exists(),
@@ -1156,6 +1173,53 @@ mod tests {
assert_eq!(std::fs::read(unowned_journal).unwrap(), retained);
assert_eq!(std::fs::read(unknown_journal).unwrap(), retained);
}
+ #[tokio::test]
+ async fn retirement_reports_unlink_failure_and_keeps_pending_identity_for_retry() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let supervisor = supervisor_in(dir.path());
+ let descriptor_path = supervisor.descriptor_dir.join("w-retire.json");
+ // No recorded identity means this fixture never reaches a signal
+ // carrier. A directory forces an unlink error even when run as root.
+ let descriptor: DaemonWorkerDescriptor = serde_json::from_value(json!({
+ "version": 2,
+ "workerId": "w-retire",
+ "pid": 0,
+ "socketPath": dir.path().join("worker.sock"),
+ "recoveryJournalPath": dir.path().join("recovery.jsonl"),
+ "supervisorSocketPath": supervisor.options.socket_path,
+ "authenticationToken": "token",
+ "rootActiveSessionId": "w-retire",
+ "createdAt": "2026-10-07T00:00:00Z",
+ "updatedAt": "2026-10-07T00:00:00Z",
+ "lifecycle": "stopping",
+ "createCommand": {},
+ "consecutiveFailures": 0,
+ }))
+ .expect("descriptor");
+ std::fs::create_dir(&descriptor_path).expect("blocked descriptor path");
+ crate::descriptor::write_identity_pending(
+ &descriptor_path,
+ "moved-session",
+ "moved.jsonl",
+ "2026-10-07T00:00:01Z",
+ )
+ .expect("pending identity");
+ let resident =
+ ResidentWorker::new("w-retire".to_string(), descriptor, descriptor_path.clone());
+
+ assert!(!supervisor.retire_worker_after_stop(&resident).await);
+ assert!(descriptor_path.is_dir());
+ assert!(crate::descriptor::read_identity_pending(&descriptor_path).is_some());
+
+ std::fs::remove_dir(&descriptor_path).expect("unblock descriptor path");
+ persist_worker(&descriptor_path, &*resident.descriptor.lock().await)
+ .expect("descriptor persisted for retry");
+ assert!(supervisor.retire_worker_after_stop(&resident).await);
+ assert!(!descriptor_path.exists());
+ assert!(crate::descriptor::read_identity_pending(&descriptor_path).is_none());
+ assert!(supervisor.retire_worker_after_stop(&resident).await);
+ }
+
#[tokio::test]
async fn a_live_worker_keeps_its_orphan_journal() {
let dir = tempfile::tempdir().expect("temp dir");