A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
fix(pa-daemon): wait on the kernel's exit notification for adopted workers (#3242)
Sebastian Müller9e7b674402b4
diff --git a/.changes/fix-adopted-worker-exit-wait.md b/.changes/fix-adopted-worker-exit-wait.mdnew file mode 100644index 0000000000..88ca2cb8d1--- /dev/null+++ b/.changes/fix-adopted-worker-exit-wait.md@@ -0,0 +1 @@+- The daemon no longer wakes up every second for each session it keeps running across a restart or update: it now waits on the operating system's process-exit notification (kqueue on macOS, pidfd on Linux), which cuts idle battery drain. On macOS this also stops a `ps` process being spawned every second for each such session.diff --git a/crates/pa-core/README.md b/crates/pa-core/README.mdindex 1055520e80..5958ebacf0 100644--- a/crates/pa-core/README.md+++ b/crates/pa-core/README.md@@ -23,7 +23,7 @@ logic of its own). The RLM recursion host seam`rlm.create_session`/`rlm.list_subagents`/`rlm.collect`/`rlm.delete_subagent` host requests call into, with the roster/collect/selector-error vocabulary the daemon implements over the supervisor link.-Platform wall (`platform`): process control (signals/process groups), file locking, file permissions, and shell selection - every OS-specific behavior in the engine lives there behind cfg-gated implementations (the durable-write rename primitive is pa-telemetry's `rename_onto`, re-exported as `platform::rename_onto`). Scheduled jobs (`cron`, the `AgentCronJobStore` port of `core/cron-jobs.ts`): file-backed job state under session artifacts (`scheduled-jobs.json` partitions) with cross-process locking, plus the public read-only scan (`cron::store::read_scheduled_jobs_artifact`) the update flow's roster projection and boot re-arm read. Tools (bash, edit, ipython + internal rename/stdout), file mutation queue, truncation and rendering rules, RLM kernel lifecycle (IPython spawn/execute/revive), skills loading, system prompt assembly, compaction, harness refinement, settings/config, package manager (npm/git/local source install/remove/list/update against settings, plus `resolve()`: precedence-ranked session resource resolution over configured packages, settings arrays, auto-discovery, and bundled skills), session manager (persist/resume). origin/main+Platform wall (`platform`): process control (signals/process groups, kernel exit waits), file locking, file permissions, and shell selection - every OS-specific behavior in the engine lives there behind cfg-gated implementations (the durable-write rename primitive is pa-telemetry's `rename_onto`, re-exported as `platform::rename_onto`). Scheduled jobs (`cron`, the `AgentCronJobStore` port of `core/cron-jobs.ts`): file-backed job state under session artifacts (`scheduled-jobs.json` partitions) with cross-process locking, plus the public read-only scan (`cron::store::read_scheduled_jobs_artifact`) the update flow's roster projection and boot re-arm read. Tools (bash, edit, ipython + internal rename/stdout), file mutation queue, truncation and rendering rules, RLM kernel lifecycle (IPython spawn/execute/revive), skills loading, system prompt assembly, compaction, harness refinement, settings/config, package manager (npm/git/local source install/remove/list/update against settings, plus `resolve()`: precedence-ranked session resource resolution over configured packages, settings arrays, auto-discovery, and bundled skills), session manager (persist/resume). origin/mainProvider resilience policies in `session_engine`: the shared quick-retrypolicy (TS `provider-retry.ts`: permanent-kind classification,diff --git a/crates/pa-core/src/platform/process.rs b/crates/pa-core/src/platform/process.rsindex 541604066b..2fc4684de0 100644--- a/crates/pa-core/src/platform/process.rs+++ b/crates/pa-core/src/platform/process.rs@@ -240,25 +240,37 @@ pub fn pid_exists(pid: u32) -> bool {/// The kernel-held process handle (`pidfd_open`): pins the exact process/// behind the pid, so a signal through it ([`pidfd_signal`]) reaches that-/// process even if the numeric pid is recycled afterwards. `None` when-/// the platform or kernel has no pidfd, or the process is already gone-/// (the caller treats an unobtainable handle as never-signal: a missed-/// stop is recoverable, a wrong one is not).+/// process even if the numeric pid is recycled afterwards. The caller+/// treats an unobtainable handle as never-signal: a missed stop is+/// recoverable, a wrong one is not.+///+/// # Errors+///+/// Returns the open failure verbatim: `ESRCH` names a process that is+/// already gone, `Unsupported` a platform with no pidfd arm.#[cfg(all(unix,target_os = "linux",any(target_arch = "x86_64", target_arch = "aarch64")))]-#[must_use]-pub fn open_pidfd(pid: u32) -> Option<i32> {+pub fn open_pidfd(pid: u32) -> std::io::Result<i32> {// `SYS_pidfd_open`/`SYS_pidfd_send_signal` share their numbers across// x86_64 and aarch64 (the platforms this workspace ships) — Linux only:// pidfd is a Linux syscall family, and the macOS libc crate carries no// `SYS_pidfd_*` constants for the same cfg to compile against.let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) };- (fd >= 0).then_some(fd as i32)+ if fd < 0 {+ Err(std::io::Error::last_os_error())+ } else {+ Ok(fd as i32)+ }}+/// No pidfd arm on this platform or kernel: the handle is unobtainable.+///+/// # Errors+///+/// Always `Unsupported`.#[cfg(all(unix,not(all(@@ -266,13 +278,18 @@ pub fn open_pidfd(pid: u32) -> Option<i32> {any(target_arch = "x86_64", target_arch = "aarch64")))))]-pub fn open_pidfd(_pid: u32) -> Option<i32> {- None+pub fn open_pidfd(_pid: u32) -> std::io::Result<i32> {+ Err(std::io::ErrorKind::Unsupported.into())}+/// No pidfd arm on this platform or kernel: the handle is unobtainable.+///+/// # Errors+///+/// Always `Unsupported`.#[cfg(not(unix))]-pub fn open_pidfd(_pid: u32) -> Option<i32> {- None+pub fn open_pidfd(_pid: u32) -> std::io::Result<i32> {+ Err(std::io::ErrorKind::Unsupported.into())}/// Signal through the kernel-held handle (`pidfd_send_signal`): the@@ -326,6 +343,160 @@ pub fn close_pidfd(fd: i32) {let _ = fd;}+/// Resolve once the process behind `pid` has exited, parked on the kernel's+/// exit notification (Linux pidfd readability, macOS kqueue+/// `EVFILT_PROC`/`NOTE_EXIT`), with no timer. A pid that names no process+/// (already exited and reaped) resolves at once. The kernel handle pins+/// the process instance, so a pid recycled after registration is never+/// mistaken for it.+///+/// # Errors+///+/// The OS error when the watch cannot register (no pidfd, descriptor+/// exhaustion, an unsupported platform); the caller owns its fallback.+#[cfg(target_os = "linux")]+pub async fn wait_for_exit(pid: u32) -> std::io::Result<()> {+ use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};+ use tokio::io::{unix::AsyncFd, Interest};++ let fd = match open_pidfd(pid) {+ Ok(fd) => fd,+ Err(error) if error.raw_os_error() == Some(libc::ESRCH) => return Ok(()),+ Err(error) => return Err(error),+ };+ // SAFETY: `open_pidfd` returned a fresh descriptor this call owns+ // (and closes on drop, on every path).+ let fd = AsyncFd::with_interest(unsafe { OwnedFd::from_raw_fd(fd) }, Interest::READABLE)?;+ // Every wake is confirmed by a zero-timeout poll (AsyncFd readiness+ // can be spurious).+ loop {+ let mut ready = fd.readable().await?;+ if let Ok(result) = ready.try_io(|fd| {+ let mut probe = libc::pollfd {+ fd: fd.as_raw_fd(),+ events: libc::POLLIN,+ revents: 0,+ };+ // SAFETY: polls one descriptor this call owns, without+ // blocking.+ match unsafe { libc::poll(&raw mut probe, 1, 0) } {+ 1 => Ok(()),+ 0 => Err(std::io::ErrorKind::WouldBlock.into()),+ _ => Err(std::io::Error::last_os_error()),+ }+ }) {+ return result;+ }+ }+}++/// macOS: kqueue `EVFILT_PROC`/`NOTE_EXIT` on a pollable kqueue.+///+/// # Errors+///+/// The OS error when the watch cannot register; the caller owns its+/// fallback.+#[cfg(target_os = "macos")]+pub async fn wait_for_exit(pid: u32) -> std::io::Result<()> {+ use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};+ use tokio::io::{unix::AsyncFd, Interest};++ // SAFETY: `kqueue()` takes no arguments.+ let kq = unsafe { libc::kqueue() };+ if kq < 0 {+ return Err(std::io::Error::last_os_error());+ }+ // SAFETY: a fresh descriptor this call owns (and closes on drop, on+ // every path).+ let kq = unsafe { OwnedFd::from_raw_fd(kq) };+ // The block drops the `kevent` value (its `udata` raw pointer is+ // `!Send`) before the first `.await`.+ {+ let change = libc::kevent {+ ident: pid as libc::uintptr_t,+ filter: libc::EVFILT_PROC,+ flags: libc::EV_ADD,+ fflags: libc::NOTE_EXIT,+ data: 0,+ udata: std::ptr::null_mut(),+ };+ // SAFETY: registers one change and reads no events; `change`+ // outlives the call. With no event space, an attach failure comes+ // back as -1/errno (kevent(2)), not an `EV_ERROR` event.+ if unsafe {+ libc::kevent(+ kq.as_raw_fd(),+ &raw const change,+ 1,+ std::ptr::null_mut(),+ 0,+ std::ptr::null(),+ )+ } < 0+ {+ // `ESRCH` means the pid names no attachable process (reaped,+ // or already past its exit ref-drain - XNU runs the drain+ // before the exit knote fires): the exit already happened.+ // Every other errno (EMFILE, ENOMEM, ...) is a watch that+ // could not register; the caller falls back.+ let error = std::io::Error::last_os_error();+ return match error.raw_os_error() {+ Some(libc::ESRCH) => Ok(()),+ _ => Err(error),+ };+ }+ }+ let kq = AsyncFd::with_interest(kq, Interest::READABLE)?;+ // Only the exit knote is registered, so a drained event is the exit.+ loop {+ let mut ready = kq.readable().await?;+ if let Ok(result) = ready.try_io(|kq| {+ let zero = libc::timespec {+ tv_sec: 0,+ tv_nsec: 0,+ };+ let mut event = libc::kevent {+ ident: 0,+ filter: 0,+ flags: 0,+ fflags: 0,+ data: 0,+ udata: std::ptr::null_mut(),+ };+ // SAFETY: drains one event with a zero timeout; `event` and+ // `zero` outlive the call.+ match unsafe {+ libc::kevent(+ kq.as_raw_fd(),+ std::ptr::null(),+ 0,+ &raw mut event,+ 1,+ &raw const zero,+ )+ } {+ 1 => Ok(()),+ 0 => Err(std::io::ErrorKind::WouldBlock.into()),+ _ => Err(std::io::Error::last_os_error()),+ }+ }) {+ return result;+ }+ }+}++/// No kernel exit watch on this platform: the caller takes its own+/// liveness-poll fallback.+///+/// # Errors+///+/// Always `Unsupported`.+#[cfg(not(any(target_os = "linux", target_os = "macos")))]+#[allow(clippy::unused_async)] // one awaitable signature across the kernel-watch arms+pub async fn wait_for_exit(_pid: u32) -> std::io::Result<()> {+ Err(std::io::ErrorKind::Unsupported.into())+}+/// Windows: the shared handle probe (win32 has no unreaped-zombie state,/// so existing and running are the same predicate - Node's `kill(pid, 0)`/// checks the same `STILL_ACTIVE` exit code). A query that fails outright@@ -408,3 +579,27 @@ mod windows_tests {assert!(!kill_process_group_or_pid(-1));}}++/// The kernel-watch arms' shared exit contract: a pid that names no live+/// process resolves at once, so a worker that died before its watch+/// registered still reports an exit. Both kernel-watch platforms only+/// (the other targets take the caller's fallback by design).+#[cfg(all(test, any(target_os = "linux", target_os = "macos")))]+mod exit_wait_tests {+ use super::*;++ #[tokio::test]+ async fn wait_for_exit_resolves_at_once_for_a_reaped_pid() {+ let mut child = std::process::Command::new("sleep")+ .arg("600")+ .spawn()+ .expect("spawn sleep");+ let pid = child.id();+ child.kill().expect("kill sleep");+ child.wait().expect("reap sleep");+ assert!(+ wait_for_exit(pid).await.is_ok(),+ "a reaped pid resolves at once"+ );+ }+}diff --git a/crates/pa-daemon/src/boot_reap.rs b/crates/pa-daemon/src/boot_reap.rsindex 7651262a6d..e463906669 100644--- a/crates/pa-daemon/src/boot_reap.rs+++ b/crates/pa-daemon/src/boot_reap.rs@@ -311,7 +311,7 @@ async fn stop_target_within(// first, then verify the pid still names our process, and only then// does any signal ride the held fd (a signal through this fd can// reach the pinned process and nothing else, ever).- let Some(pidfd) = pa_core::platform::process::open_pidfd(target.pid) else {+ let Ok(pidfd) = pa_core::platform::process::open_pidfd(target.pid) else {// The kernel-held handle is unavailable (an unsupported platform,// an old kernel, or a process that just exited): the conservative// default never signals - a missed reap is recoverable, a wrong@@ -945,7 +945,7 @@ mod tests {.expect("spawn sleep");let pid = child.id();assert!(- pa_core::platform::process::open_pidfd(pid).is_some(),+ pa_core::platform::process::open_pidfd(pid).is_ok(),"the kernel-held handle opens");let outcome = stop_target(&target(pid)).await;diff --git a/crates/pa-daemon/src/supervisor/supervision.rs b/crates/pa-daemon/src/supervisor/supervision.rsindex 4b2c2e1cf7..d3da85e889 100644--- a/crates/pa-daemon/src/supervisor/supervision.rs+++ b/crates/pa-daemon/src/supervisor/supervision.rs@@ -20,6 +20,9 @@ pub(super) const MAX_CONSECUTIVE_FAILURES: u32 = 5;pub(super) const STABLE_LIFETIME_MS: u64 = 30_000;const BASE_BACKOFF_MS: u64 = 250;const MAX_BACKOFF_MS: u64 = 30_000;+/// The adopted-worker liveness poll used only where the kernel exit watch+/// cannot register (no pidfd, descriptor exhaustion, Windows).+const ADOPTED_EXIT_FALLBACK_POLL: Duration = Duration::from_secs(30);impl Supervisor {/// Watch a worker process: on unexpected exit, restart with backoff.@@ -71,24 +74,30 @@ impl Supervisor {return;}} else if adopted_pid != 0 {- // Adopted worker: poll liveness (cannot wait on a foreign- // pid). A previous relaunch that produced no worker leaves- // pid 0 here - there is nothing to watch, and polling pid 0- // would report a phantom exit; fall straight to the- // failure/backoff/relaunch arm instead.- loop {- if self.shutting_down.load(Ordering::SeqCst)- || resident.intentional_stop.load(Ordering::SeqCst)- {- return;- }- if !matches!(is_process_alive(adopted_pid as u32), Ok(true)) {- break;+ // Adopted worker (no Child handle): park on the kernel's+ // exit notification for the pid. A previous relaunch that+ // produced no worker leaves pid 0 here - there is nothing+ // to watch, and a watch on pid 0 would report a phantom+ // exit; fall straight to the failure/backoff/relaunch arm+ // instead.+ if let Err(error) =+ pa_core::platform::process::wait_for_exit(adopted_pid as u32).await+ {+ // The slow fallback, only where the kernel exit watch+ // cannot register.+ self.log_line(&format!(+ "session worker {}: kernel exit watch unavailable ({error}); polling liveness every {}s",+ resident.worker_id,+ ADOPTED_EXIT_FALLBACK_POLL.as_secs()+ ));+ while matches!(is_process_alive(adopted_pid as u32), Ok(true)) {+ tokio::time::sleep(ADOPTED_EXIT_FALLBACK_POLL).await;}- tokio::time::sleep(Duration::from_secs(1)).await;}- if resident.intentional_stop.load(Ordering::SeqCst)- || self.shutting_down.load(Ordering::SeqCst)+ // A relaunch elsewhere (update abandon, retry) took over the+ // resident with a new process.+ if self.is_stopping(&resident)+ || resident.descriptor.lock().await.pid != adopted_pid{return;}diff --git a/crates/pa-daemon/src/supervisor/tests.rs b/crates/pa-daemon/src/supervisor/tests.rsindex 910840d41d..854a4db075 100644--- a/crates/pa-daemon/src/supervisor/tests.rs+++ b/crates/pa-daemon/src/supervisor/tests.rs@@ -169,6 +169,114 @@ async fn a_stop_during_the_storm_leaves_the_terminal_state_to_the_stop() {);}+/// An adopted worker's watch parks on the kernel's exit notification and+/// hands the exit to whoever owns the resident now - a stop that landed,+/// or a relaunch elsewhere that took the resident's pid. Both cases return+/// at the exit without entering the crash arm, and with no timer in the+/// runtime the paused clock cannot advance.+#[cfg(any(target_os = "linux", target_os = "macos"))] // the kernel-watch arms+#[tokio::test(start_paused = true)]+async fn an_adopted_watch_hands_off_at_exit_without_a_timer() {+ let dir = tempfile::TempDir::new().unwrap();+ let agent_dir = dir.path().join("agent");+ std::fs::create_dir_all(&agent_dir).unwrap();+ // A hand-off the watch misses (the unfixed tree) reaches the crash+ // arm: its relaunch must fail deterministically (the logs dir cannot+ // be created - a file stands where it would go) instead of spawning+ // the test binary as a worker.+ std::fs::write(agent_dir.join("logs"), "not a directory").unwrap();+ let supervisor = Arc::new(+ Supervisor::new(SupervisorOptions {+ socket_path: dir.path().join("daemon.sock"),+ agent_dir: agent_dir.clone(),+ })+ .expect("supervisor"),+ );+ {+ // A stop that landed owns the exit's terminal state.+ let mut worker = std::process::Command::new("sleep")+ .arg("600")+ .spawn()+ .expect("spawn sleep");+ let resident = adopted_watch_resident(dir.path(), worker.id());+ let started = tokio::time::Instant::now();+ let mut watch = std::pin::pin!(Arc::clone(&supervisor).watch_worker(+ Arc::clone(&resident),+ None,+ u64::from(worker.id())+ ));+ assert!(+ futures::poll!(watch.as_mut()).is_pending(),+ "a live adopted worker keeps the watch parked"+ );+ resident.intentional_stop.store(true, Ordering::SeqCst);+ worker.kill().expect("kill sleep");+ worker.wait().expect("reap sleep");+ watch.await;+ assert_eq!(+ tokio::time::Instant::now(),+ started,+ "the kernel exit notification woke the watch; no poll timer fired"+ );+ }+ {+ // A relaunch elsewhere took the resident: the pid no longer names+ // the watched process, so the watch hands off instead of running+ // the crash arm over the replacement.+ let mut worker = std::process::Command::new("sleep")+ .arg("600")+ .spawn()+ .expect("spawn sleep");+ let resident = adopted_watch_resident(dir.path(), worker.id());+ let started = tokio::time::Instant::now();+ let mut watch = std::pin::pin!(Arc::clone(&supervisor).watch_worker(+ Arc::clone(&resident),+ None,+ u64::from(worker.id())+ ));+ assert!(+ futures::poll!(watch.as_mut()).is_pending(),+ "a live adopted worker keeps the watch parked"+ );+ resident.descriptor.lock().await.pid = u64::from(std::process::id());+ worker.kill().expect("kill sleep");+ worker.wait().expect("reap sleep");+ watch.await;+ assert_eq!(+ tokio::time::Instant::now(),+ started,+ "the kernel exit notification woke the watch; no poll timer fired"+ );+ }+}++/// The watched adopted worker's resident, with the descriptor carrying the+/// watched pid (the post-exit ownership check compares against it).+#[cfg(any(target_os = "linux", target_os = "macos"))]+fn adopted_watch_resident(dir: &std::path::Path, pid: u32) -> Arc<ResidentWorker> {+ let descriptor: DaemonWorkerDescriptor = serde_json::from_value(serde_json::json!({+ "version": 2,+ "workerId": "w-adopted-watch",+ "pid": pid,+ "socketPath": "/tmp/none.sock",+ "recoveryJournalPath": dir.join("journal.jsonl").to_string_lossy(),+ "supervisorSocketPath": "/tmp/none.sock",+ "authenticationToken": "token",+ "rootActiveSessionId": "w-adopted-watch",+ "createdAt": "t",+ "updatedAt": "t",+ "lifecycle": "ready",+ "createCommand": {},+ "consecutiveFailures": 0,+ }))+ .expect("descriptor");+ ResidentWorker::new(+ "w-adopted-watch".to_string(),+ descriptor,+ dir.join("w-adopted-watch.descriptor.json"),+ )+}+/// The saved-session surfaces (the `list --all` summary row and the/// `list_saved_sessions` catalog row) carry the persisted thinking/// level: the agents-view Model column renders "model:level" for