A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
keep a newer process's socket when an old one exits (#3291)
kta0c131566b6f
diff --git a/crates/pa-daemon/src/supervisor.rs b/crates/pa-daemon/src/supervisor.rsindex ab12c59e30..40349dad05 100644--- a/crates/pa-daemon/src/supervisor.rs+++ b/crates/pa-daemon/src/supervisor.rs@@ -121,6 +121,14 @@ use crate::{socket, util};pub struct Supervisor {pub(crate) options: SupervisorOptions,descriptor_dir: PathBuf,+ /// The bind-time filesystem identity of this supervisor's socket file+ /// (TS `DaemonSupervisor` captures `socketIdentity` right after+ /// `listen`, daemon-supervisor.ts:879): the exit cleanup passes it as+ /// the unlink's expected identity, so a file REPLACED at the path+ /// after this bind - an external sweep plus a successor's bind - is+ /// never unlinked by this process. `None` until `run` binds (named+ /// pipes keep `None`: there is no file to stat).+ bound_socket_identity: std::sync::Mutex<Option<socket::SocketIdentity>>,/// The durable session-binding table (the stale-active-id rebind/// surface): every active id the supervisor has routed stays/// addressable through its session's durable identity, so a client@@ -286,6 +294,7 @@ impl Supervisor {Ok(Supervisor {options,descriptor_dir,+ bound_socket_identity: std::sync::Mutex::new(None),session_bindings: crate::session_bindings::SessionBindingTable::new(),opening_files: std::sync::Mutex::new(std::collections::HashMap::new()),telemetry: std::sync::Mutex::new(None),@@ -391,6 +400,13 @@ impl Supervisor {self.options.socket_path.display())})?;+ // Capture the bound file's identity before anything can replace+ // it (TS daemon-supervisor.ts:879, between `listen` and+ // `restrictDaemonSocketPath`): the exit cleanup below compares+ // against THIS value, never a fresh read, so a successor's file+ // at the same path survives this supervisor's exit.+ *self.bound_socket_identity.lock().unwrap() =+ socket::socket_identity(&self.options.socket_path);socket::restrict_socket_path(&self.options.socket_path);self.log.append(&format!("supervisor started pid {}", std::process::id()));@@ -486,7 +502,7 @@ impl Supervisor {accept_loop::serve(&self, &*listener).await?;socket::cleanup_socket_path(&self.options.socket_path,- socket::socket_identity(&self.options.socket_path),+ self.bound_socket_identity.lock().unwrap().clone(),);Ok(())}diff --git a/crates/pa-daemon/src/supervisor_lost.rs b/crates/pa-daemon/src/supervisor_lost.rsindex 69d9ef1c02..4aed88d6f6 100644--- a/crates/pa-daemon/src/supervisor_lost.rs+++ b/crates/pa-daemon/src/supervisor_lost.rs@@ -126,9 +126,13 @@ async fn exit_orphaned(worker: &Worker, absent_since: tokio::time::Instant) {agent_engine.dispose_kernel().await;}let _ = worker.record_recovery(false, "shutdown");+ // The bind-time identity (captured in `serve`) is the unlink's+ // expected identity, so a REPLACED file at the path - a successor+ // worker's live socket, the deterministic-path relaunch - survives+ // this exit.crate::socket::cleanup_socket_path(&worker.config.socket_path,- crate::socket::socket_identity(&worker.config.socket_path),+ worker.bound_socket_identity.lock().unwrap().clone(),);std::process::exit(0);}diff --git a/crates/pa-daemon/src/worker.rs b/crates/pa-daemon/src/worker.rsindex 6b6f1cf2aa..cb0d7d53bd 100644--- a/crates/pa-daemon/src/worker.rs+++ b/crates/pa-daemon/src/worker.rs@@ -103,6 +103,15 @@ use crate::types::{AgentConnectionState, SessionActionSnapshot};pub struct Worker {pub(crate) config: WorkerConfig,+ /// The bind-time filesystem identity of this worker's own socket file+ /// (TS daemon-mode captures `socketIdentity` in the listen callback,+ /// daemon-mode.ts:718): the exit cleanups pass it as the unlink's+ /// expected identity, so a file REPLACED at the path after this bind -+ /// a successor worker the supervisor relaunches on the same+ /// deterministic path - is never unlinked by this process (the+ /// D-state-survivor late-exit edge). `None` until `serve` binds+ /// (named pipes keep `None`: there is no file to stat).+ pub(crate) bound_socket_identity: std::sync::Mutex<Option<crate::socket::SocketIdentity>>,/// Supervisor self-registration handle; `None` for standalone workers.registration: Option<RegistrationHandle>,/// Live connections authenticated as the supervisor role. A non-zero@@ -793,6 +802,7 @@ impl Worker {);Worker {config,+ bound_socket_identity: std::sync::Mutex::new(None),registration,supervisor_claims,core,@@ -854,10 +864,13 @@ impl Worker {// A graceful exit owns its socket file: remove it now so a respawn// does not wait out the stale-socket path (a killed worker cannot// clean up, but its killer relaunches through- // `prepare_socket_path`).+ // `prepare_socket_path`). The bind-time identity (captured in+ // `serve`) is the unlink's expected identity, so a REPLACED file+ // at the path - a successor worker's live socket - survives this+ // exit (TS daemon-mode.ts:1078-1080).crate::socket::cleanup_socket_path(&self.config.socket_path,- crate::socket::socket_identity(&self.config.socket_path),+ self.bound_socket_identity.lock().unwrap().clone(),);std::process::exit(0)}diff --git a/crates/pa-daemon/src/worker/connection.rs b/crates/pa-daemon/src/worker/connection.rsindex 9de3f2eb2b..f99e305669 100644--- a/crates/pa-daemon/src/worker/connection.rs+++ b/crates/pa-daemon/src/worker/connection.rs@@ -253,6 +253,13 @@ impl Worker {let listener = bind_transport(&self.config.socket_path).await.with_context(|| format!("bind worker socket {}", self.config.socket_path.display()))?;+ // Capture the bound file's identity before anything can replace+ // it (TS daemon-mode.ts:718, the listen callback, between the+ // identity capture and `restrictDaemonSocketPath`): the exit+ // cleanups compare against THIS value, never a fresh read, so a+ // successor's file at the same path survives this worker's exit.+ *self.bound_socket_identity.lock().unwrap() =+ crate::socket::socket_identity(&self.config.socket_path);crate::socket::restrict_socket_path(&self.config.socket_path);loop {let stream = match listener.accept().await {diff --git a/crates/pa-daemon/tests/socket_identity_guard_e2e.rs b/crates/pa-daemon/tests/socket_identity_guard_e2e.rsnew file mode 100644index 0000000000..e13d3a96de--- /dev/null+++ b/crates/pa-daemon/tests/socket_identity_guard_e2e.rs@@ -0,0 +1,394 @@+//! Bind-time socket-identity guard e2e (the bind-choreography parity+//! audit's D1 fix): the three exit cleanups used to pass a cleanup-time+//! `socket_identity(path)` read as their own "expected" guard, comparing a+//! fresh read with itself - the guard never discriminated, so any file+//! REPLACED at the path after the bind was unlinked by the dying owner's+//! exit. TS captures the identity at LISTEN time (daemon-supervisor.ts:879,+//! daemon-mode.ts:718) and compares THAT at exit (daemon-mode.ts:1078-1080),+//! so a replaced file survives the old owner's exit. The fix carries the+//! bind-time capture to the three call sites; these oracles pin the+//! replaced-file direction at each of them:+//!+//! * `supervisor_lost.rs` `exit_orphaned` - the reachable failure: a+//! force-stopped worker survives its stop verdict, the supervisor+//! relaunches its replacement on the SAME deterministic path+//! (supervision.rs reuses `descriptor.socket_path`), and the survivor's+//! late orphan exit must not unlink the replacement's live socket.+//! * `worker.rs` `exit_after_close` - the same class through the routed+//! `shutdown` arm.+//! * `supervisor.rs` `run`'s exit cleanup - the external-sweep edge: the+//! socket file is removed under a serving supervisor, a successor binds+//! the path, and the original's graceful exit must not unlink the+//! successor's live socket.+//!+//! The still-ours direction (the unlink DOES run on the same exit paths+//! when the file is the binder's own) is pinned by the in-tree controls+//! `worker_orphan_exit_e2e::orphaned_worker_exits_when_the_supervisor_socket_never_answers`+//! and+//! `supervisor_e2e::shutdown::a_sigterm_exits_the_supervisor_through_the_graceful_drain`,+//! plus the routed-shutdown control below. Each replacement is bound with+//! the original file renamed aside first: a freed inode can be handed+//! straight back to a fresh bind, which the dev+ino gate cannot see, so+//! keeping the original inode allocated guarantees the identities differ+//! and the oracle is never vacuous on inode reuse.+#![cfg(unix)]++use std::io::{Read, Write};+use std::os::unix::fs::MetadataExt;+use std::os::unix::net::{UnixListener, UnixStream};+use std::path::{Path, PathBuf};+use std::process::{Child, Command, Stdio};+use std::time::{Duration, Instant};++use serde_json::{json, Value};++/// The filesystem identity the guard compares: dev + ino of the file+/// currently at the path.+fn socket_identity(path: &Path) -> (u64, u64) {+ let metadata = std::fs::symlink_metadata(path).expect("stat socket file");+ (metadata.dev(), metadata.ino())+}++/// Bind a replacement listener at `socket` while the original file is+/// renamed aside: the original inode stays allocated, so the replacement's+/// identity is guaranteed to differ from `original_identity`. Returns the+/// replacement listener and its identity.+fn bind_replacement_at(socket: &Path, original_identity: (u64, u64)) -> (UnixListener, (u64, u64)) {+ let aside: PathBuf = socket.parent().expect("socket parent").join(format!(+ "{}.replaced-aside",+ socket+ .file_name()+ .expect("socket file name")+ .to_string_lossy()+ ));+ std::fs::rename(socket, &aside).expect("rename the original socket file aside");+ let replacement = UnixListener::bind(socket).expect("bind the replacement socket");+ let identity = socket_identity(socket);+ assert_ne!(+ identity, original_identity,+ "the replacement must be a different file than the one the daemon bound"+ );+ (replacement, identity)+}++/// A live replacement socket must answer connects after the old owner's+/// exit: the file survived AND it still serves.+fn assert_replacement_serves(socket: &Path, identity: (u64, u64)) {+ assert!(+ socket.exists(),+ "the replacement socket file survived the old owner's exit"+ );+ assert_eq!(+ socket_identity(socket),+ identity,+ "the file at the path is still the replacement's, not a new file"+ );+ let connect = UnixStream::connect(socket).expect("the replacement socket still accepts");+ drop(connect);+}++/// Kill a leftover process at scope exit (the oracle children exit on+/// their own; this is the teardown safety net).+struct ProcessGuard {+ child: Child,+}++impl Drop for ProcessGuard {+ fn drop(&mut self) {+ let _ = self.child.kill();+ let _ = self.child.wait();+ }+}++/// Wait until the child exits on its own within `budget`, asserting a+/// clean exit: the exit paths under test all end in `std::process::exit(0)`+/// (or a graceful supervisor return), so a signal death or a nonzero exit+/// means the wrong path ran.+fn wait_clean_exit(child: &mut Child, budget: Duration) -> std::process::ExitStatus {+ let deadline = Instant::now() + budget;+ loop {+ if let Some(status) = child.try_wait().expect("poll child") {+ assert!(+ status.success(),+ "the exit path must end in a clean exit 0, not {status}"+ );+ return status;+ }+ assert!(Instant::now() < deadline, "the child did not exit");+ std::thread::sleep(Duration::from_millis(50));+ }+}++fn wait_socket_ready(socket: &Path) {+ let deadline = Instant::now() + Duration::from_secs(15);+ loop {+ if UnixStream::connect(socket).is_ok() {+ return;+ }+ assert!(Instant::now() < deadline, "daemon socket never came up");+ std::thread::sleep(Duration::from_millis(20));+ }+}++fn spawn_supervisor(socket: &Path, agent_dir: &Path) -> ProcessGuard {+ std::fs::create_dir_all(agent_dir).expect("agent dir");+ let child = Command::new(env!("CARGO_BIN_EXE_pa-daemon"))+ .arg("supervisor")+ .arg("--socket")+ .arg(socket)+ .arg("--agent-dir")+ .arg(agent_dir)+ // Hermetic agent dir: the ambient environment may export a real+ // agent dir; point every fallback at the test sandbox instead.+ .env("PRIME_AGENT_CODING_AGENT_DIR", agent_dir)+ .env_remove("PA_API_KEY")+ .stdin(Stdio::null())+ .stdout(Stdio::null())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn pa-daemon supervisor");+ ProcessGuard { child }+}++/// Spawn the real `pa-daemon worker` binary on its own socket, monitoring a+/// supervisor socket that never answers. `lost_window_ms` sets the+/// supervisor-lost exit window (`None` keeps the default five minutes).+fn spawn_worker(+ dir: &Path,+ socket: &Path,+ token: &str,+ lost_window_ms: Option<&str>,+) -> ProcessGuard {+ std::fs::create_dir_all(dir.join("agent")).expect("agent dir");+ let mut command = Command::new(env!("CARGO_BIN_EXE_pa-daemon"));+ command+ .arg("worker")+ .env(pa_daemon::worker::WORKER_ROLE_ENV, "1")+ .env(pa_daemon::worker::WORKER_TOKEN_ENV, token)+ .env(+ pa_daemon::worker::WORKER_ACTIVE_SESSION_ID_ENV,+ "socket-identity-guard",+ )+ .env(pa_daemon::worker::WORKER_SOCKET_ENV, socket)+ .env(+ pa_daemon::worker::WORKER_SUPERVISOR_SOCKET_ENV,+ dir.join("absent-supervisor.sock"),+ )+ .env(+ pa_daemon::worker::WORKER_RECOVERY_JOURNAL_ENV,+ dir.join("recovery.jsonl"),+ )+ .env("PRIME_AGENT_CODING_AGENT_DIR", dir.join("agent"))+ .stdin(Stdio::null())+ .stdout(Stdio::null())+ .stderr(Stdio::null());+ if let Some(window) = lost_window_ms {+ command.env(+ pa_daemon::worker::WORKER_SUPERVISOR_LOST_EXIT_MS_ENV,+ window,+ );+ }+ let child = command.spawn().expect("spawn pa-daemon worker");+ ProcessGuard { child }+}++fn wait_worker_socket(socket: &Path) {+ let deadline = Instant::now() + Duration::from_secs(15);+ while !socket.exists() {+ assert!(Instant::now() < deadline, "worker socket never appeared");+ std::thread::sleep(Duration::from_millis(20));+ }+}++/// A raw private-frame client for the worker socket (the same wire the+/// supervisor's request pump speaks).+struct WorkerClient {+ stream: UnixStream,+}++impl WorkerClient {+ fn connect(socket: &Path, token: &str) -> Self {+ let stream = UnixStream::connect(socket).expect("connect worker socket");+ stream+ .set_read_timeout(Some(Duration::from_millis(100)))+ .expect("set timeout");+ let mut client = WorkerClient { stream };+ let (header, _payload) = client.read_frame();+ assert_eq!(header["outboundType"], "daemon_hello", "worker hello");+ let auth = client.request(+ "worker_auth",+ &json!({+ "token": token,+ "supervisorGeneration": "sup:socket-identity-guard",+ "supervisorPid": 1,+ "supervisorSocketPath": "/nonexistent/supervisor.sock",+ }),+ );+ assert_eq!(auth["success"], true, "worker auth failed: {auth}");+ client+ }++ fn send_frame(&mut self, header: &Value, payload: &Value) {+ let frame = pa_daemon::framing::encode_private_frame(+ header,+ &serde_json::to_vec(payload).expect("payload"),+ pa_daemon::framing::DEFAULT_PRIVATE_FRAME_LIMITS,+ )+ .expect("encode frame");+ self.stream.write_all(&frame).expect("write frame");+ self.stream.flush().expect("flush");+ }++ fn request(&mut self, command_type: &str, payload: &Value) -> Value {+ let request_id = format!("req-{command_type}");+ self.send_frame(+ &json!({+ "kind": "command",+ "requestId": request_id,+ "commandType": command_type,+ }),+ payload,+ );+ let deadline = Instant::now() + Duration::from_secs(20);+ loop {+ assert!(Instant::now() < deadline, "no response for {command_type}");+ let (header, body) = self.read_frame();+ if header["outboundType"].as_str() == Some("response")+ && header["requestId"].as_str() == Some(&request_id)+ {+ return serde_json::from_slice(&body).expect("response body");+ }+ }+ }++ fn read_frame(&mut self) -> (Value, Vec<u8>) {+ let deadline = Instant::now() + Duration::from_secs(20);+ let mut prefix = [0u8; 8];+ read_exact_timeout(&mut self.stream, &mut prefix, deadline);+ let header_len = u32::from_be_bytes(prefix[0..4].try_into().unwrap()) as usize;+ let payload_len = u32::from_be_bytes(prefix[4..8].try_into().unwrap()) as usize;+ let mut header = vec![0u8; header_len];+ read_exact_timeout(&mut self.stream, &mut header, deadline);+ let mut payload = vec![0u8; payload_len];+ read_exact_timeout(&mut self.stream, &mut payload, deadline);+ let header: Value = serde_json::from_slice(&header).expect("frame header");+ (header, payload)+ }+}++fn read_exact_timeout(stream: &mut UnixStream, buffer: &mut [u8], deadline: Instant) {+ let mut read = 0usize;+ while read < buffer.len() {+ assert!(Instant::now() < deadline, "worker frame read timed out");+ match stream.read(&mut buffer[read..]) {+ Ok(0) => panic!("worker closed the connection mid-frame"),+ Ok(n) => read += n,+ Err(error)+ if error.kind() == std::io::ErrorKind::WouldBlock+ || error.kind() == std::io::ErrorKind::TimedOut => {}+ Err(error) => panic!("worker read: {error}"),+ }+ }+}++/// The survivor-wakes scenario on the orphan exit (`supervisor_lost.rs`+/// `exit_orphaned`): the supervisor relaunches a stopped worker's+/// replacement on the same deterministic socket path, and the old+/// process's late orphan exit must leave the replacement's live socket+/// alone - a stale-socket unlink there bounces every route to the+/// replacement until the next relaunch cycle.+#[test]+fn orphan_exit_never_unlinks_a_replacement_bound_on_the_worker_path() {+ let dir = tempfile::TempDir::new().expect("temp dir");+ let socket = dir.path().join("worker.sock");+ // The zero window exits the worker at the first availability check+ // (1.5s after boot); the replacement is bound long before it.+ let mut worker = spawn_worker(dir.path(), &socket, "orphan-guard-token", Some("0"));+ wait_worker_socket(&socket);+ let original = socket_identity(&socket);+ let (_replacement, replacement_identity) = bind_replacement_at(&socket, original);+ // The oracle's served-path precondition: the exit must fire with the+ // replacement in place, or a pass would say nothing about the guard.+ assert!(+ worker.child.try_wait().expect("poll worker").is_none(),+ "the replacement must be bound before the orphan exit fires"+ );+ wait_clean_exit(&mut worker.child, Duration::from_secs(15));+ assert_replacement_serves(&socket, replacement_identity);+}++/// The routed `shutdown` control (the still-ours direction of+/// `exit_after_close`): a worker's graceful exit owns its socket file - it+/// must remove it so a respawn does not wait out the stale-socket path.+/// The still-ours directions of the other two call sites are pinned by the+/// in-tree controls named in the module doc.+#[test]+fn routed_shutdown_removes_the_workers_own_socket_file() {+ let dir = tempfile::TempDir::new().expect("temp dir");+ let socket = dir.path().join("worker.sock");+ let mut worker = spawn_worker(dir.path(), &socket, "shutdown-guard-token", Some("15000"));+ wait_worker_socket(&socket);+ let mut client = WorkerClient::connect(&socket, "shutdown-guard-token");+ let shutdown = client.request("shutdown", &json!({}));+ assert_eq!(shutdown["success"], true, "shutdown failed: {shutdown}");+ wait_clean_exit(&mut worker.child, Duration::from_secs(10));+ assert!(+ !socket.exists(),+ "the graceful exit removed the worker's own socket file"+ );+}++/// The replaced-socket scenario on the routed `shutdown` arm+/// (`worker.rs` `exit_after_close`): the same survivor class as the+/// orphan exit - a file replaced at the path after the bind is never+/// unlinked by the old owner's exit.+#[test]+fn routed_shutdown_never_unlinks_a_replacement_bound_on_the_worker_path() {+ let dir = tempfile::TempDir::new().expect("temp dir");+ let socket = dir.path().join("worker.sock");+ let mut worker = spawn_worker(dir.path(), &socket, "shutdown-guard-token", Some("15000"));+ wait_worker_socket(&socket);+ // Authenticate first: the supervisor role both arms the routed+ // `shutdown` arm and disarms the orphan monitor (claims > 0), so the+ // exit below is `exit_after_close` alone.+ let mut client = WorkerClient::connect(&socket, "shutdown-guard-token");+ let original = socket_identity(&socket);+ let (_replacement, replacement_identity) = bind_replacement_at(&socket, original);+ assert!(+ worker.child.try_wait().expect("poll worker").is_none(),+ "the replacement must be bound before the shutdown exit fires"+ );+ let shutdown = client.request("shutdown", &json!({}));+ assert_eq!(shutdown["success"], true, "shutdown failed: {shutdown}");+ wait_clean_exit(&mut worker.child, Duration::from_secs(10));+ assert_replacement_serves(&socket, replacement_identity);+}++/// The replaced-socket scenario on the supervisor exit (`supervisor.rs`+/// `run`'s cleanup): the socket file is removed under a serving+/// supervisor (a tmpfiles age sweep; the TS module names the same edge)+/// and a successor binds the path while the original still serves its+/// nameless listener. The original's graceful drain must not unlink the+/// successor's live socket - every new connect would fail ENOENT while+/// the socket looks alive.+#[test]+fn a_drained_supervisor_never_unlinks_a_replaced_socket_file() {+ let dir = tempfile::TempDir::new().expect("temp dir");+ let socket = dir.path().join("daemon.sock");+ let agent_dir = dir.path().join("agent");+ let mut daemon = spawn_supervisor(&socket, &agent_dir);+ wait_socket_ready(&socket);+ let original = socket_identity(&socket);+ let (_replacement, replacement_identity) = bind_replacement_at(&socket, original);+ assert!(+ daemon.child.try_wait().expect("poll supervisor").is_none(),+ "the replacement must be bound before the supervisor's exit"+ );+ let _ = Command::new("kill")+ .arg("-TERM")+ .arg(daemon.child.id().to_string())+ .status()+ .expect("SIGTERM the supervisor");+ wait_clean_exit(&mut daemon.child, Duration::from_secs(10));+ assert_replacement_serves(&socket, replacement_identity);+}