Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

keep a newer process's socket when an old one exits (#3291)

kta0c131566b6f
diff --git a/crates/pa-daemon/src/supervisor.rs b/crates/pa-daemon/src/supervisor.rs
index ab12c59e30..40349dad05 100644
--- a/crates/pa-daemon/src/supervisor.rs
+++ b/crates/pa-daemon/src/supervisor.rs
@@ -121,6 +121,14 @@ use crate::{socket, util};
pub struct Supervisor {
pub(crate) options: SupervisorOptions,
descriptor_dir: PathBuf,
+ /// The bind-time filesystem identity of this supervisor's socket file
+ /// (TS `DaemonSupervisor` captures `socketIdentity` right after
+ /// `listen`, daemon-supervisor.ts:879): the exit cleanup passes it as
+ /// the unlink's expected identity, so a file REPLACED at the path
+ /// after this bind - an external sweep plus a successor's bind - is
+ /// never unlinked by this process. `None` until `run` binds (named
+ /// pipes keep `None`: there is no file to stat).
+ bound_socket_identity: std::sync::Mutex<Option<socket::SocketIdentity>>,
/// The durable session-binding table (the stale-active-id rebind
/// surface): every active id the supervisor has routed stays
/// addressable through its session's durable identity, so a client
@@ -286,6 +294,7 @@ impl Supervisor {
Ok(Supervisor {
options,
descriptor_dir,
+ bound_socket_identity: std::sync::Mutex::new(None),
session_bindings: crate::session_bindings::SessionBindingTable::new(),
opening_files: std::sync::Mutex::new(std::collections::HashMap::new()),
telemetry: std::sync::Mutex::new(None),
@@ -391,6 +400,13 @@ impl Supervisor {
self.options.socket_path.display()
)
})?;
+ // Capture the bound file's identity before anything can replace
+ // it (TS daemon-supervisor.ts:879, between `listen` and
+ // `restrictDaemonSocketPath`): the exit cleanup below compares
+ // against THIS value, never a fresh read, so a successor's file
+ // at the same path survives this supervisor's exit.
+ *self.bound_socket_identity.lock().unwrap() =
+ socket::socket_identity(&self.options.socket_path);
socket::restrict_socket_path(&self.options.socket_path);
self.log
.append(&format!("supervisor started pid {}", std::process::id()));
@@ -486,7 +502,7 @@ impl Supervisor {
accept_loop::serve(&self, &*listener).await?;
socket::cleanup_socket_path(
&self.options.socket_path,
- socket::socket_identity(&self.options.socket_path),
+ self.bound_socket_identity.lock().unwrap().clone(),
);
Ok(())
}
diff --git a/crates/pa-daemon/src/supervisor_lost.rs b/crates/pa-daemon/src/supervisor_lost.rs
index 69d9ef1c02..4aed88d6f6 100644
--- a/crates/pa-daemon/src/supervisor_lost.rs
+++ b/crates/pa-daemon/src/supervisor_lost.rs
@@ -126,9 +126,13 @@ async fn exit_orphaned(worker: &Worker, absent_since: tokio::time::Instant) {
agent_engine.dispose_kernel().await;
}
let _ = worker.record_recovery(false, "shutdown");
+ // The bind-time identity (captured in `serve`) is the unlink's
+ // expected identity, so a REPLACED file at the path - a successor
+ // worker's live socket, the deterministic-path relaunch - survives
+ // this exit.
crate::socket::cleanup_socket_path(
&worker.config.socket_path,
- crate::socket::socket_identity(&worker.config.socket_path),
+ worker.bound_socket_identity.lock().unwrap().clone(),
);
std::process::exit(0);
}
diff --git a/crates/pa-daemon/src/worker.rs b/crates/pa-daemon/src/worker.rs
index 6b6f1cf2aa..cb0d7d53bd 100644
--- a/crates/pa-daemon/src/worker.rs
+++ b/crates/pa-daemon/src/worker.rs
@@ -103,6 +103,15 @@ use crate::types::{AgentConnectionState, SessionActionSnapshot};
pub struct Worker {
pub(crate) config: WorkerConfig,
+ /// The bind-time filesystem identity of this worker's own socket file
+ /// (TS daemon-mode captures `socketIdentity` in the listen callback,
+ /// daemon-mode.ts:718): the exit cleanups pass it as the unlink's
+ /// expected identity, so a file REPLACED at the path after this bind -
+ /// a successor worker the supervisor relaunches on the same
+ /// deterministic path - is never unlinked by this process (the
+ /// D-state-survivor late-exit edge). `None` until `serve` binds
+ /// (named pipes keep `None`: there is no file to stat).
+ pub(crate) bound_socket_identity: std::sync::Mutex<Option<crate::socket::SocketIdentity>>,
/// Supervisor self-registration handle; `None` for standalone workers.
registration: Option<RegistrationHandle>,
/// Live connections authenticated as the supervisor role. A non-zero
@@ -793,6 +802,7 @@ impl Worker {
);
Worker {
config,
+ bound_socket_identity: std::sync::Mutex::new(None),
registration,
supervisor_claims,
core,
@@ -854,10 +864,13 @@ impl Worker {
// A graceful exit owns its socket file: remove it now so a respawn
// does not wait out the stale-socket path (a killed worker cannot
// clean up, but its killer relaunches through
- // `prepare_socket_path`).
+ // `prepare_socket_path`). The bind-time identity (captured in
+ // `serve`) is the unlink's expected identity, so a REPLACED file
+ // at the path - a successor worker's live socket - survives this
+ // exit (TS daemon-mode.ts:1078-1080).
crate::socket::cleanup_socket_path(
&self.config.socket_path,
- crate::socket::socket_identity(&self.config.socket_path),
+ self.bound_socket_identity.lock().unwrap().clone(),
);
std::process::exit(0)
}
diff --git a/crates/pa-daemon/src/worker/connection.rs b/crates/pa-daemon/src/worker/connection.rs
index 9de3f2eb2b..f99e305669 100644
--- a/crates/pa-daemon/src/worker/connection.rs
+++ b/crates/pa-daemon/src/worker/connection.rs
@@ -253,6 +253,13 @@ impl Worker {
let listener = bind_transport(&self.config.socket_path)
.await
.with_context(|| format!("bind worker socket {}", self.config.socket_path.display()))?;
+ // Capture the bound file's identity before anything can replace
+ // it (TS daemon-mode.ts:718, the listen callback, between the
+ // identity capture and `restrictDaemonSocketPath`): the exit
+ // cleanups compare against THIS value, never a fresh read, so a
+ // successor's file at the same path survives this worker's exit.
+ *self.bound_socket_identity.lock().unwrap() =
+ crate::socket::socket_identity(&self.config.socket_path);
crate::socket::restrict_socket_path(&self.config.socket_path);
loop {
let stream = match listener.accept().await {
diff --git a/crates/pa-daemon/tests/socket_identity_guard_e2e.rs b/crates/pa-daemon/tests/socket_identity_guard_e2e.rs
new file mode 100644
index 0000000000..e13d3a96de
--- /dev/null
+++ b/crates/pa-daemon/tests/socket_identity_guard_e2e.rs
@@ -0,0 +1,394 @@
+//! Bind-time socket-identity guard e2e (the bind-choreography parity
+//! audit's D1 fix): the three exit cleanups used to pass a cleanup-time
+//! `socket_identity(path)` read as their own "expected" guard, comparing a
+//! fresh read with itself - the guard never discriminated, so any file
+//! REPLACED at the path after the bind was unlinked by the dying owner's
+//! exit. TS captures the identity at LISTEN time (daemon-supervisor.ts:879,
+//! daemon-mode.ts:718) and compares THAT at exit (daemon-mode.ts:1078-1080),
+//! so a replaced file survives the old owner's exit. The fix carries the
+//! bind-time capture to the three call sites; these oracles pin the
+//! replaced-file direction at each of them:
+//!
+//! * `supervisor_lost.rs` `exit_orphaned` - the reachable failure: a
+//! force-stopped worker survives its stop verdict, the supervisor
+//! relaunches its replacement on the SAME deterministic path
+//! (supervision.rs reuses `descriptor.socket_path`), and the survivor's
+//! late orphan exit must not unlink the replacement's live socket.
+//! * `worker.rs` `exit_after_close` - the same class through the routed
+//! `shutdown` arm.
+//! * `supervisor.rs` `run`'s exit cleanup - the external-sweep edge: the
+//! socket file is removed under a serving supervisor, a successor binds
+//! the path, and the original's graceful exit must not unlink the
+//! successor's live socket.
+//!
+//! The still-ours direction (the unlink DOES run on the same exit paths
+//! when the file is the binder's own) is pinned by the in-tree controls
+//! `worker_orphan_exit_e2e::orphaned_worker_exits_when_the_supervisor_socket_never_answers`
+//! and
+//! `supervisor_e2e::shutdown::a_sigterm_exits_the_supervisor_through_the_graceful_drain`,
+//! plus the routed-shutdown control below. Each replacement is bound with
+//! the original file renamed aside first: a freed inode can be handed
+//! straight back to a fresh bind, which the dev+ino gate cannot see, so
+//! keeping the original inode allocated guarantees the identities differ
+//! and the oracle is never vacuous on inode reuse.
+#![cfg(unix)]
+
+use std::io::{Read, Write};
+use std::os::unix::fs::MetadataExt;
+use std::os::unix::net::{UnixListener, UnixStream};
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command, Stdio};
+use std::time::{Duration, Instant};
+
+use serde_json::{json, Value};
+
+/// The filesystem identity the guard compares: dev + ino of the file
+/// currently at the path.
+fn socket_identity(path: &Path) -> (u64, u64) {
+ let metadata = std::fs::symlink_metadata(path).expect("stat socket file");
+ (metadata.dev(), metadata.ino())
+}
+
+/// Bind a replacement listener at `socket` while the original file is
+/// renamed aside: the original inode stays allocated, so the replacement's
+/// identity is guaranteed to differ from `original_identity`. Returns the
+/// replacement listener and its identity.
+fn bind_replacement_at(socket: &Path, original_identity: (u64, u64)) -> (UnixListener, (u64, u64)) {
+ let aside: PathBuf = socket.parent().expect("socket parent").join(format!(
+ "{}.replaced-aside",
+ socket
+ .file_name()
+ .expect("socket file name")
+ .to_string_lossy()
+ ));
+ std::fs::rename(socket, &aside).expect("rename the original socket file aside");
+ let replacement = UnixListener::bind(socket).expect("bind the replacement socket");
+ let identity = socket_identity(socket);
+ assert_ne!(
+ identity, original_identity,
+ "the replacement must be a different file than the one the daemon bound"
+ );
+ (replacement, identity)
+}
+
+/// A live replacement socket must answer connects after the old owner's
+/// exit: the file survived AND it still serves.
+fn assert_replacement_serves(socket: &Path, identity: (u64, u64)) {
+ assert!(
+ socket.exists(),
+ "the replacement socket file survived the old owner's exit"
+ );
+ assert_eq!(
+ socket_identity(socket),
+ identity,
+ "the file at the path is still the replacement's, not a new file"
+ );
+ let connect = UnixStream::connect(socket).expect("the replacement socket still accepts");
+ drop(connect);
+}
+
+/// Kill a leftover process at scope exit (the oracle children exit on
+/// their own; this is the teardown safety net).
+struct ProcessGuard {
+ child: Child,
+}
+
+impl Drop for ProcessGuard {
+ fn drop(&mut self) {
+ let _ = self.child.kill();
+ let _ = self.child.wait();
+ }
+}
+
+/// Wait until the child exits on its own within `budget`, asserting a
+/// clean exit: the exit paths under test all end in `std::process::exit(0)`
+/// (or a graceful supervisor return), so a signal death or a nonzero exit
+/// means the wrong path ran.
+fn wait_clean_exit(child: &mut Child, budget: Duration) -> std::process::ExitStatus {
+ let deadline = Instant::now() + budget;
+ loop {
+ if let Some(status) = child.try_wait().expect("poll child") {
+ assert!(
+ status.success(),
+ "the exit path must end in a clean exit 0, not {status}"
+ );
+ return status;
+ }
+ assert!(Instant::now() < deadline, "the child did not exit");
+ std::thread::sleep(Duration::from_millis(50));
+ }
+}
+
+fn wait_socket_ready(socket: &Path) {
+ let deadline = Instant::now() + Duration::from_secs(15);
+ loop {
+ if UnixStream::connect(socket).is_ok() {
+ return;
+ }
+ assert!(Instant::now() < deadline, "daemon socket never came up");
+ std::thread::sleep(Duration::from_millis(20));
+ }
+}
+
+fn spawn_supervisor(socket: &Path, agent_dir: &Path) -> ProcessGuard {
+ std::fs::create_dir_all(agent_dir).expect("agent dir");
+ let child = Command::new(env!("CARGO_BIN_EXE_pa-daemon"))
+ .arg("supervisor")
+ .arg("--socket")
+ .arg(socket)
+ .arg("--agent-dir")
+ .arg(agent_dir)
+ // Hermetic agent dir: the ambient environment may export a real
+ // agent dir; point every fallback at the test sandbox instead.
+ .env("PRIME_AGENT_CODING_AGENT_DIR", agent_dir)
+ .env_remove("PA_API_KEY")
+ .stdin(Stdio::null())
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .spawn()
+ .expect("spawn pa-daemon supervisor");
+ ProcessGuard { child }
+}
+
+/// Spawn the real `pa-daemon worker` binary on its own socket, monitoring a
+/// supervisor socket that never answers. `lost_window_ms` sets the
+/// supervisor-lost exit window (`None` keeps the default five minutes).
+fn spawn_worker(
+ dir: &Path,
+ socket: &Path,
+ token: &str,
+ lost_window_ms: Option<&str>,
+) -> ProcessGuard {
+ std::fs::create_dir_all(dir.join("agent")).expect("agent dir");
+ let mut command = Command::new(env!("CARGO_BIN_EXE_pa-daemon"));
+ command
+ .arg("worker")
+ .env(pa_daemon::worker::WORKER_ROLE_ENV, "1")
+ .env(pa_daemon::worker::WORKER_TOKEN_ENV, token)
+ .env(
+ pa_daemon::worker::WORKER_ACTIVE_SESSION_ID_ENV,
+ "socket-identity-guard",
+ )
+ .env(pa_daemon::worker::WORKER_SOCKET_ENV, socket)
+ .env(
+ pa_daemon::worker::WORKER_SUPERVISOR_SOCKET_ENV,
+ dir.join("absent-supervisor.sock"),
+ )
+ .env(
+ pa_daemon::worker::WORKER_RECOVERY_JOURNAL_ENV,
+ dir.join("recovery.jsonl"),
+ )
+ .env("PRIME_AGENT_CODING_AGENT_DIR", dir.join("agent"))
+ .stdin(Stdio::null())
+ .stdout(Stdio::null())
+ .stderr(Stdio::null());
+ if let Some(window) = lost_window_ms {
+ command.env(
+ pa_daemon::worker::WORKER_SUPERVISOR_LOST_EXIT_MS_ENV,
+ window,
+ );
+ }
+ let child = command.spawn().expect("spawn pa-daemon worker");
+ ProcessGuard { child }
+}
+
+fn wait_worker_socket(socket: &Path) {
+ let deadline = Instant::now() + Duration::from_secs(15);
+ while !socket.exists() {
+ assert!(Instant::now() < deadline, "worker socket never appeared");
+ std::thread::sleep(Duration::from_millis(20));
+ }
+}
+
+/// A raw private-frame client for the worker socket (the same wire the
+/// supervisor's request pump speaks).
+struct WorkerClient {
+ stream: UnixStream,
+}
+
+impl WorkerClient {
+ fn connect(socket: &Path, token: &str) -> Self {
+ let stream = UnixStream::connect(socket).expect("connect worker socket");
+ stream
+ .set_read_timeout(Some(Duration::from_millis(100)))
+ .expect("set timeout");
+ let mut client = WorkerClient { stream };
+ let (header, _payload) = client.read_frame();
+ assert_eq!(header["outboundType"], "daemon_hello", "worker hello");
+ let auth = client.request(
+ "worker_auth",
+ &json!({
+ "token": token,
+ "supervisorGeneration": "sup:socket-identity-guard",
+ "supervisorPid": 1,
+ "supervisorSocketPath": "/nonexistent/supervisor.sock",
+ }),
+ );
+ assert_eq!(auth["success"], true, "worker auth failed: {auth}");
+ client
+ }
+
+ fn send_frame(&mut self, header: &Value, payload: &Value) {
+ let frame = pa_daemon::framing::encode_private_frame(
+ header,
+ &serde_json::to_vec(payload).expect("payload"),
+ pa_daemon::framing::DEFAULT_PRIVATE_FRAME_LIMITS,
+ )
+ .expect("encode frame");
+ self.stream.write_all(&frame).expect("write frame");
+ self.stream.flush().expect("flush");
+ }
+
+ fn request(&mut self, command_type: &str, payload: &Value) -> Value {
+ let request_id = format!("req-{command_type}");
+ self.send_frame(
+ &json!({
+ "kind": "command",
+ "requestId": request_id,
+ "commandType": command_type,
+ }),
+ payload,
+ );
+ let deadline = Instant::now() + Duration::from_secs(20);
+ loop {
+ assert!(Instant::now() < deadline, "no response for {command_type}");
+ let (header, body) = self.read_frame();
+ if header["outboundType"].as_str() == Some("response")
+ && header["requestId"].as_str() == Some(&request_id)
+ {
+ return serde_json::from_slice(&body).expect("response body");
+ }
+ }
+ }
+
+ fn read_frame(&mut self) -> (Value, Vec<u8>) {
+ let deadline = Instant::now() + Duration::from_secs(20);
+ let mut prefix = [0u8; 8];
+ read_exact_timeout(&mut self.stream, &mut prefix, deadline);
+ let header_len = u32::from_be_bytes(prefix[0..4].try_into().unwrap()) as usize;
+ let payload_len = u32::from_be_bytes(prefix[4..8].try_into().unwrap()) as usize;
+ let mut header = vec![0u8; header_len];
+ read_exact_timeout(&mut self.stream, &mut header, deadline);
+ let mut payload = vec![0u8; payload_len];
+ read_exact_timeout(&mut self.stream, &mut payload, deadline);
+ let header: Value = serde_json::from_slice(&header).expect("frame header");
+ (header, payload)
+ }
+}
+
+fn read_exact_timeout(stream: &mut UnixStream, buffer: &mut [u8], deadline: Instant) {
+ let mut read = 0usize;
+ while read < buffer.len() {
+ assert!(Instant::now() < deadline, "worker frame read timed out");
+ match stream.read(&mut buffer[read..]) {
+ Ok(0) => panic!("worker closed the connection mid-frame"),
+ Ok(n) => read += n,
+ Err(error)
+ if error.kind() == std::io::ErrorKind::WouldBlock
+ || error.kind() == std::io::ErrorKind::TimedOut => {}
+ Err(error) => panic!("worker read: {error}"),
+ }
+ }
+}
+
+/// The survivor-wakes scenario on the orphan exit (`supervisor_lost.rs`
+/// `exit_orphaned`): the supervisor relaunches a stopped worker's
+/// replacement on the same deterministic socket path, and the old
+/// process's late orphan exit must leave the replacement's live socket
+/// alone - a stale-socket unlink there bounces every route to the
+/// replacement until the next relaunch cycle.
+#[test]
+fn orphan_exit_never_unlinks_a_replacement_bound_on_the_worker_path() {
+ let dir = tempfile::TempDir::new().expect("temp dir");
+ let socket = dir.path().join("worker.sock");
+ // The zero window exits the worker at the first availability check
+ // (1.5s after boot); the replacement is bound long before it.
+ let mut worker = spawn_worker(dir.path(), &socket, "orphan-guard-token", Some("0"));
+ wait_worker_socket(&socket);
+ let original = socket_identity(&socket);
+ let (_replacement, replacement_identity) = bind_replacement_at(&socket, original);
+ // The oracle's served-path precondition: the exit must fire with the
+ // replacement in place, or a pass would say nothing about the guard.
+ assert!(
+ worker.child.try_wait().expect("poll worker").is_none(),
+ "the replacement must be bound before the orphan exit fires"
+ );
+ wait_clean_exit(&mut worker.child, Duration::from_secs(15));
+ assert_replacement_serves(&socket, replacement_identity);
+}
+
+/// The routed `shutdown` control (the still-ours direction of
+/// `exit_after_close`): a worker's graceful exit owns its socket file - it
+/// must remove it so a respawn does not wait out the stale-socket path.
+/// The still-ours directions of the other two call sites are pinned by the
+/// in-tree controls named in the module doc.
+#[test]
+fn routed_shutdown_removes_the_workers_own_socket_file() {
+ let dir = tempfile::TempDir::new().expect("temp dir");
+ let socket = dir.path().join("worker.sock");
+ let mut worker = spawn_worker(dir.path(), &socket, "shutdown-guard-token", Some("15000"));
+ wait_worker_socket(&socket);
+ let mut client = WorkerClient::connect(&socket, "shutdown-guard-token");
+ let shutdown = client.request("shutdown", &json!({}));
+ assert_eq!(shutdown["success"], true, "shutdown failed: {shutdown}");
+ wait_clean_exit(&mut worker.child, Duration::from_secs(10));
+ assert!(
+ !socket.exists(),
+ "the graceful exit removed the worker's own socket file"
+ );
+}
+
+/// The replaced-socket scenario on the routed `shutdown` arm
+/// (`worker.rs` `exit_after_close`): the same survivor class as the
+/// orphan exit - a file replaced at the path after the bind is never
+/// unlinked by the old owner's exit.
+#[test]
+fn routed_shutdown_never_unlinks_a_replacement_bound_on_the_worker_path() {
+ let dir = tempfile::TempDir::new().expect("temp dir");
+ let socket = dir.path().join("worker.sock");
+ let mut worker = spawn_worker(dir.path(), &socket, "shutdown-guard-token", Some("15000"));
+ wait_worker_socket(&socket);
+ // Authenticate first: the supervisor role both arms the routed
+ // `shutdown` arm and disarms the orphan monitor (claims > 0), so the
+ // exit below is `exit_after_close` alone.
+ let mut client = WorkerClient::connect(&socket, "shutdown-guard-token");
+ let original = socket_identity(&socket);
+ let (_replacement, replacement_identity) = bind_replacement_at(&socket, original);
+ assert!(
+ worker.child.try_wait().expect("poll worker").is_none(),
+ "the replacement must be bound before the shutdown exit fires"
+ );
+ let shutdown = client.request("shutdown", &json!({}));
+ assert_eq!(shutdown["success"], true, "shutdown failed: {shutdown}");
+ wait_clean_exit(&mut worker.child, Duration::from_secs(10));
+ assert_replacement_serves(&socket, replacement_identity);
+}
+
+/// The replaced-socket scenario on the supervisor exit (`supervisor.rs`
+/// `run`'s cleanup): the socket file is removed under a serving
+/// supervisor (a tmpfiles age sweep; the TS module names the same edge)
+/// and a successor binds the path while the original still serves its
+/// nameless listener. The original's graceful drain must not unlink the
+/// successor's live socket - every new connect would fail ENOENT while
+/// the socket looks alive.
+#[test]
+fn a_drained_supervisor_never_unlinks_a_replaced_socket_file() {
+ let dir = tempfile::TempDir::new().expect("temp dir");
+ let socket = dir.path().join("daemon.sock");
+ let agent_dir = dir.path().join("agent");
+ let mut daemon = spawn_supervisor(&socket, &agent_dir);
+ wait_socket_ready(&socket);
+ let original = socket_identity(&socket);
+ let (_replacement, replacement_identity) = bind_replacement_at(&socket, original);
+ assert!(
+ daemon.child.try_wait().expect("poll supervisor").is_none(),
+ "the replacement must be bound before the supervisor's exit"
+ );
+ let _ = Command::new("kill")
+ .arg("-TERM")
+ .arg(daemon.child.id().to_string())
+ .status()
+ .expect("SIGTERM the supervisor");
+ wait_clean_exit(&mut daemon.child, Duration::from_secs(10));
+ assert_replacement_serves(&socket, replacement_identity);
+}