A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
protect the successor socket on listener close and tombstoned adoption (#3337)
kt93241d9c7235
diff --git a/Cargo.lock b/Cargo.lockindex 5ec6ea6641..69e03714a8 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1340,6 +1340,7 @@ dependencies = ["serde","serde_json","sha2",+ "socket2","tempfile","thiserror","tokio",@@ -1415,6 +1416,7 @@ dependencies = ["regex","serde","serde_json",+ "socket2","tempfile","thiserror","tokio",diff --git a/crates/pa-daemon/Cargo.toml b/crates/pa-daemon/Cargo.tomlindex d0b67631c4..edec141847 100644--- a/crates/pa-daemon/Cargo.toml+++ b/crates/pa-daemon/Cargo.toml@@ -24,6 +24,7 @@ uuid = { version = "1", features = ["v4", "v7"] }[dev-dependencies]filetime = "0.2"tempfile = "3"+socket2 = "0.6"# The supervisor property suite draws sibling rosters for the live# rename ladder; the plain std features are enough (no fork/timeout).proptest = { workspace = true }diff --git a/crates/pa-daemon/src/socket.rs b/crates/pa-daemon/src/socket.rsindex 72e8698a95..340b60afd8 100644--- a/crates/pa-daemon/src/socket.rs+++ b/crates/pa-daemon/src/socket.rs@@ -189,12 +189,65 @@ pub fn cleanup_socket_path(path: &Path, expected_identity: Option<SocketIdentitylet _ = std::fs::remove_file(path);}+/// The exit cleanup after the owner's own listener is closed (the TS+/// graceful-shutdown sequence closes before cleanup). A successor's live+/// listener must survive even when a poisoned bind-time identity capture+/// names the successor's inode. A nonblocking connect can distinguish a+/// definitely closed listener (`ECONNREFUSED`) from a saturated backlog+/// (`EAGAIN` on Linux); unknown outcomes preserve the socket path. Only+/// after definite refusal may the existing cleanup lock and identity gate+/// unlink the stale, still-ours socket. TS cleanup checks identity alone,+/// so the poisoned-capture case remains a disclosed TS difference.+///+/// A caller without a captured identity never unlinks: `None` skips+/// `cleanup_socket_path`'s inode gate, so a replacement that binds the+/// path between this probe and that remove would lose its live file to+/// an identity-less unlink. The worker's exit paths wait for the serve+/// handshake's confirmation - which always follows the identity capture -+/// so a registration-refusal exit inside the bind->capture window still+/// reads its own captured identity; `None` reaches here only from exits+/// before the bind (no listener, no file) or from platforms without a+/// file identity (named pipes), and the cleanup stays a no-op.+#[cfg(unix)]+pub fn cleanup_socket_path_after_close(path: &Path, expected_identity: Option<SocketIdentity>) {+ if expected_identity.is_none()+ || !path.exists()+ || !pa_types::platform::transport::unix_listener_definitely_closed(path)+ {+ return;+ }+ cleanup_socket_path(path, expected_identity);+}++#[cfg(not(unix))]+pub fn cleanup_socket_path_after_close(_path: &Path, _expected_identity: Option<SocketIdentity>) {}+/// Restrict the bound socket file to its owner (Unix mode 0o600; Windows/// named pipes use ACLs on the pipe object instead).pub fn restrict_socket_path(path: &Path) {let _ = pa_core::platform::perms::restrict_file(path);}+/// The bind-capture gap seam (the `PA_DAEMON_EVENT_LOG` seam family): a+/// replacement landing between the bind and the bind-time identity capture+/// poisons the captured identity - the exact residual the+/// close-listener exit cleanup exists to survive. Production leaves the+/// gap unset, so the bind and the capture stay back-to-back; the+/// poisoned-capture oracle sets the gap so the replacement provably lands+/// in the window instead of racing microseconds.+pub const BIND_CAPTURE_GAP_ENV: &str = "PA_DAEMON_BIND_CAPTURE_GAP_MS";++/// Sleep the bounded bind-capture fault-injection gap in debug builds only.+/// Production binaries never pause startup between bind and identity capture.+pub async fn bind_capture_gap() {+ #[cfg(debug_assertions)]+ if let Ok(raw) = std::env::var(BIND_CAPTURE_GAP_ENV) {+ if let Ok(ms @ 1..=2_000) = raw.parse::<u64>() {+ tokio::time::sleep(Duration::from_millis(ms)).await;+ }+ }+}+#[cfg(all(test, unix))]mod tests {use super::*;@@ -246,6 +299,201 @@ mod tests {drop(listener);}+ /// Dropping the bound listener is the graceful close the exit+ /// cleanups run before their unlink (the TS `server.close` step,+ /// daemon-mode.ts:8011-8018): the bind releases (a fresh connect is+ /// refused) while the socket FILE survives (the close never+ /// unlinks), an in-flight accepted stream keeps serving across the+ /// close, and the listener's own fd is closed while the in-flight+ /// stream's fd stays open - no fd is leaked on the bound socket+ /// across the exit sequence.+ /// The fd-leak oracle reads `/proc/self/fd`, so it runs where that+ /// exists: the neighbors' `target_os = "linux"` gate, not bare `unix`+ /// (macOS has no `/proc` and the fd probes would always read false).+ #[cfg(target_os = "linux")]+ #[tokio::test]+ async fn dropping_the_listener_is_the_graceful_exit_close() {+ use std::io::Write;+ use std::os::fd::AsRawFd;+ use tokio::io::{AsyncReadExt, AsyncWriteExt};++ fn fd_exists(fd: std::os::fd::RawFd) -> bool {+ std::fs::read_link(format!("/proc/self/fd/{fd}")).is_ok()+ }++ let dir = tempfile::TempDir::new().unwrap();+ let socket = dir.path().join("daemon.sock");+ let listener = tokio::net::UnixListener::bind(&socket).unwrap();+ // One accepted connection in flight: the client connected, the+ // listener accepted; the stream must survive the listener's close.+ let mut client = std::os::unix::net::UnixStream::connect(&socket).unwrap();+ let mut accepted = listener.accept().await.unwrap().0;+ let listener_fd = listener.as_raw_fd();+ let accepted_fd = accepted.as_raw_fd();+ assert!(fd_exists(listener_fd));+ // The leak check compares the fd's /proc TARGET (the socket's+ // anon inode), never the bare fd number: parallel tests recycle+ // fd numbers the moment a close lands, so a bare-number probe+ // misreads another test's fresh fd as a leak. The exact socket+ // object is what a leak would still reference.+ let listener_target = std::fs::read_link(format!("/proc/self/fd/{listener_fd}"))+ .expect("the bound listener's fd target before the drop");+ drop(listener);+ let leaked_at_fd = std::fs::read_link(format!("/proc/self/fd/{listener_fd}"));+ assert!(+ !matches!(&leaked_at_fd, Ok(target) if *target == listener_target),+ "the listener's socket object is no longer referenced at the dropped fd: no fd leaked on the bound socket"+ );+ assert!(+ fd_exists(accepted_fd),+ "the in-flight accepted stream's fd survives the close"+ );+ assert!(socket.exists(), "the close never unlinks the file");+ assert!(+ !can_connect(&socket, Duration::from_millis(250)).await,+ "the bind released at the drop"+ );+ // The in-flight stream still moves bytes across the close.+ client.write_all(b"ping").unwrap();+ let mut buffer = [0u8; 4];+ accepted.read_exact(&mut buffer).await.unwrap();+ assert_eq!(&buffer, b"ping");+ assert!(accepted.flush().await.is_ok());+ }++ /// The exit cleanup after the owner's listener closed spares a LIVE+ /// successor even when the expected identity matches that+ /// successor's file exactly - the poisoned capture a replacement+ /// landing in the bind->capture window produces - and still unlinks+ /// the dead file the matching identity describes (the still-ours+ /// direction: a respawn does not wait out the stale-socket path).+ #[tokio::test]+ async fn exit_cleanup_after_close_spares_a_live_successor_with_a_matching_identity() {+ let dir = tempfile::TempDir::new().unwrap();+ let socket = dir.path().join("daemon.sock");+ // The owner's own bind, closed exactly like the exit sequences+ // close it before their cleanup.+ let owner = bind_transport(&socket).await.unwrap();+ drop(owner);+ // The poisoned capture: the successor binds the path after the+ // owner's file is renamed aside, and the "captured" identity is+ // the successor's own file (a replacement landing in the+ // bind->capture window stores exactly this).+ let aside = dir.path().join("owner.sock");+ std::fs::rename(&socket, &aside).unwrap();+ let successor = bind_transport(&socket).await.unwrap();+ let poisoned = socket_identity(&socket).unwrap();+ cleanup_socket_path_after_close(&socket, Some(poisoned.clone()));+ assert!(+ socket.exists(),+ "a live successor is never unlinked, even with a matching identity"+ );+ assert!(can_connect(&socket, Duration::from_millis(250)).await);+ drop(successor);+ std::fs::remove_file(&aside).unwrap();+ }++ /// The still-ours direction of the close cleanup: the matching+ /// identity now describes a dead file the probe passed as definitely+ /// closed. Linux-only because `unix_listener_definitely_closed` only+ /// rules `ECONNREFUSED` definitive there (a saturated BSD/macOS+ /// backlog also refuses, so those platforms never reach this arm).+ #[cfg(target_os = "linux")]+ #[tokio::test]+ async fn exit_cleanup_after_close_unlinks_the_dead_still_ours_file() {+ let dir = tempfile::TempDir::new().unwrap();+ let socket = dir.path().join("daemon.sock");+ let owner = bind_transport(&socket).await.unwrap();+ let identity = socket_identity(&socket).unwrap();+ drop(owner);+ cleanup_socket_path_after_close(&socket, Some(identity));+ assert!(!socket.exists(), "the dead still-ours file is unlinked");+ }++ /// Off Linux the refused probe is ambiguous (a saturated backlog+ /// also refuses), so the close cleanup preserves the dead still-ours+ /// file; the next bind's stale-socket prepare clears it instead.+ #[cfg(not(target_os = "linux"))]+ #[tokio::test]+ async fn exit_cleanup_off_linux_preserves_the_dead_file_until_the_next_bind() {+ let dir = tempfile::TempDir::new().unwrap();+ let socket = dir.path().join("daemon.sock");+ let owner = bind_transport(&socket).await.unwrap();+ let identity = socket_identity(&socket).unwrap();+ drop(owner);+ cleanup_socket_path_after_close(&socket, Some(identity));+ assert!(+ socket.exists(),+ "off Linux the close cleanup never claims a dead file from the probe alone"+ );+ prepare_socket_path(&socket).await.unwrap();+ assert!(+ !socket.exists(),+ "the next bind's stale-socket prepare clears the preserved dead file"+ );+ }++ /// A full accept queue is not proof of a dead listener: the successor's+ /// inode can exactly match a poisoned bind-time identity capture.+ #[cfg(target_os = "linux")]+ #[tokio::test]+ async fn exit_cleanup_preserves_a_backlogged_successor_with_a_matching_identity() {+ let dir = tempfile::TempDir::new().unwrap();+ let socket = dir.path().join("daemon.sock");+ let owner = bind_transport(&socket).await.unwrap();+ drop(owner);+ let aside = dir.path().join("owner.sock");+ std::fs::rename(&socket, &aside).unwrap();+ let successor = std::os::unix::net::UnixListener::bind(&socket).unwrap();+ socket2::SockRef::from(&successor).listen(1).unwrap();+ let poisoned = socket_identity(&socket).unwrap();+ let mut queued = Vec::new();+ // Never accept: hold each successful connection until the queue fills.+ for _ in 0..4 {+ match tokio::time::timeout(+ Duration::from_millis(100),+ tokio::net::UnixStream::connect(&socket),+ )+ .await+ {+ Ok(Ok(stream)) => queued.push(stream),+ _ => break,+ }+ }+ assert!(!queued.is_empty());+ assert!(+ !can_connect(&socket, Duration::from_millis(100)).await,+ "the successor's queue must be saturated for this oracle"+ );+ cleanup_socket_path_after_close(&socket, Some(poisoned.clone()));+ assert!(+ socket.exists(),+ "a live backlogged successor must not be unlinked"+ );+ drop(successor);+ cleanup_socket_path_after_close(&socket, Some(poisoned));+ assert!(+ !socket.exists(),+ "the same inode unlinks after its listener closes"+ );+ drop(queued);+ std::fs::remove_file(&aside).unwrap();+ }++ #[cfg(target_os = "linux")]+ #[tokio::test]+ async fn exit_cleanup_unlinks_a_closed_listener_on_a_long_socket_path() {+ let dir = tempfile::TempDir::new().unwrap();+ let deep = dir.path().join("a".repeat(80)).join("b".repeat(80));+ std::fs::create_dir_all(&deep).unwrap();+ let socket = deep.join("daemon.sock");+ let owner = bind_transport(&socket).await.unwrap();+ let identity = socket_identity(&socket).unwrap();+ drop(owner);+ cleanup_socket_path_after_close(&socket, Some(identity));+ assert!(!socket.exists(), "deep stale path still unlinks");+ }+#[tokio::test]async fn stale_socket_file_is_removed_and_the_path_rebinds() {let dir = tempfile::TempDir::new().unwrap();diff --git a/crates/pa-daemon/src/supervisor.rs b/crates/pa-daemon/src/supervisor.rsindex ea746d8090..4acfaeef7c 100644--- a/crates/pa-daemon/src/supervisor.rs+++ b/crates/pa-daemon/src/supervisor.rs@@ -352,6 +352,7 @@ impl Supervisor {self.options.socket_path.display())})?;+ socket::bind_capture_gap().await;// Capture the bound file's identity before anything can replace// it (TS daemon-supervisor.ts:879, between `listen` and// `restrictDaemonSocketPath`): the exit cleanup below compares@@ -467,11 +468,15 @@ impl Supervisor {});}- accept_loop::serve(&self, &*listener).await?;- socket::cleanup_socket_path(- &self.options.socket_path,- self.bound_socket_identity.lock().unwrap().clone(),- );+ accept_loop::serve(&self, listener).await?;+ // The accept loop OWNED the listener, so its return already+ // closed it (TS daemon-supervisor.ts:7436-7491 awaits the+ // "daemon server" close step before the "daemon socket" cleanup+ // step): the cleanup below probes the path with the owner's+ // listener provably closed, so a successor's live socket at the+ // path survives even a poisoned bind-time capture.+ let expected_identity = self.bound_socket_identity.lock().unwrap().clone();+ socket::cleanup_socket_path_after_close(&self.options.socket_path, expected_identity);self.flush_telemetry_on_exit().await;Ok(())}diff --git a/crates/pa-daemon/src/supervisor/accept_loop.rs b/crates/pa-daemon/src/supervisor/accept_loop.rsindex c7bb05d765..91a2333081 100644--- a/crates/pa-daemon/src/supervisor/accept_loop.rs+++ b/crates/pa-daemon/src/supervisor/accept_loop.rs@@ -34,13 +34,20 @@ async fn retry_backoff(supervisor: &Supervisor) {/// Serve clients until `begin_shutdown` completes its stop pass and sets the exit flag.///+/// Owns the bound listener: every return path drops it, so the listener+/// is closed before the caller's exit cleanup runs (the TS graceful-+/// shutdown order - daemon-supervisor.ts:7436-7491 awaits the "daemon+/// server" close step, then runs the "daemon socket" cleanup step). A+/// live listener at the socket path after this returns can only be a+/// successor's, which is what makes the cleanup's liveness probe sound.+////// # Errors////// Returns the transport's accept error once [`GIVE_UP_AFTER`] consecutive failures/// exhaust the give-up budget; the caller exits, releasing the socket bind.pub(super) async fn serve(supervisor: &Arc<Supervisor>,- listener: &dyn TransportListener,+ listener: Box<dyn TransportListener>,) -> Result<()> {let mut consecutive_failures = 0u32;let mut recoverable_streak = 0u32;@@ -117,10 +124,15 @@ mod tests {use super::*;+ /// The scripted accept results, shared so the test can read what the+ /// loop consumed after the stand-in is moved into `serve` (the+ /// owned-listener shape the production loop takes now).+ type ScriptedResults = Arc<Mutex<VecDeque<io::Result<Box<dyn TransportStream>>>>>;+/// A stand-in listener replaying a scripted accept sequence; when the script drains,/// it fires the shutdown wake and parks, so `serve` falls out like a real listener.struct ScriptedAccepts {- results: Mutex<VecDeque<io::Result<Box<dyn TransportStream>>>>,+ results: ScriptedResults,supervisor: Arc<Supervisor>,}@@ -180,7 +192,7 @@ mod tests {let dir = TempDir::new().unwrap();let supervisor = test_supervisor(&dir);let scripted = ScriptedAccepts {- results: Mutex::new(+ results: Arc::new(Mutex::new(vec![accept_error(ErrorKind::ConnectionAborted, "aborted"),accept_error(ErrorKind::ConnectionReset, "reset"),@@ -188,11 +200,12 @@ mod tests {Ok(accepted_stream()),].into(),- ),+ )),supervisor: Arc::clone(&supervisor),};+ let results = Arc::clone(&scripted.results);let started = tokio::time::Instant::now();- serve(&supervisor, &scripted)+ serve(&supervisor, Box::new(scripted)).await.expect("the loop must survive recoverable accept errors");assert!(@@ -201,7 +214,7 @@ mod tests {started.elapsed());assert!(- scripted.results.lock().unwrap().is_empty(),+ results.lock().unwrap().is_empty(),"the loop kept accepting past every error and served a client");}@@ -216,11 +229,12 @@ mod tests {}results.push_back(Ok(accepted_stream()));let scripted = ScriptedAccepts {- results: Mutex::new(results),+ results: Arc::new(Mutex::new(results)),supervisor: Arc::clone(&supervisor),};+ let results = Arc::clone(&scripted.results);let started = tokio::time::Instant::now();- serve(&supervisor, &scripted)+ serve(&supervisor, Box::new(scripted)).await.expect("a recoverable storm must never exit the loop");let elapsed = started.elapsed();@@ -234,7 +248,7 @@ mod tests {"exactly one backoff per storm (elapsed {elapsed:?})");assert!(- scripted.results.lock().unwrap().is_empty(),+ results.lock().unwrap().is_empty(),"the client behind the storm was served");}@@ -244,17 +258,18 @@ mod tests {let dir = TempDir::new().unwrap();let supervisor = test_supervisor(&dir);let scripted = ScriptedAccepts {- results: Mutex::new(+ results: Arc::new(Mutex::new(vec![accept_error(ErrorKind::Other, "too many open files"),Ok(accepted_stream()),].into(),- ),+ )),supervisor: Arc::clone(&supervisor),};+ let results = Arc::clone(&scripted.results);let started = tokio::time::Instant::now();- serve(&supervisor, &scripted)+ serve(&supervisor, Box::new(scripted)).await.expect("one hard error must not exit the loop");let elapsed = started.elapsed();@@ -267,7 +282,7 @@ mod tests {"exactly one backoff for one error (elapsed {elapsed:?})");assert!(- scripted.results.lock().unwrap().is_empty(),+ results.lock().unwrap().is_empty(),"the loop kept serving after the error");}@@ -286,14 +301,15 @@ mod tests {results.push_back(accept_error(ErrorKind::Other, "transient"));}let scripted = ScriptedAccepts {- results: Mutex::new(results),+ results: Arc::new(Mutex::new(results)),supervisor: Arc::clone(&supervisor),};- serve(&supervisor, &scripted)+ let results = Arc::clone(&scripted.results);+ serve(&supervisor, Box::new(scripted)).await.expect("two sub-budget bursts with a served client between them never escalate");assert!(- scripted.results.lock().unwrap().is_empty(),+ results.lock().unwrap().is_empty(),"every scripted accept was served");}@@ -310,10 +326,11 @@ mod tests {results.push_back(accept_error(ErrorKind::Other, "listener broken"));}let scripted = ScriptedAccepts {- results: Mutex::new(results),+ results: Arc::new(Mutex::new(results)),supervisor: Arc::clone(&supervisor),};- let error = serve(&supervisor, &scripted)+ let results = Arc::clone(&scripted.results);+ let error = serve(&supervisor, Box::new(scripted)).await.expect_err("a permanently broken listener must escalate");assert_eq!(@@ -322,7 +339,7 @@ mod tests {"the escalation carries the transport error");assert_eq!(- scripted.results.lock().unwrap().len(),+ results.lock().unwrap().len(),overflow as usize,"the loop gave up at exactly the budget");@@ -338,11 +355,12 @@ mod tests {results.push_back(accept_error(ErrorKind::Other, "shutting down"));}let scripted = ScriptedAccepts {- results: Mutex::new(results),+ results: Arc::new(Mutex::new(results)),supervisor: Arc::clone(&supervisor),};+ let results = Arc::clone(&scripted.results);let started = tokio::time::Instant::now();- serve(&supervisor, &scripted)+ serve(&supervisor, Box::new(scripted)).await.expect("shutdown-window errors never escalate");assert!(@@ -351,7 +369,7 @@ mod tests {started.elapsed());assert!(- scripted.results.lock().unwrap().is_empty(),+ results.lock().unwrap().is_empty(),"every error continued immediately");}@@ -361,7 +379,9 @@ mod tests {let dir = TempDir::new().unwrap();let supervisor = test_supervisor(&dir);let scripted = ScriptedAccepts {- results: Mutex::new(vec![accept_error(ErrorKind::Other, "hard error")].into()),+ results: Arc::new(Mutex::new(+ vec![accept_error(ErrorKind::Other, "hard error")].into(),+ )),supervisor: Arc::clone(&supervisor),};// The terminal wake fires mid-backoff: after a 100ms pause the exit flag and@@ -373,7 +393,7 @@ mod tests {shutting_down.shutdown_notify.notify_one();});let started = tokio::time::Instant::now();- serve(&supervisor, &scripted)+ serve(&supervisor, Box::new(scripted)).await.expect("the shutdown wake ends the loop");assert!(diff --git a/crates/pa-daemon/src/supervisor/adoption.rs b/crates/pa-daemon/src/supervisor/adoption.rsindex d22f092c32..5165b46f6c 100644--- a/crates/pa-daemon/src/supervisor/adoption.rs+++ b/crates/pa-daemon/src/supervisor/adoption.rs@@ -10,6 +10,39 @@ use super::{WorkerRegistration,};+/// An unobservable start identity does not prove that a live pid was recycled.+/// Only an observed mismatch proves the descriptor's process is gone.+fn recorded_process_alive(+ alive: Option<bool>,+ expected: Option<&str>,+ observed: Option<&str>,+) -> bool {+ alive != Some(false) && (expected.is_none() || observed.is_none() || expected == observed)+}++#[cfg(test)]+mod recorded_process_tests {+ use super::recorded_process_alive;++ #[test]+ fn unobservable_start_id_keeps_live_tombstone_owned_by_recorded_pid() {+ assert!(recorded_process_alive(Some(true), Some("original"), None));+ assert!(recorded_process_alive(None, Some("original"), None));+ assert!(recorded_process_alive(Some(true), None, None));+ assert!(recorded_process_alive(+ Some(true),+ Some("original"),+ Some("original")+ ));+ assert!(!recorded_process_alive(+ Some(true),+ Some("original"),+ Some("recycled")+ ));+ assert!(!recorded_process_alive(Some(false), Some("original"), None));+ }+}+/// The boot the descriptor-adoption pass runs under. An update boot relaunches kept/// workers before the roster restore walks the rows (spec §6 step 2's create-or-adopt/// order). A plain startup revives only genuinely interrupted ones — no mass-revival@@ -159,7 +192,41 @@ impl Supervisor {return AdoptionOutcome::AdoptedLive;}let socket_path = PathBuf::from(&descriptor.socket_path);- let alive = socket::can_connect(&socket_path, Duration::from_millis(500)).await;+ // A tombstoned descriptor belongs to its recorded process, not+ // whichever listener now owns its pathname. A dead pid (or a+ // recycled one with a different start id) must bypass auth and+ // finish the stop; a foreign listener can otherwise keep adoption+ // waiting behind the worker-auth budget. TS adoptOrRecoverWorker+ // checks the recorded pid before connecting a stopped worker.+ // Ordinary descriptors retain the existing socket-based revival+ // decision, which also handles descriptors without a start id.+ // The identity probes are the tombstone's alone: an ordinary+ // descriptor's stop term is `stop_requested_at.is_none()` and the+ // OR never reads the probes, so running them for every descriptor+ // pays a process spawn (`ps` on Unix platforms without /proc or+ // sysctl) inside the async adoption task - blocking an executor+ // worker at boot. The tombstoned path runs the probes off the+ // runtime through `spawn_blocking`; a join failure conservatively+ // treats the recorded process as alive (the graceful IPC leg+ // below degrades to the same finalize a dead verdict runs).+ let tombstoned = descriptor.stop_requested_at.is_some();+ let recorded_process_alive = if tombstoned {+ let pid = descriptor.pid as u32;+ let expected = descriptor.process_start_id.clone();+ tokio::task::spawn_blocking(move || {+ recorded_process_alive(+ crate::lease::is_process_alive(pid).ok(),+ expected.as_deref(),+ crate::lease::get_process_start_id(pid).as_deref(),+ )+ })+ .await+ .unwrap_or(true)+ } else {+ true+ };+ let alive = (descriptor.stop_requested_at.is_none() || recorded_process_alive)+ && socket::can_connect(&socket_path, Duration::from_millis(500)).await;let pid = descriptor.pid;let journal_path = PathBuf::from(&descriptor.recovery_journal_path);let resident = ResidentWorker::new(worker_id.clone(), descriptor, path);diff --git a/crates/pa-daemon/src/supervisor/handshake_tests.rs b/crates/pa-daemon/src/supervisor/handshake_tests.rsindex 45035baebf..19a2c97696 100644--- a/crates/pa-daemon/src/supervisor/handshake_tests.rs+++ b/crates/pa-daemon/src/supervisor/handshake_tests.rs@@ -150,6 +150,87 @@ async fn handshake_channel_stays_private_until_auth_answers() {);}+/// A tombstoned stop's entire silent-peer authentication uses the strict+/// one-second budget, closes both transport halves, and drops its pending slot.+#[tokio::test]+async fn silent_peer_stop_auth_closes_socket_without_leaving_pending() {+ run_silent_peer_auth(false).await;+}++#[tokio::test]+async fn cancelled_auth_closes_socket_without_leaving_pending() {+ run_silent_peer_auth(true).await;+}++async fn run_silent_peer_auth(cancel_connect: bool) {+ let dir = tempfile::TempDir::new().expect("temp dir");+ let socket_path = dir.path().join("worker.sock");+ let agent_dir = dir.path().join("agent");+ std::fs::create_dir_all(&agent_dir).unwrap();+ let supervisor = Arc::new(+ Supervisor::new(SupervisorOptions {+ socket_path: dir.path().join("daemon.sock"),+ agent_dir,+ })+ .expect("supervisor"),+ );+ let descriptor: DaemonWorkerDescriptor = serde_json::from_value(json!({+ "version": 2, "workerId": "w-silent", "pid": 4242,+ "socketPath": socket_path.to_string_lossy(),+ "recoveryJournalPath": "/tmp/none.jsonl", "supervisorSocketPath": "/tmp/none.sock",+ "authenticationToken": "silent-token", "rootActiveSessionId": "none",+ "createdAt": "2026-09-23T00:00:00Z", "updatedAt": "2026-09-23T00:00:00Z",+ "lifecycle": "ready", "createCommand": {}, "consecutiveFailures": 0,+ }))+ .expect("descriptor");+ let resident = ResidentWorker::new(+ "w-silent".to_string(),+ descriptor,+ dir.path().join("w-silent.json"),+ );+ let listener = bind_fake_worker(&socket_path).await;+ let deadline = tokio::time::Instant::now() + Duration::from_millis(150);+ let connect = {+ let supervisor = Arc::clone(&supervisor);+ let resident = Arc::clone(&resident);+ tokio::spawn(async move {+ supervisor+ .connect_worker_for_stop(&resident, deadline)+ .await+ })+ };+ let mut fake = accept_fake_worker(listener).await;+ let frame = read_supervisor_frame(&mut fake).await;+ assert_eq!(frame.header.get("commandType"), Some(&json!("worker_auth")));+ if cancel_connect {+ connect.abort();+ assert!(connect.await.is_err(), "connect future must be cancelled");+ tokio::time::timeout(Duration::from_secs(1), async {+ while !resident.pending.lock().await.is_empty() {+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("cancelled auth pending cleanup");+ } else {+ let result = tokio::time::timeout(Duration::from_secs(1), connect)+ .await+ .expect("strict deadline")+ .expect("join");+ assert!(result.is_err(), "silent peer must time out");+ }+ assert!(+ resident.pending.lock().await.is_empty(),+ "auth pending slot must be cleared"+ );+ let mut reader = PrivateFrameReader::new(&mut fake.read_half, DEFAULT_PRIVATE_FRAME_LIMITS);+ let closed = tokio::time::timeout(Duration::from_secs(1), reader.read_frame())+ .await+ .expect("peer must close promptly")+ .expect("clean EOF");+ assert!(closed.is_none(), "failed auth must close the socket");+}+/// A registration that lands mid-handshake must not kill the launch:/// the registration's roster refresh routes onto its own authenticated/// channel once the handshake installs it, while the handshake'sdiff --git a/crates/pa-daemon/src/supervisor/supervision.rs b/crates/pa-daemon/src/supervisor/supervision.rsindex 8ada8bdce8..949e160531 100644--- a/crates/pa-daemon/src/supervisor/supervision.rs+++ b/crates/pa-daemon/src/supervisor/supervision.rs@@ -18,6 +18,36 @@ pub(super) const MAX_CONSECUTIVE_FAILURES: u32 = 5;pub(super) const STABLE_LIFETIME_MS: u64 = 30_000;const BASE_BACKOFF_MS: u64 = 250;const MAX_BACKOFF_MS: u64 = 30_000;++/// A failed/cancelled authentication must close both pump-owned socket halves.+struct PreAuthPumps {+ writer: tokio::task::JoinHandle<()>,+ reader: tokio::task::JoinHandle<()>,+ resident: Arc<ResidentWorker>,+ epoch: u64,+ installed: bool,+}++impl Drop for PreAuthPumps {+ fn drop(&mut self) {+ if !self.installed {+ self.writer.abort();+ self.reader.abort();+ self.resident.note_connection_lost(self.epoch);+ // A cancelled auth future skips route_command_on's timed-out+ // cleanup. Its dropped oneshot receiver identifies only this+ // attempt's pending slot, without disturbing newer routes.+ let resident = Arc::clone(&self.resident);+ tokio::spawn(async move {+ resident+ .pending+ .lock()+ .await+ .retain(|_, reply| !reply.is_closed());+ });+ }+ }+}/// The adopted-worker liveness poll used only where the kernel exit watch/// cannot register (no pidfd, descriptor exhaustion, Windows).const ADOPTED_EXIT_FALLBACK_POLL: Duration = Duration::from_secs(30);@@ -470,6 +500,25 @@ impl Supervisor {self: &Arc<Self>,resident: &Arc<ResidentWorker>,connect_deadline: tokio::time::Instant,+ ) -> Result<()> {+ self.connect_worker_with_auth_floor(resident, connect_deadline, true)+ .await+ }++ pub(super) async fn connect_worker_for_stop(+ self: &Arc<Self>,+ resident: &Arc<ResidentWorker>,+ connect_deadline: tokio::time::Instant,+ ) -> Result<()> {+ self.connect_worker_with_auth_floor(resident, connect_deadline, false)+ .await+ }++ async fn connect_worker_with_auth_floor(+ self: &Arc<Self>,+ resident: &Arc<ResidentWorker>,+ connect_deadline: tokio::time::Instant,+ auth_floor: bool,) -> Result<()> {let (socket_path, token) = {let descriptor = resident.descriptor.lock().await;@@ -478,9 +527,14 @@ impl Supervisor {descriptor.authentication_token.clone(),)};- let stream = connect_transport(&socket_path)- .await- .with_context(|| format!("connect worker socket {}", socket_path.display()))?;+ let stream = if auth_floor {+ connect_transport(&socket_path).await+ } else {+ tokio::time::timeout_at(connect_deadline, connect_transport(&socket_path))+ .await+ .map_err(|_| anyhow!("Session worker timed out"))?+ }+ .with_context(|| format!("connect worker socket {}", socket_path.display()))?;let (reader, mut writer) = stream.split();// Bounded at the in-flight capacity (the admission seam in `route_command` refuses// or waits before enqueueing): no unbounded channel on the request path.@@ -495,7 +549,7 @@ impl Supervisor {// Writer pump: holds only a weak resident reference — a strong one would keep// `cmd_tx` alive and park this pump forever after the resident is dropped.let writer_resident = Arc::downgrade(resident);- tokio::spawn(async move {+ let writer_pump = tokio::spawn(async move {while let Some(request) = cmd_rx.recv().await {let header = json!({"kind": "command",@@ -540,7 +594,7 @@ impl Supervisor {}});// Reader: route responses to pending requests, forward session events.- {+ let reader_pump = {let reader_resident = Arc::clone(resident);let reader_supervisor = Arc::clone(self);tokio::spawn(async move {@@ -703,8 +757,15 @@ impl Supervisor {}).await;}- });- }+ })+ };+ let mut pumps = PreAuthPumps {+ writer: writer_pump,+ reader: reader_pump,+ resident: Arc::clone(resident),+ epoch: connection_epoch,+ installed: false,+ };// The handshake owns the channel privately (TS `pendingClient`): any non-`worker_auth`// route winning the enqueue race would strand the handshake for the whole connect// budget. A pre-auth route finds no installed channel; the install is epoch-guarded.@@ -713,10 +774,17 @@ impl Supervisor {// Authenticate within the remaining connect budget (TS `handshakeBudgetMs`: probes,// connect, and auth share one deadline). A worker whose probes ate the whole budget// still proved it is alive, so the handshake always gets at least the auth floor.- let auth_budget_ms = connect_deadline+ let remaining_ms = connect_deadline.saturating_duration_since(tokio::time::Instant::now())- .as_millis()- .max(WORKER_AUTH_FLOOR_MS.into()) as u64;+ .as_millis() as u64;+ let auth_budget_ms = if auth_floor {+ remaining_ms.max(WORKER_AUTH_FLOOR_MS)+ } else {+ if remaining_ms == 0 {+ return Err(anyhow!("Session worker timed out"));+ }+ remaining_ms+ };let response = self.route_command_on_typed(resident,@@ -778,6 +846,7 @@ impl Supervisor {resident.peer_transport_capable.store(peer_transport_capable, Ordering::SeqCst);+ pumps.installed = true;Ok(())}}diff --git a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs b/crates/pa-daemon/src/supervisor/worker_lifecycle.rsindex 77813a3bac..3ee7d9a05b 100644--- a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs+++ b/crates/pa-daemon/src/supervisor/worker_lifecycle.rs@@ -34,21 +34,30 @@ impl Supervisor {// kill for the kill stop, shutdown for the resumable stop. A failed connect// degrades to the finalize below.resident.intentional_stop.store(true, Ordering::SeqCst);- if self- .connect_worker(resident, worker_connect_deadline())- .await- .is_ok()- {+ // TS stopWorker(force) bounds the graceful IPC leg to one+ // second before process escalation. Auth and stop share that+ // budget: the ordinary route's 30s timeout begins only AFTER+ // auth and cannot bound a silent listener in the auth phase.+ let deadline = tokio::time::Instant::now() + Duration::from_secs(1);+ if matches!(+ self.connect_worker_for_stop(resident, deadline).await,+ Ok(())+ ) {let command = if kill_stop { "kill" } else { "shutdown" };- let _ = self- .route_command_typed(- resident,- command,- json!({}),- ROUTE_TIMEOUT_MS,- RouteAdmission::SupervisorInternal,- )- .await;+ let remaining_ms = deadline+ .saturating_duration_since(tokio::time::Instant::now())+ .as_millis() as u64;+ if remaining_ms > 0 {+ let _ = self+ .route_command_typed(+ resident,+ command,+ json!({}),+ remaining_ms,+ RouteAdmission::SupervisorInternal,+ )+ .await;+ }}}// TS `scheduleWorkerStopFinalization`: the interrupted stop'sdiff --git a/crates/pa-daemon/src/supervisor_lost.rs b/crates/pa-daemon/src/supervisor_lost.rsindex b44a448bfe..cb8147e7ca 100644--- a/crates/pa-daemon/src/supervisor_lost.rs+++ b/crates/pa-daemon/src/supervisor_lost.rs@@ -86,9 +86,14 @@ async fn monitor(worker: Arc<Worker>) {}}-/// The TS give-up exit: dispose the session's kernel, persist the recovery journal,-/// remove the socket file, and end the process. The exit runs no destructors, so an-/// undisposed kernel would be orphaned here.+/// The TS give-up exit: dispose the session's kernel, persist the recovery+/// journal, close the listener and remove the worker's own socket file,+/// and end the process (the same sequence the routed `shutdown` command+/// runs; the monitor only reaches this with no session work in flight,+/// so nothing else needs to settle first). The kernel dispose is the TS+/// `shutdown(0)` close pass (`closeSession` -> runtime dispose ->+/// `IpythonKernelProvisioner.dispose`): the process exit runs no+/// destructors, so an undisposed kernel would be orphaned here.async fn exit_orphaned(worker: &Worker, absent_since: tokio::time::Instant) {eprintln!("pa-daemon worker: supervisor {} unreachable for {}s; exiting orphaned worker",@@ -99,14 +104,14 @@ async fn exit_orphaned(worker: &Worker, absent_since: tokio::time::Instant) {agent_engine.dispose_kernel().await;}let _ = worker.record_recovery(false, "shutdown");- // The bind-time identity (captured in `serve`) is the unlink's- // expected identity, so a REPLACED file at the path - a successor- // worker's live socket, the deterministic-path relaunch - survives- // this exit.- crate::socket::cleanup_socket_path(- &worker.config.socket_path,- worker.bound_socket_identity.lock().unwrap().clone(),- );+ // The listener closes FIRST and the cleanup probes the path with the+ // owner's bind provably released (the TS graceful-shutdown sequence,+ // shared with `exit_after_close`), so both a REPLACED file and a+ // POISONED capture (a replacement landing in the bind->capture+ // window) spare a successor worker's live socket - the+ // deterministic-path relaunch - while the worker's own dead file+ // still unlinks through the identity gate.+ worker.close_listener_then_cleanup_socket().await;std::process::exit(0);}diff --git a/crates/pa-daemon/src/worker.rs b/crates/pa-daemon/src/worker.rsindex 20a8f1f56d..2d5ff90246 100644--- a/crates/pa-daemon/src/worker.rs+++ b/crates/pa-daemon/src/worker.rs@@ -106,6 +106,17 @@ pub struct Worker {/// D-state-survivor late-exit edge). `None` until `serve` binds/// (named pipes keep `None`: there is no file to stat).pub(crate) bound_socket_identity: std::sync::Mutex<Option<crate::socket::SocketIdentity>>,+ /// The bound-listener close handshake (the TS graceful-shutdown+ /// sequence, daemon-mode.ts:8011-8018: `server.close()` is awaited+ /// FIRST, the socket cleanup runs after): an exiting path requests+ /// the close, the accept loop drops the listener it owns, and the+ /// exit proceeds only once the bind is provably released - which is+ /// what makes the exit cleanup's liveness probe sound (a live+ /// listener at the path afterwards can only be a successor's).+ pub(crate) listener_close_requested: tokio::sync::Notify,+ /// The accept loop's confirmation that it dropped the bound+ /// listener; see [`Worker::listener_close_requested`].+ pub(crate) listener_closed: tokio::sync::Notify,/// Supervisor self-registration handle; `None` for standalone workers.registration: Option<RegistrationHandle>,/// Live connections authenticated as the supervisor role; a non-zero@@ -704,6 +715,8 @@ impl Worker {Worker {config,bound_socket_identity: std::sync::Mutex::new(None),+ listener_close_requested: tokio::sync::Notify::new(),+ listener_closed: tokio::sync::Notify::new(),registration,supervisor_claims,core,@@ -753,26 +766,52 @@ impl Worker {)}- /// The durable tail of a successful close: the resume entry, the- /// worker's own socket cleanup, and the process exit. The routed- /// `shutdown` arm and the registration-retirement path share it- /// (`std::process::exit` runs no destructors, so the caller must- /// have settled the close first).- fn exit_after_close(&self) -> ! {+ /// Close the bound listener, then clean up the socket path: the TS+ /// graceful-shutdown sequence (daemon-mode.ts:8011-8018 awaits+ /// `server.close()` FIRST and runs `cleanupSocketPath()` after). The+ /// accept loop drops the listener it owns on the close request and+ /// confirms, so the cleanup below probes the path with the owner's+ /// listener provably closed - a live listener at the path can only+ /// be a successor's, and even a poisoned bind-time capture (a+ /// replacement landing in the bind->capture window) never unlinks+ /// the successor's live socket. The still-ours direction is+ /// unchanged: the worker's own closed file passes the probe dead and+ /// the identity gate unlinks exactly what it captured, so a respawn+ /// does not wait out the stale-socket path.+ ///+ /// The close confirmation is awaited UNCONDITIONALLY: a bound-flag+ /// check cannot close the check-then-act window between `serve`'s+ /// bind and the flag store (an exit landing exactly there would+ /// skip the wait and leave this worker's own dead socket behind -+ /// the refusal-exit variant of the stale-file bug). `serve` instead+ /// confirms exactly once on every path: after the accept loop drops+ /// the listener, or - with no listener - on the prepare/bind error+ /// returns. An exit that fires before the bind therefore waits out+ /// the whole setup and then unlinks only what the identity gate+ /// still owns, and a booting `serve` never strands a waiting exit+ /// path on a handshake that will not come. The parked confirmation+ /// also orders the identity read for the exits that fire inside+ /// `serve`'s setup (the registration-refusal exit racing the+ /// bind->capture gap): the capture precedes the accept loop's arm,+ /// which precedes this confirmation, so a bound listener's own+ /// cleanup always reads a captured identity, never the gap's+ /// `None`.+ pub(crate) async fn close_listener_then_cleanup_socket(&self) {+ self.listener_close_requested.notify_one();+ self.listener_closed.notified().await;+ let expected_identity = self.bound_socket_identity.lock().unwrap().clone();+ crate::socket::cleanup_socket_path_after_close(&self.config.socket_path, expected_identity);+ }++ /// listener close and the worker's own socket cleanup, and the+ /// process exit. The routed `shutdown` arm and the+ /// registration-retirement path share it (`std::process::exit` runs+ /// no destructors, so the caller must have settled the close first).+ async fn exit_after_close(&self) -> ! {// Shutdown keeps the resume entry and exits the process (TS// `closeKeepsResumeEntry("shutdown")`).let _ = self.record_recovery(false, "shutdown");- // A graceful exit owns its socket file: remove it now so a respawn- // does not wait out the stale-socket path (a killed worker cannot- // clean up, but its killer relaunches through- // `prepare_socket_path`). The bind-time identity (captured in- // `serve`) is the unlink's expected identity, so a REPLACED file- // at the path - a successor worker's live socket - survives this- // exit (TS daemon-mode.ts:1078-1080).- crate::socket::cleanup_socket_path(- &self.config.socket_path,- self.bound_socket_identity.lock().unwrap().clone(),- );+ self.close_listener_then_cleanup_socket().await;std::process::exit(0)}@@ -784,7 +823,7 @@ impl Worker {std::process::id());let _ = self.handle_shutdown().await;- self.exit_after_close();+ self.exit_after_close().await;}}diff --git a/crates/pa-daemon/src/worker/connection.rs b/crates/pa-daemon/src/worker/connection.rsindex a490b9af08..93555e2caa 100644--- a/crates/pa-daemon/src/worker/connection.rs+++ b/crates/pa-daemon/src/worker/connection.rs@@ -218,10 +218,37 @@ impl Worker {if !self.config.supervisor_socket_path.as_os_str().is_empty() {crate::supervisor_lost::start(self.clone());}- crate::socket::prepare_socket_path(&self.config.socket_path).await?;- let listener = bind_transport(&self.config.socket_path)- .await- .with_context(|| format!("bind worker socket {}", self.config.socket_path.display()))?;+ if let Err(error) = crate::socket::prepare_socket_path(&self.config.socket_path).await {+ // The no-listener confirmation of the close handshake: an+ // exit path already parked on the confirmation proceeds (its+ // `None` identity makes the cleanup a no-op) instead of+ // waiting on a listener that will never exist.+ self.listener_closed.notify_one();+ return Err(error);+ }+ let listener = match bind_transport(&self.config.socket_path).await {+ Ok(listener) => listener,+ Err(error) => {+ let error = error.context(format!(+ "bind worker socket {}",+ self.config.socket_path.display()+ ));+ self.listener_closed.notify_one();+ return Err(error);+ }+ };+ // From the bind on, the close handshake is fully asynchronous:+ // an exit path firing anywhere in the setup below (a+ // registration refusal parked in the capture gap, the orphan+ // monitor, a routed shutdown) parks on the confirmation, which+ // only this task emits - after the accept loop has consumed the+ // stored close request and dropped the listener. The close+ // request's permit stays stored while the loop has not armed+ // yet, so the loop consumes it as its first event, and the+ // confirmation always orders an exit path's identity read+ // behind the capture below: a bound listener's own cleanup+ // never reads the gap's `None` identity.+ crate::socket::bind_capture_gap().await;// Capture the bound file's identity before anything can replace// it (TS daemon-mode.ts:718, the listen callback, between the// identity capture and `restrictDaemonSocketPath`): the exit@@ -230,15 +257,28 @@ impl Worker {*self.bound_socket_identity.lock().unwrap() =crate::socket::socket_identity(&self.config.socket_path);crate::socket::restrict_socket_path(&self.config.socket_path);- loop {- let stream = match listener.accept().await {- Ok(accepted) => {- if std::env::var("PA_DAEMON_DEBUG").is_ok() {- eprintln!("[worker {}] accepted connection", std::process::id());+ // This loop owns the bound listener and hands its close to the+ // exit paths (TS daemon-mode.ts:8011-8018 awaits `server.close()`+ // FIRST and cleans the socket path after): an exiting path+ // requests the close, the loop drops the listener - releasing+ // the bind while every already-accepted connection keeps its own+ // socket - confirms, and parks; the exiting path owns the+ // process from its confirmation on. The arm order is safe+ // against wake loss: a close request that fires while an accept+ // is being handed off leaves its permit stored, and the next+ // loop iteration consumes it.+ let accept_error = loop {+ let stream = tokio::select! {+ accepted = listener.accept() => match accepted {+ Ok(accepted) => {+ if std::env::var("PA_DAEMON_DEBUG").is_ok() {+ eprintln!("[worker {}] accepted connection", std::process::id());+ }+ accepted}- accepted- }- Err(error) => return Err(anyhow!("worker accept: {error}")),+ Err(error) => break Some(anyhow!("worker accept: {error}")),+ },+ () = self.listener_close_requested.notified() => break None,};let worker = Arc::clone(&self);tokio::spawn(async move {@@ -246,7 +286,18 @@ impl Worker {eprintln!("pa-daemon worker connection error: {error:#}");}});+ };+ // The graceful close: drop the bound listener so the bind+ // releases, confirm to the exiting path, and park forever (the+ // parked task keeps the runtime - and so the in-flight+ // connections - alive until the exiting path's `process::exit`+ // ends them all).+ drop(listener);+ self.listener_closed.notify_one();+ if let Some(error) = accept_error {+ return Err(error);}+ std::future::pending::<Result<()>>().await}async fn handle_connection(self: Arc<Self>, stream: Box<dyn TransportStream>) -> Result<()> {@@ -439,7 +490,7 @@ impl Worker {self.write_response_frame(&sink, &request_id, response).await;if success {- self.exit_after_close();+ self.exit_after_close().await;}continue;}diff --git a/crates/pa-daemon/tests/close_listener_cleanup_e2e.rs b/crates/pa-daemon/tests/close_listener_cleanup_e2e.rsnew file mode 100644index 0000000000..b1c73df9c0--- /dev/null+++ b/crates/pa-daemon/tests/close_listener_cleanup_e2e.rs@@ -0,0 +1,502 @@+//! Poisoned-capture exit oracles (the #3291 residual, the registered+//! follow-up this lane closes): the bind-time identity capture sits AFTER+//! the bind it guards - a replacement landing in the MICROSECOND+//! bind->capture window poisons the capture (it stores the successor's+//! file identity as the owner's own), and the poisoned identity gate+//! later unlinks the SUCCESSOR'S live socket at the owner's exit. The+//! replacement-window direction (a replacement landing after the+//! capture) is pinned by `socket_identity_guard_e2e`; these oracles pin+//! the poisoned direction at the two process shapes whose exit paths+//! run the gate:+//!+//! * the worker's routed `shutdown` (`worker.rs` `exit_after_close`), and+//! * the supervisor's SIGTERM graceful drain (`supervisor.rs` `run`'s+//! tail; the worker's orphan exit shares the same cleanup choreography).+//!+//! The fix's shape is the TS graceful-shutdown precedent: TS awaits+//! `server.close()` FIRST and cleans the socket path after+//! (daemon-mode.ts:8011-8018, and the supervisor's+//! daemon-supervisor.ts:7436-7491 awaits its "daemon server" close step+//! before its "daemon socket" cleanup step). The Rust port ran all three+//! exit cleanups with the owner's listener still live, which is why a+//! cleanup-side liveness probe was ruled out there (it would refuse the+//! still-ours unlink too); once the listener closes first, a LIVE+//! listener at the path can only be a successor's, so the probe-then-gate+//! cleanup never removes a live socket even under a poisoned capture.+//!+//! The poisoning is made deterministic with the bind-capture gap seam+//! (`PA_DAEMON_BIND_CAPTURE_GAP_MS`, the `PA_DAEMON_EVENT_LOG` seam+//! family): the oracle parks the owner between its bind and its capture+//! with the successor bound at the path, so the capture provably stores+//! the successor's identity (the same file the race would poison, no+//! timing luck involved). The owner is then driven through its REAL exit+//! path - the routed `shutdown` reply, the SIGTERM graceful drain - and+//! the successor's live socket must survive.+//!+//! On a tree WITHOUT the close-then-cleanup fix these oracles fail:+//! the poisoned gate matches the successor's live file and unlinks it.+#![cfg(unix)]++use std::io::{BufRead, BufReader, Read, Write};+use std::os::unix::fs::MetadataExt;+use std::os::unix::net::{UnixListener, UnixStream};+use std::path::{Path, PathBuf};+use std::process::{Child, Command, Stdio};+use std::time::{Duration, Instant};++use serde_json::{json, Value};++/// The bind-capture gap the oracles set: wide enough that the replacement+/// provably lands inside the window, small enough to keep the oracle fast.+const BIND_CAPTURE_GAP_MS: &str = "2000";++/// The filesystem identity the gate compares: dev + ino of the file+/// currently at the path.+fn socket_identity(path: &Path) -> (u64, u64) {+ let metadata = std::fs::symlink_metadata(path).expect("stat socket file");+ (metadata.dev(), metadata.ino())+}++/// Move the bound socket file aside, returning the aside path: the bound+/// listener keeps serving through the rename (the kernel tracks the inode,+/// not the name), so the oracle can still drive the owner through its+/// real exit path while the successor owns the original path.+fn rename_bound_socket_aside(socket: &Path) -> PathBuf {+ let aside: PathBuf = socket.parent().expect("socket parent").join(format!(+ "{}.poisoned-aside",+ socket+ .file_name()+ .expect("socket file name")+ .to_string_lossy()+ ));+ std::fs::rename(socket, &aside).expect("rename the bound socket file aside");+ aside+}++/// Bind the successor at the owner's original path while the owner's own+/// capture is still pending: the capture then reads THIS file's identity+/// as the owner's own - the poisoned state the microsecond race+/// produces. The original inode stays allocated (renamed aside), so the+/// identities provably differ.+fn bind_poisoning_successor(socket: &Path) -> (UnixListener, (u64, u64)) {+ let successor = UnixListener::bind(socket).expect("bind the poisoning successor");+ let identity = socket_identity(socket);+ (successor, identity)+}++/// A live successor's socket must answer connects after the poisoned+/// owner's exit: the file survived AND it still serves.+fn assert_successor_serves(socket: &Path, identity: (u64, u64)) {+ assert!(+ socket.exists(),+ "the successor's live socket survived the poisoned owner's exit"+ );+ assert_eq!(+ socket_identity(socket),+ identity,+ "the file at the path is still the successor's, not a new file"+ );+ let connect = UnixStream::connect(socket).expect("the successor socket still accepts");+ drop(connect);+}++/// Kill a leftover process at scope exit (the oracle children exit on+/// their own; this is the teardown safety net).+struct ProcessGuard {+ child: Child,+}++impl Drop for ProcessGuard {+ fn drop(&mut self) {+ let _ = self.child.kill();+ let _ = self.child.wait();+ }+}++/// Wait until the child exits on its own within `budget`, asserting a+/// clean exit: the exit paths under test end in `std::process::exit(0)`+/// (or a graceful supervisor return), so a signal death or a nonzero+/// exit means the wrong path ran.+fn wait_clean_exit(child: &mut Child, budget: Duration) -> std::process::ExitStatus {+ let deadline = Instant::now() + budget;+ loop {+ if let Some(status) = child.try_wait().expect("poll child") {+ assert!(+ status.success(),+ "the exit path must end in a clean exit 0, not {status}"+ );+ return status;+ }+ assert!(Instant::now() < deadline, "the child did not exit");+ std::thread::sleep(Duration::from_millis(50));+ }+}++fn wait_socket_file(socket: &Path) {+ let deadline = Instant::now() + Duration::from_secs(15);+ while !socket.exists() {+ assert!(Instant::now() < deadline, "socket file never appeared");+ std::thread::sleep(Duration::from_millis(20));+ }+}++fn spawn_supervisor(socket: &Path, agent_dir: &Path) -> ProcessGuard {+ std::fs::create_dir_all(agent_dir).expect("agent dir");+ let child = Command::new(env!("CARGO_BIN_EXE_pa-daemon"))+ .arg("supervisor")+ .arg("--socket")+ .arg(socket)+ .arg("--agent-dir")+ .arg(agent_dir)+ // Hermetic agent dir: the ambient environment may export a real+ // agent dir; point every fallback at the test sandbox instead.+ .env("PRIME_AGENT_CODING_AGENT_DIR", agent_dir)+ .env_remove("PA_API_KEY")+ .env(pa_daemon::socket::BIND_CAPTURE_GAP_ENV, BIND_CAPTURE_GAP_MS)+ .stdin(Stdio::null())+ .stdout(Stdio::null())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn pa-daemon supervisor");+ ProcessGuard { child }+}++/// Spawn the real `pa-daemon worker` binary on its own socket, monitoring+/// a supervisor socket that never answers, with the bind-capture gap armed+/// so the oracle can poison the worker's capture.+fn spawn_worker(dir: &Path, socket: &Path, supervisor_socket: &Path, token: &str) -> ProcessGuard {+ std::fs::create_dir_all(dir.join("agent")).expect("agent dir");+ let child = Command::new(env!("CARGO_BIN_EXE_pa-daemon"))+ .arg("worker")+ .env(pa_daemon::worker::WORKER_ROLE_ENV, "1")+ .env(pa_daemon::worker::WORKER_TOKEN_ENV, token)+ .env(+ pa_daemon::worker::WORKER_ACTIVE_SESSION_ID_ENV,+ "poisoned-capture",+ )+ .env(pa_daemon::worker::WORKER_SOCKET_ENV, socket)+ .env(+ pa_daemon::worker::WORKER_SUPERVISOR_SOCKET_ENV,+ supervisor_socket,+ )+ .env(+ pa_daemon::worker::WORKER_RECOVERY_JOURNAL_ENV,+ dir.join("recovery.jsonl"),+ )+ .env("PRIME_AGENT_CODING_AGENT_DIR", dir.join("agent"))+ .env(pa_daemon::socket::BIND_CAPTURE_GAP_ENV, BIND_CAPTURE_GAP_MS)+ .stdin(Stdio::null())+ .stdout(Stdio::null())+ .stderr(Stdio::null())+ .spawn()+ .expect("spawn pa-daemon worker");+ ProcessGuard { child }+}++/// A supervisor stand-in that definitively rejects the worker's one+/// registration (the `Unknown session worker` verdict, TS+/// `adopt_registered_worker`'s unknown-worker error): the registration+/// loop treats it as terminal, the refused-registration self-heal+/// retires the worker, and the retirement exit runs the real+/// close-then-cleanup choreography under test. The socket must exist+/// before the worker spawns; the exchange runs on its own thread. The+/// registration protocol is hello first, register second, reply last+/// (`connect_and_register` reads the hello before it writes).+fn spawn_rejecting_supervisor(socket: &Path) {+ let listener = UnixListener::bind(socket).expect("bind the rejecting supervisor");+ std::thread::spawn(move || {+ let (stream, _) = listener.accept().expect("accept the registration");+ let mut write_half = stream.try_clone().expect("clone the registration");+ let _ = write_half.write_all(br#"{"type":"daemon_hello"}"#);+ let _ = write_half.write_all(b"\n");+ let _ = write_half.flush();+ let mut line = String::new();+ let mut reader = BufReader::new(stream);+ reader+ .read_line(&mut line)+ .expect("read the worker registration");+ let request_id = serde_json::from_str::<Value>(&line)+ .expect("registration envelope")+ .get("id")+ .and_then(Value::as_str)+ .expect("registration id")+ .to_string();+ let rejection = json!({+ "id": request_id,+ "success": false,+ "error": "Unknown session worker: the supervisor holds no descriptor for this identity",+ });+ let _ = write_half.write_all(format!("{rejection}\n").as_bytes());+ let _ = write_half.flush();+ });+}++/// Read one JSONL line from the supervisor's aside socket within the+/// deadline: a served hello proves the supervisor's accept loop (and so+/// the capture before it) has completed - the poisoned state is armed.+fn supervisor_hello_line(socket: &Path) -> Value {+ let stream = UnixStream::connect(socket).expect("connect the supervisor aside socket");+ stream+ .set_read_timeout(Some(Duration::from_millis(100)))+ .expect("set timeout");+ let mut reader = BufReader::new(stream);+ let deadline = Instant::now() + Duration::from_secs(15);+ let mut line = String::new();+ loop {+ assert!(+ Instant::now() < deadline,+ "the poisoned supervisor never served its hello"+ );+ match reader.read_line(&mut line) {+ Ok(0) | Err(_) => {+ line.clear();+ std::thread::sleep(Duration::from_millis(20));+ }+ Ok(_) => return serde_json::from_str(&line).expect("hello line"),+ }+ }+}++/// A raw private-frame client for the worker's aside socket (the same+/// wire the supervisor's request pump speaks).+struct WorkerClient {+ stream: UnixStream,+}++impl WorkerClient {+ fn connect(socket: &Path, token: &str) -> Self {+ let stream = UnixStream::connect(socket).expect("connect the worker aside socket");+ stream+ .set_read_timeout(Some(Duration::from_millis(100)))+ .expect("set timeout");+ let mut client = WorkerClient { stream };+ let (header, _payload) = client.read_frame();+ assert_eq!(header["outboundType"], "daemon_hello", "worker hello");+ // The supervisor role both arms the routed `shutdown` arm and+ // disarms the orphan monitor (claims > 0), so the exit below is+ // `exit_after_close` alone.+ let auth = client.request(+ "worker_auth",+ &json!({+ "token": token,+ "supervisorGeneration": "sup:poisoned-capture",+ "supervisorPid": 1,+ "supervisorSocketPath": "/nonexistent/supervisor.sock",+ }),+ );+ assert_eq!(auth["success"], true, "worker auth failed: {auth}");+ client+ }++ fn send_frame(&mut self, header: &Value, payload: &Value) {+ let frame = pa_daemon::framing::encode_private_frame(+ header,+ &serde_json::to_vec(payload).expect("payload"),+ pa_daemon::framing::DEFAULT_PRIVATE_FRAME_LIMITS,+ )+ .expect("encode frame");+ self.stream.write_all(&frame).expect("write frame");+ self.stream.flush().expect("flush");+ }++ fn request(&mut self, command_type: &str, payload: &Value) -> Value {+ let request_id = format!("req-{command_type}");+ self.send_frame(+ &json!({+ "kind": "command",+ "requestId": request_id,+ "commandType": command_type,+ }),+ payload,+ );+ let deadline = Instant::now() + Duration::from_secs(20);+ loop {+ assert!(Instant::now() < deadline, "no response for {command_type}");+ let (header, body) = self.read_frame();+ if header["outboundType"].as_str() == Some("response")+ && header["requestId"].as_str() == Some(&request_id)+ {+ return serde_json::from_slice(&body).expect("response body");+ }+ }+ }++ fn read_frame(&mut self) -> (Value, Vec<u8>) {+ let deadline = Instant::now() + Duration::from_secs(20);+ let mut prefix = [0u8; 8];+ read_exact_timeout(&mut self.stream, &mut prefix, deadline);+ let header_len = u32::from_be_bytes(prefix[0..4].try_into().unwrap()) as usize;+ let payload_len = u32::from_be_bytes(prefix[4..8].try_into().unwrap()) as usize;+ let mut header = vec![0u8; header_len];+ read_exact_timeout(&mut self.stream, &mut header, deadline);+ let mut payload = vec![0u8; payload_len];+ read_exact_timeout(&mut self.stream, &mut payload, deadline);+ let header: Value = serde_json::from_slice(&header).expect("frame header");+ (header, payload)+ }+}++fn read_exact_timeout(stream: &mut UnixStream, buffer: &mut [u8], deadline: Instant) {+ let mut read = 0usize;+ while read < buffer.len() {+ assert!(Instant::now() < deadline, "worker frame read timed out");+ match stream.read(&mut buffer[read..]) {+ Ok(0) => panic!("worker closed the connection mid-frame"),+ Ok(n) => read += n,+ Err(error)+ if error.kind() == std::io::ErrorKind::WouldBlock+ || error.kind() == std::io::ErrorKind::TimedOut => {}+ Err(error) => panic!("worker read: {error}"),+ }+ }+}++/// The poisoned worker's routed shutdown: a replacement bound at the path+/// inside the worker's bind->capture window poisons the captured+/// identity, the worker's real exit path runs, and the successor's LIVE+/// socket at the path must survive it.+#[test]+fn a_poisoned_workers_shutdown_exit_spares_the_successors_live_socket() {+ let dir = tempfile::TempDir::new().expect("temp dir");+ let socket = dir.path().join("worker.sock");+ let mut worker = spawn_worker(+ dir.path(),+ &socket,+ &dir.path().join("absent-supervisor.sock"),+ "poisoned-shutdown-token",+ );+ // The socket file exists: the bind succeeded and the worker is+ // parked in the bind-capture gap. Poison the capture with the+ // successor bound at the original path, then drive the worker+ // through its aside path (the bound listener keeps serving the+ // renamed inode).+ wait_socket_file(&socket);+ let aside = rename_bound_socket_aside(&socket);+ let (_successor, successor_identity) = bind_poisoning_successor(&socket);+ // The served hello through the aside path proves the capture has+ // completed on the poisoned file (the accept loop starts after it):+ // the poisoned state is armed, not assumed.+ let mut client = WorkerClient::connect(&aside, "poisoned-shutdown-token");+ let shutdown = client.request("shutdown", &json!({}));+ assert_eq!(shutdown["success"], true, "shutdown failed: {shutdown}");+ wait_clean_exit(&mut worker.child, Duration::from_secs(10));+ assert_successor_serves(&socket, successor_identity);+}++/// The poisoned supervisor's SIGTERM graceful drain: a replacement bound+/// at the path inside the supervisor's bind->capture window poisons the+/// captured identity, the real drain exit runs, and the successor's LIVE+/// socket at the path must survive it.+#[test]+fn a_poisoned_supervisors_drain_spares_the_successors_live_socket() {+ let dir = tempfile::TempDir::new().expect("temp dir");+ let socket = dir.path().join("daemon.sock");+ let agent_dir = dir.path().join("agent");+ let mut daemon = spawn_supervisor(&socket, &agent_dir);+ wait_socket_file(&socket);+ let aside = rename_bound_socket_aside(&socket);+ let (_successor, successor_identity) = bind_poisoning_successor(&socket);+ // A served hello through the aside path proves the capture has+ // completed on the poisoned file before the drain fires.+ let hello = supervisor_hello_line(&aside);+ assert_eq!(hello["type"], "daemon_hello", "supervisor hello: {hello}");+ let _ = Command::new("kill")+ .arg("-TERM")+ .arg(daemon.child.id().to_string())+ .status()+ .expect("SIGTERM the supervisor");+ wait_clean_exit(&mut daemon.child, Duration::from_secs(10));+ assert_successor_serves(&socket, successor_identity);+}++/// The refused-registration retirement inside the bind->capture gap (the+/// registration rejection lands while the worker is parked in the+/// fault-injection seam, so the exit path's identity is still `None`+/// when it fires): the exit must WAIT for the serve handshake to resume,+/// capture, arm, and drop the listener - and then remove the worker's+/// OWN now-dead socket file (the registration-refusal arm of the+/// close-then-cleanup choreography; a flag-gated wait that skips the+/// handshake in that window preserves the live file and strands a stale+/// socket). The refusal is driven through the real+/// `exit_refused_registration` path by a supervisor stand-in that+/// definitively rejects the registration.+///+/// Linux-only: `unix_listener_definitely_closed` rules `ECONNREFUSED`+/// definitive there alone (the unit-test platform split mirrors this),+/// so the refusal exit unlinks its own dead file on Linux and+/// conservatively preserves it everywhere else.+#[cfg(target_os = "linux")]+#[test]+fn a_refused_registrations_exit_removes_the_workers_own_dead_socket() {+ let dir = tempfile::TempDir::new().expect("temp dir");+ let socket = dir.path().join("worker.sock");+ let supervisor_socket = dir.path().join("rejecting-supervisor.sock");+ spawn_rejecting_supervisor(&supervisor_socket);+ let mut worker = spawn_worker(+ dir.path(),+ &socket,+ &supervisor_socket,+ "refused-registration-token",+ );+ wait_socket_file(&socket);+ // The rejection retires the worker while it is parked in the gap;+ // the exit parks on the close confirmation until the serve task+ // resumes, then unlinks the worker's own dead file.+ wait_clean_exit(&mut worker.child, Duration::from_secs(10));+ assert!(+ !socket.exists(),+ "the refused worker's own dead socket file is removed, not left stale"+ );+}++/// Off Linux the refused worker's dead file is CONSERVATIVELY+/// PRESERVED (a saturated BSD/macOS backlog also refuses the probe+/// connect, so `ECONNREFUSED` never proves closure there): the refusal+/// exit must leave the file alone rather than guess - the next bind's+/// stale-socket prepare is what clears it.+#[cfg(not(target_os = "linux"))]+#[test]+fn a_refused_registrations_exit_preserves_its_dead_file_off_linux() {+ let dir = tempfile::TempDir::new().expect("temp dir");+ let socket = dir.path().join("worker.sock");+ let supervisor_socket = dir.path().join("rejecting-supervisor.sock");+ spawn_rejecting_supervisor(&supervisor_socket);+ let mut worker = spawn_worker(+ dir.path(),+ &socket,+ &supervisor_socket,+ "refused-registration-token",+ );+ wait_socket_file(&socket);+ wait_clean_exit(&mut worker.child, Duration::from_secs(10));+ assert!(+ socket.exists(),+ "off Linux the refusal exit never claims the dead file from the probe alone"+ );+}++/// The same refusal exit under a poisoned capture: a successor bound at+/// the path inside the bind->capture gap poisons the worker's captured+/// identity (it names the successor's live inode), and the retirement+/// exit must still spare that successor - the liveness probe refuses the+/// unlink even though the poisoned identity gate would match. This pins+/// the refusal exit to the same successor-protection contract the+/// routed-shutdown exit carries.+#[test]+fn a_refused_registrations_exit_spares_the_poisoning_successor() {+ let dir = tempfile::TempDir::new().expect("temp dir");+ let socket = dir.path().join("worker.sock");+ let supervisor_socket = dir.path().join("rejecting-supervisor.sock");+ spawn_rejecting_supervisor(&supervisor_socket);+ let mut worker = spawn_worker(+ dir.path(),+ &socket,+ &supervisor_socket,+ "refused-poisoned-token",+ );+ wait_socket_file(&socket);+ let _aside = rename_bound_socket_aside(&socket);+ let (_successor, successor_identity) = bind_poisoning_successor(&socket);+ wait_clean_exit(&mut worker.child, Duration::from_secs(10));+ assert_successor_serves(&socket, successor_identity);+}diff --git a/crates/pa-daemon/tests/session_stop_lifecycle_e2e.rs b/crates/pa-daemon/tests/session_stop_lifecycle_e2e.rsindex fc54b145fd..21c4994a28 100644--- a/crates/pa-daemon/tests/session_stop_lifecycle_e2e.rs+++ b/crates/pa-daemon/tests/session_stop_lifecycle_e2e.rs@@ -24,6 +24,7 @@#![cfg(unix)]use std::io::{BufRead, BufReader, Write};+use std::os::unix::net::UnixListener;use std::path::{Path, PathBuf};use std::process::{Child, Command, Stdio};use std::time::{Duration, Instant};@@ -432,6 +433,12 @@ fn kill_cancels_goal_and_heartbeat_and_no_wake_revives_the_session() {let _ = Command::new("kill").arg("-9").arg(pid.to_string()).status();}std::fs::write(&planted_path, planted_content).expect("replant the tombstoned descriptor");+ // An unrelated process can claim the stopped worker's stale socket+ // pathname. Its connectable listener must not make the DEAD descriptor+ // a live worker: the recorded pid/start identity is the authority.+ let stopped_socket = PathBuf::from(tombstoned["socketPath"].as_str().expect("socket path"));+ std::fs::remove_file(&stopped_socket).ok();+ let _silent_impostor = UnixListener::bind(&stopped_socket).expect("bind silent impostor");std::fs::remove_file(&socket).ok();let supervisor = spawn_supervisor(&socket, &agent_dir);diff --git a/crates/pa-types/Cargo.toml b/crates/pa-types/Cargo.tomlindex dc623282f7..5e70d17e4b 100644--- a/crates/pa-types/Cargo.toml+++ b/crates/pa-types/Cargo.toml@@ -12,6 +12,7 @@ thiserror = { workspace = true }anyhow = { workspace = true }tokio = { workspace = true }libc = "0.2"+socket2 = "0.6"# The incident classifier's log-message patterns are the TS regexes# (src/cli/incident.ts); the `regex` crate keeps their semantics 1:1.regex = "1"diff --git a/crates/pa-types/src/platform/transport.rs b/crates/pa-types/src/platform/transport.rsindex 72e85573cb..3607489101 100644--- a/crates/pa-types/src/platform/transport.rs+++ b/crates/pa-types/src/platform/transport.rs@@ -161,6 +161,47 @@ pub async fn connect_transport(path: &Path) -> Result<Box<dyn TransportStream>>Ok(Box::new(stream))}+/// Definitive nonblocking Unix listener refusal, used by the exit cleanup.+/// A saturated backlog can yield `EAGAIN` on Linux rather than a completed+/// connect. Only `ConnectionRefused` permits unlink; all unknown outcomes fail+/// closed. Reuse the transport's long-path re-anchor so deep socket paths+/// retain their stale-cleanup behavior.+///+/// The refused verdict is definitive on Linux only: on macOS and the BSDs a+/// saturated listen queue ALSO refuses the connect, so a live successor that+/// filled its backlog is indistinguishable from a dead path by this probe.+/// There the function never claims closure (the exit cleanup preserves the+/// path; the next bind's stale-socket prepare cleans a file nothing serves).+#[cfg(unix)]+#[must_use]+pub fn unix_listener_definitely_closed(path: &Path) -> bool {+ #[cfg(not(target_os = "linux"))]+ {+ let _ = path;+ return false;+ }+ #[cfg(target_os = "linux")]+ {+ let Ok(effective) = UnixSocketAddress::new(path) else {+ return false;+ };+ let Ok(address) = socket2::SockAddr::unix(effective.effective()) else {+ return false;+ };+ let Ok(socket) = socket2::Socket::new(socket2::Domain::UNIX, socket2::Type::STREAM, None)+ else {+ return false;+ };+ if socket.set_nonblocking(true).is_err() {+ return false;+ }+ matches!(+ socket.connect(&address),+ Err(error) if error.kind() == std::io::ErrorKind::ConnectionRefused+ )+ }+}+#[cfg(windows)]impl TransportListener for super::windows_pipe::NamedPipeListener {fn accept(&self) -> AcceptFuture<'_> {