A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
settle user bash process trees and command slots (#3541)
kt53e13b89e14b
diff --git a/crates/pa-daemon/src/supervisor_lost.rs b/crates/pa-daemon/src/supervisor_lost.rsindex 035ece10f8..6822256f4d 100644--- a/crates/pa-daemon/src/supervisor_lost.rs+++ b/crates/pa-daemon/src/supervisor_lost.rs@@ -76,10 +76,12 @@ async fn monitor(worker: Arc<Worker>) {if since.elapsed() < window {continue;}- let ongoing = worker.core.lock().unwrap().has_ongoing_work();+ let ongoing =+ worker.core.lock().unwrap().has_ongoing_work() || worker.user_bash.is_running();if ongoing {- // TS `hasOngoingSessionWork`: an active run owns the worker a little- // longer; its turn end lets the next check reconsider.+ // TS `hasOngoingSessionWork` (`isSessionActive`, which counts+ // `isBashRunning`): an active run or a running user bash owns the+ // worker a little longer; its end lets the next check reconsider.continue;}exit_orphaned(&worker, since).await;diff --git a/crates/pa-daemon/src/user_bash.rs b/crates/pa-daemon/src/user_bash.rsindex 0bd56a7443..8cb48ea304 100644--- a/crates/pa-daemon/src/user_bash.rs+++ b/crates/pa-daemon/src/user_bash.rs@@ -10,11 +10,18 @@ use std::sync::Arc;use serde_json::{json, Map, Value};use tokio::io::{AsyncRead, AsyncReadExt};-use tokio::sync::Mutex;+use tokio::sync::Notify;+use tokio_util::sync::CancellationToken;use crate::protocol::{response_failure, response_success, DaemonResponse};use crate::worker::Worker;+/// How long a run still reads output after its shell exits (TS `EXIT_STDIO_GRACE_MS`): a+/// background process holding the inherited pipes must not hold the run open.+const EXIT_STDIO_GRACE: std::time::Duration = std::time::Duration::from_millis(100);+/// How long a closing session waits for its aborted runs to settle (TS+/// `UPDATE_RESTART_ABORT_BASH_TIMEOUT_MS`).+const CLOSE_SETTLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);/// Streaming window: chunks are retained (spilled to disk beyond this).const DEFAULT_MAX_BYTES: usize = 50 * 1024;/// Streaming retention cap: the in-memory tail (TS `maxOutputBytes`).@@ -32,10 +39,10 @@ pub(crate) struct UserBash {/// Awaited bash runs in flight (TS `_bashAbortControllers.size`):/// they count toward `isBashRunning` without claiming the slot.awaited: AtomicUsize,- /// An abort was requested: the settled result reports cancelled.- abort_requested: AtomicBool,- /// The in-flight process (user bash or an awaited run): `abort_bash` kills it.- child: Mutex<Option<tokio::process::Child>>,+ /// Each invocation owns its cancellation state; a later start cannot clear an earlier abort.+ controllers: std::sync::Mutex<Vec<std::sync::Weak<CancellationToken>>>,+ /// Signals each settle (the slot's release or an awaited run's end).+ settled: Notify,}impl UserBash {@@ -43,26 +50,26 @@ impl UserBash {Self {running: AtomicBool::new(false),awaited: AtomicUsize::new(0),- abort_requested: AtomicBool::new(false),- child: Mutex::new(None),+ controllers: std::sync::Mutex::new(Vec::new()),+ settled: Notify::new(),}}- /// Claim the slot; `false` when a user command is already running. A fresh claim clears a stale- /// abort request (TS clears at each start), so a leftover flag cannot cancel a later run.- fn claim(&self) -> bool {- let claimed = self- .running- .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)- .is_ok();- if claimed {- self.abort_requested.store(false, Ordering::SeqCst);+ /// Claim and register synchronously, before the spawned task can race an abort.+ fn claim(&self) -> Option<Arc<CancellationToken>> {+ let mut controllers = self.controllers.lock().unwrap();+ if self.running.swap(true, Ordering::SeqCst) {+ return None;}- claimed+ let cancellation = Arc::new(CancellationToken::new());+ controllers.retain(|controller| controller.strong_count() > 0);+ controllers.push(Arc::downgrade(&cancellation));+ Some(cancellation)}fn release(&self) {self.running.store(false, Ordering::SeqCst);+ self.settled.notify_waiters();}/// Whether a user bash or an awaited bash run is in flight (TS `isBashRunning`).@@ -73,18 +80,48 @@ impl UserBash {/// Count one awaited run (`execute_bash_and_wait`) toward [`Self::is_running`]; the returned/// bracket decrements on drop, so a dropped future cannot leave the count stuck on.pub(crate) fn begin_awaited(&self) -> AwaitedRun<'_> {+ let mut controllers = self.controllers.lock().unwrap();+ let cancellation = Arc::new(CancellationToken::new());+ controllers.retain(|controller| controller.strong_count() > 0);+ controllers.push(Arc::downgrade(&cancellation));self.awaited.fetch_add(1, Ordering::SeqCst);- AwaitedRun { user_bash: self }+ AwaitedRun {+ user_bash: self,+ cancellation,+ }}- /// Kill the in-flight process (TS `abortBash` aborts every- /// controller); the settled runs report cancelled.- pub(crate) async fn abort(&self) {- self.abort_requested.store(true, Ordering::SeqCst);- let mut child = self.child.lock().await;- if let Some(child) = child.as_mut() {- let _ = child.start_kill();- }+ /// Notify every in-flight run to kill its process tree (TS `abortBash`+ /// aborts every controller); the settled runs report cancelled.+ pub(crate) fn abort(&self) {+ let mut controllers = self.controllers.lock().unwrap();+ controllers.retain(|controller| {+ if let Some(controller) = controller.upgrade() {+ controller.cancel();+ true+ } else {+ false+ }+ });+ }++ /// [`Self::abort`], then wait (bounded) until no run is in flight: each run kills its tree+ /// in its own task, so a close must not exit ahead of the kills (TS `abortBashForClose`).+ pub(crate) async fn abort_and_settle(&self) {+ self.abort();+ let settled = async {+ loop {+ // Armed before the check, so a settle in between still wakes this wait.+ let notified = self.settled.notified();+ tokio::pin!(notified);+ notified.as_mut().enable();+ if !self.is_running() {+ return;+ }+ notified.await;+ }+ };+ let _ = tokio::time::timeout(CLOSE_SETTLE_TIMEOUT, settled).await;}}@@ -165,15 +202,14 @@ impl Worker {);};// Claim the slot synchronously (TS claims before the async dispatch- // could slip a second command through).- if !self.user_bash.claim() {- return response_failure(- None,- "execute_bash",- "A bash command is already running",- None,- );- }+ // could slip a second command through), through the admission gate:+ // the gate re-check and the claim share the lock, so the close's+ // abort cannot miss the new token (a claim its `abort_and_settle`+ // missed would leave the child running past the exit).+ let cancellation = match self.admit_streamed_bash() {+ Ok(cancellation) => cancellation,+ Err(response) => return response,+ };let exclude_from_context = payload.get("excludeFromContext").and_then(Value::as_bool)@@ -215,7 +251,7 @@ impl Worker {cwd: &cwd,prefix: prefix.as_deref(),shell_path: shell_path.as_deref(),- user_bash: &user_bash,+ cancellation: &cancellation,on_chunk: Some((Arc::clone(&core), Arc::clone(&events))),}).await;@@ -257,6 +293,55 @@ impl Worker {response_success(None, "execute_bash", None)}+ /// `require_created`'s shutdown gate re-checked with the streamed slot+ /// claim under the same lock: the claim cannot interleave with the gate's+ /// close, so the close's abort always sees — and cancels — the new token+ /// (a claim its `abort_and_settle` missed would leave the child running+ /// past the exit).+ // DaemonResponse is the wire response struct and is deliberately wide; the+ // error channel here carries the whole response, so allow the large-err lint.+ #[allow(clippy::result_large_err)]+ pub(crate) fn admit_streamed_bash(&self) -> Result<Arc<CancellationToken>, DaemonResponse> {+ let core = self.core.lock().unwrap();+ if core.shutdown_requested {+ return Err(response_failure(+ None,+ "execute_bash",+ "Session is shutting down",+ None,+ ));+ }+ self.user_bash.claim().ok_or_else(|| {+ response_failure(+ None,+ "execute_bash",+ "A bash command is already running",+ None,+ )+ })+ }++ /// `require_created`'s shutdown gate re-checked and the awaited run+ /// registered under the same lock: the registration cannot interleave+ /// with the gate's close, so the close's abort always sees — and cancels —+ /// the new token (a registration its `abort_and_settle` missed would let+ /// a child outlive the exit).+ // DaemonResponse is the wire response struct and is deliberately wide; the+ // error channel here carries the whole response, so allow the large-err lint.+ #[allow(clippy::result_large_err)]+ pub(crate) fn admit_awaited_bash(&self) -> Result<AwaitedRun<'_>, DaemonResponse> {+ let core = self.core.lock().unwrap();+ if core.shutdown_requested {+ return Err(response_failure(+ None,+ "execute_bash_and_wait",+ "Session is shutting down",+ None,+ ));+ }+ Ok(self.user_bash.begin_awaited())+ }+/// `execute_bash_and_wait { command }` (TS `executeBash`): run to/// completion, record the row, answer the `BashResult` wire shape.pub(crate) async fn handle_execute_bash_and_wait(&self, payload: &Value) -> DaemonResponse {@@ -271,11 +356,6 @@ impl Worker {None,);};- // A fresh run owns a fresh abort state (each TS invocation owns its- // own controller).- self.user_bash- .abort_requested- .store(false, Ordering::SeqCst);let (cwd, prefix, shell_path) = {let core = self.core.lock().unwrap();let settings =@@ -287,26 +367,35 @@ impl Worker {settings.shell_path.clone(),)};- let user_bash = Arc::clone(&self.user_bash);let command = command.to_string();// The awaited run counts toward `isBashRunning` without- // blocking a streamed user bash.- let awaited = user_bash.begin_awaited();+ // blocking a streamed user bash. The admission re-checks the+ // shutdown gate and registers under the same lock, so a close+ // cannot slip between them: it either sees the gate shut (and+ // the dispatch is rejected) or aborts a token that is already+ // registered (a registration after its abort would leave a+ // child the exit orphans).+ let awaited = match self.admit_awaited_bash() {+ Ok(awaited) => awaited,+ Err(response) => return response,+ };let end = run_bash(RunBash {command: &command,cwd: &cwd,prefix: prefix.as_deref(),shell_path: shell_path.as_deref(),- user_bash: &user_bash,+ cancellation: &awaited.cancellation,// The awaited path emits nothing (TS passes no `onChunk`).on_chunk: None,}).await;- drop(awaited);- // The awaited path emits no session events, so the port enqueues- // the roster flush TS's `execute_bash_and_wait` does in `finally`.+ // The durable row lands while the run still counts: the orphan+ // monitor treats `!is_running` as exit permission, so an awaited+ // result released before its row (the streamed path persists,+ // then releases) could be lost to a racing orphan exit.self.roster_pushes.push();if let Some(error) = &end.error_message {+ drop(awaited);return response_failure(None, "execute_bash_and_wait", error, None);}let result = BashResult {@@ -317,6 +406,7 @@ impl Worker {full_output_path: end.full_output_path,};record_bash_result(&self.core, &command, &result, false);+ drop(awaited);response_success(None,"execute_bash_and_wait",@@ -326,11 +416,11 @@ impl Worker {/// `abort_bash` (TS `abortBash`): kill the in-flight command; the/// settled run reports cancelled. Always a success.- pub(crate) async fn handle_abort_bash(&self) -> DaemonResponse {+ pub(crate) fn handle_abort_bash(&self) -> DaemonResponse {if let Err(response) = self.require_created("abort_bash") {return response;}- self.user_bash.abort().await;+ self.user_bash.abort();response_success(None, "abort_bash", None)}}@@ -360,11 +450,13 @@ fn merge_identity(mut event: Value, identity: &Value) -> Value {/// `isBashRunning` flag, released even when the run's future is dropped mid-await.pub(crate) struct AwaitedRun<'a> {user_bash: &'a UserBash,+ cancellation: Arc<CancellationToken>,}impl Drop for AwaitedRun<'_> {fn drop(&mut self) {self.user_bash.awaited.fetch_sub(1, Ordering::SeqCst);+ self.user_bash.settled.notify_waiters();}}@@ -399,7 +491,7 @@ struct RunBash<'a> {cwd: &'a str,prefix: Option<&'a str>,shell_path: Option<&'a str>,- user_bash: &'a UserBash,+ cancellation: &'a CancellationToken,/// The streaming emit target for each sanitized chunk; `None` on/// the awaited path, which emits nothing.on_chunk: Option<(@@ -499,20 +591,33 @@ async fn run_bash(run: RunBash<'_>) -> BashEnd {.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped());+ // Its own process group on POSIX, so a kill reaches every process the+ // command started (TS `detached: true`); Windows tree kills go through+ // `taskkill /T`.+ #[cfg(unix)]+ pa_core::platform::process::set_new_process_group(command.as_std_mut());let mut child = match command.spawn() {Ok(child) => child,Err(error) => return spawn_failure(error.to_string()),};let stdout = child.stdout.take();let stderr = child.stderr.take();- {- let mut slot = run.user_bash.child.lock().await;- // A late abort that arrived before the process spawned kills it- // immediately; the flag still settles the run cancelled.- if run.user_bash.abort_requested.load(Ordering::SeqCst) {+ // Only this run kills its tree, and only before it reaps the child: a+ // reaped pid can be reused. A failed tree kill still stops the shell.+ let kill_tree = |child: &mut tokio::process::Child| {+ let killed = child+ .id()+ .is_some_and(|pid| pa_core::platform::process::kill_process_group_or_pid(pid as i32));+ if !killed {let _ = child.start_kill();}- *slot = Some(child);+ };+ let abort = run.cancellation.cancelled();+ tokio::pin!(abort);+ // Cancellation remains sticky even if another command starts before this task runs.+ let mut killed = run.cancellation.is_cancelled();+ if killed {+ kill_tree(&mut child);}let stream = std::sync::Arc::new(std::sync::Mutex::new(OutputStream {chunks: Vec::new(),@@ -537,30 +642,39 @@ async fn run_bash(run: RunBash<'_>) -> BashEnd {};let stdout_pump = stdout.map(|pipe| pump(Box::new(pipe)));let stderr_pump = stderr.map(|pipe| pump(Box::new(pipe)));- match (stdout_pump, stderr_pump) {- (Some(stdout_pump), Some(stderr_pump)) => {- futures::future::join(stdout_pump, stderr_pump).await;+ let mut pumps = Box::pin(async move {+ match (stdout_pump, stderr_pump) {+ (Some(stdout_pump), Some(stderr_pump)) => {+ futures::future::join(stdout_pump, stderr_pump).await;+ }+ (Some(pump), None) | (None, Some(pump)) => pump.await,+ (None, None) => {}+ }+ });+ let mut pumps_done = false;+ // The run settles on the shell's exit, not on its pipes closing+ // (abort_bash may kill it meanwhile; a signal death reports no exit code).+ let status = loop {+ tokio::select! {+ // If both are ready, kill while the unreaped child still pins its PID.+ biased;+ () = &mut abort, if !killed => {+ killed = true;+ kill_tree(&mut child);+ }+ status = child.wait() => break status,+ () = &mut pumps, if !pumps_done => pumps_done = true,}- (Some(pump), None) | (None, Some(pump)) => pump.await,- (None, None) => {}- }- // Take the child back for the exit status (abort_bash may have killed- // it meanwhile; a signal death reports no exit code).- let mut child = {- let mut slot = run.user_bash.child.lock().await;- slot.take()- };- let exit_code = match child.as_mut() {- Some(child) => child- .wait()- .await- .ok()- .and_then(|status| status.code())- .map(i64::from),- None => None,};- let cancelled = run.user_bash.abort_requested.load(Ordering::SeqCst);- // The reader tasks joined, so this is the last reference; a poisoned+ // Descendants still holding the pipes (a `cmd &`) get the grace, then the+ // pipes close (TS `waitForChildProcess`).+ if !pumps_done {+ let _ = tokio::time::timeout(EXIT_STDIO_GRACE, &mut pumps).await;+ }+ drop(pumps);+ let exit_code = status.ok().and_then(|status| status.code()).map(i64::from);+ let cancelled = run.cancellation.is_cancelled();+ // The pumps are dropped, so this is the last reference; a poisoned// lock (a reader panicked) keeps whatever it collected.let mut stream = match Arc::try_unwrap(stream) {Ok(mutex) => mutex@@ -916,6 +1030,30 @@ mod tests {);}+ #[test]+ fn an_abort_stays_with_every_admitted_run_when_another_run_starts() {+ let user_bash = UserBash::new();+ let streamed = user_bash.claim().expect("claim slot");+ let first = user_bash.begin_awaited();+ let second = user_bash.begin_awaited();+ user_bash.abort();+ let later = user_bash.begin_awaited();+ assert_eq!(+ [+ streamed.is_cancelled(),+ first.cancellation.is_cancelled(),+ second.cancellation.is_cancelled(),+ later.cancellation.is_cancelled()+ ],+ [true, true, true, false],+ );+ user_bash.abort();+ assert!(later.cancellation.is_cancelled());+ user_bash.release();+ drop((first, second, later));+ assert!(!user_bash.is_running());+ }+#[test]fn an_abandoned_awaited_run_releases_the_flag() {let user_bash = UserBash::new();@@ -1043,6 +1181,182 @@ mod tests {);}+ /// The next `session_event` of `event_type` (bounded: a missing event fails the test).+ #[cfg(unix)]+ async fn next_session_event(+ events: &mut tokio::sync::broadcast::Receiver<Arc<crate::worker::OutboundFrame>>,+ event_type: &str,+ ) -> Value {+ let next = async {+ loop {+ let frame = events.recv().await.expect("the event stream stays live");+ if frame.outbound_type != "session_event" {+ continue;+ }+ let outbound: Value = serde_json::from_slice(&frame.payload).expect("event JSON");+ if outbound["event"]["type"] == json!(event_type) {+ return outbound["event"].clone();+ }+ }+ };+ tokio::time::timeout(std::time::Duration::from_secs(10), next)+ .await+ .unwrap_or_else(|_| panic!("no {event_type} event arrived"))+ }++ /// The child a running user bash command forked (the shell runs `sh`, which execs `sleep`).+ /// The child holds a FIFO's write end, so the FIFO reader's EOF is the child's death.+ #[cfg(unix)]+ struct ForkedChild {+ /// Killed on drop until [`Self::assert_killed`] saw the death: a failing run leaks nothing.+ pid: Option<i32>,+ gone: Option<tokio::task::JoinHandle<()>>,+ }++ #[cfg(unix)]+ impl ForkedChild {+ async fn start(+ worker: &Worker,+ cwd: &std::path::Path,+ events: &mut tokio::sync::broadcast::Receiver<Arc<crate::worker::OutboundFrame>>,+ ) -> Self {+ let fifo = cwd.join("child-alive");+ let made = std::process::Command::new("mkfifo")+ .arg(&fifo)+ .status()+ .expect("run mkfifo");+ assert!(made.success(), "mkfifo failed: {made}");+ let response = worker+ .dispatch(+ "execute_bash",+ &json!({+ "activeSessionId": "bash-session",+ "command": r#"sh -c 'exec 3<>child-alive; echo "ready $$"; exec sleep 60'; echo after"#,+ }),+ )+ .await;+ assert!(response.success, "failed: {response:?}");+ let ready = next_session_event(events, "bash_output").await;+ let pid = ready["chunk"]+ .as_str()+ .and_then(|chunk| chunk.trim().strip_prefix("ready "))+ .and_then(|pid| pid.parse().ok());+ assert!(pid.is_some(), "the child reports its pid: {ready}");+ // The child opened its end before `ready`, so this open returns at once.+ let mut reader = std::fs::File::open(&fifo).expect("open the fifo");+ let gone = tokio::task::spawn_blocking(move || {+ std::io::copy(&mut reader, &mut std::io::sink()).expect("drain the fifo");+ });+ ForkedChild {+ pid,+ gone: Some(gone),+ }+ }++ async fn assert_killed(&mut self) {+ let gone = self.gone.take().expect("awaited once");+ tokio::time::timeout(std::time::Duration::from_secs(10), gone)+ .await+ .expect("the command's child is still running")+ .expect("the fifo reader joins");+ self.pid = None;+ }+ }++ #[cfg(unix)]+ impl Drop for ForkedChild {+ fn drop(&mut self) {+ if let Some(pid) = self.pid {+ let _ = pa_core::platform::process::kill_pid(+ pid,+ pa_core::platform::process::Signal::Kill,+ );+ }+ }+ }++ /// `abort_bash` kills every process the command started, not just the shell: a surviving+ /// child would keep running and hold the output pipes open.+ #[cfg(unix)]+ #[tokio::test]+ async fn abort_bash_kills_the_commands_child_processes() {+ let cwd = tempfile::tempdir().expect("tempdir");+ let worker = created_worker(cwd.path()).await;+ let mut events = worker.events.subscribe();+ let mut child = ForkedChild::start(&worker, cwd.path(), &mut events).await;+ let aborted = worker+ .dispatch("abort_bash", &json!({ "activeSessionId": "bash-session" }))+ .await;+ assert!(aborted.success, "failed: {aborted:?}");+ child.assert_killed().await;+ assert_eq!(+ next_session_event(&mut events, "bash_end").await,+ json!({+ "type": "bash_end",+ "exitCode": null,+ "cancelled": true,+ "truncated": false,+ })+ );+ }++ /// The graceful stop's abort reaches the command's children too: the worker's exit never+ /// leaves the user's processes running.+ #[cfg(unix)]+ #[tokio::test]+ async fn shutdown_kills_the_running_commands_child_processes() {+ let cwd = tempfile::tempdir().expect("tempdir");+ let worker = created_worker(cwd.path()).await;+ let mut events = worker.events.subscribe();+ let mut child = ForkedChild::start(&worker, cwd.path(), &mut events).await;+ let stopped = worker.dispatch("shutdown", &json!({})).await;+ assert!(stopped.success, "failed: {stopped:?}");+ assert!(+ !worker.user_bash.is_running(),+ "the stop answered before the aborted run settled"+ );+ child.assert_killed().await;+ }++ /// A command that backgrounds a process settles when its shell exits: the background process+ /// holding the inherited output pipes does not hold the run open (TS `waitForChildProcess`).+ #[cfg(unix)]+ #[tokio::test]+ async fn a_backgrounded_process_does_not_hold_the_run_open() {+ let cwd = tempfile::tempdir().expect("tempdir");+ let worker = created_worker(cwd.path()).await;+ let settled = tokio::time::timeout(+ std::time::Duration::from_secs(10),+ worker.dispatch(+ "execute_bash_and_wait",+ &json!({+ "activeSessionId": "bash-session",+ "command": "sleep 60 & echo $! > background.pid; echo started",+ }),+ ),+ )+ .await;+ // The background process outlives the run (TS leaves it running too).+ if let Some(pid) = std::fs::read_to_string(cwd.path().join("background.pid"))+ .ok()+ .and_then(|pid| pid.trim().parse().ok())+ {+ let _ =+ pa_core::platform::process::kill_pid(pid, pa_core::platform::process::Signal::Kill);+ }+ let response = settled.expect("the run waited on the backgrounded process");+ assert!(response.success, "failed: {response:?}");+ assert_eq!(+ response.data,+ Some(json!({+ "output": "started\n",+ "exitCode": 0,+ "cancelled": false,+ "truncated": false,+ }))+ );+ }+#[tokio::test]async fn transient_execute_bash_records_nothing() {let cwd = tempfile::tempdir().expect("tempdir");@@ -1128,4 +1442,154 @@ mod tests {assert!(kept.len() <= DEFAULT_MAX_BYTES);assert!(big.ends_with(&kept), "the tail wins");}++ /// Readiness wait for a racing thread's entered signal: the signal is+ /// what the caller proceeds on — the deadline only bounds failure, it can+ /// never make an assertion pass.+ fn wait_for(signal: &AtomicBool) {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);+ while !signal.load(Ordering::SeqCst) {+ assert!(+ std::time::Instant::now() < deadline,+ "the racing admission never started"+ );+ std::hint::spin_loop();+ std::thread::yield_now();+ }+ }++ #[tokio::test]+ async fn the_shutdown_gate_cannot_close_between_a_streamed_claim_and_its_registration() {+ let cwd = tempfile::tempdir().expect("tempdir");+ let worker = created_worker(cwd.path()).await;++ // Gate shut: the admission's own re-check rejects, nothing claims.+ worker.core.lock().unwrap().shutdown_requested = true;+ assert!(worker.admit_streamed_bash().is_err());+ assert!(!worker.user_bash.is_running());++ // A gate that closes while a claim waits on the admission lock: the+ // gate check and the claim are one critical section, so the claim+ // cannot slip past the close (a claim the close's abort missed would+ // leave the child running past the exit).+ {+ let mut core = worker.core.lock().unwrap();+ core.shutdown_requested = false;+ }+ let mut gate = worker.core.lock().unwrap();+ // Readiness is the racing admission's own signal, not a fixed sleep:+ // once the thread is inside its call, the held gate pins it to enter+ // the critical section only after the close.+ let entered = Arc::new(AtomicBool::new(false));+ let racing = std::thread::spawn({+ let worker = Arc::clone(&worker);+ let entered = Arc::clone(&entered);+ move || {+ entered.store(true, Ordering::SeqCst);+ worker.admit_streamed_bash().is_ok()+ }+ });+ wait_for(&entered);+ assert!(+ !worker.user_bash.is_running(),+ "the gated claim must not register"+ );+ gate.shutdown_requested = true;+ drop(gate);+ let claimed = racing.join().expect("claim thread");+ assert!(!claimed, "the closed gate must reject it");+ assert!(!worker.user_bash.is_running(), "nothing may register");+ }++ #[tokio::test]+ async fn the_shutdown_gate_cannot_close_between_an_awaited_admission_and_its_registration() {+ let cwd = tempfile::tempdir().expect("tempdir");+ let worker = created_worker(cwd.path()).await;++ // Gate shut: the admission's own re-check rejects, nothing registers.+ worker.core.lock().unwrap().shutdown_requested = true;+ assert!(worker.admit_awaited_bash().is_err());+ assert!(!worker.user_bash.is_running());++ // A gate that closes while an admission waits on the admission+ // lock: the gate check and the registration are one critical+ // section, so the admission cannot slip past the close (a+ // registration the close's abort missed would leave the child+ // running past the exit).+ {+ let mut core = worker.core.lock().unwrap();+ core.shutdown_requested = false;+ }+ let mut gate = worker.core.lock().unwrap();+ // Readiness is the racing admission's own signal, not a fixed sleep+ // (same as the streamed claim's gate test).+ let entered = Arc::new(AtomicBool::new(false));+ let racing = std::thread::spawn({+ let worker = Arc::clone(&worker);+ let entered = Arc::clone(&entered);+ move || {+ entered.store(true, Ordering::SeqCst);+ worker.admit_awaited_bash().is_ok()+ }+ });+ wait_for(&entered);+ assert!(+ !worker.user_bash.is_running(),+ "the gated admission must not register"+ );+ gate.shutdown_requested = true;+ drop(gate);+ let admitted = racing.join().expect("admission thread");+ assert!(!admitted, "the closed gate must reject it");+ assert!(!worker.user_bash.is_running(), "nothing may register");+ }++ #[tokio::test]+ async fn the_close_abort_cancels_an_admitted_awaited_run_and_settles_after_it() {+ let cwd = tempfile::tempdir().expect("tempdir");+ let worker = created_worker(cwd.path()).await;+ let awaited = worker.admit_awaited_bash().expect("gate open");+ assert!(worker.user_bash.is_running());+ let settle = tokio::spawn({+ let worker = Arc::clone(&worker);+ async move { worker.user_bash.abort_and_settle().await }+ });+ // The close's abort cancels the admitted run's token: the token's+ // own cancellation is the observable signal, and the bound only+ // bounds failure.+ tokio::time::timeout(+ std::time::Duration::from_secs(2),+ awaited.cancellation.cancelled(),+ )+ .await+ .expect("the abort never reached the admitted token");+ // The settle still waits: the run counts until it settles.+ assert!(!settle.is_finished());+ drop(awaited);+ tokio::time::timeout(std::time::Duration::from_secs(2), settle)+ .await+ .expect("the settle completes once the run settles")+ .expect("the settle task joins");+ }++ #[tokio::test]+ async fn an_awaited_dispatch_against_a_closing_gate_records_no_row() {+ let cwd = tempfile::tempdir().expect("tempdir");+ let worker = created_worker(cwd.path()).await;+ worker.core.lock().unwrap().shutdown_requested = true;+ let response = worker+ .dispatch(+ "execute_bash_and_wait",+ &json!({ "activeSessionId": "bash-session", "command": "echo no" }),+ )+ .await;+ assert!(!response.success);+ assert_eq!(response.error.as_deref(), Some("Session is shutting down"));+ assert_eq!(+ bash_rows(&worker).len(),+ 0,+ "a rejected run records nothing"+ );+ assert!(!worker.user_bash.is_running());+ }}diff --git a/crates/pa-daemon/src/worker/commands.rs b/crates/pa-daemon/src/worker/commands.rsindex b34f47d08f..26ae1c203e 100644--- a/crates/pa-daemon/src/worker/commands.rs+++ b/crates/pa-daemon/src/worker/commands.rs@@ -152,7 +152,7 @@ impl Worker {"factory_activity" => self.handle_factory_activity(payload).await,"execute_bash" => self.handle_execute_bash(payload),"execute_bash_and_wait" => self.handle_execute_bash_and_wait(payload).await,- "abort_bash" => self.handle_abort_bash().await,+ "abort_bash" => self.handle_abort_bash(),"list_kernel_bash" | "tail_kernel_bash" | "kill_kernel_bash" => {self.handle_kernel_bash_activity(command_type, payload).awaitdiff --git a/crates/pa-daemon/src/worker/lifecycle.rs b/crates/pa-daemon/src/worker/lifecycle.rsindex 9818776773..1b59668546 100644--- a/crates/pa-daemon/src/worker/lifecycle.rs+++ b/crates/pa-daemon/src/worker/lifecycle.rs@@ -76,14 +76,17 @@ impl Worker {{let mut core = self.core.lock().unwrap();// The shutdown gate closes FIRST: a racing execute_bash must see the- // stop before the abort runs, or the fresh claim clears the abort and- // spawns a child the exit leaves running.+ // stop before the abort runs — a claim that slipped in after the+ // abort would register a controller nothing later cancels, and the+ // exit leaves its child running.core.shutdown_requested = true;core.abort_requested = true;}// The running user bash goes with the stop: the passivation stop must- // never leave the user's process running after the worker exits.- self.user_bash.abort().await;+ // never leave the user's process running after the worker exits. Each+ // run kills its tree in its own task, so the stop waits (bounded) for+ // the runs to settle (TS `abortBashForClose`).+ self.user_bash.abort_and_settle().await;// TS `shutdown` closes through `session.abort()` -> `requestAbort()`:// the in-flight turn's fetch cancels now, not at its next event.self.engine.abort_in_flight_turn();diff --git a/crates/pa-daemon/src/worker/turn_stream_tests/interleave.rs b/crates/pa-daemon/src/worker/turn_stream_tests/interleave.rsindex 83207ea2ee..dcf3b8bc92 100644--- a/crates/pa-daemon/src/worker/turn_stream_tests/interleave.rs+++ b/crates/pa-daemon/src/worker/turn_stream_tests/interleave.rs@@ -77,12 +77,11 @@//! coverage. The departure release (a closed connection's release//! re-arming the window) is the park family's window battery too. The//! gate-before-abort ORDER inside `handle_shutdown` (one critical section-//! sets `shutdown_requested` before `user_bash.abort()` runs) has no seam-//! a test can hold between the two steps, so no test here pins it - the-//! tests pin each step's effect. The base's `UserBash` open corners (the single-slot overlap,-//! the forked-grandchild survival) are TS-anchored follow-ups on the-//! base's bash surface — out of this lane's scope (the round-9 rebuttal-//! carries them).+//! sets `shutdown_requested` before `user_bash.abort_and_settle()` runs) has+//! no seam a test can hold between the two steps, so no test here pins it -+//! the tests pin each step's effect. The forked-child kill and the stop's+//! settle wait are pinned in `user_bash`'s own tests; the single-slot overlap+//! stays a TS-anchored follow-up on the base's bash surface.// Unix-only: the probe harness stands in for the supervisor side of// the passivation ask over a real unix domain socket// (tokio::net::UnixListener), which has no counterpart on the windowsdiff --git a/crates/pa-daemon/tests/worker_orphan_exit_e2e.rs b/crates/pa-daemon/tests/worker_orphan_exit_e2e.rsindex d4b6f4c5d2..deb5bc1306 100644--- a/crates/pa-daemon/tests/worker_orphan_exit_e2e.rs+++ b/crates/pa-daemon/tests/worker_orphan_exit_e2e.rs@@ -274,3 +274,59 @@ fn the_orphan_exit_preserves_the_session_and_allows_dead_owner_lease_reclaim() {"the session file survives the lease release");}++/// TS `isSessionActive` counts `isBashRunning`: an orphaned worker waits for a+/// running user bash instead of exiting under it, so the command finishes and+/// its row lands in the session file. The command is the work being waited on,+/// not a readiness signal: an idle orphan exits at the second availability+/// check after the client's close (checks every 5s, a 4s window), so a worker+/// that ignored the run would exit while the 20s command still runs.+#[test]+fn an_orphaned_worker_waits_for_a_running_user_bash() {+ let dir = tempfile::tempdir().expect("temp dir");+ let canonical_dir = std::fs::canonicalize(dir.path()).expect("canonical dir");+ let supervisor_socket = dir.path().join("never-bound-supervisor.sock");+ let mut worker = spawn_worker_with_window(dir.path(), &supervisor_socket, "4000");+ let worker_socket = dir.path().join("worker.sock");+ wait_worker_socket(&worker_socket);+ let session_file = canonical_dir.join("orphan-bash-session.jsonl");+ let mut client = WorkerClient::connect(&worker_socket, "orphan-e2e-bootstrap-token");+ let created = client.request(+ "create",+ &serde_json::json!({"sessionPath": session_file.to_string_lossy()}),+ );+ assert_eq!(created["success"], true, "create failed: {created}");+ let command = "sleep 20; echo finished";+ let started = client.request("execute_bash", &serde_json::json!({ "command": command }));+ assert_eq!(started["success"], true, "execute_bash failed: {started}");+ drop(client);++ assert!(+ wait_exit(&mut worker.child, Duration::from_secs(40)),+ "the orphaned worker exited once the command settled"+ );+ let rows: Vec<serde_json::Value> = std::fs::read_to_string(&session_file)+ .expect("session file")+ .lines()+ .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())+ .map(|entry| entry["message"].clone())+ .filter(|message| message["role"] == "bashExecution")+ .map(|row| {+ serde_json::json!({+ "command": row["command"],+ "output": row["output"],+ "exitCode": row["exitCode"],+ "cancelled": row["cancelled"],+ })+ })+ .collect();+ assert_eq!(+ rows,+ vec![serde_json::json!({+ "command": command,+ "output": "finished\n",+ "exitCode": 0,+ "cancelled": false,+ })]+ );+}