Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

settle user bash process trees and command slots (#3541)

kt53e13b89e14b
diff --git a/crates/pa-daemon/src/supervisor_lost.rs b/crates/pa-daemon/src/supervisor_lost.rs
index 035ece10f8..6822256f4d 100644
--- a/crates/pa-daemon/src/supervisor_lost.rs
+++ b/crates/pa-daemon/src/supervisor_lost.rs
@@ -76,10 +76,12 @@ async fn monitor(worker: Arc<Worker>) {
if since.elapsed() < window {
continue;
}
- let ongoing = worker.core.lock().unwrap().has_ongoing_work();
+ let ongoing =
+ worker.core.lock().unwrap().has_ongoing_work() || worker.user_bash.is_running();
if ongoing {
- // TS `hasOngoingSessionWork`: an active run owns the worker a little
- // longer; its turn end lets the next check reconsider.
+ // TS `hasOngoingSessionWork` (`isSessionActive`, which counts
+ // `isBashRunning`): an active run or a running user bash owns the
+ // worker a little longer; its end lets the next check reconsider.
continue;
}
exit_orphaned(&worker, since).await;
diff --git a/crates/pa-daemon/src/user_bash.rs b/crates/pa-daemon/src/user_bash.rs
index 0bd56a7443..8cb48ea304 100644
--- a/crates/pa-daemon/src/user_bash.rs
+++ b/crates/pa-daemon/src/user_bash.rs
@@ -10,11 +10,18 @@ use std::sync::Arc;
use serde_json::{json, Map, Value};
use tokio::io::{AsyncRead, AsyncReadExt};
-use tokio::sync::Mutex;
+use tokio::sync::Notify;
+use tokio_util::sync::CancellationToken;
use crate::protocol::{response_failure, response_success, DaemonResponse};
use crate::worker::Worker;
+/// How long a run still reads output after its shell exits (TS `EXIT_STDIO_GRACE_MS`): a
+/// background process holding the inherited pipes must not hold the run open.
+const EXIT_STDIO_GRACE: std::time::Duration = std::time::Duration::from_millis(100);
+/// How long a closing session waits for its aborted runs to settle (TS
+/// `UPDATE_RESTART_ABORT_BASH_TIMEOUT_MS`).
+const CLOSE_SETTLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
/// Streaming window: chunks are retained (spilled to disk beyond this).
const DEFAULT_MAX_BYTES: usize = 50 * 1024;
/// Streaming retention cap: the in-memory tail (TS `maxOutputBytes`).
@@ -32,10 +39,10 @@ pub(crate) struct UserBash {
/// Awaited bash runs in flight (TS `_bashAbortControllers.size`):
/// they count toward `isBashRunning` without claiming the slot.
awaited: AtomicUsize,
- /// An abort was requested: the settled result reports cancelled.
- abort_requested: AtomicBool,
- /// The in-flight process (user bash or an awaited run): `abort_bash` kills it.
- child: Mutex<Option<tokio::process::Child>>,
+ /// Each invocation owns its cancellation state; a later start cannot clear an earlier abort.
+ controllers: std::sync::Mutex<Vec<std::sync::Weak<CancellationToken>>>,
+ /// Signals each settle (the slot's release or an awaited run's end).
+ settled: Notify,
}
impl UserBash {
@@ -43,26 +50,26 @@ impl UserBash {
Self {
running: AtomicBool::new(false),
awaited: AtomicUsize::new(0),
- abort_requested: AtomicBool::new(false),
- child: Mutex::new(None),
+ controllers: std::sync::Mutex::new(Vec::new()),
+ settled: Notify::new(),
}
}
- /// Claim the slot; `false` when a user command is already running. A fresh claim clears a stale
- /// abort request (TS clears at each start), so a leftover flag cannot cancel a later run.
- fn claim(&self) -> bool {
- let claimed = self
- .running
- .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
- .is_ok();
- if claimed {
- self.abort_requested.store(false, Ordering::SeqCst);
+ /// Claim and register synchronously, before the spawned task can race an abort.
+ fn claim(&self) -> Option<Arc<CancellationToken>> {
+ let mut controllers = self.controllers.lock().unwrap();
+ if self.running.swap(true, Ordering::SeqCst) {
+ return None;
}
- claimed
+ let cancellation = Arc::new(CancellationToken::new());
+ controllers.retain(|controller| controller.strong_count() > 0);
+ controllers.push(Arc::downgrade(&cancellation));
+ Some(cancellation)
}
fn release(&self) {
self.running.store(false, Ordering::SeqCst);
+ self.settled.notify_waiters();
}
/// Whether a user bash or an awaited bash run is in flight (TS `isBashRunning`).
@@ -73,18 +80,48 @@ impl UserBash {
/// Count one awaited run (`execute_bash_and_wait`) toward [`Self::is_running`]; the returned
/// bracket decrements on drop, so a dropped future cannot leave the count stuck on.
pub(crate) fn begin_awaited(&self) -> AwaitedRun<'_> {
+ let mut controllers = self.controllers.lock().unwrap();
+ let cancellation = Arc::new(CancellationToken::new());
+ controllers.retain(|controller| controller.strong_count() > 0);
+ controllers.push(Arc::downgrade(&cancellation));
self.awaited.fetch_add(1, Ordering::SeqCst);
- AwaitedRun { user_bash: self }
+ AwaitedRun {
+ user_bash: self,
+ cancellation,
+ }
}
- /// Kill the in-flight process (TS `abortBash` aborts every
- /// controller); the settled runs report cancelled.
- pub(crate) async fn abort(&self) {
- self.abort_requested.store(true, Ordering::SeqCst);
- let mut child = self.child.lock().await;
- if let Some(child) = child.as_mut() {
- let _ = child.start_kill();
- }
+ /// Notify every in-flight run to kill its process tree (TS `abortBash`
+ /// aborts every controller); the settled runs report cancelled.
+ pub(crate) fn abort(&self) {
+ let mut controllers = self.controllers.lock().unwrap();
+ controllers.retain(|controller| {
+ if let Some(controller) = controller.upgrade() {
+ controller.cancel();
+ true
+ } else {
+ false
+ }
+ });
+ }
+
+ /// [`Self::abort`], then wait (bounded) until no run is in flight: each run kills its tree
+ /// in its own task, so a close must not exit ahead of the kills (TS `abortBashForClose`).
+ pub(crate) async fn abort_and_settle(&self) {
+ self.abort();
+ let settled = async {
+ loop {
+ // Armed before the check, so a settle in between still wakes this wait.
+ let notified = self.settled.notified();
+ tokio::pin!(notified);
+ notified.as_mut().enable();
+ if !self.is_running() {
+ return;
+ }
+ notified.await;
+ }
+ };
+ let _ = tokio::time::timeout(CLOSE_SETTLE_TIMEOUT, settled).await;
}
}
@@ -165,15 +202,14 @@ impl Worker {
);
};
// Claim the slot synchronously (TS claims before the async dispatch
- // could slip a second command through).
- if !self.user_bash.claim() {
- return response_failure(
- None,
- "execute_bash",
- "A bash command is already running",
- None,
- );
- }
+ // could slip a second command through), through the admission gate:
+ // the gate re-check and the claim share the lock, so the close's
+ // abort cannot miss the new token (a claim its `abort_and_settle`
+ // missed would leave the child running past the exit).
+ let cancellation = match self.admit_streamed_bash() {
+ Ok(cancellation) => cancellation,
+ Err(response) => return response,
+ };
let exclude_from_context = payload
.get("excludeFromContext")
.and_then(Value::as_bool)
@@ -215,7 +251,7 @@ impl Worker {
cwd: &cwd,
prefix: prefix.as_deref(),
shell_path: shell_path.as_deref(),
- user_bash: &user_bash,
+ cancellation: &cancellation,
on_chunk: Some((Arc::clone(&core), Arc::clone(&events))),
})
.await;
@@ -257,6 +293,55 @@ impl Worker {
response_success(None, "execute_bash", None)
}
+ /// `require_created`'s shutdown gate re-checked with the streamed slot
+ /// claim under the same lock: the claim cannot interleave with the gate's
+ /// close, so the close's abort always sees — and cancels — the new token
+ /// (a claim its `abort_and_settle` missed would leave the child running
+ /// past the exit).
+ // DaemonResponse is the wire response struct and is deliberately wide; the
+ // error channel here carries the whole response, so allow the large-err lint.
+ #[allow(clippy::result_large_err)]
+ pub(crate) fn admit_streamed_bash(&self) -> Result<Arc<CancellationToken>, DaemonResponse> {
+ let core = self.core.lock().unwrap();
+ if core.shutdown_requested {
+ return Err(response_failure(
+ None,
+ "execute_bash",
+ "Session is shutting down",
+ None,
+ ));
+ }
+ self.user_bash.claim().ok_or_else(|| {
+ response_failure(
+ None,
+ "execute_bash",
+ "A bash command is already running",
+ None,
+ )
+ })
+ }
+
+ /// `require_created`'s shutdown gate re-checked and the awaited run
+ /// registered under the same lock: the registration cannot interleave
+ /// with the gate's close, so the close's abort always sees — and cancels —
+ /// the new token (a registration its `abort_and_settle` missed would let
+ /// a child outlive the exit).
+ // DaemonResponse is the wire response struct and is deliberately wide; the
+ // error channel here carries the whole response, so allow the large-err lint.
+ #[allow(clippy::result_large_err)]
+ pub(crate) fn admit_awaited_bash(&self) -> Result<AwaitedRun<'_>, DaemonResponse> {
+ let core = self.core.lock().unwrap();
+ if core.shutdown_requested {
+ return Err(response_failure(
+ None,
+ "execute_bash_and_wait",
+ "Session is shutting down",
+ None,
+ ));
+ }
+ Ok(self.user_bash.begin_awaited())
+ }
+
/// `execute_bash_and_wait { command }` (TS `executeBash`): run to
/// completion, record the row, answer the `BashResult` wire shape.
pub(crate) async fn handle_execute_bash_and_wait(&self, payload: &Value) -> DaemonResponse {
@@ -271,11 +356,6 @@ impl Worker {
None,
);
};
- // A fresh run owns a fresh abort state (each TS invocation owns its
- // own controller).
- self.user_bash
- .abort_requested
- .store(false, Ordering::SeqCst);
let (cwd, prefix, shell_path) = {
let core = self.core.lock().unwrap();
let settings =
@@ -287,26 +367,35 @@ impl Worker {
settings.shell_path.clone(),
)
};
- let user_bash = Arc::clone(&self.user_bash);
let command = command.to_string();
// The awaited run counts toward `isBashRunning` without
- // blocking a streamed user bash.
- let awaited = user_bash.begin_awaited();
+ // blocking a streamed user bash. The admission re-checks the
+ // shutdown gate and registers under the same lock, so a close
+ // cannot slip between them: it either sees the gate shut (and
+ // the dispatch is rejected) or aborts a token that is already
+ // registered (a registration after its abort would leave a
+ // child the exit orphans).
+ let awaited = match self.admit_awaited_bash() {
+ Ok(awaited) => awaited,
+ Err(response) => return response,
+ };
let end = run_bash(RunBash {
command: &command,
cwd: &cwd,
prefix: prefix.as_deref(),
shell_path: shell_path.as_deref(),
- user_bash: &user_bash,
+ cancellation: &awaited.cancellation,
// The awaited path emits nothing (TS passes no `onChunk`).
on_chunk: None,
})
.await;
- drop(awaited);
- // The awaited path emits no session events, so the port enqueues
- // the roster flush TS's `execute_bash_and_wait` does in `finally`.
+ // The durable row lands while the run still counts: the orphan
+ // monitor treats `!is_running` as exit permission, so an awaited
+ // result released before its row (the streamed path persists,
+ // then releases) could be lost to a racing orphan exit.
self.roster_pushes.push();
if let Some(error) = &end.error_message {
+ drop(awaited);
return response_failure(None, "execute_bash_and_wait", error, None);
}
let result = BashResult {
@@ -317,6 +406,7 @@ impl Worker {
full_output_path: end.full_output_path,
};
record_bash_result(&self.core, &command, &result, false);
+ drop(awaited);
response_success(
None,
"execute_bash_and_wait",
@@ -326,11 +416,11 @@ impl Worker {
/// `abort_bash` (TS `abortBash`): kill the in-flight command; the
/// settled run reports cancelled. Always a success.
- pub(crate) async fn handle_abort_bash(&self) -> DaemonResponse {
+ pub(crate) fn handle_abort_bash(&self) -> DaemonResponse {
if let Err(response) = self.require_created("abort_bash") {
return response;
}
- self.user_bash.abort().await;
+ self.user_bash.abort();
response_success(None, "abort_bash", None)
}
}
@@ -360,11 +450,13 @@ fn merge_identity(mut event: Value, identity: &Value) -> Value {
/// `isBashRunning` flag, released even when the run's future is dropped mid-await.
pub(crate) struct AwaitedRun<'a> {
user_bash: &'a UserBash,
+ cancellation: Arc<CancellationToken>,
}
impl Drop for AwaitedRun<'_> {
fn drop(&mut self) {
self.user_bash.awaited.fetch_sub(1, Ordering::SeqCst);
+ self.user_bash.settled.notify_waiters();
}
}
@@ -399,7 +491,7 @@ struct RunBash<'a> {
cwd: &'a str,
prefix: Option<&'a str>,
shell_path: Option<&'a str>,
- user_bash: &'a UserBash,
+ cancellation: &'a CancellationToken,
/// The streaming emit target for each sanitized chunk; `None` on
/// the awaited path, which emits nothing.
on_chunk: Option<(
@@ -499,20 +591,33 @@ async fn run_bash(run: RunBash<'_>) -> BashEnd {
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
+ // Its own process group on POSIX, so a kill reaches every process the
+ // command started (TS `detached: true`); Windows tree kills go through
+ // `taskkill /T`.
+ #[cfg(unix)]
+ pa_core::platform::process::set_new_process_group(command.as_std_mut());
let mut child = match command.spawn() {
Ok(child) => child,
Err(error) => return spawn_failure(error.to_string()),
};
let stdout = child.stdout.take();
let stderr = child.stderr.take();
- {
- let mut slot = run.user_bash.child.lock().await;
- // A late abort that arrived before the process spawned kills it
- // immediately; the flag still settles the run cancelled.
- if run.user_bash.abort_requested.load(Ordering::SeqCst) {
+ // Only this run kills its tree, and only before it reaps the child: a
+ // reaped pid can be reused. A failed tree kill still stops the shell.
+ let kill_tree = |child: &mut tokio::process::Child| {
+ let killed = child
+ .id()
+ .is_some_and(|pid| pa_core::platform::process::kill_process_group_or_pid(pid as i32));
+ if !killed {
let _ = child.start_kill();
}
- *slot = Some(child);
+ };
+ let abort = run.cancellation.cancelled();
+ tokio::pin!(abort);
+ // Cancellation remains sticky even if another command starts before this task runs.
+ let mut killed = run.cancellation.is_cancelled();
+ if killed {
+ kill_tree(&mut child);
}
let stream = std::sync::Arc::new(std::sync::Mutex::new(OutputStream {
chunks: Vec::new(),
@@ -537,30 +642,39 @@ async fn run_bash(run: RunBash<'_>) -> BashEnd {
};
let stdout_pump = stdout.map(|pipe| pump(Box::new(pipe)));
let stderr_pump = stderr.map(|pipe| pump(Box::new(pipe)));
- match (stdout_pump, stderr_pump) {
- (Some(stdout_pump), Some(stderr_pump)) => {
- futures::future::join(stdout_pump, stderr_pump).await;
+ let mut pumps = Box::pin(async move {
+ match (stdout_pump, stderr_pump) {
+ (Some(stdout_pump), Some(stderr_pump)) => {
+ futures::future::join(stdout_pump, stderr_pump).await;
+ }
+ (Some(pump), None) | (None, Some(pump)) => pump.await,
+ (None, None) => {}
+ }
+ });
+ let mut pumps_done = false;
+ // The run settles on the shell's exit, not on its pipes closing
+ // (abort_bash may kill it meanwhile; a signal death reports no exit code).
+ let status = loop {
+ tokio::select! {
+ // If both are ready, kill while the unreaped child still pins its PID.
+ biased;
+ () = &mut abort, if !killed => {
+ killed = true;
+ kill_tree(&mut child);
+ }
+ status = child.wait() => break status,
+ () = &mut pumps, if !pumps_done => pumps_done = true,
}
- (Some(pump), None) | (None, Some(pump)) => pump.await,
- (None, None) => {}
- }
- // Take the child back for the exit status (abort_bash may have killed
- // it meanwhile; a signal death reports no exit code).
- let mut child = {
- let mut slot = run.user_bash.child.lock().await;
- slot.take()
- };
- let exit_code = match child.as_mut() {
- Some(child) => child
- .wait()
- .await
- .ok()
- .and_then(|status| status.code())
- .map(i64::from),
- None => None,
};
- let cancelled = run.user_bash.abort_requested.load(Ordering::SeqCst);
- // The reader tasks joined, so this is the last reference; a poisoned
+ // Descendants still holding the pipes (a `cmd &`) get the grace, then the
+ // pipes close (TS `waitForChildProcess`).
+ if !pumps_done {
+ let _ = tokio::time::timeout(EXIT_STDIO_GRACE, &mut pumps).await;
+ }
+ drop(pumps);
+ let exit_code = status.ok().and_then(|status| status.code()).map(i64::from);
+ let cancelled = run.cancellation.is_cancelled();
+ // The pumps are dropped, so this is the last reference; a poisoned
// lock (a reader panicked) keeps whatever it collected.
let mut stream = match Arc::try_unwrap(stream) {
Ok(mutex) => mutex
@@ -916,6 +1030,30 @@ mod tests {
);
}
+ #[test]
+ fn an_abort_stays_with_every_admitted_run_when_another_run_starts() {
+ let user_bash = UserBash::new();
+ let streamed = user_bash.claim().expect("claim slot");
+ let first = user_bash.begin_awaited();
+ let second = user_bash.begin_awaited();
+ user_bash.abort();
+ let later = user_bash.begin_awaited();
+ assert_eq!(
+ [
+ streamed.is_cancelled(),
+ first.cancellation.is_cancelled(),
+ second.cancellation.is_cancelled(),
+ later.cancellation.is_cancelled()
+ ],
+ [true, true, true, false],
+ );
+ user_bash.abort();
+ assert!(later.cancellation.is_cancelled());
+ user_bash.release();
+ drop((first, second, later));
+ assert!(!user_bash.is_running());
+ }
+
#[test]
fn an_abandoned_awaited_run_releases_the_flag() {
let user_bash = UserBash::new();
@@ -1043,6 +1181,182 @@ mod tests {
);
}
+ /// The next `session_event` of `event_type` (bounded: a missing event fails the test).
+ #[cfg(unix)]
+ async fn next_session_event(
+ events: &mut tokio::sync::broadcast::Receiver<Arc<crate::worker::OutboundFrame>>,
+ event_type: &str,
+ ) -> Value {
+ let next = async {
+ loop {
+ let frame = events.recv().await.expect("the event stream stays live");
+ if frame.outbound_type != "session_event" {
+ continue;
+ }
+ let outbound: Value = serde_json::from_slice(&frame.payload).expect("event JSON");
+ if outbound["event"]["type"] == json!(event_type) {
+ return outbound["event"].clone();
+ }
+ }
+ };
+ tokio::time::timeout(std::time::Duration::from_secs(10), next)
+ .await
+ .unwrap_or_else(|_| panic!("no {event_type} event arrived"))
+ }
+
+ /// The child a running user bash command forked (the shell runs `sh`, which execs `sleep`).
+ /// The child holds a FIFO's write end, so the FIFO reader's EOF is the child's death.
+ #[cfg(unix)]
+ struct ForkedChild {
+ /// Killed on drop until [`Self::assert_killed`] saw the death: a failing run leaks nothing.
+ pid: Option<i32>,
+ gone: Option<tokio::task::JoinHandle<()>>,
+ }
+
+ #[cfg(unix)]
+ impl ForkedChild {
+ async fn start(
+ worker: &Worker,
+ cwd: &std::path::Path,
+ events: &mut tokio::sync::broadcast::Receiver<Arc<crate::worker::OutboundFrame>>,
+ ) -> Self {
+ let fifo = cwd.join("child-alive");
+ let made = std::process::Command::new("mkfifo")
+ .arg(&fifo)
+ .status()
+ .expect("run mkfifo");
+ assert!(made.success(), "mkfifo failed: {made}");
+ let response = worker
+ .dispatch(
+ "execute_bash",
+ &json!({
+ "activeSessionId": "bash-session",
+ "command": r#"sh -c 'exec 3<>child-alive; echo "ready $$"; exec sleep 60'; echo after"#,
+ }),
+ )
+ .await;
+ assert!(response.success, "failed: {response:?}");
+ let ready = next_session_event(events, "bash_output").await;
+ let pid = ready["chunk"]
+ .as_str()
+ .and_then(|chunk| chunk.trim().strip_prefix("ready "))
+ .and_then(|pid| pid.parse().ok());
+ assert!(pid.is_some(), "the child reports its pid: {ready}");
+ // The child opened its end before `ready`, so this open returns at once.
+ let mut reader = std::fs::File::open(&fifo).expect("open the fifo");
+ let gone = tokio::task::spawn_blocking(move || {
+ std::io::copy(&mut reader, &mut std::io::sink()).expect("drain the fifo");
+ });
+ ForkedChild {
+ pid,
+ gone: Some(gone),
+ }
+ }
+
+ async fn assert_killed(&mut self) {
+ let gone = self.gone.take().expect("awaited once");
+ tokio::time::timeout(std::time::Duration::from_secs(10), gone)
+ .await
+ .expect("the command's child is still running")
+ .expect("the fifo reader joins");
+ self.pid = None;
+ }
+ }
+
+ #[cfg(unix)]
+ impl Drop for ForkedChild {
+ fn drop(&mut self) {
+ if let Some(pid) = self.pid {
+ let _ = pa_core::platform::process::kill_pid(
+ pid,
+ pa_core::platform::process::Signal::Kill,
+ );
+ }
+ }
+ }
+
+ /// `abort_bash` kills every process the command started, not just the shell: a surviving
+ /// child would keep running and hold the output pipes open.
+ #[cfg(unix)]
+ #[tokio::test]
+ async fn abort_bash_kills_the_commands_child_processes() {
+ let cwd = tempfile::tempdir().expect("tempdir");
+ let worker = created_worker(cwd.path()).await;
+ let mut events = worker.events.subscribe();
+ let mut child = ForkedChild::start(&worker, cwd.path(), &mut events).await;
+ let aborted = worker
+ .dispatch("abort_bash", &json!({ "activeSessionId": "bash-session" }))
+ .await;
+ assert!(aborted.success, "failed: {aborted:?}");
+ child.assert_killed().await;
+ assert_eq!(
+ next_session_event(&mut events, "bash_end").await,
+ json!({
+ "type": "bash_end",
+ "exitCode": null,
+ "cancelled": true,
+ "truncated": false,
+ })
+ );
+ }
+
+ /// The graceful stop's abort reaches the command's children too: the worker's exit never
+ /// leaves the user's processes running.
+ #[cfg(unix)]
+ #[tokio::test]
+ async fn shutdown_kills_the_running_commands_child_processes() {
+ let cwd = tempfile::tempdir().expect("tempdir");
+ let worker = created_worker(cwd.path()).await;
+ let mut events = worker.events.subscribe();
+ let mut child = ForkedChild::start(&worker, cwd.path(), &mut events).await;
+ let stopped = worker.dispatch("shutdown", &json!({})).await;
+ assert!(stopped.success, "failed: {stopped:?}");
+ assert!(
+ !worker.user_bash.is_running(),
+ "the stop answered before the aborted run settled"
+ );
+ child.assert_killed().await;
+ }
+
+ /// A command that backgrounds a process settles when its shell exits: the background process
+ /// holding the inherited output pipes does not hold the run open (TS `waitForChildProcess`).
+ #[cfg(unix)]
+ #[tokio::test]
+ async fn a_backgrounded_process_does_not_hold_the_run_open() {
+ let cwd = tempfile::tempdir().expect("tempdir");
+ let worker = created_worker(cwd.path()).await;
+ let settled = tokio::time::timeout(
+ std::time::Duration::from_secs(10),
+ worker.dispatch(
+ "execute_bash_and_wait",
+ &json!({
+ "activeSessionId": "bash-session",
+ "command": "sleep 60 & echo $! > background.pid; echo started",
+ }),
+ ),
+ )
+ .await;
+ // The background process outlives the run (TS leaves it running too).
+ if let Some(pid) = std::fs::read_to_string(cwd.path().join("background.pid"))
+ .ok()
+ .and_then(|pid| pid.trim().parse().ok())
+ {
+ let _ =
+ pa_core::platform::process::kill_pid(pid, pa_core::platform::process::Signal::Kill);
+ }
+ let response = settled.expect("the run waited on the backgrounded process");
+ assert!(response.success, "failed: {response:?}");
+ assert_eq!(
+ response.data,
+ Some(json!({
+ "output": "started\n",
+ "exitCode": 0,
+ "cancelled": false,
+ "truncated": false,
+ }))
+ );
+ }
+
#[tokio::test]
async fn transient_execute_bash_records_nothing() {
let cwd = tempfile::tempdir().expect("tempdir");
@@ -1128,4 +1442,154 @@ mod tests {
assert!(kept.len() <= DEFAULT_MAX_BYTES);
assert!(big.ends_with(&kept), "the tail wins");
}
+
+ /// Readiness wait for a racing thread's entered signal: the signal is
+ /// what the caller proceeds on — the deadline only bounds failure, it can
+ /// never make an assertion pass.
+ fn wait_for(signal: &AtomicBool) {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
+ while !signal.load(Ordering::SeqCst) {
+ assert!(
+ std::time::Instant::now() < deadline,
+ "the racing admission never started"
+ );
+ std::hint::spin_loop();
+ std::thread::yield_now();
+ }
+ }
+
+ #[tokio::test]
+ async fn the_shutdown_gate_cannot_close_between_a_streamed_claim_and_its_registration() {
+ let cwd = tempfile::tempdir().expect("tempdir");
+ let worker = created_worker(cwd.path()).await;
+
+ // Gate shut: the admission's own re-check rejects, nothing claims.
+ worker.core.lock().unwrap().shutdown_requested = true;
+ assert!(worker.admit_streamed_bash().is_err());
+ assert!(!worker.user_bash.is_running());
+
+ // A gate that closes while a claim waits on the admission lock: the
+ // gate check and the claim are one critical section, so the claim
+ // cannot slip past the close (a claim the close's abort missed would
+ // leave the child running past the exit).
+ {
+ let mut core = worker.core.lock().unwrap();
+ core.shutdown_requested = false;
+ }
+ let mut gate = worker.core.lock().unwrap();
+ // Readiness is the racing admission's own signal, not a fixed sleep:
+ // once the thread is inside its call, the held gate pins it to enter
+ // the critical section only after the close.
+ let entered = Arc::new(AtomicBool::new(false));
+ let racing = std::thread::spawn({
+ let worker = Arc::clone(&worker);
+ let entered = Arc::clone(&entered);
+ move || {
+ entered.store(true, Ordering::SeqCst);
+ worker.admit_streamed_bash().is_ok()
+ }
+ });
+ wait_for(&entered);
+ assert!(
+ !worker.user_bash.is_running(),
+ "the gated claim must not register"
+ );
+ gate.shutdown_requested = true;
+ drop(gate);
+ let claimed = racing.join().expect("claim thread");
+ assert!(!claimed, "the closed gate must reject it");
+ assert!(!worker.user_bash.is_running(), "nothing may register");
+ }
+
+ #[tokio::test]
+ async fn the_shutdown_gate_cannot_close_between_an_awaited_admission_and_its_registration() {
+ let cwd = tempfile::tempdir().expect("tempdir");
+ let worker = created_worker(cwd.path()).await;
+
+ // Gate shut: the admission's own re-check rejects, nothing registers.
+ worker.core.lock().unwrap().shutdown_requested = true;
+ assert!(worker.admit_awaited_bash().is_err());
+ assert!(!worker.user_bash.is_running());
+
+ // A gate that closes while an admission waits on the admission
+ // lock: the gate check and the registration are one critical
+ // section, so the admission cannot slip past the close (a
+ // registration the close's abort missed would leave the child
+ // running past the exit).
+ {
+ let mut core = worker.core.lock().unwrap();
+ core.shutdown_requested = false;
+ }
+ let mut gate = worker.core.lock().unwrap();
+ // Readiness is the racing admission's own signal, not a fixed sleep
+ // (same as the streamed claim's gate test).
+ let entered = Arc::new(AtomicBool::new(false));
+ let racing = std::thread::spawn({
+ let worker = Arc::clone(&worker);
+ let entered = Arc::clone(&entered);
+ move || {
+ entered.store(true, Ordering::SeqCst);
+ worker.admit_awaited_bash().is_ok()
+ }
+ });
+ wait_for(&entered);
+ assert!(
+ !worker.user_bash.is_running(),
+ "the gated admission must not register"
+ );
+ gate.shutdown_requested = true;
+ drop(gate);
+ let admitted = racing.join().expect("admission thread");
+ assert!(!admitted, "the closed gate must reject it");
+ assert!(!worker.user_bash.is_running(), "nothing may register");
+ }
+
+ #[tokio::test]
+ async fn the_close_abort_cancels_an_admitted_awaited_run_and_settles_after_it() {
+ let cwd = tempfile::tempdir().expect("tempdir");
+ let worker = created_worker(cwd.path()).await;
+ let awaited = worker.admit_awaited_bash().expect("gate open");
+ assert!(worker.user_bash.is_running());
+ let settle = tokio::spawn({
+ let worker = Arc::clone(&worker);
+ async move { worker.user_bash.abort_and_settle().await }
+ });
+ // The close's abort cancels the admitted run's token: the token's
+ // own cancellation is the observable signal, and the bound only
+ // bounds failure.
+ tokio::time::timeout(
+ std::time::Duration::from_secs(2),
+ awaited.cancellation.cancelled(),
+ )
+ .await
+ .expect("the abort never reached the admitted token");
+ // The settle still waits: the run counts until it settles.
+ assert!(!settle.is_finished());
+ drop(awaited);
+ tokio::time::timeout(std::time::Duration::from_secs(2), settle)
+ .await
+ .expect("the settle completes once the run settles")
+ .expect("the settle task joins");
+ }
+
+ #[tokio::test]
+ async fn an_awaited_dispatch_against_a_closing_gate_records_no_row() {
+ let cwd = tempfile::tempdir().expect("tempdir");
+ let worker = created_worker(cwd.path()).await;
+ worker.core.lock().unwrap().shutdown_requested = true;
+ let response = worker
+ .dispatch(
+ "execute_bash_and_wait",
+ &json!({ "activeSessionId": "bash-session", "command": "echo no" }),
+ )
+ .await;
+ assert!(!response.success);
+ assert_eq!(response.error.as_deref(), Some("Session is shutting down"));
+ assert_eq!(
+ bash_rows(&worker).len(),
+ 0,
+ "a rejected run records nothing"
+ );
+ assert!(!worker.user_bash.is_running());
+ }
}
diff --git a/crates/pa-daemon/src/worker/commands.rs b/crates/pa-daemon/src/worker/commands.rs
index b34f47d08f..26ae1c203e 100644
--- a/crates/pa-daemon/src/worker/commands.rs
+++ b/crates/pa-daemon/src/worker/commands.rs
@@ -152,7 +152,7 @@ impl Worker {
"factory_activity" => self.handle_factory_activity(payload).await,
"execute_bash" => self.handle_execute_bash(payload),
"execute_bash_and_wait" => self.handle_execute_bash_and_wait(payload).await,
- "abort_bash" => self.handle_abort_bash().await,
+ "abort_bash" => self.handle_abort_bash(),
"list_kernel_bash" | "tail_kernel_bash" | "kill_kernel_bash" => {
self.handle_kernel_bash_activity(command_type, payload)
.await
diff --git a/crates/pa-daemon/src/worker/lifecycle.rs b/crates/pa-daemon/src/worker/lifecycle.rs
index 9818776773..1b59668546 100644
--- a/crates/pa-daemon/src/worker/lifecycle.rs
+++ b/crates/pa-daemon/src/worker/lifecycle.rs
@@ -76,14 +76,17 @@ impl Worker {
{
let mut core = self.core.lock().unwrap();
// The shutdown gate closes FIRST: a racing execute_bash must see the
- // stop before the abort runs, or the fresh claim clears the abort and
- // spawns a child the exit leaves running.
+ // stop before the abort runs — a claim that slipped in after the
+ // abort would register a controller nothing later cancels, and the
+ // exit leaves its child running.
core.shutdown_requested = true;
core.abort_requested = true;
}
// The running user bash goes with the stop: the passivation stop must
- // never leave the user's process running after the worker exits.
- self.user_bash.abort().await;
+ // never leave the user's process running after the worker exits. Each
+ // run kills its tree in its own task, so the stop waits (bounded) for
+ // the runs to settle (TS `abortBashForClose`).
+ self.user_bash.abort_and_settle().await;
// TS `shutdown` closes through `session.abort()` -> `requestAbort()`:
// the in-flight turn's fetch cancels now, not at its next event.
self.engine.abort_in_flight_turn();
diff --git a/crates/pa-daemon/src/worker/turn_stream_tests/interleave.rs b/crates/pa-daemon/src/worker/turn_stream_tests/interleave.rs
index 83207ea2ee..dcf3b8bc92 100644
--- a/crates/pa-daemon/src/worker/turn_stream_tests/interleave.rs
+++ b/crates/pa-daemon/src/worker/turn_stream_tests/interleave.rs
@@ -77,12 +77,11 @@
//! coverage. The departure release (a closed connection's release
//! re-arming the window) is the park family's window battery too. The
//! gate-before-abort ORDER inside `handle_shutdown` (one critical section
-//! sets `shutdown_requested` before `user_bash.abort()` runs) has no seam
-//! a test can hold between the two steps, so no test here pins it - the
-//! tests pin each step's effect. The base's `UserBash` open corners (the single-slot overlap,
-//! the forked-grandchild survival) are TS-anchored follow-ups on the
-//! base's bash surface — out of this lane's scope (the round-9 rebuttal
-//! carries them).
+//! sets `shutdown_requested` before `user_bash.abort_and_settle()` runs) has
+//! no seam a test can hold between the two steps, so no test here pins it -
+//! the tests pin each step's effect. The forked-child kill and the stop's
+//! settle wait are pinned in `user_bash`'s own tests; the single-slot overlap
+//! stays a TS-anchored follow-up on the base's bash surface.
// Unix-only: the probe harness stands in for the supervisor side of
// the passivation ask over a real unix domain socket
// (tokio::net::UnixListener), which has no counterpart on the windows
diff --git a/crates/pa-daemon/tests/worker_orphan_exit_e2e.rs b/crates/pa-daemon/tests/worker_orphan_exit_e2e.rs
index d4b6f4c5d2..deb5bc1306 100644
--- a/crates/pa-daemon/tests/worker_orphan_exit_e2e.rs
+++ b/crates/pa-daemon/tests/worker_orphan_exit_e2e.rs
@@ -274,3 +274,59 @@ fn the_orphan_exit_preserves_the_session_and_allows_dead_owner_lease_reclaim() {
"the session file survives the lease release"
);
}
+
+/// TS `isSessionActive` counts `isBashRunning`: an orphaned worker waits for a
+/// running user bash instead of exiting under it, so the command finishes and
+/// its row lands in the session file. The command is the work being waited on,
+/// not a readiness signal: an idle orphan exits at the second availability
+/// check after the client's close (checks every 5s, a 4s window), so a worker
+/// that ignored the run would exit while the 20s command still runs.
+#[test]
+fn an_orphaned_worker_waits_for_a_running_user_bash() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let canonical_dir = std::fs::canonicalize(dir.path()).expect("canonical dir");
+ let supervisor_socket = dir.path().join("never-bound-supervisor.sock");
+ let mut worker = spawn_worker_with_window(dir.path(), &supervisor_socket, "4000");
+ let worker_socket = dir.path().join("worker.sock");
+ wait_worker_socket(&worker_socket);
+ let session_file = canonical_dir.join("orphan-bash-session.jsonl");
+ let mut client = WorkerClient::connect(&worker_socket, "orphan-e2e-bootstrap-token");
+ let created = client.request(
+ "create",
+ &serde_json::json!({"sessionPath": session_file.to_string_lossy()}),
+ );
+ assert_eq!(created["success"], true, "create failed: {created}");
+ let command = "sleep 20; echo finished";
+ let started = client.request("execute_bash", &serde_json::json!({ "command": command }));
+ assert_eq!(started["success"], true, "execute_bash failed: {started}");
+ drop(client);
+
+ assert!(
+ wait_exit(&mut worker.child, Duration::from_secs(40)),
+ "the orphaned worker exited once the command settled"
+ );
+ let rows: Vec<serde_json::Value> = std::fs::read_to_string(&session_file)
+ .expect("session file")
+ .lines()
+ .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
+ .map(|entry| entry["message"].clone())
+ .filter(|message| message["role"] == "bashExecution")
+ .map(|row| {
+ serde_json::json!({
+ "command": row["command"],
+ "output": row["output"],
+ "exitCode": row["exitCode"],
+ "cancelled": row["cancelled"],
+ })
+ })
+ .collect();
+ assert_eq!(
+ rows,
+ vec![serde_json::json!({
+ "command": command,
+ "output": "finished\n",
+ "exitCode": 0,
+ "cancelled": false,
+ })]
+ );
+}