Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

add race tests for idle session stops and fix two bugs (#3195)

ktb98481c351a6
diff --git a/crates/pa-daemon/src/worker/connection.rs b/crates/pa-daemon/src/worker/connection.rs
index f99e305669..be64e44a3b 100644
--- a/crates/pa-daemon/src/worker/connection.rs
+++ b/crates/pa-daemon/src/worker/connection.rs
@@ -823,9 +823,6 @@ impl Worker {
.and_then(|value| serde_json::from_value::<DaemonResumeCursor>(value).ok());
let mut core = self.core.lock().unwrap();
- if !core.attached_client_ids.iter().any(|id| id == &client_id) {
- core.attached_client_ids.push(client_id.clone());
- }
// The connection-scoped registry (the fresh bots' release
// findings): the attach's retention is keyed by the connection
// token so the release on ANY return path (the guard's Drop)
@@ -833,8 +830,20 @@ impl Worker {
// clientId's `anonymous` fallback included. The core lock stays
// held (the registry's lock nests inside it — the same order
// the release path uses).
- if let Some(token) = payload.get("connectionToken").and_then(Value::as_str) {
- self.register_session_attach(token, &client_id);
+ //
+ // The core retain rides the registration's verdict: a token the
+ // guard already released (the close beat the detached handler)
+ // must not recreate an unowned hold - the registry entry stays
+ // empty, so nothing would ever release it and the idle
+ // passivation's unattached gate closes forever. A routed attach
+ // without a connection token (the supervisor's shape) owns its
+ // lifecycle on the routed detach path, so its retain stands.
+ let retained = match payload.get("connectionToken").and_then(Value::as_str) {
+ Some(token) => self.register_session_attach(token, &client_id),
+ None => true,
+ };
+ if retained && !core.attached_client_ids.iter().any(|id| id == &client_id) {
+ core.attached_client_ids.push(client_id.clone());
}
let summary = self.summary_locked(&core);
let mut messages: Vec<Value> = core
@@ -961,16 +970,22 @@ impl Worker {
/// already released is REJECTED (the round-8 bots' finding: the
/// attach dispatch is detached, so the connection's close can beat
/// the handler's registration - a late registration would recreate
- /// an unowned attachment that leaks the hold forever).
- pub(crate) fn register_session_attach(&self, token: &str, client_id: &str) {
+ /// an unowned attachment that leaks the hold forever). The verdict
+ /// rides back to the caller: the attach's CORE retain is taken only
+ /// on an accepted registration (the interleave harness's residual
+ /// finding - the round-8 belt closed the registry entry, but the
+ /// ungated core push still leaked the id in `attached_client_ids`
+ /// with no registry entry left to release it).
+ pub(crate) fn register_session_attach(&self, token: &str, client_id: &str) -> bool {
let mut attachments = self.session_attachments.lock().unwrap();
if self.released_attach_tokens.lock().unwrap().contains(token) {
- return;
+ return false;
}
let ids = attachments.entry(token.to_string()).or_default();
if !ids.iter().any(|id| id == client_id) {
ids.push(client_id.to_string());
}
+ true
}
/// Release one connection's retained attaches (the registry's
@@ -989,10 +1004,13 @@ impl Worker {
// and leak the hold).
if final_release {
let mut released = self.released_attach_tokens.lock().unwrap();
- released.insert(token.to_string());
- if released.len() > 8192 {
+ // Clear before the insert: the token released right now is
+ // the one most likely to race a late registration, so the
+ // overflow must never forget it.
+ if released.len() >= 8192 {
released.clear();
}
+ released.insert(token.to_string());
}
let mut core = self.core.lock().unwrap();
let ids = {
diff --git a/crates/pa-daemon/src/worker/turn.rs b/crates/pa-daemon/src/worker/turn.rs
index 6f38a4ee9f..554667e99e 100644
--- a/crates/pa-daemon/src/worker/turn.rs
+++ b/crates/pa-daemon/src/worker/turn.rs
@@ -353,14 +353,19 @@ impl TurnRunner {
// The post-await revalidation (the fresh bots' race findings):
// the engine gate's await opened a window - a bash admitted, a
// prompt parked in a lane, a replay prefix restored, a manual
- // compaction started, or a CLIENT ATTACHED during it must all
- // cancel the stop (the shutdown would cancel the compaction and
- // disconnect the new client); the bash that started in the
- // window keeps the worker resident exactly like the pre-gate
- // check.
+ // compaction started, a CLIENT ATTACHED, or the worker's own
+ // graceful SHUTDOWN starting during it must all cancel the stop
+ // (the shutdown would cancel the compaction and disconnect the
+ // new client; a stop ask under a running shutdown races the
+ // worker's own exit). The shutdown arm is stricter than TS:
+ // daemon-mode.ts `passivateSession` checks `shuttingDown` only
+ // BEFORE its fresh-snapshot await, not after it. The bash that
+ // started in the window keeps the worker resident exactly like
+ // the pre-gate check.
{
let core = self.core.lock().unwrap();
if core.compacting
+ || core.shutdown_requested
|| core.queued_input_suspended
|| !core.attached_client_ids.is_empty()
|| !core.steering.is_empty()
diff --git a/crates/pa-daemon/src/worker/turn_stream_tests.rs b/crates/pa-daemon/src/worker/turn_stream_tests.rs
index 44b2df5822..397cb264ed 100644
--- a/crates/pa-daemon/src/worker/turn_stream_tests.rs
+++ b/crates/pa-daemon/src/worker/turn_stream_tests.rs
@@ -14,6 +14,7 @@ mod abort_idle_race;
mod broadcast;
mod burst;
mod feed;
+mod interleave;
mod park;
mod queue;
diff --git a/crates/pa-daemon/src/worker/turn_stream_tests/interleave.rs b/crates/pa-daemon/src/worker/turn_stream_tests/interleave.rs
new file mode 100644
index 0000000000..c1b1f09ebc
--- /dev/null
+++ b/crates/pa-daemon/src/worker/turn_stream_tests/interleave.rs
@@ -0,0 +1,942 @@
+//! The interleave harness (the passivation lane's named follow-up from the
+//! nine-round bar): the race windows between concurrent lifecycle events —
+//! the class every fresh bot pass found a new instance of (the round-1
+//! concurrent-wake join, the round-5 departure release, the round-6
+//! post-await windows, the round-7 compacting/attach revalidation, the
+//! round-8 shutdown/admission and attach/close orders, the round-9 token
+//! lifecycle) — pinned as tests, so the class is closed by construction,
+//! not by the next bot pass.
+//!
+//! THE HARNESS CONTRACT (the abort-idle-race lane's rate-harness precedents,
+//! made deterministic):
+//! - THE DIRECT-DRIVE FIRE: the raced window lives inside
+//! [`TurnRunner::maybe_request_idle_passivation`] — the engine gate's
+//! await, between the fresh-snapshot fence and the post-await
+//! revalidation. The harness drives THAT path directly (the same
+//! discipline the product's own e2e uses: `idle_passivation_e2e.rs`
+//! drives the worker->supervisor passivation request directly so the
+//! e2e stays gate-fast). The park->window->timer leg cannot be driven
+//! deterministically: the park arm re-stamps `last_activity_ms` from
+//! `SystemTime` and the fire's fresh-clock gate re-checks the same wall
+//! clock, so no in-process clock fake can carry the fire past its own
+//! gate (and `idleEvictionMinutes: 0` is not a zero threshold — the
+//! settings grammar falls back to the default). The timer leg stays
+//! pinned at rest by the park family's window battery; this harness
+//! pins the RACE — the events landing while the fire holds.
+//! - PROBE-TRUE-AT-THE-SEAM: every raced test parks the fire at a
+//! PROVABLE point before landing the racing event — the fire holds
+//! inside the engine gate's await (the [`GateHoldEngine`] seam, the
+//! exact window the post-await revalidation covers), reached exactly
+//! once (the gate counter). The racing event never lands blind.
+//! - THE JOIN BARRIER: the settle is the fire task's own join — the
+//! raced continuation (the post-await revalidation, the ask) runs to
+//! completion before any assertion reads the transcript. No test
+//! sleeps; no assertion reads a wall latency.
+//! - THE TRANSCRIPT ASSERTION: the settled state names the winner —
+//! the recording supervisor probe OBSERVES the ask as the exact
+//! command the product sends (`worker_idle_passivation` with the
+//! worker token and the threshold), never inferred from a side
+//! effect (the served-path oracle, the `window_serves` pattern: an
+//! oracle that cannot prove the request left is vacuous exactly
+//! where it matters).
+//! - THE SERVED-PATH PAIRING: every cancel arm shares the positive
+//! arm's fixture, so a broken harness (a link that never answers, a
+//! gate that never releases) fails the positive arm first — the
+//! cancel arms cannot green themselves by never asking.
+//!
+//! THE PINS (the nine rounds' windows, each named by the round that found
+//! it, plus the two residual windows the harness itself found — the lane's
+//! product fixes):
+//! - THE POST-AWAIT REVALIDATION (rounds 6/7/9): a client attaching, lane
+//! work parking, a manual compaction starting, an input suspension
+//! engaging, a user bash admitted, or a shutdown starting WHILE the fire
+//! holds at the engine gate each cancel the stop and keep the racing
+//! work intact. The shutdown arm is the harness's residual finding: the
+//! round-9 "uniform predicate set" claim did not hold at the post-await
+//! site (the arm ships with the harness).
+//! - THE SHUTDOWN'S BASH ADMISSION (rounds 7/8): a bash dispatched while
+//! the stop is closing is refused ("Session is shutting down"), and a
+//! bash admitted before the stop is aborted by it (the settled run
+//! reports cancelled).
+//! - THE ATTACH/CLOSE TOKEN LIFECYCLE (rounds 8/9): both orders pinned —
+//! the close beating the detached handler's registration leaves no
+//! unowned hold (the harness's residual finding: the round-8 belt
+//! closed the registry entry but the core retain was still ungated; the
+//! retain now rides the registration's verdict), the registration
+//! beating the close releases cleanly, the non-final detach re-registers,
+//! the shared client id survives one connection's close, the
+//! released-token set caps.
+//!
+//! DISCLOSED COVERAGE: the input-pause lease arm of the suspension gates
+//! shares the window/fence predicate with `queued_input_suspended` (one
+//! `||` term at every site); the pause table's lease arms are not
+//! runner-seizable in-crate, so the arm stays covered at rest by the park
+//! family's predicate-set test. The timer leg (the park loop's sleep to
+//! the threshold) is wall-clock-bound by design and stays pinned at rest
+//! by the park family's window battery plus the product's real-time
+//! coverage. The departure release (a closed connection's release
+//! re-arming the window) is the park family's window battery too. The
+//! gate-before-abort ORDER inside `handle_shutdown` (one critical section
+//! sets `shutdown_requested` before `user_bash.abort()` runs) has no seam
+//! a test can hold between the two steps, so no test here pins it - the
+//! tests pin each step's effect. The base's `UserBash` open corners (the single-slot overlap,
+//! the forked-grandchild survival) are TS-anchored follow-ups on the
+//! base's bash surface — out of this lane's scope (the round-9 rebuttal
+//! carries them).
+use super::park::passivation_settings;
+use super::*;
+
+// ---------------------------------------------------------------------------
+// The fixtures: the recording supervisor probe, the engine-gate seam, and
+// the runner/worker builders.
+// ---------------------------------------------------------------------------
+
+/// The recording supervisor-side probe: a real private-framed JSONL socket
+/// the harness owns, standing in for the supervisor's stop path. The fire's
+/// ask is OBSERVED as the exact command the product sends (the
+/// `worker_idle_passivation` shape with the worker token and the threshold)
+/// and answered with a success — never inferred from a side effect.
+struct StopAskProbe {
+ socket: PathBuf,
+ asks: Arc<Mutex<Vec<Value>>>,
+ _dir: tempfile::TempDir,
+}
+
+impl StopAskProbe {
+ /// Bind the probe socket and spawn its accept loop (one connection per
+ /// ask — the product link dials a fresh socket per request).
+ fn spawn() -> Self {
+ let dir = tempfile::TempDir::new().expect("probe dir");
+ let socket = dir.path().join("probe.sock");
+ let listener = tokio::net::UnixListener::bind(&socket).expect("bind probe socket");
+ let asks: Arc<Mutex<Vec<Value>>> = Arc::new(Mutex::new(Vec::new()));
+ let sink = Arc::clone(&asks);
+ tokio::spawn(async move {
+ loop {
+ let Ok((stream, _)) = listener.accept().await else {
+ break;
+ };
+ let asks = Arc::clone(&sink);
+ tokio::spawn(async move {
+ use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
+ let (read_half, mut write_half) = stream.into_split();
+ // The link's handshake: the probe is the supervisor
+ // side, so the hello goes first (the link only checks
+ // the `daemon_hello` tag).
+ if write_half
+ .write_all(
+ b"{\"type\":\"daemon_hello\",\"protocol\":{\"name\":\"prime-agent.daemon\",\"version\":7}}\n",
+ )
+ .await
+ .is_err()
+ {
+ return;
+ }
+ let mut reader = BufReader::new(read_half);
+ let mut line = String::new();
+ loop {
+ line.clear();
+ match reader.read_line(&mut line).await {
+ Ok(0) | Err(_) => return,
+ Ok(_) => {}
+ }
+ let Ok(value) = serde_json::from_str::<Value>(line.trim()) else {
+ continue;
+ };
+ if value.get("type").and_then(Value::as_str) != Some("command") {
+ continue;
+ }
+ let Some(command) = value.get("command").cloned() else {
+ continue;
+ };
+ let id = value
+ .get("id")
+ .and_then(Value::as_str)
+ .unwrap_or_default()
+ .to_string();
+ let kind = command
+ .get("type")
+ .and_then(Value::as_str)
+ .unwrap_or_default()
+ .to_string();
+ asks.lock().unwrap().push(command);
+ let response = crate::protocol::response_line(
+ &crate::protocol::response_success(Some(&id), &kind, None),
+ );
+ let mut answer = serde_json::to_string(&response).unwrap_or_default();
+ answer.push('\n');
+ if write_half.write_all(answer.as_bytes()).await.is_err() {
+ return;
+ }
+ }
+ });
+ }
+ });
+ Self {
+ socket,
+ asks,
+ _dir: dir,
+ }
+ }
+
+ /// The recorded stop asks (the served-path oracle's read side).
+ fn asks(&self) -> Vec<Value> {
+ self.asks.lock().unwrap().clone()
+ }
+}
+
+/// The engine-gate seam: `can_passivate_worker` signals entry and
+/// parks on its release Notify, so the harness holds the fire at the EXACT
+/// await the post-await revalidation covers — probe-true-at-the-seam. The
+/// engine counts gate entries (the cancel arms must reach it exactly once).
+struct GateHoldEngine {
+ entered: Arc<Notify>,
+ release: Arc<Notify>,
+ gates: Arc<std::sync::atomic::AtomicUsize>,
+}
+
+impl GateHoldEngine {
+ fn new() -> Self {
+ Self {
+ entered: Arc::new(Notify::new()),
+ release: Arc::new(Notify::new()),
+ gates: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
+ }
+ }
+}
+
+impl SessionEngine for GateHoldEngine {
+ fn run_prompt(
+ &self,
+ _prompt_index: usize,
+ _request: PromptRequest,
+ _aborted: &dyn Fn() -> bool,
+ emit: &mut dyn FnMut(EngineEvent) -> bool,
+ ) {
+ emit(EngineEvent::Done(Ok(())));
+ }
+
+ fn run_side_question(
+ &self,
+ _request: SideQuestionRequest,
+ _signal: &pa_agent::abort::AbortSignal,
+ _sink: &pa_core::session_engine::side_question::SideQuestionSink,
+ ) -> SideQuestionOutcome {
+ SideQuestionOutcome::Failed {
+ answer: String::new(),
+ error: "unsupported".to_string(),
+ }
+ }
+
+ fn run_compaction(
+ &self,
+ _request: CompactionRequest,
+ _signal: &pa_agent::abort::AbortSignal,
+ ) -> CompactionOutcome {
+ CompactionOutcome::Skipped {
+ message: "nothing to compact".to_string(),
+ }
+ }
+
+ fn run_branch_summary(
+ &self,
+ _request: crate::engine::BranchSummaryRequest,
+ _signal: &pa_agent::abort::AbortSignal,
+ ) -> crate::engine::BranchSummaryOutcome {
+ crate::engine::BranchSummaryOutcome::Failed {
+ error: "unsupported".to_string(),
+ }
+ }
+
+ fn rebuild_session_context(
+ &self,
+ _branch_entries: Vec<pa_types::session::FileEntry>,
+ _goal_reload: pa_core::session_engine::goal_driver::GoalBranchReload,
+ ) -> anyhow::Result<()> {
+ Ok(())
+ }
+
+ fn can_passivate_worker(
+ &self,
+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send + '_>> {
+ let entered = Arc::clone(&self.entered);
+ let release = Arc::clone(&self.release);
+ let gates = Arc::clone(&self.gates);
+ Box::pin(async move {
+ gates.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ entered.notify_waiters();
+ release.notified().await;
+ true
+ })
+ }
+}
+
+/// The raced-fire runner fixture: an unattached session (roots and children
+/// alike reach the fire since the depth fence left) under the one-minute
+/// eviction threshold, its idle clock already two minutes
+/// past the stamp (the park loop's own re-stamp shape — `now_ms() - 120s`,
+/// wall-clock elapsed, so the fire's fresh-clock gate passes
+/// deterministically; no test waits a real threshold), the probe link in
+/// place of the supervisor socket, and the gate engine as the session's
+/// engine (the fire can reach the gate and hold).
+fn interleave_runner(
+ engine: &Arc<GateHoldEngine>,
+ probe: &StopAskProbe,
+) -> (TurnRunner, tempfile::TempDir) {
+ let dir = tempfile::TempDir::new().expect("runner dir");
+ let agent_dir = dir.path().join("agent");
+ passivation_settings(&agent_dir, &json!(1));
+ let mut runner = burst_runner(Arc::clone(engine) as Arc<dyn SessionEngine>);
+ {
+ let mut core = runner.core.lock().unwrap();
+ core.cwd = dir.path().to_string_lossy().to_string();
+ // The idle clock crossed the 1-minute threshold two minutes ago:
+ // the fresh-snapshot fence and the fresh-clock gate both pass, and
+ // the 120s margin dwarfs any test-time jitter.
+ core.last_activity_ms = crate::util::now_ms().saturating_sub(120_000);
+ }
+ runner.passivation.agent_dir = agent_dir;
+ runner.passivation.link = Arc::new(crate::supervisor_link::SupervisorLink::new(
+ probe.socket.clone(),
+ ));
+ runner.passivation.worker_token = "interleave-worker-token".to_string();
+ (runner, dir)
+}
+
+/// Drive the fire itself to the engine-gate seam and prove it: the spawned
+/// fire runs the fresh-snapshot fence and the fresh-clock gate, and the
+/// returned seam completes only once the fire PROVABLY reached the engine
+/// gate's await (the racing event's landing zone). The park->window->timer
+/// leg is the park family's at-rest battery (see the module docs).
+async fn drive_the_fire_to_the_gate(
+ runner: TurnRunner,
+ dir: tempfile::TempDir,
+ gate: &GateHoldEngine,
+) -> FiredSeam {
+ let runner = Arc::new(runner);
+ let core = Arc::clone(&runner.core);
+ let user_bash = Arc::clone(&runner.user_bash);
+ let entered = gate.entered.notified();
+ let fired = {
+ let fire = Arc::clone(&runner);
+ tokio::spawn(async move {
+ fire.maybe_request_idle_passivation().await;
+ })
+ };
+ entered.await;
+ assert_eq!(
+ gate.gates.load(std::sync::atomic::Ordering::SeqCst),
+ 1,
+ "the fire must reach the engine gate (the seam probe)"
+ );
+ FiredSeam {
+ core,
+ user_bash,
+ fired,
+ _dir: dir,
+ }
+}
+
+/// The reached seam: the fire is parked inside the engine gate's await,
+/// and these are the handles the racing events land through.
+struct FiredSeam {
+ core: Arc<Mutex<SessionCore>>,
+ user_bash: Arc<crate::user_bash::UserBash>,
+ /// The in-flight fire: the settle barrier — the raced continuation
+ /// (the post-await revalidation, the ask) runs to completion when
+ /// this handle joins.
+ fired: tokio::task::JoinHandle<()>,
+ /// The runner's temp tree (the settings the fire reads): it stays
+ /// alive with the seam so the fire never reads a gone file.
+ _dir: tempfile::TempDir,
+}
+
+/// Release the held fire and settle on its own completion: the post-await
+/// revalidation and the ask run to their end before any assertion reads
+/// the transcript — the join barrier, never a timing window. The settle
+/// TAKES the fire handle (the caller's last use of the seam's settle
+/// arm; the survivors stay readable through the seam's own handles).
+async fn release_the_gate_and_settle(gate: &GateHoldEngine, fired: tokio::task::JoinHandle<()>) {
+ gate.release.notify_one();
+ fired.await.expect("the fire settles at its own completion");
+}
+
+// ---------------------------------------------------------------------------
+// THE POST-AWAIT REVALIDATION (the fire's window, rounds 6/7/9 + the
+// shutdown arm the harness found).
+// ---------------------------------------------------------------------------
+
+/// The positive arm (the served-path oracle): an unraced fire passes the
+/// fresh-snapshot fence, the fresh-clock gate, and the engine gate, then
+/// asks the supervisor to stop — exactly one `worker_idle_passivation` ask,
+/// carrying the worker token and the live threshold. Every cancel arm
+/// below shares this fixture; without this proof a missing ask would be
+/// vacuous.
+#[tokio::test]
+async fn the_unraced_fire_passes_the_gate_and_asks_the_supervisor_to_stop() {
+ let probe = StopAskProbe::spawn();
+ let gate = Arc::new(GateHoldEngine::new());
+ let (runner, dir) = interleave_runner(&gate, &probe);
+ let seam = drive_the_fire_to_the_gate(runner, dir, &gate).await;
+ release_the_gate_and_settle(&gate, seam.fired).await;
+ let asks = probe.asks();
+ assert_eq!(
+ asks.len(),
+ 1,
+ "the unraced fire must ask exactly once: {asks:?}"
+ );
+ assert_eq!(
+ asks[0].get("type").and_then(Value::as_str),
+ Some("worker_idle_passivation")
+ );
+ assert_eq!(
+ asks[0].get("workerToken").and_then(Value::as_str),
+ Some("interleave-worker-token")
+ );
+ assert_eq!(asks[0].get("idleMinutes").and_then(Value::as_u64), Some(1));
+}
+
+/// A client attaching while the fire holds at the engine gate (the round-7
+/// window): the post-await revalidation cancels the stop, and the racing
+/// client's hold survives intact (the stop would have disconnected it).
+#[tokio::test]
+async fn a_client_attaching_at_the_gate_cancels_the_stop() {
+ let probe = StopAskProbe::spawn();
+ let gate = Arc::new(GateHoldEngine::new());
+ let (runner, dir) = interleave_runner(&gate, &probe);
+ let seam = drive_the_fire_to_the_gate(runner, dir, &gate).await;
+ seam.core
+ .lock()
+ .unwrap()
+ .attached_client_ids
+ .push("late-client".to_string());
+ release_the_gate_and_settle(&gate, seam.fired).await;
+ assert!(
+ probe.asks().is_empty(),
+ "a client attaching at the gate must cancel the stop: {:?}",
+ probe.asks()
+ );
+ assert!(
+ seam.core
+ .lock()
+ .unwrap()
+ .attached_client_ids
+ .iter()
+ .any(|id| id.as_str() == "late-client"),
+ "the racing client's hold must survive the canceled stop"
+ );
+}
+
+/// Lane work parking while the fire holds at the engine gate (the round-6
+/// B1/B3 windows: a racing steer, and a replayed `pending_next_turn` row):
+/// the stop cancels — the queued work lives only on the resident worker,
+/// so the graceful stop would have discarded it — and both rows survive.
+#[tokio::test]
+async fn lane_work_parking_at_the_gate_cancels_the_stop() {
+ let probe = StopAskProbe::spawn();
+ let gate = Arc::new(GateHoldEngine::new());
+ let (runner, dir) = interleave_runner(&gate, &probe);
+ let seam = drive_the_fire_to_the_gate(runner, dir, &gate).await;
+ let parked_steer = QueuedItem {
+ priority: QueuePriority::Human,
+ preview: None,
+ message: "the racing steer".to_string(),
+ custom_message: None,
+ agent_message: None,
+ queue_key: None,
+ admission_id: None,
+ images: Vec::new(),
+ done: None,
+ queue_visible: true,
+ policy: TurnPolicy::Queued,
+ forced_batch: false,
+ };
+ {
+ let mut core = seam.core.lock().unwrap();
+ core.steering.push_back(parked_steer);
+ core.pending_next_turn
+ .push(json!({ "type": "user", "text": "the replayed prefix row" }));
+ }
+ release_the_gate_and_settle(&gate, seam.fired).await;
+ assert!(
+ probe.asks().is_empty(),
+ "lane work parking at the gate must cancel the stop: {:?}",
+ probe.asks()
+ );
+ let core = seam.core.lock().unwrap();
+ assert_eq!(
+ core.steering.len(),
+ 1,
+ "the racing steer must survive the canceled stop"
+ );
+ assert_eq!(
+ core.pending_next_turn.len(),
+ 1,
+ "the racing replay prefix must survive the canceled stop"
+ );
+}
+
+/// A manual compaction starting while the fire holds at the engine gate
+/// (the round-7 window): the stop cancels — the shutdown would have
+/// cancelled the compaction mid-run.
+#[tokio::test]
+async fn a_manual_compaction_starting_at_the_gate_cancels_the_stop() {
+ let probe = StopAskProbe::spawn();
+ let gate = Arc::new(GateHoldEngine::new());
+ let (runner, dir) = interleave_runner(&gate, &probe);
+ let seam = drive_the_fire_to_the_gate(runner, dir, &gate).await;
+ seam.core.lock().unwrap().compacting = true;
+ release_the_gate_and_settle(&gate, seam.fired).await;
+ assert!(
+ probe.asks().is_empty(),
+ "a compaction starting at the gate must cancel the stop: {:?}",
+ probe.asks()
+ );
+}
+
+/// An input suspension engaging while the fire holds at the engine gate
+/// (the round-8 window): the stop cancels — the revival initializes the
+/// fresh core un-suspended, so the paused pump's suspension state would be
+/// lost. The `queued_input_suspended` arm stands for the suspension
+/// predicate set (the input-pause lease arm shares the same `||` term).
+#[tokio::test]
+async fn an_input_suspension_engaging_at_the_gate_cancels_the_stop() {
+ let probe = StopAskProbe::spawn();
+ let gate = Arc::new(GateHoldEngine::new());
+ let (runner, dir) = interleave_runner(&gate, &probe);
+ let seam = drive_the_fire_to_the_gate(runner, dir, &gate).await;
+ seam.core.lock().unwrap().queued_input_suspended = true;
+ release_the_gate_and_settle(&gate, seam.fired).await;
+ assert!(
+ probe.asks().is_empty(),
+ "a suspension engaging at the gate must cancel the stop: {:?}",
+ probe.asks()
+ );
+}
+
+/// A user bash admitted while the fire holds at the engine gate (the
+/// round-6/7 windows): the stop cancels — the worker's exit would have
+/// left the user's process running (the awaited-run bracket holds the same
+/// `isBashRunning` contribution the exclusive slot does).
+#[tokio::test]
+async fn a_user_bash_admitted_at_the_gate_cancels_the_stop() {
+ let probe = StopAskProbe::spawn();
+ let gate = Arc::new(GateHoldEngine::new());
+ let (runner, dir) = interleave_runner(&gate, &probe);
+ let seam = drive_the_fire_to_the_gate(runner, dir, &gate).await;
+ // The awaited-run bracket: `begin_awaited` is the same
+ // `isBashRunning` contribution `execute_bash_and_wait` makes.
+ let awaited = seam.user_bash.begin_awaited();
+ assert!(
+ seam.user_bash.is_running(),
+ "the probe: the bash hold is live at the seam"
+ );
+ release_the_gate_and_settle(&gate, seam.fired).await;
+ assert!(
+ probe.asks().is_empty(),
+ "a bash admitted at the gate must cancel the stop: {:?}",
+ probe.asks()
+ );
+ drop(awaited);
+}
+
+/// A shutdown starting while the fire holds at the engine gate: the stop
+/// cancels (the round-9 "uniform predicate set" claim, made true at the
+/// post-await site — the harness's residual finding: the ask would have
+/// raced the worker's own exit). Stricter than TS: daemon-mode.ts
+/// `passivateSession` checks `shuttingDown` only before its fresh-snapshot
+/// await.
+#[tokio::test]
+async fn a_shutdown_starting_at_the_gate_cancels_the_stop() {
+ let probe = StopAskProbe::spawn();
+ let gate = Arc::new(GateHoldEngine::new());
+ let (runner, dir) = interleave_runner(&gate, &probe);
+ let seam = drive_the_fire_to_the_gate(runner, dir, &gate).await;
+ seam.core.lock().unwrap().shutdown_requested = true;
+ release_the_gate_and_settle(&gate, seam.fired).await;
+ assert!(
+ probe.asks().is_empty(),
+ "a shutdown starting at the gate must cancel the stop ask: {:?}",
+ probe.asks()
+ );
+}
+
+// ---------------------------------------------------------------------------
+// THE SHUTDOWN'S BASH ADMISSION (rounds 7/8): the gate refuses, the abort kills.
+// ---------------------------------------------------------------------------
+
+/// A created worker over the plain scripted engine: the register/release
+/// and shutdown/admission seams need the worker's own dispatch surface.
+async fn interleave_worker() -> (Arc<Worker>, tempfile::TempDir) {
+ let dir = tempfile::TempDir::new().expect("worker dir");
+ let config = WorkerConfig {
+ socket_path: dir.path().join("worker.sock"),
+ supervisor_socket_path: PathBuf::new(),
+ token: "token".to_string(),
+ worker_instance_id: String::new(),
+ active_session_id: "interleave-worker".to_string(),
+ agent_dir: dir.path().join("agent"),
+ recovery_journal_path: dir.path().join("recovery.jsonl"),
+ telemetry_disabled: None,
+ script: Some(json!({ "responses": [] })),
+ };
+ let worker = Arc::new(Worker::new(config, None));
+ let created = worker
+ .dispatch(
+ "create",
+ &json!({ "noSession": true, "cwd": dir.path().to_string_lossy(), "name": "interleave" }),
+ )
+ .await;
+ assert!(created.success, "the worker's create failed: {created:?}");
+ (worker, dir)
+}
+
+/// A bounded state probe (the repo's standard wait shape): poll until the
+/// predicate holds or the deadline passes — never a fixed sleep.
+async fn wait_for(mut predicate: impl FnMut() -> bool) {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
+ while !predicate() {
+ assert!(
+ std::time::Instant::now() < deadline,
+ "the probed state never arrived"
+ );
+ tokio::time::sleep(std::time::Duration::from_millis(10)).await;
+ }
+}
+
+/// Read the worker's event stream until one `session_event` frame of the
+/// given type lands (the positive signal the raced flow ran).
+async fn wait_for_event(
+ events: &mut tokio::sync::broadcast::Receiver<Arc<OutboundFrame>>,
+ event_type: &str,
+) -> Value {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15);
+ loop {
+ let frame =
+ tokio::time::timeout_at(tokio::time::Instant::from_std(deadline), events.recv())
+ .await
+ .expect("the event never arrived")
+ .expect("the event stream stays live");
+ if frame.outbound_type != "session_event" {
+ continue;
+ }
+ let Ok(outbound) = serde_json::from_slice::<Value>(&frame.payload) else {
+ continue;
+ };
+ if outbound["event"]["type"].as_str() == Some(event_type) {
+ return outbound["event"].clone();
+ }
+ }
+}
+
+/// The round-8 admission pin: a bash dispatched while the graceful stop
+/// holds at its work-settled park (a compaction in flight keeps it parked
+/// there) is REFUSED by the shutdown admission gate and spawns no child.
+/// This pins the gate, not its order against the bash abort (see the
+/// module docs).
+#[tokio::test]
+async fn the_shutdown_gate_refuses_a_bash_admitted_inside_the_stop() {
+ let (worker, _dir) = interleave_worker().await;
+ // The settle seam: a compaction in flight parks the graceful stop at
+ // `await_session_work_settled` — AFTER the admission gate closed and
+ // the bash abort ran.
+ worker.core.lock().unwrap().compacting = true;
+ let stopping = {
+ let worker = Arc::clone(&worker);
+ tokio::spawn(async move { worker.dispatch("shutdown", &json!({})).await })
+ };
+ // THE SEAM PROBE: the gate is closed (the stop reached its park).
+ {
+ let core_set = Arc::clone(&worker.core);
+ wait_for(move || core_set.lock().unwrap().shutdown_requested).await;
+ }
+ // THE RACED ADMISSION: a bash dispatched inside the stop.
+ let raced = worker
+ .dispatch("execute_bash", &json!({ "command": "sleep 30" }))
+ .await;
+ assert!(
+ !raced.success,
+ "a bash admitted inside the stop must be refused: {raced:?}"
+ );
+ assert!(
+ raced
+ .error
+ .as_deref()
+ .unwrap_or_default()
+ .contains("shutting down"),
+ "the refusal names the shutdown gate: {raced:?}"
+ );
+ assert!(
+ !worker.user_bash.is_running(),
+ "no bash child may spawn behind the stop's abort"
+ );
+ // The stop completes once the compaction window ends.
+ worker.core.lock().unwrap().compacting = false;
+ worker.idle_notify.notify_waiters();
+ let stopped = tokio::time::timeout(std::time::Duration::from_secs(15), stopping)
+ .await
+ .expect("the shutdown settles")
+ .expect("the shutdown task joins");
+ assert!(stopped.success, "the graceful stop answers: {stopped:?}");
+}
+
+/// The round-7 abort pin: a bash admitted BEFORE the gate closed (the
+/// pre-window interleave) is aborted by the stop — the settled run reports
+/// cancelled, so the worker's exit never leaves the user's process running.
+#[tokio::test]
+async fn a_bash_admitted_before_the_gate_is_aborted_by_the_stop() {
+ let (worker, _dir) = interleave_worker().await;
+ let mut events = worker.events.subscribe();
+ let started = worker
+ .dispatch("execute_bash", &json!({ "command": "sleep 30" }))
+ .await;
+ assert!(
+ started.success,
+ "the bash admits before the gate: {started:?}"
+ );
+ // THE POSITIVE SIGNAL: the bash_start frame proves the child is live.
+ wait_for_event(&mut events, "bash_start").await;
+ assert!(
+ worker.user_bash.is_running(),
+ "the probe: the bash child is live before the stop"
+ );
+ let stopped = worker.dispatch("shutdown", &json!({})).await;
+ assert!(stopped.success, "the graceful stop answers: {stopped:?}");
+ let ended = wait_for_event(&mut events, "bash_end").await;
+ assert_eq!(
+ ended.get("cancelled").and_then(Value::as_bool),
+ Some(true),
+ "the stop must abort the running user bash: {ended}"
+ );
+ assert!(
+ !worker.user_bash.is_running(),
+ "the aborted bash releases the slot"
+ );
+}
+
+// ---------------------------------------------------------------------------
+// THE ATTACH/CLOSE TOKEN LIFECYCLE (rounds 8/9): both orders pinned.
+// ---------------------------------------------------------------------------
+
+/// The round-8 order the harness re-found: the connection's close beats the
+/// detached attach handler's registration — the token is dead, and the late
+/// handler must not recreate an unowned hold (the round-8 belt closed the
+/// REGISTRY entry; the core retain was still ungated, so the id leaked in
+/// `attached_client_ids` with no registry entry to release it — the idle
+/// gate disabled forever. The retain now rides the registration's
+/// verdict).
+#[tokio::test]
+async fn the_close_beats_the_late_attach_registration() {
+ let (worker, _dir) = interleave_worker().await;
+ // THE CLOSE: the attach guard's Drop ran first (the connection's EOF
+ // beat the detached handler).
+ worker.release_session_attachments("late-conn", true);
+ assert!(
+ worker
+ .released_attach_tokens
+ .lock()
+ .unwrap()
+ .contains("late-conn"),
+ "the probe: the token is dead before the late registration"
+ );
+ // THE LATE REGISTRATION: the detached attach handler runs now, on the
+ // dead connection's token.
+ let response = worker.handle_attach(&json!({
+ "clientId": "late-client",
+ "connectionToken": "late-conn",
+ }));
+ assert!(
+ response.success,
+ "the attach itself still answers: {response:?}"
+ );
+ assert!(
+ !worker
+ .core
+ .lock()
+ .unwrap()
+ .attached_client_ids
+ .iter()
+ .any(|id| id.as_str() == "late-client"),
+ "the late registration must not recreate the unowned core hold"
+ );
+ assert!(
+ worker
+ .session_attachments
+ .lock()
+ .unwrap()
+ .get("late-conn")
+ .is_none(),
+ "the dead token's registry entry stays empty"
+ );
+}
+
+/// The routed attach (no connection token — the supervisor's shape):
+/// its core retain stands — its release rides the routed detach, so the
+/// token-gated verdict must not hold it.
+#[tokio::test]
+async fn a_routed_attach_without_a_token_keeps_the_core_retain() {
+ let (worker, _dir) = interleave_worker().await;
+ let response = worker.handle_attach(&json!({ "clientId": "supervisor-routed" }));
+ assert!(response.success, "the routed attach answers: {response:?}");
+ assert!(
+ worker
+ .core
+ .lock()
+ .unwrap()
+ .attached_client_ids
+ .iter()
+ .any(|id| id.as_str() == "supervisor-routed"),
+ "the supervisor-routed attach keeps its core retain"
+ );
+}
+
+/// The opposite order (green at the base): the registration beats the
+/// close — the live connection's attach registers, and the close releases
+/// both the registry entry and the core hold.
+#[tokio::test]
+async fn the_registration_beats_the_close_and_the_release_frees_the_hold() {
+ let (worker, _dir) = interleave_worker().await;
+ let response = worker.handle_attach(&json!({
+ "clientId": "client-1",
+ "connectionToken": "conn-1",
+ }));
+ assert!(response.success, "the live attach answers: {response:?}");
+ assert!(
+ worker
+ .core
+ .lock()
+ .unwrap()
+ .attached_client_ids
+ .iter()
+ .any(|id| id.as_str() == "client-1"),
+ "the live attach retains the core hold"
+ );
+ worker.release_session_attachments("conn-1", true);
+ assert!(
+ worker.core.lock().unwrap().attached_client_ids.is_empty(),
+ "the close frees the core hold"
+ );
+ assert!(
+ worker
+ .session_attachments
+ .lock()
+ .unwrap()
+ .get("conn-1")
+ .is_none(),
+ "the close frees the registry entry"
+ );
+ assert!(
+ worker
+ .released_attach_tokens
+ .lock()
+ .unwrap()
+ .contains("conn-1"),
+ "the close marks the token dead"
+ );
+}
+
+/// The round-9 non-final corner: the explicit detach is NOT final — the
+/// connection lives on, so the token stays live, a later re-attach on the
+/// same connection re-registers, and the eventual close releases it (no
+/// leak either way).
+#[tokio::test]
+async fn the_detach_is_not_final_the_reattach_registers_and_the_close_releases() {
+ let (worker, _dir) = interleave_worker().await;
+ let attach = worker.handle_attach(&json!({
+ "clientId": "client-1",
+ "connectionToken": "conn-1",
+ }));
+ assert!(attach.success, "the first attach answers: {attach:?}");
+ // The explicit detach (NOT final — the connection lives on).
+ worker.release_session_attachments("conn-1", false);
+ assert!(
+ !worker
+ .released_attach_tokens
+ .lock()
+ .unwrap()
+ .contains("conn-1"),
+ "the probe: the detach leaves the token live"
+ );
+ // The re-attach on the same connection re-registers.
+ let again = worker.handle_attach(&json!({
+ "clientId": "client-1",
+ "connectionToken": "conn-1",
+ }));
+ assert!(again.success, "the re-attach answers: {again:?}");
+ assert!(
+ worker
+ .session_attachments
+ .lock()
+ .unwrap()
+ .get("conn-1")
+ .is_some_and(|ids| ids.iter().any(|id| id == "client-1")),
+ "the re-attach re-registers on the live token"
+ );
+ // The eventual close releases everything (the re-attach leak corner).
+ worker.release_session_attachments("conn-1", true);
+ assert!(
+ worker.core.lock().unwrap().attached_client_ids.is_empty(),
+ "the close after the re-attach still frees the core hold"
+ );
+}
+
+/// The shared-client-id reconnect shape (the round-6/9 sibling holds): the
+/// id leaves the core only when the LAST live connection holding it goes.
+#[tokio::test]
+async fn a_shared_client_id_survives_one_connection_close_while_the_other_holds() {
+ let (worker, _dir) = interleave_worker().await;
+ let first = worker.handle_attach(&json!({
+ "clientId": "shared-client",
+ "connectionToken": "conn-a",
+ }));
+ let second = worker.handle_attach(&json!({
+ "clientId": "shared-client",
+ "connectionToken": "conn-b",
+ }));
+ assert!(first.success && second.success, "both attaches answer");
+ worker.release_session_attachments("conn-a", true);
+ assert!(
+ worker
+ .core
+ .lock()
+ .unwrap()
+ .attached_client_ids
+ .iter()
+ .any(|id| id.as_str() == "shared-client"),
+ "the shared id survives one connection's close"
+ );
+ worker.release_session_attachments("conn-b", true);
+ assert!(
+ worker.core.lock().unwrap().attached_client_ids.is_empty(),
+ "the last holder's close frees the shared id"
+ );
+}
+
+/// The round-9 overflow belt: the released-token set caps at 8192 with the
+/// clear-on-overflow arm — a long-lived worker cannot grow it unbounded —
+/// and the clear runs BEFORE the insert, so the overflowing release still
+/// rejects its own late registration.
+#[tokio::test]
+async fn the_released_token_set_caps_at_8192() {
+ let (worker, _dir) = interleave_worker().await;
+ for index in 0..8193 {
+ worker.release_session_attachments(&format!("cap-{index}"), true);
+ }
+ let len = worker.released_attach_tokens.lock().unwrap().len();
+ assert!(
+ len <= 8192,
+ "the released-token set must cap on overflow (len {len})"
+ );
+ assert!(
+ worker
+ .released_attach_tokens
+ .lock()
+ .unwrap()
+ .contains("cap-8192"),
+ "the overflowing release must survive the clear"
+ );
+ assert!(
+ !worker.register_session_attach("cap-8192", "late-client"),
+ "a late registration on the overflowing token is rejected"
+ );
+}
diff --git a/crates/pa-daemon/src/worker/turn_stream_tests/park.rs b/crates/pa-daemon/src/worker/turn_stream_tests/park.rs
index 88cc4debee..df4dce6476 100644
--- a/crates/pa-daemon/src/worker/turn_stream_tests/park.rs
+++ b/crates/pa-daemon/src/worker/turn_stream_tests/park.rs
@@ -292,7 +292,7 @@ async fn compacting_window_parks_a_cleared_suspension_until_it_ends() {
/// A settings fixture: the agent dir's `settings.json` carries the
/// `idleEvictionMinutes` value under test.
-fn passivation_settings(dir: &std::path::Path, value: &serde_json::Value) {
+pub(super) fn passivation_settings(dir: &std::path::Path, value: &serde_json::Value) {
std::fs::create_dir_all(dir).unwrap();
let settings = serde_json::json!({ "idleEvictionMinutes": value });
std::fs::write(