A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
make queued agent messages survive every abort shape (#3558)
Sebastian Müller7d191b47ece0
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 738050247a..a546d5a451 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -371,17 +371,17 @@ jobs:test -x "${PRIME_AGENT_KERNEL_PYTHON}"mkdir -p "${RUNNER_TEMP}/lifecycle-parity-receipts"comparison_status=0- for scenario in graceful_shutdown goal_backoff draft_disconnect restore_notice live_mcp; do- verifier="scripts/review_${scenario}_parity.py"- if [ -f "${verifier}" ]; then- if [ "${scenario}" = "draft_disconnect" ]; then- python3 scripts/test_review_draft_disconnect_parity.py || comparison_status=1- fi- if [ "${scenario}" = "live_mcp" ]; then- python3 scripts/test_review_live_mcp_parity.py || comparison_status=1- fi- python3 "${verifier}" --receipt "${RUNNER_TEMP}/lifecycle-parity-receipts/${scenario}.json" || comparison_status=1+ # Discover independently added verifiers so feature PRs do not+ # overwrite or conflict with each other's comparison selection.+ for verifier in scripts/review_*_parity.py; do+ [ -f "${verifier}" ] || continue+ scenario="${verifier#scripts/review_}"+ scenario="${scenario%_parity.py}"+ verifier_test="scripts/test_review_${scenario}_parity.py"+ if [ -f "${verifier_test}" ]; then+ python3 "${verifier_test}" || comparison_status=1fi+ python3 "${verifier}" --receipt "${RUNNER_TEMP}/lifecycle-parity-receipts/${scenario}.json" || comparison_status=1doneexit "${comparison_status}"- name: Upload the binary comparison receiptsdiff --git a/crates/pa-daemon/src/messaging.rs b/crates/pa-daemon/src/messaging.rsindex f8495ad84e..c8901f2150 100644--- a/crates/pa-daemon/src/messaging.rs+++ b/crates/pa-daemon/src/messaging.rs@@ -17,7 +17,7 @@ use crate::supervisor::Supervisor;/// Worker round-trip budget for the sender-summary read and the delivery/// route (TS `WORKER_REQUEST_TIMEOUT_MS`).-const WORKER_REQUEST_TIMEOUT_MS: u64 = 30_000;+pub(crate) const WORKER_REQUEST_TIMEOUT_MS: u64 = 30_000;impl Supervisor {/// `send_message`: route to the target worker as `worker_deliver_message`;diff --git a/crates/pa-daemon/src/worker.rs b/crates/pa-daemon/src/worker.rsindex 5aaa081b64..1fbd77fd77 100644--- a/crates/pa-daemon/src/worker.rs+++ b/crates/pa-daemon/src/worker.rs@@ -128,6 +128,8 @@ pub struct Worker {/// The concrete agent engine behind `engine` for the create command's/// eager session build (the kernel prewarm).pub(crate) agent_engine: Option<std::sync::Arc<crate::agent_engine::AgentSessionEngine>>,+ /// The worker's supervisor link (roster pushes, passivation asks, bounces).+ pub(crate) roster_link: std::sync::Arc<crate::supervisor_link::SupervisorLink>,/// The monotonic roster-delta counter the roster pushes stamp and the/// authoritative pulls embed: a push can race a fresher pull, so the/// supervisor's gate drops the stale snapshot.@@ -277,6 +279,7 @@ impl Worker {/// # Panics////// The wired closures panic on a poisoned session-core mutex.+ #[must_use]pub fn new(config: WorkerConfig, registration: Option<RegistrationHandle>) -> Self {let events = Arc::new(EventPump::new());let supervisor_claims = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));@@ -341,11 +344,10 @@ impl Worker {let idle_notify = Arc::new(Notify::new());// The supervisor link and worker token for roster pushes: one// construction shared by the turn runner and the command arms- // (identical dial path).+ // (identical dial path). The config field is the one supervisor-+ // socket source.let roster_link = std::sync::Arc::new(crate::supervisor_link::SupervisorLink::new(- std::env::var_os(WORKER_SUPERVISOR_SOCKET_ENV)- .map(std::path::PathBuf::from)- .unwrap_or_default(),+ config.supervisor_socket_path.clone(),));let worker_token = std::env::var(WORKER_TOKEN_ENV).unwrap_or_default();let roster_delta_sequence = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));@@ -755,6 +757,7 @@ impl Worker {core,engine,agent_engine,+ roster_link: Arc::clone(&roster_link),roster_delta_sequence,work_notify,idle_notify,diff --git a/crates/pa-daemon/src/worker/agent_message_tests.rs b/crates/pa-daemon/src/worker/agent_message_tests.rsindex 97ace33bd0..778882c59c 100644--- a/crates/pa-daemon/src/worker/agent_message_tests.rs+++ b/crates/pa-daemon/src/worker/agent_message_tests.rs@@ -380,3 +380,139 @@ async fn attach_response_wire_bytes_keep_the_ts_key_order() {}}}++/// `clear_queue` bounces each dropped agent-message row to its sender over+/// the supervisor `send_message` route; the cleared user prompt sends+/// nothing (unix: the link endpoint is an `AF_UNIX` socket).+#[cfg(unix)]+#[tokio::test]+async fn clear_queue_bounces_dropped_agent_messages_to_their_senders() {+ assert_clear_queue_bounces(/*restore_queue*/ false).await;+}++#[cfg(unix)]+#[tokio::test]+async fn clear_queue_bounces_recovered_agent_messages_to_their_senders() {+ assert_clear_queue_bounces(/*restore_queue*/ true).await;+}++#[cfg(unix)]+async fn assert_clear_queue_bounces(restore_queue: bool) {+ let dir = std::env::temp_dir().join(format!("pa-worker-am-bounce-{}", uuid::Uuid::new_v4()));+ std::fs::create_dir_all(&dir).unwrap();+ let socket = dir.join("supervisor.sock");+ let (sends, _server) = crate::worker::tests::fake_supervisor(&socket);+ let config = WorkerConfig {+ socket_path: dir.join("worker.sock"),+ supervisor_socket_path: socket,+ token: "token".to_string(),+ worker_instance_id: String::new(),+ active_session_id: "target-session".to_string(),+ agent_dir: dir.join("agent"),+ recovery_journal_path: dir.join("recovery.jsonl"),+ telemetry_disabled: None,+ script: Some(json!({ "responses": ["ack"] })),+ };+ let worker = Arc::new(Worker::new(config, None));+ // Production opens the recovery sink in serve(); direct-dispatch fixtures+ // must install it before admission so checkpoints persist synchronously.+ *worker.recovery.lock().unwrap() =+ Some(WorkerRecoveryJournal::open(&worker.config.recovery_journal_path).unwrap());+ let created = worker+ .dispatch(+ "create",+ &json!({ "noSession": true, "cwd": "/tmp", "name": "target" }),+ )+ .await;+ assert!(created.success, "create failed: {created:?}");+ // Busy parks both rows in their lanes.+ worker.core.lock().unwrap().busy = true;+ let delivered = worker+ .dispatch(+ "worker_deliver_message",+ &json!({+ "targetActiveSessionId": "target-session",+ "message": "please review the plan",+ "sender": {+ "activeSessionId": "source-session",+ "sessionName": "source-agent",+ },+ }),+ )+ .await;+ assert!(delivered.success, "deliver failed: {delivered:?}");+ let queued = worker+ .dispatch("follow_up", &json!({ "message": "plain queued prompt" }))+ .await;+ assert!(queued.success, "follow_up failed: {queued:?}");+ if restore_queue {+ // Reopen the persisted journal as a respawn would: the marker must+ // survive alongside the custom row for clear/pause/status handling.+ // A respawn acquires ownership after the old worker releases it.+ // Dropping the sink releases the journal lock; the durable admissions+ // above have already completed their synchronous checkpoint writes.+ drop(worker.recovery.lock().unwrap().take());+ let journal = WorkerRecoveryJournal::open(&worker.config.recovery_journal_path).unwrap();+ let (steering, follow_up) = restore_queue_snapshot(&journal, "target-session");+ assert_eq!(steering.len(), 1, "agent row checkpoint must be durable");+ assert_eq!(follow_up.len(), 1, "prompt checkpoint must be durable");+ // The replacement worker owns the reopened sink for clear's checkpoint.+ *worker.recovery.lock().unwrap() = Some(journal);+ let mut core = worker.core.lock().unwrap();+ core.steering = steering;+ core.follow_up = follow_up;+ assert_eq!(+ core.steering[0].agent_message.as_deref(),+ Some("please review the plan")+ );+ }+ let cleared = worker.dispatch("clear_queue", &json!({})).await;+ assert!(cleared.success, "clear_queue failed: {cleared:?}");+ let data = cleared.data.expect("the clear data");+ assert_eq!(+ data["steering"],+ json!(["[agent-message from source-agent]\n\nplease review the plan"]),+ "the response shape is unchanged: {data}"+ );+ assert_eq!(+ data["followUp"],+ json!(["plain queued prompt"]),+ "the response shape is unchanged: {data}"+ );+ // Exactly one bounce leaves for the agent row's sender; the cleared+ // user prompt sends nothing.+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ let sends = loop {+ let recorded: Vec<Value> = sends+ .lock()+ .unwrap()+ .iter()+ .filter(|command| command["type"] == "send_message")+ .cloned()+ .collect();+ if !recorded.is_empty() {+ break recorded;+ }+ assert!(+ std::time::Instant::now() < deadline,+ "the bounce never reached the supervisor link"+ );+ tokio::time::sleep(std::time::Duration::from_millis(25)).await;+ };+ assert_eq!(+ sends.len(),+ 1,+ "one bounce per agent row, none for the prompt: {sends:?}"+ );+ let command = &sends[0];+ assert_eq!(command["type"], "send_message");+ assert_eq!(command["targetActiveSessionId"], "source-session");+ assert_eq!(command["fromActiveSessionId"], "target-session");+ assert_eq!(command["agentOrigin"], true);+ assert_eq!(+ command["message"],+ "Your agent message was cleared from target's queue before delivery: please review the plan",+ "the bounce names the receiver and quotes the dropped body: {command}"+ );+ let _ = std::fs::remove_dir_all(&dir);+}diff --git a/crates/pa-daemon/src/worker/commands.rs b/crates/pa-daemon/src/worker/commands.rsindex a53c01ca93..e02ec07832 100644--- a/crates/pa-daemon/src/worker/commands.rs+++ b/crates/pa-daemon/src/worker/commands.rs@@ -313,13 +313,15 @@ impl Worker {/// flowing — the armed steering rows co-deliver, then the follow-up drains.////// SANCTIONED DIVERGENCE (operator ruling 2026-09-25): TS parks the queue- /// whenever nothing is armable; here the abort resumes. Returns whether- /// the queue was resumed.+ /// whenever nothing is armable; here the abort resumes. The same divergence+ /// covers a held admission pause: the abort clears the suspension and the+ /// pump's own pause gate holds the delivery until the release re-notifies+ /// it. Returns whether the queue was resumed.pub(crate) fn abort_and_send_queued(&self) -> bool {- // TS `canResume`: no admission pause held — the arm only- // fires in the send arm.- let can_resume =- !self.input_pauses.paused() && !self.core.lock().unwrap().shutdown_requested;+ // TS `canResume` also parks under an admission pause; the pump's own+ // pause gate is the one hold that matters, so only a shutting-down+ // worker (whose queue survives via the restart journal) stays parked.+ let can_resume = !self.core.lock().unwrap().shutdown_requested;if !can_resume {self.request_abort();return false;@@ -527,18 +529,25 @@ impl Worker {return response;}let mut core = self.core.lock().unwrap();- let drain_lane = |lane: &mut VecDeque<QueuedItem>| -> Vec<String> {- lane.drain(..)- .map(|item| {- if let Some(id) = item.admission_id.as_deref() {- self.prompt_admissions.clear(id);- }- item.message- })- .collect()- };- let steering: Vec<String> = drain_lane(&mut core.steering);- let follow_up: Vec<String> = drain_lane(&mut core.follow_up);+ // The dropped agent-message rows bounce to their senders: a queued+ // row's sender otherwise never learns its message will not deliver.+ let mut dropped_agent_rows = Vec::new();+ let drain_lane =+ |lane: &mut VecDeque<QueuedItem>, dropped_agent_rows: &mut Vec<Value>| -> Vec<String> {+ lane.drain(..)+ .map(|item| {+ if let Some(id) = item.admission_id.as_deref() {+ self.prompt_admissions.clear(id);+ }+ if item.agent_message.is_some() {+ dropped_agent_rows.extend(item.custom_message);+ }+ item.message+ })+ .collect()+ };+ let steering: Vec<String> = drain_lane(&mut core.steering, &mut dropped_agent_rows);+ let follow_up: Vec<String> = drain_lane(&mut core.follow_up, &mut dropped_agent_rows);clear_withdrawn_progress_note(&core, self.engine.as_ref());let snapshot = Self::snapshot_locked(&core);drop(core);@@ -548,6 +557,9 @@ impl Worker {operation: "queue_cleared",});let _ = self.emit_action_update(&snapshot);+ for row in &dropped_agent_rows {+ self.bounce_cleared_agent_message(row);+ }response_success(None,"clear_queue",@@ -555,6 +567,54 @@ impl Worker {)}+ /// One dropped agent-message row's sender notice: the row's own+ /// `details` name the sender and the body, and the bounce rides the+ /// supervisor `send_message` route from this session. Spawned off the+ /// clear's response path — a failed bounce is logged, never retried.+ fn bounce_cleared_agent_message(&self, row: &Value) {+ let Some(details) = row.get("details") else {+ return;+ };+ let sender = details+ .get("from")+ .and_then(|from| from.get("activeSessionId"))+ .and_then(Value::as_str)+ .filter(|id| !id.is_empty());+ let Some(sender) = sender else {+ // CLI-origin senders carry no session to bounce to.+ return;+ };+ let message = details+ .get("message")+ .and_then(Value::as_str)+ .unwrap_or_default();+ // The cleared queue is this worker's own; its identity — not the+ // row's delivery-time copy — names the receiver.+ let receiver = {+ let core = self.core.lock().unwrap();+ core.store+ .as_ref()+ .and_then(|store| store.session_name().map(str::to_string))+ .filter(|name| !name.is_empty())+ .unwrap_or_else(|| self.config.active_session_id.clone())+ };+ let command = json!({+ "type": "send_message",+ "targetActiveSessionId": sender,+ "message": cleared_agent_message_bounce_text(&receiver, message),+ "fromActiveSessionId": self.config.active_session_id,+ "agentOrigin": true,+ });+ let link = std::sync::Arc::clone(&self.roster_link);+ let sender = sender.to_string();+ let timeout = std::time::Duration::from_millis(crate::messaging::WORKER_REQUEST_TIMEOUT_MS);+ tokio::spawn(async move {+ if let Err(error) = link.request_success(command, timeout).await {+ eprintln!("pa-daemon: cleared agent-message bounce to {sender} failed: {error:#}");+ }+ });+ }+fn handle_abort_and_clear_queue(&self) -> DaemonResponse {let cleared = self.handle_clear_queue();if !cleared.success {@@ -782,3 +842,49 @@ impl Worker {)}}++/// The bounce body: the receiver label, the fact, and the dropped body —+/// truncated so the whole notice stays inside the delivery size limit the+/// receiving `worker_deliver_message` enforces.+fn cleared_agent_message_bounce_text(receiver: &str, message: &str) -> String {+ let limit = pa_core::session_engine::agent_messaging::DEFAULT_AGENT_MESSAGE_MAX_CHARS;+ let opening = "Your agent message was cleared from ";+ let ending = "'s queue before delivery: ";+ // Session names are unbounded. Reserve the notice itself before fitting+ // the receiver label, then spend the remaining characters on the body.+ let receiver_budget = limit.saturating_sub(opening.chars().count() + ending.chars().count());+ let receiver: String = receiver.chars().take(receiver_budget).collect();+ let prefix = format!("{opening}{receiver}{ending}");+ let budget = limit.saturating_sub(prefix.chars().count());+ let body: String = message.chars().take(budget).collect();+ format!("{prefix}{body}")+}++#[cfg(test)]+mod bounce_text_tests {+ use super::cleared_agent_message_bounce_text;+ use pa_core::session_engine::agent_messaging::DEFAULT_AGENT_MESSAGE_MAX_CHARS;++ #[test]+ fn bounce_text_limits_receiver_and_body_without_losing_the_notice() {+ for (receiver, message) in [+ (+ "target".to_string(),+ "界".repeat(DEFAULT_AGENT_MESSAGE_MAX_CHARS),+ ),+ (+ "界".repeat(DEFAULT_AGENT_MESSAGE_MAX_CHARS + 1),+ "body".to_string(),+ ),+ (+ "界".repeat(DEFAULT_AGENT_MESSAGE_MAX_CHARS + 1),+ String::new(),+ ),+ ] {+ let text = cleared_agent_message_bounce_text(&receiver, &message);+ assert_eq!(text.chars().count(), DEFAULT_AGENT_MESSAGE_MAX_CHARS);+ assert!(text.starts_with("Your agent message was cleared from "));+ assert!(text.contains("'s queue before delivery: "));+ }+ }+}diff --git a/crates/pa-daemon/src/worker/queue.rs b/crates/pa-daemon/src/worker/queue.rsindex 1a251c473f..0f0de62176 100644--- a/crates/pa-daemon/src/worker/queue.rs+++ b/crates/pa-daemon/src/worker/queue.rs@@ -2,7 +2,8 @@//! and queue recovery.use super::{emit_worker_event_with, json, oneshot, Arc, Duration, EventPump, Mutex, Notify, Result,- SessionCore, Value, VecDeque, WorkerRecoveryJournal, AUTONOMOUS_QUEUE_KEY,+ SessionCore, Value, VecDeque, WorkerRecoveryJournal, AGENT_MESSAGE_SOURCE,+ AUTONOMOUS_QUEUE_KEY,};#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -372,6 +373,18 @@ pub(crate) fn restore_queue_snapshot(.into_iter().map(|record| {let policy = record.policy();+ // The journal retains the custom row, including the original+ // body. Reconstruct the marker used by clear/pause/status.+ let agent_message = record.custom_message.as_ref().and_then(|row| {+ (row.get("customType").and_then(Value::as_str) == Some(AGENT_MESSAGE_SOURCE))+ .then(|| {+ row.get("details")?+ .get("message")?+ .as_str()+ .map(str::to_string)+ })+ .flatten()+ });QueuedItem {preview: record.preview,message: record.message,@@ -383,7 +396,7 @@ pub(crate) fn restore_queue_snapshot(}}),custom_message: record.custom_message,- agent_message: None,+ agent_message,queue_key: record.queue_key,admission_id: None,images: Vec::new(),diff --git a/crates/pa-daemon/src/worker/tests.rs b/crates/pa-daemon/src/worker/tests.rsindex 0f32946651..8f74d8a56e 100644--- a/crates/pa-daemon/src/worker/tests.rs+++ b/crates/pa-daemon/src/worker/tests.rs@@ -43,6 +43,68 @@ fn session_events_since(events}+/// A fake supervisor link endpoint (unix): greets the link handshake,+/// answers every request line with success, and records the full command+/// values in arrival order. The callers read the field they need (roster+/// deltas read `command["summary"]`, message probes filter the type).+#[cfg(unix)]+pub(crate) fn fake_supervisor(+ socket: &std::path::Path,+) -> (+ std::sync::Arc<std::sync::Mutex<Vec<Value>>>,+ tokio::task::JoinHandle<()>,+) {+ use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};++ let listener = tokio::net::UnixListener::bind(socket).unwrap();+ let recorded = std::sync::Arc::new(std::sync::Mutex::new(Vec::<Value>::new()));+ let sink = std::sync::Arc::clone(&recorded);+ let server = tokio::spawn(async move {+ loop {+ let Ok((stream, _)) = listener.accept().await else {+ break;+ };+ let sink = std::sync::Arc::clone(&sink);+ tokio::spawn(async move {+ let (reader, mut writer) = stream.into_split();+ writer+ .write_all(+ b"{\"type\":\"daemon_hello\",\"protocol\":{\"name\":\"prime-agent.daemon\",\"version\":7}}\n",+ )+ .await+ .unwrap();+ let mut lines = BufReader::new(reader);+ loop {+ let mut line = String::new();+ if lines.read_line(&mut line).await.unwrap_or(0) == 0 {+ return;+ }+ if line.trim().is_empty() {+ continue;+ }+ let Ok(request) = serde_json::from_str::<Value>(line.trim()) else {+ continue;+ };+ let command = request["command"].clone();+ sink.lock().unwrap().push(command.clone());+ let response =+ crate::protocol::response_line(&crate::protocol::response_success(+ request["id"].as_str(),+ command["type"].as_str().unwrap_or_default(),+ None,+ ));+ writer+ .write_all(serde_json::to_string(&response).unwrap().as_bytes())+ .await+ .unwrap();+ writer.write_all(b"\n").await.unwrap();+ }+ });+ }+ });+ (recorded, server)+}+mod abort_boundary_tests;mod connection_state_tests;mod goal_tests;diff --git a/crates/pa-daemon/src/worker/tests/abort_boundary_tests.rs b/crates/pa-daemon/src/worker/tests/abort_boundary_tests.rsindex fe42266670..5bc4ed3af4 100644--- a/crates/pa-daemon/src/worker/tests/abort_boundary_tests.rs+++ b/crates/pa-daemon/src/worker/tests/abort_boundary_tests.rs@@ -682,3 +682,307 @@ async fn cancel_owned_admission_aborts_the_running_prompt() {);let _ = std::fs::remove_dir_all(&dir);}++/// The faux-script worker for the abort-and-send probes: a fresh created+/// session whose first response holds for 60s (the streaming turn the+/// abort cuts) and whose next turn answers `reply`. The faux registry is+/// process-global: hold `FAUX_TEST_LOCK` across the flow.+async fn held_turn_worker(+ active_session_id: &str,+ name: &str,+ reply: &str,+) -> (std::sync::Arc<Worker>, std::path::PathBuf) {+ let dir = std::env::temp_dir().join(format!("pa-worker-abort-am-{}", uuid::Uuid::new_v4()));+ std::fs::create_dir_all(&dir).unwrap();+ let config = WorkerConfig {+ socket_path: dir.join("worker.sock"),+ supervisor_socket_path: PathBuf::new(),+ token: "token".to_string(),+ worker_instance_id: String::new(),+ active_session_id: active_session_id.to_string(),+ agent_dir: dir.join("agent"),+ recovery_journal_path: dir.join("recovery.jsonl"),+ telemetry_disabled: None,+ script: Some(json!({+ "engine": "faux",+ "responses": [+ { "text": "held reply", "delayMs": 60000 },+ reply,+ ],+ })),+ };+ let worker = std::sync::Arc::new(Worker::new(config, None));+ let created = worker+ .dispatch(+ "create",+ &json!({ "noSession": true, "cwd": "/tmp", "name": name }),+ )+ .await;+ assert!(created.success, "create failed: {created:?}");+ (worker, dir)+}++/// Start the held turn and wait until its run registers (ENG-5991's+/// `waitForToolStart` discipline): the abort must land on the live run.+async fn start_registered_held_turn(worker: &Worker, active_session_id: &str, prompt: &str) {+ let held = worker+ .dispatch(+ "prompt",+ &json!({ "activeSessionId": active_session_id, "message": prompt }),+ )+ .await;+ assert!(held.success, "prompt failed: {held:?}");+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);+ loop {+ let registered = worker+ .agent_engine+ .as_ref()+ .and_then(|engine| {+ engine+ .turn_agent+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner)+ .clone()+ })+ .and_then(|agent| agent.signal())+ .is_some();+ if registered {+ return;+ }+ assert!(+ std::time::Instant::now() < deadline,+ "the held turn's run never registered"+ );+ tokio::time::sleep(std::time::Duration::from_millis(25)).await;+ }+}++/// Queue one agent message from `source-agent` on the busy session.+async fn queue_agent_message(worker: &Worker, active_session_id: &str, body: &str) {+ let delivered = worker+ .dispatch(+ "worker_deliver_message",+ &json!({+ "targetActiveSessionId": active_session_id,+ "message": body,+ "sender": {+ "activeSessionId": "source-session",+ "sessionName": "source-agent",+ },+ }),+ )+ .await;+ assert!(+ delivered.success,+ "worker_deliver_message failed: {delivered:?}"+ );+}++/// The delivered `agent_message` card (the custom row the turn renders) and+/// the transcript's assistant texts, in one read.+async fn agent_message_transcript(worker: &Worker) -> (Option<Value>, Vec<String>) {+ let messages = worker.dispatch("get_messages", &json!({})).await;+ assert!(messages.success, "get_messages failed: {messages:?}");+ let wire = messages+ .data+ .as_ref()+ .and_then(|data| data.get("messages"))+ .and_then(Value::as_array)+ .cloned()+ .unwrap_or_default();+ let row = wire+ .iter()+ .find(|row| row.get("customType").and_then(Value::as_str) == Some("agent_message"))+ .cloned();+ let replies: Vec<String> = wire+ .iter()+ .filter(|message| crate::types::message_role(message) == Some("assistant"))+ .map(crate::types::message_text)+ .collect();+ (row, replies)+}++/// The Esc shape: a queued agent message rides the abort-and-send funnel —+/// the aborted turn settles, then the parked row delivers as the next turn+/// (TS's ctrl+c must never strand agent mail behind the suspension).+#[tokio::test]+// the faux registry is process-global: the guard must span the async flow+#[allow(clippy::await_holding_lock)]+async fn abort_and_send_queued_delivers_the_parked_agent_message() {+ let _faux = crate::agent_engine::tests::FAUX_TEST_LOCK+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner);+ let (worker, dir) =+ held_turn_worker("abort-am-session", "abort-am", "agent message reply").await;+ let mut subscription = worker.events.subscribe();+ start_registered_held_turn(+ &worker,+ "abort-am-session",+ "held turn for the agent-message abort probe",+ )+ .await;+ queue_agent_message(&worker, "abort-am-session", "the parked agent body").await;+ let aborted = worker.dispatch("abort_and_send_queued", &json!({})).await;+ assert!(aborted.success, "abort_and_send_queued failed: {aborted:?}");+ assert_eq!(aborted.command, "abort_and_send_queued");+ let idle = tokio::time::timeout(+ std::time::Duration::from_secs(15),+ worker.dispatch("wait_for_idle", &json!({})),+ )+ .await;+ assert!(idle.is_ok(), "the session never went idle after the abort");+ assert!(idle.unwrap().success, "wait_for_idle failed");+ // The next turn's prompt IS the agent-message prompt: the delivered card+ // renders the sender-labeled body, and the turn ran to its reply.+ let (row, replies) = agent_message_transcript(&worker).await;+ let row = row.expect("the parked agent message delivered as the agent_message row");+ assert_eq!(+ crate::types::message_text(&row),+ "[agent-message from source-agent]\n\nthe parked agent body",+ "the delivered row renders the sender-labeled prompt: {row:?}"+ );+ assert!(+ replies.iter().any(|reply| reply == "agent message reply"),+ "the agent-message turn ran to its reply: {replies:?}"+ );+ // The aborted held turn ends BEFORE the agent message's turn starts.+ let events = session_events_since(&mut subscription);+ let agent_starts: Vec<usize> = events+ .iter()+ .enumerate()+ .filter(|(_, event)| event.get("type").and_then(Value::as_str) == Some("agent_start"))+ .map(|(index, _)| index)+ .collect();+ assert_eq!(agent_starts.len(), 2, "one turn per delivery: {events:?}");+ let aborted_row = events+ .iter()+ .position(|event| {+ event.get("type").and_then(Value::as_str) == Some("message_end")+ && event["message"]["role"] == "assistant"+ && event["message"].get("stopReason").and_then(Value::as_str) == Some("aborted")+ })+ .expect("the held turn never surfaced its aborted row");+ assert!(+ aborted_row < agent_starts[1],+ "the abort must end the held turn before the agent message's turn starts: {events:?}"+ );+ // The queue drained and the suspension is gone.+ {+ let core = worker.core.lock().unwrap();+ assert!(+ core.steering.is_empty() && core.follow_up.is_empty(),+ "the queue must fully drain"+ );+ assert!(+ !core.queued_input_suspended,+ "the abort-and-send suspension never cleared"+ );+ }+ let _ = std::fs::remove_dir_all(&dir);+}++/// A held input pause defers, never parks, the abort-and-send funnel: the+/// abort clears the suspension, the pump holds the delivery while the pause+/// stands, and the pause release delivers the parked agent message.+#[tokio::test]+// the faux registry is process-global: the guard must span the async flow+#[allow(clippy::await_holding_lock)]+async fn abort_and_send_queued_under_a_held_pause_delivers_on_the_release() {+ let _faux = crate::agent_engine::tests::FAUX_TEST_LOCK+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner);+ let (worker, dir) =+ held_turn_worker("abort-pause-session", "abort-pause", "agent message reply").await;+ start_registered_held_turn(+ &worker,+ "abort-pause-session",+ "held turn for the pause abort probe",+ )+ .await;+ // The input pause holds the pump while the abort lands.+ let pause = worker+ .dispatch(+ "acquire_session_input_pause",+ &json!({ "leaseKey": "abort-pause-lease" }),+ )+ .await;+ assert!(+ pause.success,+ "acquire_session_input_pause failed: {pause:?}"+ );+ let pause_id = pause+ .data+ .as_ref()+ .and_then(|data| data.get("pauseId"))+ .and_then(Value::as_str)+ .expect("the pause id")+ .to_string();+ queue_agent_message(&worker, "abort-pause-session", "the parked agent body").await;+ let aborted = worker.dispatch("abort_and_send_queued", &json!({})).await;+ assert!(aborted.success, "abort_and_send_queued failed: {aborted:?}");+ // The aborted turn settles; the pause holds the parked row.+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15);+ loop {+ let (busy, queued) = {+ let core = worker.core.lock().unwrap();+ (core.busy, core.steering.len())+ };+ if !busy && queued == 1 {+ break;+ }+ assert!(+ std::time::Instant::now() < deadline,+ "the aborted turn never settled with the agent row parked"+ );+ tokio::time::sleep(std::time::Duration::from_millis(25)).await;+ }+ let (row, _) = agent_message_transcript(&worker).await;+ assert!(+ row.is_none(),+ "the agent message must not deliver while the pause holds"+ );+ // The release arms notify the pump; the parked row delivers.+ let released = worker+ .dispatch(+ "release_session_input_pause",+ &json!({ "pauseId": pause_id }),+ )+ .await;+ assert!(+ released.success,+ "release_session_input_pause failed: {released:?}"+ );+ let idle = tokio::time::timeout(+ std::time::Duration::from_secs(15),+ worker.dispatch("wait_for_idle", &json!({})),+ )+ .await;+ assert!(+ idle.is_ok(),+ "the session never went idle after the pause released"+ );+ assert!(idle.unwrap().success, "wait_for_idle failed");+ let (row, replies) = agent_message_transcript(&worker).await;+ let row = row.expect("the parked agent message delivered after the release");+ assert_eq!(+ crate::types::message_text(&row),+ "[agent-message from source-agent]\n\nthe parked agent body",+ "the delivered row renders the sender-labeled prompt: {row:?}"+ );+ assert!(+ replies.iter().any(|reply| reply == "agent message reply"),+ "the agent-message turn ran to its reply: {replies:?}"+ );+ {+ let core = worker.core.lock().unwrap();+ assert!(+ core.steering.is_empty() && core.follow_up.is_empty(),+ "the queue must fully drain"+ );+ assert!(+ !core.queued_input_suspended,+ "the abort must clear the suspension under the pause"+ );+ }+ let _ = std::fs::remove_dir_all(&dir);+}diff --git a/crates/pa-daemon/src/worker/turn_stream_tests/feed.rs b/crates/pa-daemon/src/worker/turn_stream_tests/feed.rsindex f6239c7de3..84842838c3 100644--- a/crates/pa-daemon/src/worker/turn_stream_tests/feed.rs+++ b/crates/pa-daemon/src/worker/turn_stream_tests/feed.rs@@ -50,61 +50,6 @@ async fn the_waiting_prompt_resolves_only_after_the_turn_settles() {turn.await.expect("the turn task panicked");}-/// A fake supervisor link endpoint: every `worker_roster_delta`-/// command's summary is recorded in arrival order.-#[cfg(unix)]-fn fake_supervisor(- socket: &std::path::Path,-) -> (Arc<Mutex<Vec<Value>>>, tokio::task::JoinHandle<()>) {- let listener = tokio::net::UnixListener::bind(socket).unwrap();- let recorded = Arc::new(Mutex::new(Vec::<Value>::new()));- let sink = Arc::clone(&recorded);- let server = tokio::spawn(async move {- loop {- let Ok((stream, _)) = listener.accept().await else {- break;- };- let sink = Arc::clone(&sink);- tokio::spawn(async move {- use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};- let (reader, mut writer) = stream.into_split();- writer- .write_all(b"{\"type\":\"daemon_hello\"}\n")- .await- .unwrap();- let mut lines = BufReader::new(reader);- loop {- let mut line = String::new();- if lines.read_line(&mut line).await.unwrap_or(0) == 0 {- return;- }- if line.trim().is_empty() {- continue;- }- let Ok(request) = serde_json::from_str::<Value>(line.trim()) else {- continue;- };- sink.lock()- .unwrap()- .push(request["command"]["summary"].clone());- let response =- crate::protocol::response_line(&crate::protocol::response_success(- request["id"].as_str(),- "worker_roster_delta",- None,- ));- writer- .write_all(serde_json::to_string(&response).unwrap().as_bytes())- .await- .unwrap();- writer.write_all(b"\n").await.unwrap();- }- });- }- });- (recorded, server)-}-/// A turn runner whose roster pushes ship to a live supervisor link/// (the burst runner keeps them disabled). Unix only.#[cfg(unix)]@@ -154,7 +99,7 @@ fn live_feed_runner(engine: Arc<dyn SessionEngine>, socket: std::path::PathBuf)async fn roster_feed_publishes_live_tool_activity() {let dir = tempfile::TempDir::new().unwrap();let socket = dir.path().join("sup.sock");- let (recorded, server) = fake_supervisor(&socket);+ let (recorded, server) = crate::worker::tests::fake_supervisor(&socket);let engine: Arc<dyn SessionEngine> = Arc::new(ScriptedEngine::from_value(&json!({"responses": [{@@ -200,13 +145,18 @@ async fn roster_feed_publishes_live_tool_activity() {.lock().unwrap().last()- .is_some_and(|summary| summary["isStreaming"] == json!(false));+ .is_some_and(|command| command["summary"]["isStreaming"] == json!(false));if settled || std::time::Instant::now() > deadline {break;}tokio::time::sleep(std::time::Duration::from_millis(20)).await;}- let summaries = recorded.lock().unwrap().clone();+ let summaries: Vec<Value> = recorded+ .lock()+ .unwrap()+ .iter()+ .map(|command| command["summary"].clone())+ .collect();let statuses = |key: &str| {summaries.iter()diff --git a/crates/pa-daemon/tests/registration_refusal_heal_e2e.rs b/crates/pa-daemon/tests/registration_refusal_heal_e2e.rsindex 4adbe5c5e0..b5bb5d1d13 100644--- a/crates/pa-daemon/tests/registration_refusal_heal_e2e.rs+++ b/crates/pa-daemon/tests/registration_refusal_heal_e2e.rs@@ -269,6 +269,10 @@ fn a_refused_registration_retires_the_worker() {let agent_dir = dir.path().join("agent");std::fs::create_dir_all(agent_dir.join("sessions")).expect("sessions dir");let _daemon = spawn_daemon(&socket, &agent_dir);+ // Binding the socket precedes the boot predecessor census. Wait for the+ // accept loop's hello before spawning our intentionally descriptorless+ // worker, so startup cannot mistake it for a predecessor to reap.+ drop(Client::connect(&socket));// A bare worker with an identity no descriptor backs: the supervisor can only refuse it.// The scripted engine keeps the runtime self-contained, so the retire path under testdiff --git a/scripts/review_agent_bounce_parity.py b/scripts/review_agent_bounce_parity.pynew file mode 100644index 0000000000..db6be5651a--- /dev/null+++ b/scripts/review_agent_bounce_parity.py@@ -0,0 +1,244 @@+#!/usr/bin/env python3+"""Real-daemon #3558 comparison: baseline queue clear and explicit Rust bounce addition.++TS 0.9.8 clears queued actions but never sends a sender bounce (daemon-mode.ts+clear_queue; agent-session.ts clearQueue). The receipt records that difference,+not exact behavioral parity. Journal recovery is covered by the separate Rust+clear_queue_bounces_recovered_agent_messages_to_their_senders regression.+All processes, sockets, provider requests and session state belong to this fixture.+"""+import argparse+import http.server+import json+import os+from pathlib import Path+import socket+import subprocess+import tempfile+import threading+import time++from review_draft_disconnect_parity import digest, env_for, stop, TS_ARCHIVE_SHA256++LIMIT = 35+MESSAGE = 'AGENT_BOUNCE_PARITY_MESSAGE'+USER = 'AGENT_BOUNCE_PARITY_USER'+OPENING = 'Your agent message was cleared from '+ENDING = "'s queue before delivery: "+++class Provider(http.server.ThreadingHTTPServer):+ daemon_threads = True++ def __init__(self):+ super().__init__(('127.0.0.1', 0), Handler)+ self.arrived = threading.Condition()+ self.requests = []+ self.release = threading.Event()++ def wait_for(self, marker):+ with self.arrived:+ assert self.arrived.wait_for(lambda: any(marker in json.dumps(body)+ for body in self.requests), timeout=LIMIT), 'busy turn never reached provider: ' + marker+++class Handler(http.server.BaseHTTPRequestHandler):+ def log_message(self, *_args):+ pass++ def do_POST(self):+ if self.path != '/v1/chat/completions':+ self.send_error(404)+ return+ body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))+ with self.server.arrived:+ self.server.requests.append(body)+ self.server.arrived.notify_all()+ self.server.release.wait(LIMIT * 3)+ self.send_response(200)+ self.send_header('Content-Type', 'text/event-stream')+ self.end_headers()+ try:+ self.wfile.write(b'data: [DONE]\n\n')+ except (BrokenPipeError, ConnectionResetError):+ pass+++class Client:+ def __init__(self, path, process):+ deadline = time.monotonic() + LIMIT+ while True:+ self.sock = socket.socket(socket.AF_UNIX)+ try:+ self.sock.connect(str(path))+ break+ except (FileNotFoundError, ConnectionRefusedError):+ self.sock.close()+ assert process.poll() is None, 'daemon exited before socket readiness'+ assert time.monotonic() < deadline, 'daemon socket never became ready'+ threading.Event().wait(0.025)+ self.sock.settimeout(LIMIT)+ self.reader = self.sock.makefile('rb')+ self.hello = json.loads(self.reader.readline())+ assert self.hello['type'] == 'daemon_hello', self.hello+ self.counter = 0+ self.wire = []++ def call(self, kind, **payload):+ self.counter += 1+ ident = str(self.counter)+ envelope = {'type': 'command', 'id': ident,+ 'protocol': {'name': 'prime-agent.daemon', 'version': 7},+ 'command': {'type': kind, **payload}}+ self.sock.sendall((json.dumps(envelope) + '\n').encode())+ deadline = time.monotonic() + LIMIT+ while time.monotonic() < deadline:+ line = self.reader.readline()+ assert line, 'daemon closed connection'+ response = json.loads(line)+ self.wire.append(response)+ if response.get('type') == 'response' and response.get('id') == ident:+ assert response.get('success'), response+ return response.get('data')+ raise TimeoutError('command never answered: ' + kind)++ def close(self):+ self.reader.close()+ self.sock.close()+++def run_case(binary, kind, root, long_name, observed):+ env = env_for(root, kind)+ provider = Provider()+ thread = threading.Thread(target=provider.serve_forever, daemon=True)+ thread.start()+ clients = []+ process = None+ try:+ models = {'providers': {'parity-local': {'api': 'openai-completions',+ 'apiKey': 'fixture-only', 'baseUrl': 'http://127.0.0.1:%s/v1' % provider.server_port,+ 'models': [{'id': 'parity-1', 'name': 'Parity 1', 'reasoning': False,+ 'contextWindow': 128000, 'maxTokens': 4096}]}}}+ (root / 'agent/models.json').write_text(json.dumps(models))+ (root / 'agent/settings.json').write_text(json.dumps({'prewarmPythonKernel': False,+ 'defaultProvider': 'parity-local', 'defaultModel': 'parity-1',+ 'onboardingCompleted': True, 'telemetry': {'noticeShown': True}}))+ path = root / 'd.sock'+ log = open(root / 'daemon.log', 'wb')+ try:+ process = subprocess.Popen([str(binary), '--mode', 'daemon', '--daemon-socket', str(path)],+ cwd=root, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)+ receiver_name = '界' * 16385 if long_name else 'bounce-target'+ sessions = []+ for name in ('bounce-source', receiver_name):+ client = Client(path, process)+ clients.append(client)+ config = {'cwd': str(root), 'agentDir': str(root / 'agent'),+ 'provider': 'parity-local', 'model': 'parity-1',+ 'noExtensions': True, 'noSkills': True, 'noTools': True,+ 'noContextFiles': True, 'telemetryDisabled': True}+ created = client.call('create', name=name, config=config)+ active = created.get('activeSessionId') or created.get('id')+ assert active, created+ client.call('attach', activeSessionId=active)+ sessions.append(active)+ source, receiver = sessions+ for client, active, marker in zip(clients, sessions, ('HOLD_SOURCE', 'HOLD_RECEIVER')):+ client.call('prompt', activeSessionId=active, message=marker)+ provider.wait_for(marker)+ receipt = clients[0].call('send_message', targetActiveSessionId=receiver,+ fromActiveSessionId=source, message=MESSAGE)+ observed['delivery_receipt'] = receipt+ clients[1].call('follow_up', activeSessionId=receiver, message=USER)+ cleared = clients[1].call('clear_queue', activeSessionId=receiver)+ observed['cleared'] = cleared+ assert cleared == {'steering': ['[agent-message from bounce-source]\n\n' + MESSAGE],+ 'followUp': [USER]}, cleared+ empty = clients[1].call('get_queue', activeSessionId=receiver)+ observed['receiver_after_clear'] = empty+ assert empty == {'steering': [], 'followUp': []}, empty+ # TS source has no asynchronous bounce operation. Record the actual+ # immediate baseline; do not claim a bounded observation proves absence forever.+ queue = clients[0].call('get_queue', activeSessionId=source)+ if kind == 'rust':+ deadline = time.monotonic() + LIMIT+ while not queue['steering'] and not queue['followUp']:+ assert time.monotonic() < deadline, 'sender bounce never arrived'+ threading.Event().wait(0.025)+ queue = clients[0].call('get_queue', activeSessionId=source)+ dropped = clients[0].call('clear_queue', activeSessionId=source)+ texts = dropped['steering'] + dropped['followUp']+ assert len(texts) == 1, dropped+ prefix, body = texts[0].split('\n\n', 1)+ assert prefix.startswith('[agent-message from '), prefix+ budget = 16384 - len(OPENING) - len(ENDING)+ expected = OPENING + receiver_name[:budget] + ENDING+ expected += MESSAGE[:16384 - len(expected)]+ assert body == expected, repr(body[:200])+ assert len(body) <= 16384, len(body)+ observed['bounce'] = {'message': texts[0], 'body_characters': len(body)}+ else:+ assert queue == {'steering': [], 'followUp': []}, queue+ observed['bounce'] = None+ observed['sender_queue_observed'] = queue+ observed['hello'] = clients[0].hello+ observed['provider_request_count'] = len(provider.requests)+ observed['wire'] = [client.wire for client in clients]+ finally:+ # The supervisor owns detached workers. Ask it to shut those down+ # before ending its process group or removing their state tree.+ provider.release.set()+ if process and process.poll() is None and clients:+ try:+ clients[0].call('shutdown', force=True)+ process.wait(timeout=15)+ observed['shutdown'] = 'owned supervisor and workers stopped'+ except Exception as error:+ observed['shutdown_error'] = str(error)+ if process:+ stop(process)+ log.close()+ observed['daemon_log'] = (root / 'daemon.log').read_text(errors='replace')[-6000:]+ finally:+ observed['wire'] = [client.wire for client in clients]+ for client in clients:+ client.close()+ provider.release.set()+ provider.shutdown()+ provider.server_close()+++def main():+ parser = argparse.ArgumentParser()+ parser.add_argument('--receipt', required=True, type=Path)+ args = parser.parse_args()+ receipt = {'scenario': 'agent_bounce', 'status': 'failed',+ 'scope': 'actual clear_queue response and emptied receiver baseline; added Rust sender bounce',+ 'intentional_difference': 'TS 0.9.8 has no sender bounce; Rust adds it',+ 'recovery_scope': 'separate Rust journal reopen/clear regression, not TS binary parity',+ 'cases': {}}+ try:+ archive = Path(os.environ['PA_TS_ARCHIVE'])+ assert digest(archive) == TS_ARCHIVE_SHA256, 'TS reference archive digest mismatch'+ receipt['ts_archive_sha256'] = digest(archive)+ with tempfile.TemporaryDirectory(prefix='pa-bounce-', dir='/tmp') as folder:+ for kind in ('ts', 'rust'):+ binary = Path(os.environ['PA_' + kind.upper() + '_BINARY']).resolve()+ receipt[kind + '_binary_sha256'] = digest(binary)+ for long_name in (False, True):+ key = kind + ('_long_receiver' if long_name else '_ordinary')+ observed = receipt['cases'][key] = {}+ run_case(binary, kind, Path(folder) / key, long_name, observed)+ for suffix in ('_ordinary', '_long_receiver'):+ assert receipt['cases']['ts' + suffix]['cleared'] == receipt['cases']['rust' + suffix]['cleared']+ receipt['status'] = 'passed'+ except Exception as error:+ receipt['error'] = str(error)+ raise+ finally:+ args.receipt.parent.mkdir(parents=True, exist_ok=True)+ args.receipt.write_text(json.dumps(receipt, ensure_ascii=False, indent=2))+++if __name__ == '__main__':+ main()diff --git a/scripts/test_review_agent_bounce_parity.py b/scripts/test_review_agent_bounce_parity.pynew file mode 100644index 0000000000..dda00f0115--- /dev/null+++ b/scripts/test_review_agent_bounce_parity.py@@ -0,0 +1,93 @@+#!/usr/bin/env python3+"""Check the binary verifier's transport and provider hold, not product behavior."""+import http.client+import json+from pathlib import Path+import socket+import tempfile+import threading+import unittest++from review_agent_bounce_parity import Client, Provider+++class LiveProcess:+ @staticmethod+ def poll():+ return None+++class FixtureTests(unittest.TestCase):+ def test_client_skips_events_but_checks_correlated_command_failure(self):+ with tempfile.TemporaryDirectory(prefix='bounce-test-', dir='/tmp') as root:+ path = Path(root) / 'd.sock'+ with socket.socket(socket.AF_UNIX) as listener:+ listener.bind(str(path))+ listener.listen()+ received = []+ errors = []+ def serve():+ try:+ conn, _ = listener.accept()+ with conn, conn.makefile('rb') as stream:+ conn.sendall(b'{"type":"daemon_hello"}\n')+ received.append(json.loads(stream.readline()))+ conn.sendall(b'{"type":"session_event"}\n')+ conn.sendall(b'{"type":"response","id":"1","success":true,"data":{"steering":[],"followUp":[]}}\n')+ received.append(json.loads(stream.readline()))+ conn.sendall(b'{"type":"response","id":"2","success":false,"error":"fixture refusal"}\n')+ except Exception as error:+ errors.append(error)+ thread = threading.Thread(target=serve, daemon=True)+ thread.start()+ client = Client(path, LiveProcess())+ try:+ self.assertEqual(client.call('get_queue', activeSessionId='fixture'),+ {'steering': [], 'followUp': []})+ with self.assertRaisesRegex(AssertionError, 'fixture refusal'):+ client.call('clear_queue', activeSessionId='fixture')+ finally:+ client.close()+ thread.join(5)+ self.assertFalse(thread.is_alive())+ self.assertEqual(errors, [])+ self.assertEqual(received[0]['command'], {'type': 'get_queue', 'activeSessionId': 'fixture'})+ self.assertEqual(received[0]['protocol'], {'name': 'prime-agent.daemon', 'version': 7})++ def test_provider_observes_actual_request_before_releasing_response(self):+ provider = Provider()+ server = threading.Thread(target=provider.serve_forever, daemon=True)+ server.start()+ done = threading.Event()+ errors = []+ def request():+ try:+ conn = http.client.HTTPConnection('127.0.0.1', provider.server_port, timeout=5)+ conn.request('POST', '/v1/chat/completions',+ json.dumps({'messages': [{'content': 'HOLD_TEST'}]}),+ {'Content-Type': 'application/json'})+ response = conn.getresponse()+ self.assertEqual(response.status, 200)+ self.assertEqual(response.read(), b'data: [DONE]\n\n')+ conn.close()+ except Exception as error:+ errors.append(error)+ finally:+ done.set()+ client = threading.Thread(target=request, daemon=True)+ client.start()+ try:+ provider.wait_for('HOLD_TEST')+ self.assertFalse(done.is_set(), 'response escaped the busy-turn gate')+ provider.release.set()+ self.assertTrue(done.wait(5))+ self.assertEqual(errors, [])+ finally:+ provider.release.set()+ provider.shutdown()+ provider.server_close()+ client.join(5)+++if __name__ == '__main__':+ unittest.main()