Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

fix(acp): fail pending link requests on close and drop the turn cap (#3249)

Sebastian Müllercf285dce271f
diff --git a/crates/pa-cli/tests/acp_mode_e2e.rs b/crates/pa-cli/tests/acp_mode_e2e.rs
index bb60b2c0fb..9cb427ceaf 100644
--- a/crates/pa-cli/tests/acp_mode_e2e.rs
+++ b/crates/pa-cli/tests/acp_mode_e2e.rs
@@ -186,6 +186,28 @@ impl AcpChild {
}
}
}
+
+ /// Read frames until one `sessionUpdate` of `kind` arrives — the
+ /// observed-event readiness signal, never a timer. The frames
+ /// before it are dropped.
+ fn wait_update(&mut self, kind: &str, timeout: Duration) {
+ let deadline = Instant::now() + timeout;
+ loop {
+ let timeout_left = deadline.saturating_duration_since(Instant::now());
+ assert!(
+ !timeout_left.is_zero(),
+ "the stream never published a {kind} update"
+ );
+ let line = self
+ .lines
+ .recv_timeout(timeout_left)
+ .expect("the ACP stream stayed open");
+ let frame: Value = serde_json::from_str(&line).expect("valid JSON line");
+ if frame["params"]["update"]["sessionUpdate"] == kind {
+ return;
+ }
+ }
+ }
}
impl Drop for AcpChild {
@@ -852,22 +874,7 @@ fn acp_daemon_attached_prompt_after_cancel_runs() {
&json!({ "sessionId": session_id, "prompt": [{ "type": "text", "text": "a slow question" }] }),
);
// Readiness is the turn's own first streamed chunk, not a timer.
- let deadline = Instant::now() + TIMEOUT;
- loop {
- let timeout_left = deadline.saturating_duration_since(Instant::now());
- assert!(
- !timeout_left.is_zero(),
- "the paced turn never streamed a chunk"
- );
- let line = client
- .lines
- .recv_timeout(timeout_left)
- .expect("the ACP stream stayed open");
- let frame: Value = serde_json::from_str(&line).expect("valid JSON line");
- if frame["params"]["update"]["sessionUpdate"] == "agent_message_chunk" {
- break;
- }
- }
+ client.wait_update("agent_message_chunk", TIMEOUT);
client.notify("session/cancel", &json!({ "sessionId": session_id }));
let (first_response, _) = client.wait_response(first, TIMEOUT);
assert_eq!(
@@ -893,6 +900,66 @@ fn acp_daemon_attached_prompt_after_cancel_runs() {
);
}
+#[cfg(unix)]
+#[test]
+fn acp_daemon_attached_supervisor_loss_fails_the_prompt() {
+ // The supervisor dies while a prompt is in flight: the pending
+ // request must fail fast with an error response, never hang, because
+ // the link close is the turn's liveness bound (turn-long requests
+ // carry no timer).
+ let (mut client, socket) = AcpChild::spawn_daemon_attached(
+ &["--mode", "acp", "--no-session"],
+ &json!({ "engine": "faux", "tokensPerSecond": 2, "responses": [
+ { "text": "a paced answer that streams slowly enough to outlive the supervisor" },
+ ] }),
+ );
+ let init = client.request("initialize", &initialize_params());
+ let _ = client.wait_response(init, TIMEOUT);
+ let new = client.request("session/new", &json!({ "mcpServers": [] }));
+ let (new_response, _) = client.wait_response(new, TIMEOUT);
+ let session_id = new_response["result"]["sessionId"]
+ .as_str()
+ .unwrap()
+ .to_string();
+ let prompt = client.request(
+ "session/prompt",
+ &json!({ "sessionId": session_id, "prompt": [{ "type": "text", "text": "a slow question" }] }),
+ );
+ // Readiness is the turn's own first streamed chunk: the prompt is
+ // provably in flight before the supervisor goes away.
+ client.wait_update("agent_message_chunk", TIMEOUT);
+ // Crash the supervisor (no graceful drain, so the in-flight response
+ // can never arrive). Its pid comes from the hello frame every new
+ // connection receives.
+ let probe = pa_types::platform::transport::connect_blocking(&socket).expect("daemon socket");
+ let reader = probe.try_clone_box().expect("daemon socket clone");
+ let _ = reader.set_read_timeout(Duration::from_mins(2));
+ let hello = BufReader::new(reader)
+ .lines()
+ .map(|line| {
+ serde_json::from_str::<Value>(&line.expect("daemon line")).expect("daemon JSON")
+ })
+ .find(|frame| frame["type"] == json!("daemon_hello"))
+ .expect("the daemon closed without a hello");
+ let pid = hello["supervisorPid"].as_u64().expect("supervisor pid");
+ drop(probe);
+ let killed = std::process::Command::new("kill")
+ .args(["-9", &pid.to_string()])
+ .status()
+ .expect("kill the supervisor");
+ assert!(killed.success(), "the supervisor crash did not run");
+ let (prompt_response, _) = client.wait_response(prompt, TIMEOUT);
+ assert_eq!(
+ prompt_response["error"],
+ json!({
+ "code": -32603,
+ "message": "Internal error",
+ "data": { "details": "the daemon connection closed mid-request" }
+ }),
+ "the in-flight prompt fails fast on supervisor loss: {prompt_response}"
+ );
+}
+
#[test]
fn acp_daemon_attached_image_only_prompt_reaches_the_session() {
let (mut client, socket) = AcpChild::spawn_daemon_attached(
diff --git a/crates/pa-daemon/src/acp/daemon.rs b/crates/pa-daemon/src/acp/daemon.rs
index 17f37ed35d..6b57128018 100644
--- a/crates/pa-daemon/src/acp/daemon.rs
+++ b/crates/pa-daemon/src/acp/daemon.rs
@@ -40,10 +40,8 @@ use super::wire_config::{
};
use super::wire_events::{self, WireMappingState};
-/// Response timeout for turn-long commands (the turn itself bounds them).
-pub(crate) const TURN_TIMEOUT_MS: u64 = 600_000;
/// Response timeout for session-scoped commands.
-pub(crate) const REQUEST_TIMEOUT_MS: u64 = 30_000;
+const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
/// One inbound supervisor frame, classified by the reader.
enum LinkFrame {
@@ -56,6 +54,10 @@ enum LinkFrame {
pub(crate) struct DaemonLink {
writer: mpsc::UnboundedSender<String>,
pending: Arc<std::sync::Mutex<HashMap<String, oneshot::Sender<DaemonResponse>>>>,
+ /// Set once the frame channel ends (the socket closed and every
+ /// frame the reader read is delivered): every new request fails
+ /// fast instead of waiting for a response no one will send.
+ closed: Arc<std::sync::atomic::AtomicBool>,
frames: Mutex<mpsc::UnboundedReceiver<LinkFrame>>,
protocol_version: u64,
next_request_id: std::sync::atomic::AtomicU64,
@@ -76,6 +78,8 @@ impl DaemonLink {
let (hello_tx, hello_rx) = oneshot::channel::<Value>();
let pending: Arc<std::sync::Mutex<HashMap<String, oneshot::Sender<DaemonResponse>>>> =
Arc::new(std::sync::Mutex::new(HashMap::new()));
+ let closed: Arc<std::sync::atomic::AtomicBool> =
+ Arc::new(std::sync::atomic::AtomicBool::new(false));
tokio::spawn(async move {
let mut writer = writer_half;
@@ -146,17 +150,33 @@ impl DaemonLink {
Ok(DaemonLink {
writer: line_tx,
pending,
+ closed,
frames: Mutex::new(frame_rx),
protocol_version: version,
next_request_id: std::sync::atomic::AtomicU64::new(0),
})
}
- /// Send one command envelope and wait for the matching response.
- pub(crate) async fn request(
+ /// Send one command envelope and wait for the matching response
+ /// under the session-scoped response timeout.
+ pub(crate) async fn request(&self, command: DaemonCommand) -> anyhow::Result<DaemonResponse> {
+ self.exchange(command, Some(REQUEST_TIMEOUT)).await
+ }
+
+ /// Send one command envelope with no fixed cap: the response or the
+ /// link close ends the wait (turn-long commands — declared
+ /// divergence, TS caps them at 24 h; the link-close signal is the
+ /// liveness bound here, not a timer).
+ async fn request_until_close(&self, command: DaemonCommand) -> anyhow::Result<DaemonResponse> {
+ self.exchange(command, None).await
+ }
+
+ /// One exchange: register the response slot, send the envelope, and
+ /// wait for the answer.
+ async fn exchange(
&self,
command: DaemonCommand,
- timeout_ms: u64,
+ timeout: Option<std::time::Duration>,
) -> anyhow::Result<DaemonResponse> {
use std::sync::atomic::Ordering;
let id = format!(
@@ -176,6 +196,14 @@ impl DaemonLink {
let line = serde_json::to_string(&envelope)?;
let (tx, rx) = oneshot::channel::<DaemonResponse>();
self.pending.lock().unwrap().insert(id.clone(), tx);
+ // A request that races the close loses either way: the flag
+ // fails it here, or the pending clear already dropped its
+ // sender. The flag is stored before the clear, so no request
+ // parks unnoticed.
+ if self.closed.load(Ordering::SeqCst) {
+ self.pending.lock().unwrap().remove(&id);
+ anyhow::bail!("the daemon connection is closed");
+ }
if self.writer.send(line).is_err() {
// A closed writer leaves the pending slot behind otherwise; a
// link that never answers again would grow one entry per
@@ -183,13 +211,14 @@ impl DaemonLink {
self.pending.lock().unwrap().remove(&id);
anyhow::bail!("the daemon connection is closed");
}
- tokio::time::timeout(std::time::Duration::from_millis(timeout_ms), rx)
- .await
- .map_err(|_| {
+ match timeout {
+ Some(timeout) => tokio::time::timeout(timeout, rx).await.map_err(|_| {
self.pending.lock().unwrap().remove(&id);
anyhow::anyhow!("timed out waiting for the daemon response")
- })?
- .map_err(|_| anyhow::anyhow!("the daemon connection closed mid-request"))
+ })?,
+ None => rx.await,
+ }
+ .map_err(|_| anyhow::anyhow!("the daemon connection closed mid-request"))
}
}
@@ -370,6 +399,13 @@ pub async fn run_daemon_attached_acp_mode(options: DaemonAcpOptions) -> anyhow::
}
}
}
+ // The reader hit the socket close and every frame it read
+ // has been delivered: fail the still-pending requests
+ // (dropping a sender answers with the mid-request error)
+ // and fail every later request fast. The flag goes up
+ // before the clear; `request` covers the race.
+ link.closed.store(true, std::sync::atomic::Ordering::SeqCst);
+ link.pending.lock().unwrap().clear();
});
}
@@ -558,7 +594,7 @@ async fn handle_session_new(
launch_env: None,
rest: Map::default(),
};
- let create_response = match link.request(create, REQUEST_TIMEOUT_MS).await {
+ let create_response = match link.request(create).await {
Ok(response) => response,
Err(error) => {
*state.lock().await = DaemonAcpState::default();
@@ -593,20 +629,17 @@ async fn handle_session_new(
launch_env: None,
rest: Map::default(),
};
- if let Ok(response) = link.request(attach, REQUEST_TIMEOUT_MS).await {
+ if let Ok(response) = link.request(attach).await {
if !response.success {
let failure = response
.error
.unwrap_or_else(|| "unknown error".to_string());
let _ = link
- .request(
- DaemonCommand::Kill {
- id: None,
- active_session_id: daemon_active_session_id.clone(),
- rest: Map::default(),
- },
- REQUEST_TIMEOUT_MS,
- )
+ .request(DaemonCommand::Kill {
+ id: None,
+ active_session_id: daemon_active_session_id.clone(),
+ rest: Map::default(),
+ })
.await;
*state.lock().await = DaemonAcpState::default();
let _ = tx.send(super::internal_error(&id, &failure));
@@ -649,14 +682,11 @@ async fn handle_session_new(
if let Err(error) = replace_session_servers(link, &hosted, &resolved).await {
let failure = error.to_string();
let _ = link
- .request(
- DaemonCommand::Kill {
- id: None,
- active_session_id: hosted.daemon_active_session_id.clone(),
- rest: Map::default(),
- },
- REQUEST_TIMEOUT_MS,
- )
+ .request(DaemonCommand::Kill {
+ id: None,
+ active_session_id: hosted.daemon_active_session_id.clone(),
+ rest: Map::default(),
+ })
.await;
*state.lock().await = DaemonAcpState::default();
let _ = tx.send(super::internal_error(&id, &failure));
@@ -710,16 +740,13 @@ async fn replace_session_servers(
resolved: &[pa_core::mcp::AcpMcpServerConfig],
) -> anyhow::Result<()> {
let response = link
- .request(
- DaemonCommand::ReplaceAcpMcpServers {
- id: None,
- active_session_id: hosted.daemon_active_session_id.clone(),
- owner_id: hosted.mcp_owner_id.clone(),
- servers: serde_json::to_value(resolved)?,
- rest: Map::default(),
- },
- REQUEST_TIMEOUT_MS,
- )
+ .request(DaemonCommand::ReplaceAcpMcpServers {
+ id: None,
+ active_session_id: hosted.daemon_active_session_id.clone(),
+ owner_id: hosted.mcp_owner_id.clone(),
+ servers: serde_json::to_value(resolved)?,
+ rest: Map::default(),
+ })
.await?;
if !response.success {
anyhow::bail!(response
@@ -823,7 +850,7 @@ async fn handle_session_prompt(
},
rest: Map::default(),
};
- let response = match link.request(prompt, TURN_TIMEOUT_MS).await {
+ let response = match link.request_until_close(prompt).await {
Ok(response) => response,
Err(error) => {
producer.finish_prompt(turn_id).await;
@@ -988,15 +1015,12 @@ async fn fetch_autonomous_status(
active_session_id: &str,
) -> Option<pa_core::autonomous::AgentAutonomousStatus> {
let response = link
- .request(
- DaemonCommand::WaitForHeadlessCompletion {
- id: None,
- active_session_id: active_session_id.to_string(),
- wait_for_rlm_quiescence: None,
- rest: Map::default(),
- },
- TURN_TIMEOUT_MS,
- )
+ .request_until_close(DaemonCommand::WaitForHeadlessCompletion {
+ id: None,
+ active_session_id: active_session_id.to_string(),
+ wait_for_rlm_quiescence: None,
+ rest: Map::default(),
+ })
.await
.ok()?;
if !response.success {
@@ -1031,7 +1055,7 @@ async fn session_cancel(
rest: Map::default(),
};
drop(guard);
- let _ = link.request(abort, REQUEST_TIMEOUT_MS).await;
+ let _ = link.request(abort).await;
Ok(())
}
@@ -1067,32 +1091,26 @@ async fn handle_session_close(
return;
};
let _ = link
- .request(
- DaemonCommand::Abort {
- id: None,
- active_session_id: hosted.daemon_active_session_id.clone(),
- rest: Map::default(),
- },
- REQUEST_TIMEOUT_MS,
- )
+ .request(DaemonCommand::Abort {
+ id: None,
+ active_session_id: hosted.daemon_active_session_id.clone(),
+ rest: Map::default(),
+ })
.await;
if !hosted.mcp_server_names.is_empty() {
let _ = replace_session_servers(link, &hosted, &[]).await;
}
// The worker dies before the config queue drains: a stalled
// `set_model`/`set_thinking_level` (holding the queue on a wire
- // request) fails fast once the worker is gone instead of parking the
- // close for the turn timeout. Then the serialized config work settles
- // before the producer fences (TS `await configTask`).
+ // request) fails fast once the worker is gone instead of parking
+ // the close on the request timeout. Then the serialized config work
+ // settles before the producer fences (TS `await configTask`).
let _ = link
- .request(
- DaemonCommand::Kill {
- id: None,
- active_session_id: hosted.daemon_active_session_id.clone(),
- rest: Map::default(),
- },
- REQUEST_TIMEOUT_MS,
- )
+ .request(DaemonCommand::Kill {
+ id: None,
+ active_session_id: hosted.daemon_active_session_id.clone(),
+ rest: Map::default(),
+ })
.await;
let _ = hosted.config.queue.lock().await;
hosted.producer.close().await;
@@ -1111,24 +1129,18 @@ async fn teardown(link: &Arc<DaemonLink>, state: &Arc<Mutex<DaemonAcpState>>) {
// operation holding the queue releases once the wire peer is gone),
// then the serialized config work settles before the producer fences.
let _ = link
- .request(
- DaemonCommand::Abort {
- id: None,
- active_session_id: hosted.daemon_active_session_id.clone(),
- rest: Map::default(),
- },
- REQUEST_TIMEOUT_MS,
- )
+ .request(DaemonCommand::Abort {
+ id: None,
+ active_session_id: hosted.daemon_active_session_id.clone(),
+ rest: Map::default(),
+ })
.await;
let _ = link
- .request(
- DaemonCommand::Kill {
- id: None,
- active_session_id: hosted.daemon_active_session_id.clone(),
- rest: Map::default(),
- },
- REQUEST_TIMEOUT_MS,
- )
+ .request(DaemonCommand::Kill {
+ id: None,
+ active_session_id: hosted.daemon_active_session_id.clone(),
+ rest: Map::default(),
+ })
.await;
let _ = hosted.config.queue.lock().await;
hosted.producer.close().await;
diff --git a/crates/pa-daemon/src/acp/wire_config.rs b/crates/pa-daemon/src/acp/wire_config.rs
index a0606aad6d..abd552c7b3 100644
--- a/crates/pa-daemon/src/acp/wire_config.rs
+++ b/crates/pa-daemon/src/acp/wire_config.rs
@@ -17,7 +17,7 @@ use super::config_options::{
config_options_value, model_value, publish_config_options, session_config_options, PickerModel,
SessionConfigOption,
};
-use super::daemon::{DaemonAcpState, DaemonLink, REQUEST_TIMEOUT_MS, TURN_TIMEOUT_MS};
+use super::daemon::{DaemonAcpState, DaemonLink};
use super::jsonrpc;
use super::producer::{self, UpdateProducer};
use super::types;
@@ -189,16 +189,13 @@ async fn apply_wire_config(
WireConfigError::invalid_params(format!("Unavailable model: {value}"))
})?;
let response = link
- .request(
- DaemonCommand::SetModel {
- id: None,
- active_session_id: daemon_session_id.to_string(),
- provider: model.provider.clone(),
- model_id: model.id.clone(),
- rest: Map::default(),
- },
- TURN_TIMEOUT_MS,
- )
+ .request(DaemonCommand::SetModel {
+ id: None,
+ active_session_id: daemon_session_id.to_string(),
+ provider: model.provider.clone(),
+ model_id: model.id.clone(),
+ rest: Map::default(),
+ })
.await
.map_err(|error| WireConfigError::internal(error.to_string()))?;
if !response.success {
@@ -250,15 +247,12 @@ async fn apply_wire_config(
)));
}
let response = link
- .request(
- DaemonCommand::SetThinkingLevel {
- id: None,
- active_session_id: daemon_session_id.to_string(),
- level: value.to_string(),
- rest: Map::default(),
- },
- TURN_TIMEOUT_MS,
- )
+ .request(DaemonCommand::SetThinkingLevel {
+ id: None,
+ active_session_id: daemon_session_id.to_string(),
+ level: value.to_string(),
+ rest: Map::default(),
+ })
.await
.map_err(|error| WireConfigError::internal(error.to_string()))?;
if !response.success {
@@ -283,14 +277,11 @@ pub(super) async fn fetch_connection_state(
active_session_id: &str,
) -> Option<Value> {
let response = link
- .request(
- DaemonCommand::GetConnectionState {
- id: None,
- active_session_id: active_session_id.to_string(),
- rest: Map::default(),
- },
- REQUEST_TIMEOUT_MS,
- )
+ .request(DaemonCommand::GetConnectionState {
+ id: None,
+ active_session_id: active_session_id.to_string(),
+ rest: Map::default(),
+ })
.await
.ok()?;
if !response.success {
@@ -305,14 +296,11 @@ pub(super) async fn fetch_available_models(
active_session_id: &str,
) -> anyhow::Result<Vec<pa_types::ai::Model>> {
let response = link
- .request(
- DaemonCommand::GetAvailableModels {
- id: None,
- active_session_id: active_session_id.to_string(),
- rest: Map::default(),
- },
- REQUEST_TIMEOUT_MS,
- )
+ .request(DaemonCommand::GetAvailableModels {
+ id: None,
+ active_session_id: active_session_id.to_string(),
+ rest: Map::default(),
+ })
.await?;
if !response.success {
anyhow::bail!(response