Skip to content

Use Claude Code with your ChatGPT, Kimi, Cursor or Grok subscription via a local Anthropic-compatible proxy

Rust

keep websocket streams alive during long responses

Raine Virta7075c6b7e11f
diff --git a/src/providers/codex/client.rs b/src/providers/codex/client.rs
index 3eeb453..bebaeed 100644
--- a/src/providers/codex/client.rs
+++ b/src/providers/codex/client.rs
@@ -2851,6 +2851,9 @@ fn is_retryable_transport_error(err: &CodexError) -> bool {
if err.detail.as_deref() == Some("websocket_pre_request") {
return err.status == 0 || should_retry_codex_status(err.status);
}
+ if err.detail.as_deref() == Some(super::websocket::WEBSOCKET_KEEPALIVE_FAILURE_DETAIL) {
+ return true;
+ }
if err.status != 0 {
return false;
}
@@ -2863,6 +2866,8 @@ fn is_retryable_transport_error(err: &CodexError) -> bool {
|| message.contains("timed out")
|| message.contains("econnreset")
|| message.contains("etimedout")
+ || message.contains("broken pipe")
+ || message.contains("epipe")
}
fn is_retryable_reqwest_error(err: &reqwest::Error) -> bool {
@@ -2962,6 +2967,7 @@ pub(super) fn is_continuation_retry_error(err: &CodexError) -> bool {
| Some(super::websocket::WEBSOCKET_CONTINUATION_SOCKET_MISSING_DETAIL)
| Some(super::websocket::WEBSOCKET_RESPONSE_START_TIMEOUT_DETAIL)
| Some(super::websocket::WEBSOCKET_MISSING_TERMINAL_DETAIL)
+ | Some(super::websocket::WEBSOCKET_KEEPALIVE_FAILURE_DETAIL)
)
}
@@ -4896,6 +4902,33 @@ mod tests {
assert!(is_retryable_transport_error(&err));
}
+ #[test]
+ fn keepalive_failure_is_retryable_with_full_context() {
+ let err = CodexError {
+ status: 0,
+ message: "WebSocket keepalive error: test write failed".to_string(),
+ detail: Some(super::super::websocket::WEBSOCKET_KEEPALIVE_FAILURE_DETAIL.to_string()),
+ retry_after: None,
+ origin: CodexErrorOrigin::WebSocket,
+ };
+
+ assert!(is_retryable_transport_error(&err));
+ assert!(is_continuation_retry_error(&err));
+ }
+
+ #[test]
+ fn statusless_broken_pipe_is_retryable() {
+ let err = CodexError {
+ status: 0,
+ message: "WebSocket stream error: IO error: Broken pipe (os error 32)".to_string(),
+ detail: None,
+ retry_after: None,
+ origin: CodexErrorOrigin::WebSocket,
+ };
+
+ assert!(is_retryable_transport_error(&err));
+ }
+
#[test]
fn image_headers_reuse_oauth_without_responses_beta_headers() {
let auth = StoredAuth {
diff --git a/src/providers/codex/mod.rs b/src/providers/codex/mod.rs
index 664c2aa..54aaad7 100644
--- a/src/providers/codex/mod.rs
+++ b/src/providers/codex/mod.rs
@@ -1308,6 +1308,9 @@ fn retryable_live_start_codex_error(err: &client::CodexError) -> bool {
}
return err.status == 0 || matches!(err.status, 429 | 500 | 502 | 503 | 504 | 529);
}
+ if err.detail.as_deref() == Some(websocket::WEBSOCKET_KEEPALIVE_FAILURE_DETAIL) {
+ return true;
+ }
matches!(err.status, 429 | 500 | 502 | 503 | 504 | 529)
|| (err.status == 0 && retryable_live_message(codex_error_message(err)))
}
@@ -1345,6 +1348,8 @@ fn retryable_live_message(message: &str) -> bool {
"timed out",
"connection closed",
"connection reset",
+ "broken pipe",
+ "epipe",
]
.iter()
.any(|needle| lower.contains(needle))
@@ -2110,6 +2115,19 @@ mod tests {
assert!(!retryable_live_start_codex_error(&err));
}
+ #[test]
+ fn live_start_keepalive_failure_is_retryable() {
+ let err = client::CodexError {
+ status: 0,
+ message: "WebSocket keepalive error: test write failed".to_string(),
+ detail: Some(websocket::WEBSOCKET_KEEPALIVE_FAILURE_DETAIL.to_string()),
+ retry_after: None,
+ origin: client::CodexErrorOrigin::WebSocket,
+ };
+
+ assert!(retryable_live_start_codex_error(&err));
+ }
+
#[test]
fn live_start_payload_retry_detection_covers_rate_limit_and_overload() {
assert!(retryable_live_start_payload(
diff --git a/src/providers/codex/websocket.rs b/src/providers/codex/websocket.rs
index 5606f15..f0fa6f2 100644
--- a/src/providers/codex/websocket.rs
+++ b/src/providers/codex/websocket.rs
@@ -42,6 +42,7 @@ pub const WEBSOCKET_CONNECT_TIMEOUT_MS: u64 = 15_000;
pub const WEBSOCKET_IDLE_TIMEOUT_MS: u64 = 300_000;
pub const WEBSOCKET_RESPONSE_START_TIMEOUT_DETAIL: &str = "websocket_response_start_timeout";
pub const WEBSOCKET_MISSING_TERMINAL_DETAIL: &str = "websocket_missing_terminal";
+pub const WEBSOCKET_KEEPALIVE_FAILURE_DETAIL: &str = "websocket_keepalive_failure";
pub const WEBSOCKET_CONTINUATION_SOCKET_MISSING_DETAIL: &str =
"websocket_continuation_socket_missing";
pub(super) const WEBSOCKET_PROXY_TUNNEL_REJECTED_DETAIL: &str = "websocket_proxy_tunnel_rejected";
@@ -53,6 +54,8 @@ const POOL_CONNECT_CLEANUP_TARGET: usize = 40;
const MAX_CONNECT_RESPONSE_HEADER_BYTES: usize = 8 * 1024;
const WEBSOCKET_CONNECT_START_SPACING: Duration = Duration::from_secs(1);
const WEBSOCKET_CONNECT_FORBIDDEN_COOLDOWN: Duration = Duration::from_secs(3);
+const WEBSOCKET_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30);
+const WEBSOCKET_KEEPALIVE_SEND_TIMEOUT: Duration = Duration::from_secs(10);
// Terminal WebSocket event types that signal the request is done
const TERMINAL_EVENTS: &[&str] = &[
@@ -1945,6 +1948,55 @@ async fn connect_with_timeout(
// Event collection
// ---------------------------------------------------------------------------
+enum WebSocketRead {
+ Frame(Option<Result<Message, tokio_tungstenite::tungstenite::Error>>),
+ Timeout,
+ KeepaliveError(String),
+}
+
+async fn read_ws_frame_with_keepalive<S>(
+ ws: &mut WebSocketStream<S>,
+ read_timeout: Duration,
+ keepalive_interval: Duration,
+) -> WebSocketRead
+where
+ S: AsyncRead + AsyncWrite + Unpin,
+{
+ let timeout = tokio::time::sleep(read_timeout);
+ tokio::pin!(timeout);
+ let mut keepalive = tokio::time::interval(keepalive_interval);
+ keepalive.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ keepalive.tick().await;
+
+ loop {
+ tokio::select! {
+ biased;
+ frame = ws.next() => return WebSocketRead::Frame(frame),
+ _ = &mut timeout => return WebSocketRead::Timeout,
+ _ = keepalive.tick() => {
+ let send = ws.send(Message::Ping(Vec::new()));
+ let send_timeout = tokio::time::sleep(WEBSOCKET_KEEPALIVE_SEND_TIMEOUT);
+ tokio::pin!(send_timeout);
+ tokio::select! {
+ biased;
+ result = send => {
+ if let Err(error) = result {
+ return WebSocketRead::KeepaliveError(error.to_string());
+ }
+ }
+ _ = &mut timeout => return WebSocketRead::Timeout,
+ _ = &mut send_timeout => {
+ return WebSocketRead::KeepaliveError(format!(
+ "send timed out after {}ms",
+ WEBSOCKET_KEEPALIVE_SEND_TIMEOUT.as_millis(),
+ ));
+ }
+ }
+ }
+ }
+ }
+}
+
struct WsEvent {
event_type: String,
payload: serde_json::Value,
@@ -1957,6 +2009,28 @@ async fn collect_ws_events<S>(
pool_entry: Option<&Arc<PoolEntry>>,
traffic: Option<&TrafficCapture>,
) -> Result<(Vec<u8>, Option<WsEvent>), CodexError>
+where
+ S: AsyncRead + AsyncWrite + Unpin,
+{
+ collect_ws_events_with_keepalive_interval(
+ ws,
+ idle_timeout_ms,
+ pool_owner,
+ pool_entry,
+ traffic,
+ WEBSOCKET_KEEPALIVE_INTERVAL,
+ )
+ .await
+}
+
+async fn collect_ws_events_with_keepalive_interval<S>(
+ ws: &mut WebSocketStream<S>,
+ idle_timeout_ms: u64,
+ pool_owner: Option<&ConversationIdentity>,
+ pool_entry: Option<&Arc<PoolEntry>>,
+ traffic: Option<&TrafficCapture>,
+ keepalive_interval: Duration,
+) -> Result<(Vec<u8>, Option<WsEvent>), CodexError>
where
S: AsyncRead + AsyncWrite + Unpin,
{
@@ -1997,22 +2071,33 @@ where
}
};
- let timeout = tokio::time::timeout(read_timeout, ws.next());
-
- let frame = timeout.await.map_err(|_| {
- invalidate_pool_owner(pool_owner, pool_entry);
- if response_started {
- CodexError {
+ let frame = match read_ws_frame_with_keepalive(ws, read_timeout, keepalive_interval).await {
+ WebSocketRead::Frame(frame) => frame,
+ WebSocketRead::Timeout => {
+ invalidate_pool_owner(pool_owner, pool_entry);
+ return Err(if response_started {
+ CodexError {
+ status: 0,
+ message: format!("WebSocket idle timeout after {idle_timeout_ms}ms"),
+ detail: None,
+ retry_after: None,
+ origin: CodexErrorOrigin::WebSocket,
+ }
+ } else {
+ response_start_timeout_error(idle_timeout_ms)
+ });
+ }
+ WebSocketRead::KeepaliveError(error) => {
+ invalidate_pool_owner(pool_owner, pool_entry);
+ return Err(CodexError {
status: 0,
- message: format!("WebSocket idle timeout after {idle_timeout_ms}ms"),
- detail: None,
+ message: format!("WebSocket keepalive error: {error}"),
+ detail: Some(WEBSOCKET_KEEPALIVE_FAILURE_DETAIL.to_string()),
retry_after: None,
origin: CodexErrorOrigin::WebSocket,
- }
- } else {
- response_start_timeout_error(idle_timeout_ms)
+ });
}
- })?;
+ };
match frame {
Some(Ok(Message::Text(text))) => {
@@ -2156,9 +2241,13 @@ where
let frame = tokio::select! {
biased;
_ = tx.closed() => break,
- frame = tokio::time::timeout(read_timeout, ws.next()) => match frame {
- Ok(frame) => frame,
- Err(_) => {
+ frame = read_ws_frame_with_keepalive(
+ ws,
+ read_timeout,
+ WEBSOCKET_KEEPALIVE_INTERVAL,
+ ) => match frame {
+ WebSocketRead::Frame(frame) => frame,
+ WebSocketRead::Timeout => {
terminal_item = Some(Err(if response_started {
CodexError {
status: 0,
@@ -2172,6 +2261,16 @@ where
}));
break;
}
+ WebSocketRead::KeepaliveError(error) => {
+ terminal_item = Some(Err(CodexError {
+ status: 0,
+ message: format!("WebSocket keepalive error: {error}"),
+ detail: None,
+ retry_after: None,
+ origin: CodexErrorOrigin::WebSocket,
+ }));
+ break;
+ }
},
};
@@ -2427,6 +2526,39 @@ mod tests {
}
}
+ struct FailingWriteIo;
+
+ impl AsyncRead for FailingWriteIo {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ _cx: &mut Context<'_>,
+ _buffer: &mut ReadBuf<'_>,
+ ) -> Poll<std::io::Result<()>> {
+ Poll::Pending
+ }
+ }
+
+ impl AsyncWrite for FailingWriteIo {
+ fn poll_write(
+ self: Pin<&mut Self>,
+ _cx: &mut Context<'_>,
+ _buffer: &[u8],
+ ) -> Poll<std::io::Result<usize>> {
+ Poll::Ready(Err(std::io::Error::new(
+ std::io::ErrorKind::BrokenPipe,
+ "test write failed",
+ )))
+ }
+
+ fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+ Poll::Ready(Ok(()))
+ }
+
+ fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+ Poll::Ready(Ok(()))
+ }
+ }
+
async fn raw_test_stream(
probe: Option<(Arc<AtomicBool>, Arc<AtomicBool>)>,
) -> CodexWebSocketStream {
@@ -4027,6 +4159,100 @@ mod tests {
assert!(!WS_POOL.lock().unwrap().contains_key(&owner));
}
+ #[tokio::test(start_paused = true)]
+ async fn silent_response_wait_sends_keepalive_ping() {
+ let (client_io, server_io) = tokio::io::duplex(1024);
+ let mut client = WebSocketStream::from_raw_socket(client_io, Role::Client, None).await;
+ let mut server = WebSocketStream::from_raw_socket(server_io, Role::Server, None).await;
+ let server_task = tokio::spawn(async move {
+ let frame = server.next().await.unwrap().unwrap();
+ assert!(matches!(frame, Message::Ping(_)));
+ server
+ .send(Message::Text(r#"{"type":"response.created"}"#.into()))
+ .await
+ .unwrap();
+ });
+
+ let frame = read_ws_frame_with_keepalive(
+ &mut client,
+ Duration::from_secs(60),
+ WEBSOCKET_KEEPALIVE_INTERVAL,
+ )
+ .await;
+
+ assert!(matches!(
+ frame,
+ WebSocketRead::Frame(Some(Ok(Message::Text(text))))
+ if text.contains("response.created")
+ ));
+ server_task.await.unwrap();
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn failed_keepalive_write_returns_stable_error() {
+ let mut client = WebSocketStream::from_raw_socket(FailingWriteIo, Role::Client, None).await;
+
+ let error = match collect_ws_events_with_keepalive_interval(
+ &mut client,
+ 60_000,
+ None,
+ None,
+ None,
+ WEBSOCKET_KEEPALIVE_INTERVAL,
+ )
+ .await
+ {
+ Ok(_) => panic!("expected keepalive failure"),
+ Err(error) => error,
+ };
+
+ assert_eq!(
+ error.detail.as_deref(),
+ Some(WEBSOCKET_KEEPALIVE_FAILURE_DETAIL)
+ );
+ assert!(error.message.contains("keepalive error"));
+ }
+
+ #[tokio::test]
+ async fn keepalive_pongs_do_not_extend_response_start_timeout() {
+ let (client_io, server_io) = tokio::io::duplex(1024);
+ let mut client = WebSocketStream::from_raw_socket(client_io, Role::Client, None).await;
+ let mut server = WebSocketStream::from_raw_socket(server_io, Role::Server, None).await;
+ let ping_count = Arc::new(AtomicUsize::new(0));
+ let server_ping_count = ping_count.clone();
+ let server_task = tokio::spawn(async move {
+ while let Some(Ok(frame)) = server.next().await {
+ if let Message::Ping(payload) = frame {
+ server_ping_count.fetch_add(1, Ordering::SeqCst);
+ if server.send(Message::Pong(payload)).await.is_err() {
+ break;
+ }
+ }
+ }
+ });
+
+ let error = match collect_ws_events_with_keepalive_interval(
+ &mut client,
+ 50,
+ None,
+ None,
+ None,
+ Duration::from_millis(10),
+ )
+ .await
+ {
+ Ok(_) => panic!("expected response start timeout"),
+ Err(error) => error,
+ };
+
+ assert_eq!(
+ error.detail.as_deref(),
+ Some(WEBSOCKET_RESPONSE_START_TIMEOUT_DETAIL)
+ );
+ assert!(ping_count.load(Ordering::SeqCst) >= 2);
+ server_task.abort();
+ }
+
async fn create_dummy_stream_async() -> CodexWebSocketStream {
use tokio::net::TcpListener;