Lightweight coding agent that runs in your terminal
Rust
Keep Noise relay streams alive after repeated handshake failures (#46031)
viyatb-oai105fe8761cd5
diff --git a/codex-rs/exec-server/src/relay.rs b/codex-rs/exec-server/src/relay.rsindex 40afbf0e28..3fc1324eeb 100644--- a/codex-rs/exec-server/src/relay.rs+++ b/codex-rs/exec-server/src/relay.rs@@ -12,6 +12,7 @@ use prost::Message as ProstMessage;use tokio::sync::mpsc;use tokio::sync::watch;use tokio::task::JoinSet;+use tokio::time::Instant;use tokio::time::timeout;use tokio_tungstenite::tungstenite::Message;use tracing::debug;@@ -50,6 +51,7 @@ use crate::websocket_pong_watchdog::WebSocketPongWatchdog;const RELAY_MESSAGE_FRAME_VERSION: u32 = 1;const MAX_ACTIVE_NOISE_RELAY_STREAMS: usize = 128;const MAX_FAILED_NOISE_HANDSHAKES: usize = 8;+const NOISE_HANDSHAKE_FAILURE_COOLDOWN: Duration = Duration::from_secs(10);const MAX_HARNESS_KEY_AUTHORIZATION_BYTES: usize = 4096;const MAX_PENDING_HANDSHAKE_VALIDATIONS: usize = 32;const HARNESS_KEY_VALIDATION_TIMEOUT: Duration = Duration::from_secs(10);@@ -575,6 +577,7 @@ wherelet mut pending_handshakes: HashMap<String, PendingHandshake> = HashMap::new();let mut validation_tasks: JoinSet<HarnessKeyValidationResult> = JoinSet::new();let mut failed_handshakes = 0usize;+ let mut handshake_cooldown_until = None;let mut next_validation_id = 0u64;let mut disconnect_reason = RendezvousDisconnectReason::LocalShutdown;@@ -633,10 +636,10 @@ where"Noise harness authorization failure details");send_reset(&physical_outgoing_tx, validation_result.stream_id);- if failed_handshake_budget_exhausted(&mut failed_handshakes) {- warn!("closing Noise relay after repeated handshake failures");- break;- }+ record_failed_handshake(+ &mut failed_handshakes,+ &mut handshake_cooldown_until,+ );continue;}if streams.len() >= MAX_ACTIVE_NOISE_RELAY_STREAMS {@@ -653,10 +656,10 @@ whereErr(error) => {warn!("failed to complete Noise relay handshake: {error}");send_reset(&physical_outgoing_tx, validation_result.stream_id);- if failed_handshake_budget_exhausted(&mut failed_handshakes) {- warn!("closing Noise relay after repeated handshake failures");- break;- }+ record_failed_handshake(+ &mut failed_handshakes,+ &mut handshake_cooldown_until,+ );continue;}};@@ -746,6 +749,16 @@ wherelet stream_id = frame.stream_id.clone();match kind {RelayFrameBodyKind::Handshake => {+ // Stop admitting work before parsing another hybrid handshake.+ // Existing streams and already-admitted validations keep running.+ if let Some(deadline) = handshake_cooldown_until {+ if Instant::now() < deadline {+ send_reset(&physical_outgoing_tx, stream_id);+ continue;+ }+ failed_handshakes = 0;+ handshake_cooldown_until = None;+ }// Reject duplicate or busy streams before paying for a hybrid// handshake. Malformed attempts that reach cryptography are// covered by the connection-wide failure budget below.@@ -756,10 +769,7 @@ where// Removing pending state makes the in-flight validation result stale.if pending_handshakes.remove(&stream_id).is_some() {send_reset(&physical_outgoing_tx, stream_id);- if failed_handshake_budget_exhausted(&mut failed_handshakes) {- warn!("closing Noise relay after repeated handshake failures");- break;- }+ record_failed_handshake(&mut failed_handshakes, &mut handshake_cooldown_until);continue;}if streams.len() >= MAX_ACTIVE_NOISE_RELAY_STREAMS {@@ -788,10 +798,10 @@ whereErr(error) => {warn!("failed to read Noise relay handshake request: {error}");send_reset(&physical_outgoing_tx, stream_id);- if failed_handshake_budget_exhausted(&mut failed_handshakes) {- warn!("closing Noise relay after repeated handshake failures");- break;- }+ record_failed_handshake(+ &mut failed_handshakes,+ &mut handshake_cooldown_until,+ );continue;}};@@ -815,10 +825,7 @@ where};let Some(authorization) = authorization else {send_reset(&physical_outgoing_tx, stream_id);- if failed_handshake_budget_exhausted(&mut failed_handshakes) {- warn!("closing Noise relay after repeated handshake failures");- break;- }+ record_failed_handshake(&mut failed_handshakes, &mut handshake_cooldown_until);continue;};let harness_public_key = pending.initiator_public_key.clone();@@ -860,11 +867,11 @@ wherelet canceled_pending_handshake =pending_handshakes.remove(&stream_id).is_some();send_reset(&physical_outgoing_tx, stream_id);- if canceled_pending_handshake- && failed_handshake_budget_exhausted(&mut failed_handshakes)- {- warn!("closing Noise relay after repeated handshake failures");- break;+ if canceled_pending_handshake {+ record_failed_handshake(+ &mut failed_handshakes,+ &mut handshake_cooldown_until,+ );}continue;};@@ -909,11 +916,21 @@ where/// Charge one failed authenticated-channel attempt to this physical relay.///-/// Closing after a small fixed budget prevents a peer that has not been-/// authorized from triggering unbounded hybrid handshakes or registry checks.-fn failed_handshake_budget_exhausted(failed_handshakes: &mut usize) -> bool {+/// Pause new handshakes after a small fixed budget without disconnecting+/// authenticated streams. In-flight validation failures do not extend the+/// cooldown, so they cannot indefinitely prevent new streams from connecting.+fn record_failed_handshake(+ failed_handshakes: &mut usize,+ handshake_cooldown_until: &mut Option<Instant>,+) {+ if handshake_cooldown_until.is_some() {+ return;+ }*failed_handshakes += 1;- *failed_handshakes >= MAX_FAILED_NOISE_HANDSHAKES+ if *failed_handshakes >= MAX_FAILED_NOISE_HANDSHAKES {+ *handshake_cooldown_until = Some(Instant::now() + NOISE_HANDSHAKE_FAILURE_COOLDOWN);+ warn!("pausing Noise relay handshakes after repeated failures");+ }}/// Responder state held while registry authorization is pending.diff --git a/codex-rs/exec-server/src/relay_noise_tests.rs b/codex-rs/exec-server/src/relay_noise_tests.rsindex b92d22b9bf..e066e1eb5e 100644--- a/codex-rs/exec-server/src/relay_noise_tests.rs+++ b/codex-rs/exec-server/src/relay_noise_tests.rs@@ -10,9 +10,12 @@ use codex_exec_server_protocol::RequestId;use futures::SinkExt;use futures::StreamExt;use pretty_assertions::assert_eq;+use tokio::io::AsyncRead;+use tokio::io::AsyncWrite;use tokio::net::TcpListener;use tokio::sync::Notify;use tokio::time::timeout;+use tokio_tungstenite::WebSocketStream;use tokio_tungstenite::accept_async;use tokio_tungstenite::connect_async;use tokio_tungstenite::tungstenite::Message;@@ -21,6 +24,7 @@ use tokio_util::task::AbortOnDropHandle;use super::HarnessKeyValidator;use super::MAX_FAILED_NOISE_HANDSHAKES;use super::MAX_HARNESS_KEY_AUTHORIZATION_BYTES;+use super::NOISE_HANDSHAKE_FAILURE_COOLDOWN;use super::RendezvousDisconnectReason;use super::run_multiplexed_environment;use crate::ExecServerError;@@ -31,6 +35,8 @@ use crate::noise_channel::NoiseChannelIdentity;use crate::noise_channel::NoiseChannelPublicKey;use crate::noise_channel::noise_channel_prologue;use crate::noise_relay::NoiseHarnessConnectionArgs;+use crate::noise_relay::message_framing::JsonRpcMessageDecoder;+use crate::noise_relay::message_framing::frame_jsonrpc_message;use crate::noise_relay::noise_harness_connection_from_websocket_with_readiness;use crate::noise_relay::stream_handler::NoiseOutboundMessage;use crate::noise_relay::stream_handler::NoiseStreamConnection;@@ -44,6 +50,51 @@ use crate::server::ConnectionProcessor;const ENVIRONMENT_ID: &str = "environment-1";const EXECUTOR_REGISTRATION_ID: &str = "registration-1";+async fn next_relay_frame<S>(websocket: &mut WebSocketStream<S>) -> Result<RelayMessageFrame>+where+ S: AsyncRead + AsyncWrite + Unpin,+{+ loop {+ match websocket.next().await {+ Some(Ok(Message::Binary(payload))) => {+ return Ok(decode_relay_message_frame(payload.as_ref())?);+ }+ Some(Ok(Message::Ping(payload))) => websocket.send(Message::Pong(payload)).await?,+ Some(Ok(Message::Pong(_) | Message::Frame(_))) => {}+ Some(Ok(message)) => anyhow::bail!("expected relay frame, got {message:?}"),+ Some(Err(error)) => return Err(error.into()),+ None => anyhow::bail!("environment closed before sending relay frame"),+ }+ }+}++#[derive(Clone)]+struct EchoMessages;++impl NoiseStreamHandler for EchoMessages {+ type Incoming = JSONRPCMessage;+ type Outgoing = JSONRPCMessage;++ fn decode(payload: bytes::Bytes) -> Result<Self::Incoming, ExecServerError> {+ Ok(serde_json::from_slice(&payload)?)+ }++ fn encode(message: Self::Outgoing) -> Result<NoiseOutboundMessage, ExecServerError> {+ ConnectionProcessor::encode(message)+ }++ async fn run_connection(+ self,+ mut connection: NoiseStreamConnection<Self::Incoming, Self::Outgoing>,+ ) {+ while let Some(message) = connection.incoming_rx.recv().await {+ if connection.outgoing_tx.send(message).await.is_err() {+ break;+ }+ }+ }+}+#[derive(Clone)]struct ObservedRegistration(ConnectionProcessor,@@ -304,7 +355,7 @@ async fn pending_harness_key_validation_does_not_block_new_handshakes() -> Resul}#[tokio::test]-async fn duplicate_handshakes_exhaust_failure_budget() -> Result<()> {+async fn duplicate_handshakes_pause_admission_without_disconnecting() -> Result<()> {let listener = TcpListener::bind("127.0.0.1:0").await?;let websocket_url = format!("ws://{}", listener.local_addr()?);let harness_connection = tokio::spawn(connect_async(websocket_url));@@ -366,19 +417,11 @@ async fn duplicate_handshakes_exhaust_failure_budget() -> Result<()> {harness_websocket.send(Message::Binary(encoded.clone().into())).await?;- let payload = timeout(Duration::from_secs(1), async {- loop {- match harness_websocket.next().await {- Some(Ok(Message::Binary(payload))) => break Ok(payload),- Some(Ok(Message::Ping(_) | Message::Pong(_) | Message::Frame(_))) => {}- Some(Ok(message)) => anyhow::bail!("expected reset frame, got {message:?}"),- Some(Err(error)) => break Err(error.into()),- None => anyhow::bail!("environment closed before sending reset"),- }- }- })+ let reset = timeout(+ Duration::from_secs(1),+ next_relay_frame(&mut harness_websocket),+ ).await??;- let reset = decode_relay_message_frame(payload.as_ref())?;assert_eq!(reset.stream_id, stream_id);assert_eq!(reset.validate()?, RelayFrameBodyKind::Reset);}@@ -392,10 +435,31 @@ async fn duplicate_handshakes_exhaust_failure_budget() -> Result<()> {}}).await?;+ harness_websocket+ .send(Message::Binary(encoded.clone().into()))+ .await?;+ let reset = timeout(+ Duration::from_secs(1),+ next_relay_frame(&mut harness_websocket),+ )+ .await??;+ assert_eq!(reset.validate()?, RelayFrameBodyKind::Reset);+ // The next valid request is rejected before starting another registry check.harness_websocket.send(Message::Binary(encoded.into())).await?;- timeout(Duration::from_secs(1), environment_task).await??;+ let reset = timeout(+ Duration::from_secs(1),+ next_relay_frame(&mut harness_websocket),+ )+ .await??;+ assert_eq!(reset.validate()?, RelayFrameBodyKind::Reset);+ assert_eq!(calls.load(Ordering::SeqCst), MAX_FAILED_NOISE_HANDSHAKES);+ harness_websocket.close(None).await?;+ assert_eq!(+ timeout(Duration::from_secs(1), environment_task).await??,+ RendezvousDisconnectReason::PeerClose,+ );release.notify_waiters();Ok(())}@@ -457,7 +521,13 @@ async fn oversized_harness_authorization_is_rejected_before_validation() -> Resu}#[tokio::test]-async fn repeated_malformed_handshakes_close_the_physical_relay() -> Result<()> {+async fn malformed_handshakes_preserve_active_streams_and_admission_recovers() -> Result<()> {+ tokio::time::pause();+ // Socket I/O must not auto-advance the paused cooldown or keepalive clocks.+ let (_clock_guard, clock_released) = std::sync::mpsc::channel::<()>();+ tokio::task::spawn_blocking(move || {+ let _ = clock_released.recv();+ });let listener = TcpListener::bind("127.0.0.1:0").await?;let websocket_url = format!("ws://{}", listener.local_addr()?);let harness_connection = tokio::spawn(connect_async(websocket_url));@@ -467,20 +537,46 @@ async fn repeated_malformed_handshakes_close_the_physical_relay() -> Result<()>let environment_identity = NoiseChannelIdentity::generate()?;let harness_identity = NoiseChannelIdentity::generate()?;- let environment_task = tokio::spawn(run_multiplexed_environment(+ let calls = Arc::new(AtomicUsize::new(0));+ let release = Arc::new(Notify::new());+ let environment_task = AbortOnDropHandle::new(tokio::spawn(run_multiplexed_environment(environment_websocket,- ConnectionProcessor::new(ExecServerRuntimePaths::new(- std::env::current_exe()?,- /*codex_linux_sandbox_exe*/ None,- )?),+ EchoMessages,ENVIRONMENT_ID.to_string(),EXECUTOR_REGISTRATION_ID.to_string(),environment_identity.clone(),BlockingValidator {- calls: Arc::new(AtomicUsize::new(0)),- release: Arc::new(Notify::new()),+ calls: Arc::clone(&calls),+ release: Arc::clone(&release),},- ));+ )));++ let healthy_stream_id = "healthy";+ let prologue =+ noise_channel_prologue(ENVIRONMENT_ID, EXECUTOR_REGISTRATION_ID, healthy_stream_id);+ let (handshake, request) = InitiatorHandshake::start(+ &harness_identity,+ &environment_identity.public_key(),+ &prologue,+ b"authorization",+ )?;+ release.notify_one();+ harness_websocket+ .send(Message::Binary(+ encode_relay_message_frame(&RelayMessageFrame::handshake(+ healthy_stream_id.to_string(),+ request,+ ))+ .into(),+ ))+ .await?;+ let response = timeout(+ Duration::from_secs(1),+ next_relay_frame(&mut harness_websocket),+ )+ .await??;+ assert_eq!(response.stream_id, healthy_stream_id);+ let mut healthy_transport = handshake.finish(&response.into_handshake_payload()?)?;for attempt in 0..MAX_FAILED_NOISE_HANDSHAKES {let stream_id = format!("malformed-{attempt}");@@ -497,14 +593,99 @@ async fn repeated_malformed_handshakes_close_the_physical_relay() -> Result<()>harness_websocket.send(Message::Binary(encode_relay_message_frame(&frame).into())).await?;+ let reset = timeout(+ Duration::from_secs(1),+ next_relay_frame(&mut harness_websocket),+ )+ .await??;+ assert_eq!(+ reset,+ RelayMessageFrame::reset(+ frame.stream_id,+ crate::noise_relay::NOISE_RELAY_RESET_REASON.to_string(),+ )+ );}- timeout(Duration::from_secs(1), environment_task).await??;+ let new_stream_id = "after-cooldown";+ let prologue = noise_channel_prologue(ENVIRONMENT_ID, EXECUTOR_REGISTRATION_ID, new_stream_id);+ let (new_handshake, request) = InitiatorHandshake::start(+ &harness_identity,+ &environment_identity.public_key(),+ &prologue,+ b"authorization",+ )?;+ let new_frame = RelayMessageFrame::handshake(new_stream_id.to_string(), request);+ // Check both immediate rejection and a later attempt that must not extend+ // the cooldown. Encrypted traffic on the healthy stream still round-trips.+ for seq in 0..2 {+ harness_websocket+ .send(Message::Binary(+ encode_relay_message_frame(&new_frame).into(),+ ))+ .await?;+ let reset = timeout(+ Duration::from_secs(1),+ next_relay_frame(&mut harness_websocket),+ )+ .await??;+ assert_eq!(reset.stream_id, new_stream_id);+ assert_eq!(reset.validate()?, RelayFrameBodyKind::Reset);+ assert_eq!(calls.load(Ordering::SeqCst), 1);++ let message = JSONRPCMessage::Response(JSONRPCResponse {+ id: RequestId::Integer(i64::from(seq)),+ result: serde_json::json!({"healthy": true}),+ });+ let frame = RelayMessageFrame::data(+ healthy_stream_id.to_string(),+ seq,+ healthy_transport.encrypt(&frame_jsonrpc_message(&message)?)?,+ /*trace*/ None,+ );+ harness_websocket+ .send(Message::Binary(encode_relay_message_frame(&frame).into()))+ .await?;+ let response = timeout(+ Duration::from_secs(1),+ next_relay_frame(&mut harness_websocket),+ )+ .await??;+ assert_eq!(response.stream_id, healthy_stream_id);+ let data = response.into_data()?;+ assert_eq!(data.seq, seq);+ let messages =+ JsonRpcMessageDecoder::default().push(&healthy_transport.decrypt(&data.payload)?)?;+ assert_eq!(messages, vec![message]);++ tokio::time::advance(NOISE_HANDSHAKE_FAILURE_COOLDOWN / 2).await;+ }++ release.notify_one();+ harness_websocket+ .send(Message::Binary(+ encode_relay_message_frame(&new_frame).into(),+ ))+ .await?;+ let response = timeout(+ Duration::from_secs(1),+ next_relay_frame(&mut harness_websocket),+ )+ .await??;+ assert_eq!(response.stream_id, new_stream_id);+ new_handshake.finish(&response.into_handshake_payload()?)?;+ assert_eq!(calls.load(Ordering::SeqCst), 2);++ harness_websocket.close(None).await?;+ assert_eq!(+ timeout(Duration::from_secs(1), environment_task).await??,+ RendezvousDisconnectReason::PeerClose,+ );Ok(())}#[tokio::test]-async fn repeated_early_data_during_validation_closes_the_physical_relay() -> Result<()> {+async fn early_data_during_validation_pauses_admission_without_disconnecting() -> Result<()> {let listener = TcpListener::bind("127.0.0.1:0").await?;let websocket_url = format!("ws://{}", listener.local_addr()?);let harness_connection = tokio::spawn(connect_async(websocket_url));@@ -546,8 +727,18 @@ async fn repeated_early_data_during_validation_closes_the_physical_relay() -> Re.send(Message::Binary(encode_relay_message_frame(&frame).into())).await?;}+ let reset = timeout(+ Duration::from_secs(1),+ next_relay_frame(&mut harness_websocket),+ )+ .await??;+ assert_eq!(reset.validate()?, RelayFrameBodyKind::Reset);}- timeout(Duration::from_secs(1), environment_task).await??;+ harness_websocket.close(None).await?;+ assert_eq!(+ timeout(Duration::from_secs(1), environment_task).await??,+ RendezvousDisconnectReason::PeerClose,+ );Ok(())}