Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Honor network policy in remote control and recover remote execution (#47410)

acrognale-oaidb3cb33ecafa
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock
index 448cba9d98..087b257e91 100644
--- a/codex-rs/Cargo.lock
+++ b/codex-rs/Cargo.lock
@@ -2276,6 +2276,7 @@ dependencies = [
"codex-app-server-protocol",
"codex-config",
"codex-core",
+ "codex-http-client",
"codex-login",
"codex-model-provider",
"codex-protocol",
@@ -2283,6 +2284,7 @@ dependencies = [
"codex-uds",
"codex-utils-absolute-path",
"codex-utils-rustls-provider",
+ "codex-websocket-client",
"constant_time_eq",
"futures",
"gethostname",
diff --git a/codex-rs/app-server-transport/Cargo.toml b/codex-rs/app-server-transport/Cargo.toml
index 9b91662c40..3b3eb4c668 100644
--- a/codex-rs/app-server-transport/Cargo.toml
+++ b/codex-rs/app-server-transport/Cargo.toml
@@ -13,6 +13,8 @@ doctest = false
workspace = true
[dependencies]
+codex-http-client = { workspace = true }
+codex-websocket-client = { workspace = true }
anyhow = { workspace = true }
axum = { workspace = true, default-features = false, features = [
"http1",
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/auth.rs b/codex-rs/app-server-transport/src/transport/remote_control/auth.rs
index 93c728abec..9d274627e2 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/auth.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/auth.rs
@@ -17,6 +17,7 @@ use tracing::warn;
pub(super) struct RemoteControlAuth {
manager: Arc<AuthManager>,
pub(super) owner: crate::ConnectionAuth,
+ pub(super) network_policy: codex_http_client::NetworkPolicy,
}
pub(super) struct RemoteControlRecovery {
@@ -32,7 +33,14 @@ impl RemoteControlAuth {
.auth_cached()
.is_some_and(|auth| auth.uses_codex_backend() && auth.get_account_id().is_some());
if owner.is_current() {
- return (Self { manager, owner }, authenticated);
+ return (
+ Self {
+ network_policy: manager.application_network_policy(),
+ manager,
+ owner,
+ },
+ authenticated,
+ );
}
}
}
@@ -56,6 +64,7 @@ impl RemoteControlAuth {
pub(super) const REMOTE_CONTROL_ACCOUNT_ID_HEADER: &str = "chatgpt-account-id";
pub(super) struct RemoteControlConnectionAuth {
+ pub(super) http_client_factory: codex_http_client::HttpClientFactory,
pub(super) auth_provider: SharedAuthProvider,
pub(super) account_id: String,
}
@@ -90,8 +99,9 @@ async fn load_auth_manager(
auth_manager: &Arc<AuthManager>,
) -> io::Result<RemoteControlConnectionAuth> {
let mut reloaded = false;
- let auth = loop {
- let Some(auth) = auth_manager.auth().await else {
+ let (auth, http_client_factory) = loop {
+ let Some((auth, http_client_factory)) = auth_manager.auth_with_http_client_factory().await
+ else {
if reloaded {
return Err(io::Error::new(
ErrorKind::PermissionDenied,
@@ -103,14 +113,14 @@ async fn load_auth_manager(
continue;
};
if !auth.uses_codex_backend() {
- break auth;
+ break (auth, http_client_factory);
}
if auth.get_account_id().is_none() && !reloaded {
auth_manager.reload().await;
reloaded = true;
continue;
}
- break auth;
+ break (auth, http_client_factory);
};
if !auth.uses_codex_backend() {
@@ -121,6 +131,7 @@ async fn load_auth_manager(
}
Ok(RemoteControlConnectionAuth {
+ http_client_factory,
auth_provider: codex_model_provider::auth_provider_from_auth(&auth),
account_id: auth.get_account_id().ok_or_else(|| {
io::Error::new(
@@ -131,6 +142,38 @@ async fn load_auth_manager(
})
}
+pub(super) async fn request_client(
+ factory: &codex_http_client::HttpClientFactory,
+ endpoint: &str,
+) -> io::Result<codex_http_client::HttpClient> {
+ codex_login::default_client::create_client_for_route_without_request_logging_async(
+ factory.clone(),
+ endpoint.to_string(),
+ codex_http_client::ClientRouteClass::Api,
+ )
+ .await
+}
+
+pub(super) fn is_policy_denial(error: &io::Error) -> bool {
+ let Some(cause) = error.get_ref() else {
+ return false;
+ };
+ cause.is::<codex_http_client::NetworkPolicyDenied>()
+}
+
+pub(super) fn is_auth_error(error: &io::Error) -> bool {
+ error.kind() == ErrorKind::PermissionDenied && !is_policy_denial(error)
+}
+
+pub(super) fn request_error(error: codex_http_client::HttpError) -> io::Error {
+ match error {
+ codex_http_client::HttpError::Policy(denied) => {
+ io::Error::new(ErrorKind::PermissionDenied, denied)
+ }
+ error => io::Error::other(error),
+ }
+}
+
pub(super) async fn recover_remote_control_auth(
recovery: &mut RemoteControlRecovery,
auth_change_rx: &mut watch::Receiver<u64>,
@@ -213,6 +256,9 @@ mod tests {
provider_account_ids: Vec<&'static str>,
) -> RemoteControlConnectionAuth {
RemoteControlConnectionAuth {
+ http_client_factory: codex_http_client::HttpClientFactory::new(
+ codex_http_client::OutboundProxyPolicy::ReqwestDefault,
+ ),
auth_provider: Arc::new(TestAuthProvider {
account_ids: provider_account_ids,
}),
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/clients.rs b/codex-rs/app-server-transport/src/transport/remote_control/clients.rs
index 37ec1c1a58..8d33927ee9 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/clients.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/clients.rs
@@ -2,6 +2,7 @@ use super::auth::RemoteControlAuth;
use super::auth::RemoteControlConnectionAuth;
use super::auth::load_remote_control_auth;
use super::auth::recover_remote_control_auth;
+use super::auth::request_client;
use super::enroll::format_headers;
use super::enroll::preview_remote_control_response_body;
use super::protocol::normalize_remote_control_base_url;
@@ -12,7 +13,6 @@ use codex_app_server_protocol::RemoteControlClientsListParams;
use codex_app_server_protocol::RemoteControlClientsListResponse;
use codex_app_server_protocol::RemoteControlClientsRevokeParams;
use codex_app_server_protocol::RemoteControlClientsRevokeResponse;
-use codex_login::default_client::create_client_without_request_logging;
use serde::Deserialize;
use std::io;
use std::io::ErrorKind;
@@ -185,7 +185,12 @@ async fn send_client_management_request_once(
request: &ClientManagementRequest<'_>,
action: &str,
) -> io::Result<ClientManagementResponse> {
- let client = create_client_without_request_logging();
+ let endpoint = match request {
+ ClientManagementRequest::List { url, .. } | ClientManagementRequest::Revoke { url } => {
+ url.as_str()
+ }
+ };
+ let client = request_client(&auth.http_client_factory, endpoint).await?;
let auth_headers = auth.request_headers()?;
let request = match request {
ClientManagementRequest::List { url, params } => {
@@ -206,7 +211,9 @@ async fn send_client_management_request_once(
.to_string(),
));
}
- client.get((*url).clone()).query(&query)
+ let mut url = (*url).clone();
+ url.query_pairs_mut().extend_pairs(query);
+ client.get(url)
}
ClientManagementRequest::Revoke { url } => client.delete((*url).clone()),
};
@@ -215,13 +222,16 @@ async fn send_client_management_request_once(
.headers(auth_headers)
.send()
.await
- .map_err(|err| io::Error::other(format!("failed to {action}: {err}")))?;
+ .map_err(|error| match error {
+ codex_http_client::HttpError::Policy(_) => super::auth::request_error(error),
+ error => io::Error::other(format!("failed to {action}: {error}")),
+ })?;
let headers = response.headers().clone();
let status = response.status();
let body = response
.bytes()
.await
- .map_err(|err| io::Error::other(format!("failed to read {action} response: {err}")))?
+ .map_err(super::auth::request_error)?
.to_vec();
Ok(ClientManagementResponse {
status,
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/enroll.rs b/codex-rs/app-server-transport/src/transport/remote_control/enroll.rs
index 638d797cf3..0b92c6f611 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/enroll.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/enroll.rs
@@ -1,3 +1,4 @@
+use super::auth::request_client;
use super::pairing_unavailable_error;
use super::protocol::RemoteControlPairingStatusRequest;
use super::protocol::RemoteControlPairingStatusResponse as BackendRemoteControlPairingStatusResponse;
@@ -10,7 +11,7 @@ use axum::http::HeaderMap;
use axum::http::StatusCode;
use codex_app_server_protocol::RemoteControlPairingStartResponse;
use codex_app_server_protocol::RemoteControlPairingStatusResponse;
-use codex_login::default_client::create_client_without_request_logging;
+use codex_http_client::HttpClientFactory;
use codex_state::RemoteControlEnrollmentRecord;
use codex_state::StateRuntime;
use std::io;
@@ -50,6 +51,7 @@ pub(super) enum RemoteControlServerTokenRefreshRequirement {
impl RemoteControlEnrollment {
pub(super) async fn start_pairing(
&self,
+ http_client_factory: &HttpClientFactory,
request: StartRemoteControlPairingRequest,
) -> io::Result<RemoteControlPairingStartResponse> {
if self.server_token_refresh_requirement()
@@ -62,23 +64,24 @@ impl RemoteControlEnrollment {
.as_deref()
.ok_or_else(pairing_unavailable_error)?;
- let response = create_client_without_request_logging()
+ let response = request_client(http_client_factory, &self.remote_control_target.pair_url)
+ .await?
.post(&self.remote_control_target.pair_url)
.timeout(REMOTE_CONTROL_PAIRING_TIMEOUT)
.bearer_auth(remote_control_token)
.json(&request)
.send()
.await
- .map_err(|err| {
- io::Error::other(format!(
- "failed to start remote control pairing at `{}`: {err}",
- self.remote_control_target.pair_url
- ))
- })?;
+ .map_err(super::auth::request_error)?;
let headers = response.headers().clone();
let status = response.status();
let retry_at = retry_after_with_jitter(&headers, OffsetDateTime::now_utc());
let body = response.bytes().await.map_err(|err| {
+ if matches!(err, codex_http_client::HttpError::Policy(_))
+ && !(retry_at.is_some() && matches!(status.as_u16(), 429 | 503))
+ {
+ return super::auth::request_error(err);
+ }
pairing_response_error(
format!(
"failed to read remote control pairing response from `{}`: {err}",
@@ -153,6 +156,7 @@ impl RemoteControlEnrollment {
pub(super) async fn pairing_status(
&self,
+ http_client_factory: &HttpClientFactory,
request: RemoteControlPairingStatusRequest,
) -> io::Result<RemoteControlPairingStatusResponse> {
if self.server_token_refresh_requirement()
@@ -165,23 +169,27 @@ impl RemoteControlEnrollment {
.as_deref()
.ok_or_else(pairing_unavailable_error)?;
- let response = create_client_without_request_logging()
- .post(&self.remote_control_target.pair_status_url)
- .timeout(REMOTE_CONTROL_PAIRING_TIMEOUT)
- .bearer_auth(remote_control_token)
- .json(&request)
- .send()
- .await
- .map_err(|err| {
- io::Error::other(format!(
- "failed to check remote control pairing status at `{}`: {err}",
- self.remote_control_target.pair_status_url
- ))
- })?;
+ let response = request_client(
+ http_client_factory,
+ &self.remote_control_target.pair_status_url,
+ )
+ .await?
+ .post(&self.remote_control_target.pair_status_url)
+ .timeout(REMOTE_CONTROL_PAIRING_TIMEOUT)
+ .bearer_auth(remote_control_token)
+ .json(&request)
+ .send()
+ .await
+ .map_err(super::auth::request_error)?;
let headers = response.headers().clone();
let status = response.status();
let retry_at = retry_after_with_jitter(&headers, OffsetDateTime::now_utc());
let body = response.bytes().await.map_err(|err| {
+ if matches!(err, codex_http_client::HttpError::Policy(_))
+ && !(retry_at.is_some() && matches!(status.as_u16(), 429 | 503))
+ {
+ return super::auth::request_error(err);
+ }
pairing_response_error(
format!(
"failed to read remote control pairing status response from `{}`: {err}",
@@ -710,6 +718,9 @@ mod tests {
let err = enroll_remote_control_server(
&remote_control_target,
&RemoteControlConnectionAuth {
+ http_client_factory: codex_http_client::HttpClientFactory::new(
+ codex_http_client::OutboundProxyPolicy::ReqwestDefault,
+ ),
auth_provider: codex_model_provider::unauthenticated_auth_provider(),
account_id: "account_id".to_string(),
},
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/mod.rs b/codex-rs/app-server-transport/src/transport/remote_control/mod.rs
index aa672d36ca..f08a694d9f 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/mod.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/mod.rs
@@ -488,8 +488,11 @@ impl RemoteControlSession {
manual_code: params.manual_code,
};
self.current_enrollment.check_retry_after()?;
- let pairing_response = match enrollment.start_pairing(pairing_request()).await {
- Err(err) if err.kind() == io::ErrorKind::PermissionDenied => {
+ let pairing_response = match enrollment
+ .start_pairing(&auth.http_client_factory, pairing_request())
+ .await
+ {
+ Err(err) if auth::is_auth_error(&err) => {
clear_pairing_server_token(&mut current_enrollment, &mut enrollment)?;
refresh_pairing_enrollment(
&mut current_enrollment,
@@ -500,7 +503,9 @@ impl RemoteControlSession {
)
.await?;
self.current_enrollment.check_retry_after()?;
- enrollment.start_pairing(pairing_request()).await
+ enrollment
+ .start_pairing(&auth.http_client_factory, pairing_request())
+ .await
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {
enrollment = self
@@ -514,7 +519,9 @@ impl RemoteControlSession {
)
.await?;
self.current_enrollment.check_retry_after()?;
- enrollment.start_pairing(pairing_request()).await
+ enrollment
+ .start_pairing(&auth.http_client_factory, pairing_request())
+ .await
}
pairing_response => pairing_response,
};
@@ -535,7 +542,7 @@ impl RemoteControlSession {
.await?;
return Err(pairing_unavailable_error());
}
- io::ErrorKind::PermissionDenied => {
+ io::ErrorKind::PermissionDenied if auth::is_auth_error(err) => {
clear_pairing_server_token(&mut current_enrollment, &mut enrollment)?;
return Err(pairing_unavailable_error());
}
@@ -724,23 +731,27 @@ impl RemoteControlSession {
let pairing_status_request =
|| protocol::RemoteControlPairingStatusRequest::from(status_code.clone());
self.current_enrollment.check_retry_after()?;
- let pairing_status_response =
- match enrollment.pairing_status(pairing_status_request()).await {
- Err(err) if err.kind() == io::ErrorKind::PermissionDenied => {
- clear_pairing_server_token(&mut current_enrollment, &mut enrollment)?;
- refresh_pairing_enrollment(
- &mut current_enrollment,
- &self.auth_manager,
- &mut auth,
- &installation_id,
- &mut enrollment,
- )
- .await?;
- self.current_enrollment.check_retry_after()?;
- enrollment.pairing_status(pairing_status_request()).await
- }
- pairing_status_response => pairing_status_response,
- };
+ let pairing_status_response = match enrollment
+ .pairing_status(&auth.http_client_factory, pairing_status_request())
+ .await
+ {
+ Err(err) if auth::is_auth_error(&err) => {
+ clear_pairing_server_token(&mut current_enrollment, &mut enrollment)?;
+ refresh_pairing_enrollment(
+ &mut current_enrollment,
+ &self.auth_manager,
+ &mut auth,
+ &installation_id,
+ &mut enrollment,
+ )
+ .await?;
+ self.current_enrollment.check_retry_after()?;
+ enrollment
+ .pairing_status(&auth.http_client_factory, pairing_status_request())
+ .await
+ }
+ pairing_status_response => pairing_status_response,
+ };
if let Err(err) = &pairing_status_response {
if server_api::remote_control_retry_at(err).is_some() {
return self
@@ -760,7 +771,7 @@ impl RemoteControlSession {
.await?;
return Err(pairing_unavailable_error());
}
- io::ErrorKind::PermissionDenied => {
+ io::ErrorKind::PermissionDenied if auth::is_auth_error(err) => {
clear_pairing_server_token(&mut current_enrollment, &mut enrollment)?;
return Err(pairing_unavailable_error());
}
@@ -847,7 +858,7 @@ async fn enroll_pairing_server(
.await
{
Ok(enrollment) => return Ok(enrollment),
- Err(err) if err.kind() == io::ErrorKind::PermissionDenied => {
+ Err(err) if auth::is_auth_error(&err) => {
let mut auth_recovery = auth_manager.unauthorized_recovery();
let mut auth_change_rx = auth_manager.auth_change_receiver();
if !recover_remote_control_auth(&mut auth_recovery, &mut auth_change_rx).await {
@@ -893,10 +904,7 @@ async fn refresh_pairing_enrollment(
) -> io::Result<()> {
current_enrollment.state.check_retry_after()?;
let mut refresh_result = refresh_remote_control_server(auth, installation_id, enrollment).await;
- if refresh_result
- .as_ref()
- .is_err_and(|err| err.kind() == io::ErrorKind::PermissionDenied)
- {
+ if refresh_result.as_ref().is_err_and(auth::is_auth_error) {
let mut auth_recovery = auth_manager.unauthorized_recovery();
let mut auth_change_rx = auth_manager.auth_change_receiver();
if recover_remote_control_auth(&mut auth_recovery, &mut auth_change_rx).await {
@@ -916,10 +924,7 @@ async fn refresh_pairing_enrollment(
enrollment.clear_server_token();
}
}
- if refresh_result
- .as_ref()
- .is_err_and(|err| err.kind() == io::ErrorKind::PermissionDenied)
- {
+ if refresh_result.as_ref().is_err_and(auth::is_auth_error) {
enrollment.clear_server_token();
}
if !replace_current_enrollment(current_enrollment, enrollment) {
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/server_api.rs b/codex-rs/app-server-transport/src/transport/remote_control/server_api.rs
index ab7d7fd8a0..badf35ab4a 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/server_api.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/server_api.rs
@@ -1,4 +1,5 @@
use super::auth::RemoteControlConnectionAuth;
+use super::auth::request_client;
use super::enroll::RemoteControlEnrollment;
use super::enroll::RemoteControlServerTokenRefreshRequirement;
use super::enroll::format_headers;
@@ -9,7 +10,6 @@ use super::protocol::RefreshRemoteServerRequest;
use super::protocol::RemoteControlTarget;
use axum::http::HeaderMap;
use axum::http::StatusCode;
-use codex_login::default_client::create_client_without_request_logging;
use rand::Rng;
use serde::Serialize;
use serde::de::DeserializeOwned;
@@ -241,7 +241,7 @@ where
Request: Serialize,
Response: DeserializeOwned,
{
- let client = create_client_without_request_logging();
+ let client = request_client(&auth.http_client_factory, url).await?;
let auth_headers = auth.request_headers()?;
let response = client
.post(url)
@@ -252,6 +252,9 @@ where
.send()
.await
.map_err(|err| {
+ if matches!(err, codex_http_client::HttpError::Policy(_)) {
+ return super::auth::request_error(err);
+ }
let timed_out = err.is_timeout();
RemoteControlServerRequestError::io_error(
format!("failed to {action} remote control server at `{url}`: {err}"),
@@ -264,6 +267,11 @@ where
let status = response.status();
let retry_at = retry_after_with_jitter(&headers, OffsetDateTime::now_utc());
let body = response.bytes().await.map_err(|err| {
+ if matches!(err, codex_http_client::HttpError::Policy(_))
+ && !(retry_at.is_some() && (status.as_u16() == 429 || status.is_server_error()))
+ {
+ return super::auth::request_error(err);
+ }
let timed_out = err.is_timeout();
RemoteControlServerRequestError::io_error(
format!("failed to read remote control {response_kind} response from `{url}`: {err}"),
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/server_api_tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/server_api_tests.rs
index 0c636d3c0e..cc8d93d447 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/server_api_tests.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/server_api_tests.rs
@@ -11,6 +11,9 @@ const TEST_REQUEST_TIMEOUT: Duration = Duration::from_millis(100);
fn auth() -> RemoteControlConnectionAuth {
RemoteControlConnectionAuth {
+ http_client_factory: codex_http_client::HttpClientFactory::new(
+ codex_http_client::OutboundProxyPolicy::ReqwestDefault,
+ ),
auth_provider: codex_model_provider::unauthenticated_auth_provider(),
account_id: "account-a".to_string(),
}
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/tests/pairing_tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/tests/pairing_tests.rs
index b343242c8a..3a9d152401 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/tests/pairing_tests.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/tests/pairing_tests.rs
@@ -1,10 +1,16 @@
use super::super::protocol::RemoteControlPairingStatusRequest;
use super::super::protocol::StartRemoteControlPairingRequest;
use super::*;
+use codex_http_client::HttpClientFactory;
+use codex_http_client::OutboundProxyPolicy;
use codex_login::AuthKeyringBackendKind;
use pretty_assertions::assert_eq;
use std::io;
+fn http_client_factory() -> HttpClientFactory {
+ HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault)
+}
+
fn remote_control_enrollment(
remote_control_url: &str,
remote_control_token: &str,
@@ -107,7 +113,10 @@ async fn pairing_error(status: &'static str, body: &'static str) -> (String, Str
});
let err = remote_control_enrollment(&remote_control_url, "remote-control-token")
- .start_pairing(StartRemoteControlPairingRequest { manual_code: false })
+ .start_pairing(
+ &http_client_factory(),
+ StartRemoteControlPairingRequest { manual_code: false },
+ )
.await
.expect_err("pairing should fail");
server_task.await.expect("server task should finish");
@@ -125,7 +134,10 @@ async fn pairing_response_error(body: serde_json::Value) -> String {
});
let err = remote_control_enrollment(&remote_control_url, "remote-control-token")
- .start_pairing(StartRemoteControlPairingRequest { manual_code: false })
+ .start_pairing(
+ &http_client_factory(),
+ StartRemoteControlPairingRequest { manual_code: false },
+ )
.await
.expect_err("pairing should fail");
server_task.await.expect("server task should finish");
@@ -152,10 +164,13 @@ async fn pairing_status_error(status: &'static str, body: &'static str) -> (io::
});
let err = remote_control_enrollment(&remote_control_url, "remote-control-token")
- .pairing_status(RemoteControlPairingStatusRequest {
- pairing_code: Some("pairing-code".to_string()),
- manual_pairing_code: None,
- })
+ .pairing_status(
+ &http_client_factory(),
+ RemoteControlPairingStatusRequest {
+ pairing_code: Some("pairing-code".to_string()),
+ manual_pairing_code: None,
+ },
+ )
.await
.expect_err("pairing status should fail");
server_task.await.expect("server task should finish");
@@ -395,10 +410,13 @@ async fn remote_control_pairing_status_returns_pending() {
});
let response = remote_control_enrollment(&remote_control_url, "remote-control-token")
- .pairing_status(RemoteControlPairingStatusRequest {
- pairing_code: Some("pairing-code".to_string()),
- manual_pairing_code: None,
- })
+ .pairing_status(
+ &http_client_factory(),
+ RemoteControlPairingStatusRequest {
+ pairing_code: Some("pairing-code".to_string()),
+ manual_pairing_code: None,
+ },
+ )
.await
.expect("pairing status should succeed");
server_task.await.expect("server task should finish");
@@ -427,10 +445,13 @@ async fn remote_control_pairing_status_accepts_manual_pairing_code() {
});
let response = remote_control_enrollment(&remote_control_url, "remote-control-token")
- .pairing_status(RemoteControlPairingStatusRequest {
- pairing_code: None,
- manual_pairing_code: Some("ABCD-EFGH".to_string()),
- })
+ .pairing_status(
+ &http_client_factory(),
+ RemoteControlPairingStatusRequest {
+ pairing_code: None,
+ manual_pairing_code: Some("ABCD-EFGH".to_string()),
+ },
+ )
.await
.expect("pairing status should succeed");
server_task.await.expect("server task should finish");
@@ -454,10 +475,13 @@ async fn remote_control_pairing_status_returns_claimed() {
});
let response = remote_control_enrollment(&remote_control_url, "remote-control-token")
- .pairing_status(RemoteControlPairingStatusRequest {
- pairing_code: Some("pairing-code".to_string()),
- manual_pairing_code: None,
- })
+ .pairing_status(
+ &http_client_factory(),
+ RemoteControlPairingStatusRequest {
+ pairing_code: Some("pairing-code".to_string()),
+ manual_pairing_code: None,
+ },
+ )
.await
.expect("pairing status should succeed");
server_task.await.expect("server task should finish");
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs
index d50d31771d..716270d11e 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs
@@ -51,15 +51,11 @@ use std::collections::VecDeque;
use std::io;
use std::io::ErrorKind;
use std::sync::Arc;
-use tokio::net::TcpStream;
use tokio::sync::Mutex;
use tokio::sync::mpsc;
use tokio::sync::oneshot;
use tokio::sync::watch;
use tokio::time::MissedTickBehavior;
-use tokio_tungstenite::MaybeTlsStream;
-use tokio_tungstenite::WebSocketStream;
-use tokio_tungstenite::connect_async;
use tokio_tungstenite::tungstenite;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_util::sync::CancellationToken;
@@ -296,7 +292,7 @@ struct RemoteControlEnrollmentAuthContext<'a, 'b> {
}
enum ConnectOutcome {
- Connected(Box<WebSocketStream<MaybeTlsStream<TcpStream>>>),
+ Connected(Box<codex_websocket_client::WebSocketConnection>),
Disabled,
Shutdown,
}
@@ -722,6 +718,7 @@ impl RemoteControlWebsocket {
desired_state_tx: &self.desired_state_tx,
persistence: &self.persistence,
};
+ let mut policy_changes = self.auth_manager.network_policy.changes();
let auth_context = RemoteControlAuthContext {
auth_manager: &self.auth_manager,
auth_recovery: &mut self.auth_recovery,
@@ -775,6 +772,17 @@ impl RemoteControlWebsocket {
if !self.desired_state_rx.borrow().is_enabled() {
return ConnectOutcome::Disabled;
}
+ if super::auth::is_policy_denial(&err) {
+ self.status_publisher
+ .publish_status(RemoteControlConnectionStatus::Errored);
+ if let Some(changes) = &mut policy_changes {
+ tokio::select! {
+ _ = shutdown_token.cancelled() => return ConnectOutcome::Shutdown,
+ _ = self.desired_state_rx.wait_for(|state| !state.is_enabled()) => return ConnectOutcome::Disabled,
+ _ = changes.changed() => continue,
+ }
+ }
+ }
let server_retry_delay = remote_control_retry_delay(&err);
let reconnect_delay = if err.kind() == ErrorKind::WouldBlock {
server_retry_delay
@@ -839,7 +847,7 @@ impl RemoteControlWebsocket {
async fn run_connection(
&self,
- websocket_connection: WebSocketStream<MaybeTlsStream<TcpStream>>,
+ websocket_connection: codex_websocket_client::WebSocketConnection,
shutdown_token: CancellationToken,
) -> ConnectionEndReason {
if !self.auth_manager.owner.is_current() {
@@ -917,7 +925,7 @@ impl RemoteControlWebsocket {
server_event_rx: Arc<Mutex<mpsc::Receiver<super::QueuedServerEnvelope>>>,
used_rx: watch::Receiver<usize>,
websocket_writer: SplitSink<
- WebSocketStream<MaybeTlsStream<TcpStream>>,
+ codex_websocket_client::WebSocketConnection,
tungstenite::Message,
>,
ping_interval: std::time::Duration,
@@ -948,7 +956,7 @@ impl RemoteControlWebsocket {
server_event_rx: Arc<Mutex<mpsc::Receiver<super::QueuedServerEnvelope>>>,
mut used_rx: watch::Receiver<usize>,
mut websocket_writer: SplitSink<
- WebSocketStream<MaybeTlsStream<TcpStream>>,
+ codex_websocket_client::WebSocketConnection,
tungstenite::Message,
>,
ping_interval: std::time::Duration,
@@ -1080,7 +1088,7 @@ impl RemoteControlWebsocket {
async fn run_websocket_reader(
client_tracker: Arc<Mutex<ClientTracker>>,
state: Arc<Mutex<WebsocketState>>,
- websocket_reader: SplitStream<WebSocketStream<MaybeTlsStream<TcpStream>>>,
+ websocket_reader: SplitStream<codex_websocket_client::WebSocketConnection>,
pong_timeout: std::time::Duration,
shutdown_token: CancellationToken,
) {
@@ -1106,7 +1114,7 @@ impl RemoteControlWebsocket {
async fn run_websocket_reader_inner(
client_tracker: Arc<Mutex<ClientTracker>>,
state: Arc<Mutex<WebsocketState>>,
- mut websocket_reader: SplitStream<WebSocketStream<MaybeTlsStream<TcpStream>>>,
+ mut websocket_reader: SplitStream<codex_websocket_client::WebSocketConnection>,
pong_timeout: std::time::Duration,
shutdown_token: CancellationToken,
) -> io::Result<()> {
@@ -1350,7 +1358,7 @@ pub(super) async fn connect_remote_control_websocket(
connect_options: RemoteControlConnectOptions<'_>,
status_publisher: &RemoteControlStatusPublisher,
) -> io::Result<(
- WebSocketStream<MaybeTlsStream<TcpStream>>,
+ codex_websocket_client::WebSocketConnection,
tungstenite::http::Response<()>,
)> {
ensure_rustls_crypto_provider();
@@ -1381,24 +1389,43 @@ pub(super) async fn connect_remote_control_websocket(
.await?;
current_enrollment.check_retry_after()?;
- let websocket_connect_result = tokio::time::timeout(
- REMOTE_CONTROL_WEBSOCKET_CONNECT_TIMEOUT,
- connect_async(request),
- )
- .await
- .map_err(|_| {
- io::Error::new(
- ErrorKind::TimedOut,
- format!(
- "timed out connecting to remote control websocket at `{}` after {:?}",
- remote_control_target.websocket_url, REMOTE_CONTROL_WEBSOCKET_CONNECT_TIMEOUT
- ),
- )
- })?;
+ let websocket_connect_result =
+ tokio::time::timeout(REMOTE_CONTROL_WEBSOCKET_CONNECT_TIMEOUT, async {
+ let factory = auth.http_client_factory.clone();
+ let connector = tokio::task::spawn_blocking(move || {
+ codex_websocket_client::WebSocketConnector::new(&factory).or_else(|_| {
+ codex_websocket_client::WebSocketConnector::new_with_tls_mode(
+ &factory,
+ codex_websocket_client::WebSocketTlsMode::TungsteniteDefault,
+ )
+ })
+ })
+ .await
+ .map_err(io::Error::other)?
+ .map_err(io::Error::other)?;
+ Ok::<_, io::Error>(
+ connector
+ .connect(request, tungstenite::protocol::WebSocketConfig::default())
+ .await,
+ )
+ })
+ .await
+ .map_err(|_| {
+ io::Error::new(
+ ErrorKind::TimedOut,
+ format!(
+ "timed out connecting to remote control websocket at `{}` after {:?}",
+ remote_control_target.websocket_url, REMOTE_CONTROL_WEBSOCKET_CONNECT_TIMEOUT
+ ),
+ )
+ })??;
match websocket_connect_result {
Ok((websocket_stream, response)) => Ok((websocket_stream, response.map(|_| ()))),
Err(err) => {
+ if let Some(denied) = codex_websocket_client::network_policy_denial(&err) {
+ return Err(io::Error::new(ErrorKind::PermissionDenied, denied));
+ }
match &err {
tungstenite::Error::Http(response)
if websocket_response_reports_missing_remote_app_server(response) =>
@@ -1500,7 +1527,7 @@ async fn prepare_remote_control_enrollment(
let auth = match load_remote_control_auth(auth_context.auth_manager).await {
Ok(auth) => auth,
Err(err) => {
- if err.kind() == ErrorKind::PermissionDenied {
+ if super::auth::is_auth_error(&err) {
*enrollment = None;
status_publisher.publish_environment_id(/*environment_id*/ None);
}
@@ -1603,7 +1630,7 @@ async fn prepare_remote_control_enrollment(
)
.await?;
}
- Err(err) if err.kind() == ErrorKind::PermissionDenied => {
+ Err(err) if super::auth::is_auth_error(&err) => {
if recover_remote_control_auth(
auth_context.auth_recovery,
auth_context.auth_change_rx,
@@ -1728,7 +1755,7 @@ async fn enroll_and_persist_remote_control_server(
{
Ok(new_enrollment) => new_enrollment,
Err(err)
- if err.kind() == ErrorKind::PermissionDenied
+ if super::auth::is_auth_error(&err)
&& recover_remote_control_auth(
auth_context.recovery.auth_recovery,
auth_context.recovery.auth_change_rx,
@@ -1825,6 +1852,7 @@ mod tests {
use tokio::sync::mpsc;
use tokio::time::Duration;
use tokio::time::timeout;
+ use tokio_tungstenite::WebSocketStream;
use tokio_tungstenite::accept_async;
// Windows Bazel CI can take longer than a few seconds for the websocket
@@ -2051,6 +2079,32 @@ mod tests {
}
}
+ #[test]
+ fn remote_control_websocket_reaches_server_with_invalid_custom_ca() {
+ let temp = TempDir::new().unwrap();
+ let invalid_ca = temp.path().join("invalid.pem");
+ std::fs::write(&invalid_ca, "not a certificate").unwrap();
+ for variable in ["CODEX_CA_CERTIFICATE", "SSL_CERT_FILE"] {
+ let output = std::process::Command::new(std::env::current_exe().unwrap())
+ .args([
+ "--exact",
+ "transport::remote_control::websocket::tests::connect_remote_control_websocket_includes_http_error_details",
+ ])
+ .env_remove("CODEX_CA_CERTIFICATE")
+ .env_remove("SSL_CERT_FILE")
+ .env(variable, &invalid_ca)
+ .output()
+ .unwrap();
+ assert!(
+ output.status.success(),
+ "{variable}: {}\n{}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
+ }
+ }
+
#[tokio::test]
async fn connect_remote_control_websocket_includes_http_error_details() {
let listener = TcpListener::bind("127.0.0.1:0")
@@ -3471,18 +3525,26 @@ mod tests {
}
async fn connected_websocket_pair() -> (
- WebSocketStream<MaybeTlsStream<TcpStream>>,
+ codex_websocket_client::WebSocketConnection,
WebSocketStream<TcpStream>,
) {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("listener should bind");
- let connect_task = tokio::spawn(connect_async(format!(
- "ws://{}",
- listener
- .local_addr()
- .expect("listener should have a local addr")
- )));
+ let url = format!("ws://{}", listener.local_addr().expect("local address"));
+ let connect_task = tokio::spawn(async move {
+ codex_websocket_client::WebSocketConnector::new(
+ &codex_http_client::HttpClientFactory::new(
+ codex_http_client::OutboundProxyPolicy::ReqwestDefault,
+ ),
+ )
+ .unwrap()
+ .connect(
+ url.into_client_request().unwrap(),
+ tungstenite::protocol::WebSocketConfig::default(),
+ )
+ .await
+ });
let (server_stream, _) = listener
.accept()
.await
diff --git a/codex-rs/app-server/tests/suite/v2/remote_control.rs b/codex-rs/app-server/tests/suite/v2/remote_control.rs
index 85ad2c8fae..f1fb30ccc5 100644
--- a/codex-rs/app-server/tests/suite/v2/remote_control.rs
+++ b/codex-rs/app-server/tests/suite/v2/remote_control.rs
@@ -1251,11 +1251,14 @@ impl PairingRemoteControlBackend {
)
.await?;
- let request_after_enroll = read_http_request(&listener).await?;
- let pair_http_request = if request_after_enroll.request_line.starts_with("GET ") {
- read_http_request(&listener).await?
- } else {
- request_after_enroll
+ let mut websocket_connections = Vec::new();
+ let pair_http_request = loop {
+ let request = read_http_request(&listener).await?;
+ if request.request_line.starts_with("GET ") {
+ websocket_connections.push(request);
+ } else {
+ break request;
+ }
};
respond_with_json(
pair_http_request.reader.into_inner(),
@@ -1272,7 +1275,14 @@ impl PairingRemoteControlBackend {
serde_json::json!({ "pairing_code": "pairing-code" }),
serde_json::json!({ "manual_pairing_code": "ABCD-EFGH" }),
] {
- let status_http_request = read_http_request(&listener).await?;
+ let status_http_request = loop {
+ let request = read_http_request(&listener).await?;
+ if request.request_line.starts_with("GET ") {
+ websocket_connections.push(request);
+ } else {
+ break request;
+ }
+ };
assert_eq!(
status_http_request.request_line,
"POST /backend-api/wham/remote/control/server/pair/status HTTP/1.1"
diff --git a/codex-rs/exec-server/Cargo.toml b/codex-rs/exec-server/Cargo.toml
index 50c728b04a..c1f67cc39a 100644
--- a/codex-rs/exec-server/Cargo.toml
+++ b/codex-rs/exec-server/Cargo.toml
@@ -7,8 +7,8 @@ license.workspace = true
[lib]
doctest = false
-# Compiled through #[path] in src/client.rs; cargo-shear 1.11.2 does not normalize
-# the ../tests path and incorrectly reports this linked unit-test module as unlinked.
+# cargo-shear 1.11.2 misresolves this #[path] module in src/client.rs.
+# The module is compiled and exercised by the exec-server unit test target.
[package.metadata.cargo-shear]
ignored-paths = ["tests/unit/client_provisioning_tests.rs"]
diff --git a/codex-rs/exec-server/src/capability_discovery_cache.rs b/codex-rs/exec-server/src/capability_discovery_cache.rs
index 33bbcb8c1f..94973251f1 100644
--- a/codex-rs/exec-server/src/capability_discovery_cache.rs
+++ b/codex-rs/exec-server/src/capability_discovery_cache.rs
@@ -187,7 +187,7 @@ impl ExecutorCapabilityDiscoveryCache {
let response = match environment.discover_capability_roots(params).await {
Ok(response) => response,
Err(error) => {
- let retryable = crate::client::is_retryable_recovery_error(&error);
+ let retryable = crate::client::can_retry_connection_attempt(&error);
let error = error.to_string();
return selected_roots
.into_iter()
diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs
index eced178795..edbdcb379f 100644
--- a/codex-rs/exec-server/src/client.rs
+++ b/codex-rs/exec-server/src/client.rs
@@ -148,9 +148,14 @@ mod provisioning_tests;
mod recovery;
#[path = "client_refresh.rs"]
mod refresh;
+pub(crate) use connection_failure::can_retry_connection_attempt;
#[cfg(test)]
pub(crate) use recovery::is_environment_offline_error;
pub(crate) use recovery::is_retryable_recovery_error;
+
+#[path = "client/connection_failure.rs"]
+mod connection_failure;
+use connection_failure::ConnectionFailure;
pub(crate) use recovery::is_retryable_registry_error;
pub(crate) use recovery::registry_recovery_retry_delay;
use refresh::ConnectionAttempt;
@@ -300,7 +305,7 @@ struct ConnectionState {
enum ConnectionStatus {
Connected(Arc<RpcClient>),
Recovering,
- Failed(String),
+ Failed(ConnectionFailure),
}
impl ConnectionState {
@@ -321,7 +326,8 @@ impl ConnectionState {
let _ = self
.environment_connection_state_tx
.send_if_modified(|current| {
- if *current == state {
+ // A terminal failure must wake callers waiting on recovery.
+ if *current == state && !matches!(self.status, ConnectionStatus::Failed(_)) {
false
} else {
*current = state;
@@ -539,7 +545,7 @@ impl LazyRemoteExecServerClient {
|| self.startup.result.get().is_some_and(|result| {
result
.as_ref()
- .is_err_and(|error| recovery::is_retryable_recovery_error(error))
+ .is_err_and(|error| can_retry_connection_attempt(error))
})) {
Box::pin(self.reconnect()).await
} else {
@@ -666,6 +672,8 @@ pub enum ExecServerError {
WebSocketConfiguration(String),
#[error("timed out waiting for exec-server initialize handshake after {timeout:?}")]
InitializeTimedOut { timeout: Duration },
+ #[error(transparent)]
+ ApplicationNetworkPolicy(#[from] codex_http_client::NetworkPolicyDenied),
#[error("exec-server transport closed")]
Closed,
#[error("{0}")]
@@ -744,9 +752,7 @@ impl ExecServerClient {
ConnectionStatus::Connected(_) | ConnectionStatus::Recovering => Err(
ExecServerError::Disconnected("exec-server environment is recovering".to_string()),
),
- ConnectionStatus::Failed(message) => {
- Err(ExecServerError::Disconnected(message.clone()))
- }
+ ConnectionStatus::Failed(message) => Err(message.clone().into()),
}
}
@@ -1178,9 +1184,7 @@ impl ExecServerClient {
Some(Ok(()))
}
ConnectionStatus::Connected(_) | ConnectionStatus::Recovering => None,
- ConnectionStatus::Failed(message) => {
- Some(Err(ExecServerError::Disconnected(message.clone())))
- }
+ ConnectionStatus::Failed(message) => Some(Err(message.clone().into())),
}
}
@@ -1680,8 +1684,8 @@ impl Inner {
// Do not register a process session that can never receive environment
// notifications. Without this check, remote MCP startup could create a
// dead session and wait for process output that will never arrive.
- if let Some(message) = self.failure_message() {
- return Err(ExecServerError::Disconnected(message));
+ if let Some(message) = self.connection_failure() {
+ return Err(message.into());
}
let sessions = self.sessions.load();
if sessions.contains_key(process_id) {
diff --git a/codex-rs/exec-server/src/client/accepted.rs b/codex-rs/exec-server/src/client/accepted.rs
index c7853221a0..ae6d5f84e7 100644
--- a/codex-rs/exec-server/src/client/accepted.rs
+++ b/codex-rs/exec-server/src/client/accepted.rs
@@ -196,7 +196,7 @@ impl Inner {
.unwrap_or_else(std::sync::PoisonError::into_inner);
let current_rpc_client = match &connection.status {
ConnectionStatus::Failed(message) => {
- return Err(ExecServerError::Disconnected(message.clone()));
+ return Err(message.clone().into());
}
ConnectionStatus::Connected(rpc_client) => Some(Arc::clone(rpc_client)),
ConnectionStatus::Recovering => None,
@@ -221,7 +221,7 @@ impl Inner {
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let ConnectionStatus::Failed(message) = &connection_state.status {
- return Err(ExecServerError::Disconnected(message.clone()));
+ return Err(message.clone().into());
}
replacement_submission.submit(connection)
}
diff --git a/codex-rs/exec-server/src/client/connection_failure.rs b/codex-rs/exec-server/src/client/connection_failure.rs
new file mode 100644
index 0000000000..fb26363337
--- /dev/null
+++ b/codex-rs/exec-server/src/client/connection_failure.rs
@@ -0,0 +1,66 @@
+//! Preserve policy denials through terminal connection state and allow later policy recovery.
+
+use codex_http_client::NetworkPolicyDenied;
+use codex_http_client::RouteAwareRequestError;
+
+use super::ExecServerError;
+
+#[derive(Clone, thiserror::Error, Debug)]
+pub(super) enum ConnectionFailure {
+ #[error("{0}")]
+ Disconnected(String),
+ #[error(transparent)]
+ Policy(NetworkPolicyDenied),
+}
+
+impl From<ConnectionFailure> for ExecServerError {
+ fn from(failure: ConnectionFailure) -> Self {
+ match failure {
+ ConnectionFailure::Disconnected(message) => Self::Disconnected(message),
+ ConnectionFailure::Policy(denial) => Self::ApplicationNetworkPolicy(denial),
+ }
+ }
+}
+
+impl ExecServerError {
+ pub(crate) fn application_network_policy_denial(&self) -> Option<NetworkPolicyDenied> {
+ match self {
+ Self::ConnectionAttempt(error) => error.application_network_policy_denial(),
+ Self::ApplicationNetworkPolicy(denial)
+ | Self::EnvironmentRegistryRequest(RouteAwareRequestError::Policy(denial)) => {
+ Some(*denial)
+ }
+ Self::WebSocketConnect { source, .. } => {
+ codex_websocket_client::network_policy_denial(source)
+ }
+ Self::Spawn(_)
+ | Self::WebSocketConnectTimeout { .. }
+ | Self::WebSocketConfiguration(_)
+ | Self::InitializeTimedOut { .. }
+ | Self::Closed
+ | Self::Disconnected(_)
+ | Self::ProvisioningFailed(_)
+ | Self::Json(_)
+ | Self::HttpRequest(_)
+ | Self::Protocol(_)
+ | Self::ProvisioningModeConflict { .. }
+ | Self::Server { .. }
+ | Self::EnvironmentRegistryHttp { .. }
+ | Self::EnvironmentRegistryConfig(_)
+ | Self::EnvironmentRegistryAuth(_)
+ | Self::EnvironmentRegistryRequest(
+ RouteAwareRequestError::Request(_)
+ | RouteAwareRequestError::Route(_)
+ | RouteAwareRequestError::Build(_)
+ | RouteAwareRequestError::UnsupportedRedirectScheme(_)
+ | RouteAwareRequestError::TooManyRedirects
+ | RouteAwareRequestError::Timeout,
+ ) => None,
+ }
+ }
+}
+
+pub(crate) fn can_retry_connection_attempt(error: &ExecServerError) -> bool {
+ error.application_network_policy_denial().is_some()
+ || super::recovery::is_retryable_recovery_error(error)
+}
diff --git a/codex-rs/exec-server/src/client_recovery.rs b/codex-rs/exec-server/src/client_recovery.rs
index f3363fd3ce..fa353e4a38 100644
--- a/codex-rs/exec-server/src/client_recovery.rs
+++ b/codex-rs/exec-server/src/client_recovery.rs
@@ -253,8 +253,8 @@ impl Inner {
pub(super) async fn rpc_client(self: &Arc<Self>) -> Result<Arc<RpcClient>, ExecServerError> {
let mut connection_changed = self.connection_changed.subscribe();
loop {
- if let Some(message) = self.failure_message() {
- return Err(ExecServerError::Disconnected(message));
+ if let Some(message) = self.connection_failure() {
+ return Err(message.into());
}
let rpc_client = {
@@ -310,10 +310,10 @@ impl Inner {
}
pub(super) fn is_failed(&self) -> bool {
- self.failure_message().is_some()
+ self.connection_failure().is_some()
}
- pub(super) fn failure_message(&self) -> Option<String> {
+ pub(super) fn connection_failure(&self) -> Option<super::ConnectionFailure> {
let connection = self
.connection
.lock()
@@ -372,11 +372,13 @@ impl Inner {
let message = format!(
"{disconnect_message}; failed to resume exec-server session: recovery timed out after {SESSION_RECOVERY_TIMEOUT:?}"
);
- self.fail(message).await;
+ self.fail(super::ConnectionFailure::Disconnected(message))
+ .await;
return;
}
if self.reconnect_strategy.is_none() {
- self.fail(disconnect_message).await;
+ self.fail(super::ConnectionFailure::Disconnected(disconnect_message))
+ .await;
return;
}
@@ -384,7 +386,8 @@ impl Inner {
let message = format!(
"{disconnect_message}; failed to resume exec-server session: missing session id"
);
- self.fail(message).await;
+ self.fail(super::ConnectionFailure::Disconnected(message))
+ .await;
return;
};
let uses_registry_backoff = matches!(
@@ -399,10 +402,15 @@ impl Inner {
return;
}
}
- Ok(Err(error)) if !is_retryable_recovery_error(&error) => {
- break error.to_string();
+ Ok(Err(error)) => {
+ if let Some(denial) = error.application_network_policy_denial() {
+ self.fail(super::ConnectionFailure::Policy(denial)).await;
+ return;
+ }
+ if !is_retryable_recovery_error(&error) {
+ break error.to_string();
+ }
}
- Ok(Err(_)) => {}
Err(_) => {
break format!("recovery timed out after {SESSION_RECOVERY_TIMEOUT:?}");
}
@@ -425,7 +433,8 @@ impl Inner {
let message =
format!("{disconnect_message}; failed to resume exec-server session: {last_error}");
- self.fail(message).await;
+ self.fail(super::ConnectionFailure::Disconnected(message))
+ .await;
}
async fn wait_for_process_starts(&self) {
@@ -570,7 +579,7 @@ impl Inner {
Ok(())
}
- async fn fail(self: &Arc<Self>, message: String) {
+ async fn fail(self: &Arc<Self>, message: super::ConnectionFailure) {
let (message, newly_failed) = {
let mut connection = self
.connection
@@ -586,7 +595,7 @@ impl Inner {
};
if newly_failed {
self.notify_connection_changed();
- fail_all_in_flight_work(self, message.clone()).await;
+ fail_all_in_flight_work(self, message.to_string()).await;
}
}
}
@@ -849,6 +858,9 @@ impl ExecServerClient {
}
pub(crate) fn is_retryable_recovery_error(error: &ExecServerError) -> bool {
+ if error.application_network_policy_denial().is_some() {
+ return false;
+ }
if let ExecServerError::ConnectionAttempt(error) = error {
return is_retryable_recovery_error(error.as_ref());
}
diff --git a/codex-rs/exec-server/src/client_refresh.rs b/codex-rs/exec-server/src/client_refresh.rs
index e9e31dfcfc..3579cb4f72 100644
--- a/codex-rs/exec-server/src/client_refresh.rs
+++ b/codex-rs/exec-server/src/client_refresh.rs
@@ -269,7 +269,9 @@ impl Inner {
ConnectionStatus::Recovering | ConnectionStatus::Failed(_) => None,
};
self.retired.cancel();
- connection.set_status(ConnectionStatus::Failed(message.clone()));
+ connection.set_status(ConnectionStatus::Failed(
+ super::ConnectionFailure::Disconnected(message.clone()),
+ ));
rpc_client
};
self.connection_changed.send_replace(());
diff --git a/codex-rs/exec-server/src/client_refresh_tests.rs b/codex-rs/exec-server/src/client_refresh_tests.rs
index 4beee04dee..86a3bdb517 100644
--- a/codex-rs/exec-server/src/client_refresh_tests.rs
+++ b/codex-rs/exec-server/src/client_refresh_tests.rs
@@ -881,3 +881,58 @@ async fn retirement_rejects_pending_process_start_before_stream_cleanup() -> Res
}
Ok(())
}
+
+#[tokio::test]
+async fn application_policy_registry_denial_does_not_poison_later_noise_startup() -> Result<()> {
+ let _clock = freeze_clock();
+ let executor = Executor::start(Validator::default()).await?;
+ let registry = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .and(wiremock::matchers::path(
+ "/cloud/environment/environment/connect",
+ ))
+ .respond_with(
+ wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
+ "environment_id": "environment",
+ "url": executor.target.url,
+ "security_profile": "noise_hybrid_ik_v1",
+ "executor_registration_id": executor.target.registration,
+ "executor_public_key": executor.target.identity.public_key(),
+ "harness_key_authorization": "authorization",
+ })),
+ )
+ .expect(1)
+ .mount(&registry)
+ .await;
+ let controller = codex_http_client::NetworkPolicyController::default();
+ let policy = controller.policy();
+ let factory = HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault)
+ .with_network_policy(policy.clone());
+ let provider = crate::remote::NoiseRendezvousEnvironmentConfig::new(
+ registry.uri(),
+ "environment".into(),
+ "registry-token".into(),
+ /*chatgpt_account_id*/ None,
+ )?
+ .into_connect_provider(factory.clone())?;
+ let client = LazyRemoteExecServerClient::new(
+ ExecServerTransportParams::NoiseRendezvous {
+ provider,
+ identity: NoiseChannelIdentity::generate()?,
+ },
+ factory,
+ );
+ let error = client.get().await.err().unwrap();
+ assert_eq!(
+ error.application_network_policy_denial(),
+ Some(codex_http_client::NetworkPolicyDenied::Unavailable)
+ );
+ assert!(!crate::client::is_retryable_recovery_error(&error));
+ assert!(registry.received_requests().await.unwrap().is_empty());
+ controller.publish(
+ policy.revision(),
+ codex_http_client::DestinationPolicy::Unrestricted,
+ );
+ client.get().await?;
+ Ok(())
+}
diff --git a/codex-rs/exec-server/src/environment.rs b/codex-rs/exec-server/src/environment.rs
index c2cd7384ac..12981657f1 100644
--- a/codex-rs/exec-server/src/environment.rs
+++ b/codex-rs/exec-server/src/environment.rs
@@ -1021,19 +1021,28 @@ impl Environment {
tracing::warn!(%error, "replaying capability discovery after executor recovery");
let recovered =
tokio::time::timeout(std::time::Duration::from_secs(8), async {
- while self.readiness_result().is_none_or(|result| result.is_err()) {
+ loop {
+ match self.readiness_result() {
+ Some(Ok(())) => return Some(Ok(())),
+ Some(Err(error))
+ if !crate::client::is_retryable_recovery_error(
+ &error,
+ ) =>
+ {
+ return Some(Err(error));
+ }
+ Some(Err(_)) | None => {}
+ }
if connection_state.changed().await.is_err() {
- return false;
+ return None;
}
}
- true
})
- .await
- .unwrap_or(false);
- if recovered {
- discover().await
- } else {
- Err(error)
+ .await;
+ match recovered {
+ Ok(Some(Ok(()))) => discover().await,
+ Ok(Some(Err(error))) => Err(error),
+ Ok(None) | Err(_) => Err(error),
}
}
response => response,
diff --git a/codex-rs/exec-server/src/remote/direct.rs b/codex-rs/exec-server/src/remote/direct.rs
index 02365f58bb..a1446a9ec4 100644
--- a/codex-rs/exec-server/src/remote/direct.rs
+++ b/codex-rs/exec-server/src/remote/direct.rs
@@ -2,6 +2,8 @@ use std::time::Instant;
use codex_api::AuthError;
use codex_api::AuthProvider;
+use codex_http_client::NetworkPolicy;
+use codex_http_client::NetworkPolicyDenied;
use codex_http_client::Request;
use codex_websocket_client::WebSocketConnection;
use codex_websocket_client::WebSocketConnector;
@@ -43,6 +45,26 @@ struct DirectRegistrationResponse {
}
impl EnvironmentRegistryClient {
+ async fn register_direct_environment_when_available(
+ &self,
+ environment_id: &str,
+ policy: &NetworkPolicy,
+ ) -> Result<DirectRegistrationResponse, ExecServerError> {
+ let destination = super::endpoint_url(
+ &self.base_url,
+ &format!("/cloud/environment/{environment_id}/direct/register"),
+ );
+ let mut retry_attempt = 0;
+ loop {
+ match self.register_direct_environment(environment_id).await {
+ Err(error) if is_policy_recoverable(&error, policy, &destination) => {}
+ result => return result,
+ }
+ sleep(registry_recovery_retry_delay(environment_id, retry_attempt)).await;
+ retry_attempt = retry_attempt.saturating_add(1);
+ }
+ }
+
#[tracing::instrument(
name = "codex.exec_server.remote.register",
skip_all,
@@ -122,7 +144,10 @@ pub(super) async fn run_direct_environment(
require_tls_or_loopback(&config.base_url, "https")?;
let mut retry_attempt = 0;
let mut registration = client
- .register_direct_environment(&config.environment_id)
+ .register_direct_environment_when_available(
+ &config.environment_id,
+ config.http_client_factory.network_policy(),
+ )
.await?;
loop {
@@ -155,7 +180,12 @@ pub(super) async fn run_direct_environment(
config.telemetry.remote_reconnect("disconnected");
}
Err(error)
- if is_retryable_recovery_error(&error)
+ if (is_retryable_recovery_error(&error)
+ || is_policy_recoverable(
+ &error,
+ config.http_client_factory.network_policy(),
+ &registration.url,
+ ))
&& !matches!(
&error,
ExecServerError::WebSocketConnect {
@@ -180,7 +210,10 @@ pub(super) async fn run_direct_environment(
} if response.status() == StatusCode::CONFLICT
) {
registration = client
- .register_direct_environment(&config.environment_id)
+ .register_direct_environment_when_available(
+ &config.environment_id,
+ config.http_client_factory.network_policy(),
+ )
.await?;
}
warn!("direct exec-server connection failed; retrying");
@@ -198,6 +231,24 @@ pub(super) async fn run_direct_environment(
}
}
+fn is_policy_recoverable(
+ error: &ExecServerError,
+ policy: &NetworkPolicy,
+ destination: &str,
+) -> bool {
+ match error.application_network_policy_denial() {
+ Some(NetworkPolicyDenied::Unavailable) => true,
+ Some(NetworkPolicyDenied::Revoked) => url::Url::parse(destination).is_ok_and(|url| {
+ matches!(
+ policy.acquire(&url),
+ Ok(_) | Err(NetworkPolicyDenied::Unavailable)
+ )
+ }),
+ Some(NetworkPolicyDenied::Destination | NetworkPolicyDenied::UnsupportedTransport)
+ | None => false,
+ }
+}
+
async fn connect_direct(
url: &str,
auth_provider: &dyn AuthProvider,
diff --git a/codex-rs/exec-server/src/remote/direct_tests.rs b/codex-rs/exec-server/src/remote/direct_tests.rs
index d0cbff0d08..b0dcb98d5d 100644
--- a/codex-rs/exec-server/src/remote/direct_tests.rs
+++ b/codex-rs/exec-server/src/remote/direct_tests.rs
@@ -6,7 +6,9 @@ use std::time::Instant;
use anyhow::Result;
use codex_api::AuthProvider;
+use codex_http_client::DestinationPolicy;
use codex_http_client::HttpClientFactory;
+use codex_http_client::NetworkPolicyController;
use codex_http_client::OutboundProxyPolicy;
use http::HeaderMap;
use http::HeaderValue;
@@ -282,8 +284,14 @@ async fn direct_registration_uses_proxy_policy_without_logging_secrets() -> Resu
}
#[tokio::test]
-async fn direct_registration_failure_stops_initial_and_conflict_attempts() -> Result<()> {
- for successful_registrations in [0, 1] {
+async fn direct_registration_handles_policy_outages_and_permanent_failures() -> Result<()> {
+ for (successful_registrations, policy_unavailable) in [(0, false), (1, false), (0, true)] {
+ let controller = NetworkPolicyController::default();
+ let policy = controller.policy();
+ controller.publish(policy.revision(), DestinationPolicy::Unrestricted);
+ if policy_unavailable && successful_registrations == 0 {
+ controller.unavailable(policy.revision());
+ }
let listener = TcpListener::bind("127.0.0.1:0").await?;
let websocket_url = format!("ws://{}/connect", listener.local_addr()?);
let registry = MockServer::start().await;
@@ -294,7 +302,7 @@ async fn direct_registration_failure_stops_initial_and_conflict_attempts() -> Re
))
.respond_with(move |_: &wiremock::Request| {
let attempt = registration_count.fetch_add(1, Ordering::Relaxed);
- if attempt < successful_registrations {
+ if policy_unavailable || attempt < successful_registrations {
ResponseTemplate::new(200).set_body_json(serde_json::json!({
"environment_id": "environment-requested",
"transport": DIRECT_TRANSPORT,
@@ -313,7 +321,8 @@ async fn direct_registration_failure_stops_initial_and_conflict_attempts() -> Re
"environment-requested".to_string(),
RemoteEnvironmentTransport::Direct,
Arc::new(StaticAuthProvider),
- HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
+ HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault)
+ .with_network_policy(policy.clone().for_current_account()),
)?;
let runtime_paths = ExecServerRuntimePaths::new(
std::env::current_exe()?,
@@ -329,6 +338,18 @@ async fn direct_registration_failure_stops_initial_and_conflict_attempts() -> Re
.await?;
socket.shutdown().await?;
}
+ if policy_unavailable {
+ timeout(Duration::from_secs(1), listener.accept())
+ .await
+ .expect_err("unavailable policy opened a connection");
+ controller.publish(policy.revision(), DestinationPolicy::Unrestricted);
+ let (socket, _) = timeout(Duration::from_secs(5), listener.accept()).await??;
+ let _websocket = tokio_tungstenite::accept_async(socket).await?;
+ registry.verify().await;
+ task.abort();
+ let _ = task.await;
+ continue;
+ }
let error = timeout(Duration::from_secs(5), task)
.await??
.expect_err("registration failure should stop the runner");
diff --git a/codex-rs/exec-server/tests/accepted_websocket.rs b/codex-rs/exec-server/tests/accepted_websocket.rs
index a3caf7b3de..5e0a51fda6 100644
--- a/codex-rs/exec-server/tests/accepted_websocket.rs
+++ b/codex-rs/exec-server/tests/accepted_websocket.rs
@@ -40,7 +40,10 @@ use codex_exec_server_protocol::JSONRPCMessage;
use codex_exec_server_protocol::JSONRPCNotification;
use codex_exec_server_protocol::JSONRPCRequest;
use codex_exec_server_protocol::JSONRPCResponse;
+use codex_http_client::DestinationPolicy;
use codex_http_client::HttpClientFactory;
+use codex_http_client::NetworkPolicy;
+use codex_http_client::NetworkPolicyController;
use codex_http_client::OutboundProxyPolicy;
use codex_utils_path_uri::PathUri;
use common::exec_server::DisconnectableWebSocketProxy;
@@ -132,8 +135,12 @@ async fn accepted_websocket_environment_info_uses_initialization_metadata() -> R
Ok(())
}
+#[test_case::test_case(false; "transport_disconnect")]
+#[test_case::test_case(true; "policy_unavailable")]
#[tokio::test]
-async fn accepted_websocket_interoperates_and_recovers_with_real_direct_executor() -> Result<()> {
+async fn accepted_websocket_interoperates_and_recovers_with_real_direct_executor(
+ policy_unavailable: bool,
+) -> Result<()> {
let (websocket_url, mut accepted_sockets, server_task) = start_acceptor().await?;
let proxy = DisconnectableWebSocketProxy::new(&websocket_url).await?;
let registry = MockServer::start().await;
@@ -150,7 +157,15 @@ async fn accepted_websocket_interoperates_and_recovers_with_real_direct_executor
.mount(&registry)
.await;
- let http_client_factory = HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault);
+ let controller = NetworkPolicyController::default();
+ let policy = controller.policy();
+ controller.publish(policy.revision(), DestinationPolicy::Unrestricted);
+ let http_client_factory = HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault)
+ .with_network_policy(if policy_unavailable {
+ policy.clone().for_current_account()
+ } else {
+ NetworkPolicy::unmanaged()
+ });
let config = RemoteEnvironmentConfig::new_with_transport(
registry.uri(),
"environment-1".to_string(),
@@ -289,6 +304,19 @@ async fn accepted_websocket_interoperates_and_recovers_with_real_direct_executor
EnvironmentConnectionState::Connected
);
proxy.pause_and_disconnect().await?;
+ if policy_unavailable {
+ controller.unavailable(policy.revision());
+ assert!(
+ timeout(Duration::from_secs(/*secs*/ 1), accepted_sockets.recv())
+ .await
+ .is_err(),
+ "unavailable policy reconnected"
+ );
+ assert!(
+ !executor_task.is_finished(),
+ "policy outage stopped the runner"
+ );
+ }
timeout(
TEST_TIMEOUT,
connection_state.wait_for(|state| *state == EnvironmentConnectionState::Disconnected),
@@ -319,6 +347,7 @@ async fn accepted_websocket_interoperates_and_recovers_with_real_direct_executor
.is_err(),
"process reads should wait while recovery is in progress"
);
+ controller.publish(policy.revision(), DestinationPolicy::Unrestricted);
proxy.resume()?;
let replacement = timeout(TEST_TIMEOUT, accepted_sockets.recv())
.await?
diff --git a/codex-rs/exec-server/tests/environment.rs b/codex-rs/exec-server/tests/environment.rs
index 55b6ba3c53..fbbb3d37f1 100644
--- a/codex-rs/exec-server/tests/environment.rs
+++ b/codex-rs/exec-server/tests/environment.rs
@@ -453,3 +453,52 @@ async fn capability_discovery_retries_after_executor_reconnects() -> anyhow::Res
assert!(!cache.take_recovered_discovery());
Ok(())
}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn capability_discovery_retries_after_application_network_policy_recovers()
+-> anyhow::Result<()> {
+ let server = exec_server().await?;
+ let controller = codex_http_client::NetworkPolicyController::default();
+ let policy = controller.policy();
+ let manager = Arc::new(EnvironmentManager::without_environments(
+ HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault)
+ .with_network_policy(policy.clone()),
+ ));
+ manager.upsert_environment(
+ "recovering".to_string(),
+ server.websocket_url().to_string(),
+ /*connect_timeout*/ None,
+ )?;
+ let cache = ExecutorCapabilityDiscoveryCache::new(manager);
+ let skill_root = tempfile::tempdir()?;
+ let selected_roots = vec![SelectedCapabilityRoot {
+ id: "recovering-skill".to_string(),
+ location: CapabilityRootLocation::Environment {
+ environment_id: "recovering".to_string(),
+ path: PathUri::from_host_native_path(skill_root.path())?,
+ },
+ }];
+
+ let failed = cache.snapshot(&selected_roots, &HashMap::new()).await;
+ let error = failed.roots()[0].result.as_ref().unwrap_err();
+ assert!(error.contains("application network policy"), "{error}");
+ assert!(!cache.take_recovered_discovery());
+
+ controller.publish(
+ policy.revision(),
+ codex_http_client::DestinationPolicy::Unrestricted,
+ );
+ let recovered = timeout(
+ Duration::from_secs(/*secs*/ 10),
+ cache.snapshot(&selected_roots, &HashMap::new()),
+ )
+ .await?;
+ let discovery = recovered.roots()[0]
+ .result
+ .as_ref()
+ .map_err(|error| anyhow::anyhow!("{error}"))?;
+ assert_eq!(discovery.id, "recovering-skill");
+ assert!(cache.take_recovered_discovery());
+ assert!(!cache.take_recovered_discovery());
+ Ok(())
+}
diff --git a/codex-rs/http-client/src/route_aware_client_pool.rs b/codex-rs/http-client/src/route_aware_client_pool.rs
index 7e9c795b0e..905aaf8f0d 100644
--- a/codex-rs/http-client/src/route_aware_client_pool.rs
+++ b/codex-rs/http-client/src/route_aware_client_pool.rs
@@ -57,6 +57,7 @@ pub struct RouteAwareClientPool {
default_headers: HeaderMap,
custom_ca_fallback: CustomCaFallback,
clients: Arc<Mutex<HashMap<OutboundProxyRoute, TransportClient>>>,
+ client_build: Arc<tokio::sync::Mutex<()>>,
rustls_clients: Option<RustlsClientCache>,
}
@@ -77,6 +78,8 @@ pub enum RouteAwareClientPoolError {
Resolve(#[source] io::Error),
#[error(transparent)]
Build(#[from] BuildRouteAwareHttpClientError),
+ #[error("HTTP transport construction task failed: {0}")]
+ BuildTask(#[source] tokio::task::JoinError),
}
/// Error returned while building, routing, or sending a route-aware request.
@@ -138,7 +141,8 @@ impl RouteAwareRequestError {
Self::Route(RouteAwareClientPoolError::Resolve(_)) => {
Some(RouteFailureClass::ProxyResolutionUnavailable)
}
- Self::Request(_)
+ Self::Route(RouteAwareClientPoolError::BuildTask(_))
+ | Self::Request(_)
| Self::Policy(_)
| Self::Build(_)
| Self::UnsupportedRedirectScheme(_)
@@ -286,6 +290,7 @@ impl RouteAwareClientPool {
default_headers,
custom_ca_fallback: CustomCaFallback::Disabled,
clients: Arc::new(Mutex::new(HashMap::new())),
+ client_build: Arc::default(),
rustls_clients: None,
}
}
@@ -469,57 +474,74 @@ impl RouteAwareClientPool {
{
return Ok((route, client, SelectedTlsBackend::RustlsFallback));
}
- let clients = match self.clients.lock() {
- Ok(clients) => clients,
- Err(error) => panic!("route-aware client cache lock should not be poisoned: {error}"),
- };
- if let Some(client) = clients.get(&route) {
- return Ok((route, client.clone(), SelectedTlsBackend::TransportDefault));
+ {
+ let clients = match self.clients.lock() {
+ Ok(clients) => clients,
+ Err(error) => {
+ panic!("route-aware client cache lock should not be poisoned: {error}")
+ }
+ };
+ if let Some(client) = clients.get(&route) {
+ return Ok((route, client.clone(), SelectedTlsBackend::TransportDefault));
+ }
}
- drop(clients);
+ let build_permit = Arc::clone(&self.client_build).lock_owned().await;
+ {
+ let clients = self.clients.lock().unwrap_or_else(|error| {
+ panic!("route-aware client cache lock should not be poisoned: {error}")
+ });
+ if let Some(client) = clients.get(&route) {
+ return Ok((route, client.clone(), SelectedTlsBackend::TransportDefault));
+ }
+ }
let client_builder = if self.follows_redirects_manually() {
self.client_builder.clone().without_redirects()
} else {
self.client_builder.clone()
};
- let client = match (
- self.http_client_factory.outbound_proxy_policy(),
- self.custom_ca_fallback,
- ) {
- (_, CustomCaFallback::LegacyDirect) => {
- client_builder.build_with_custom_ca_fallback(ProxyRouting::Direct)
- }
- (OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::LegacyTransportDefault) => {
- client_builder.build_with_custom_ca_fallback(ProxyRouting::TransportDefault)
- }
- (OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::Disabled)
- | (OutboundProxyPolicy::RespectSystemProxy, CustomCaFallback::Disabled)
- | (OutboundProxyPolicy::RespectSystemProxy, CustomCaFallback::LegacyTransportDefault) => {
- client_builder.build_for_resolved_route(
- &self.http_client_factory,
- self.route_class,
- &route,
- )?
+ let pool = self.clone();
+ let build_route = route.clone();
+ let client = tokio::task::spawn_blocking(move || {
+ // A timed-out caller must not release the slot or discard a successful build.
+ let _build_permit = build_permit;
+ let client = match (
+ pool.http_client_factory.outbound_proxy_policy(),
+ pool.custom_ca_fallback,
+ ) {
+ (_, CustomCaFallback::LegacyDirect) => {
+ Ok(client_builder.build_with_custom_ca_fallback(ProxyRouting::Direct))
+ }
+ (OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::LegacyTransportDefault) => {
+ Ok(
+ client_builder
+ .build_with_custom_ca_fallback(ProxyRouting::TransportDefault),
+ )
+ }
+ (OutboundProxyPolicy::ReqwestDefault, CustomCaFallback::Disabled)
+ | (OutboundProxyPolicy::RespectSystemProxy, CustomCaFallback::Disabled)
+ | (
+ OutboundProxyPolicy::RespectSystemProxy,
+ CustomCaFallback::LegacyTransportDefault,
+ ) => client_builder.build_for_resolved_route(
+ &pool.http_client_factory,
+ pool.route_class,
+ &build_route,
+ ),
+ }?;
+ let mut clients = pool.clients.lock().unwrap_or_else(|error| {
+ panic!("route-aware client cache lock should not be poisoned: {error}")
+ });
+ if clients.len() >= MAX_CACHED_ROUTES
+ && let Some(route_to_evict) = clients.keys().next().cloned()
+ {
+ clients.remove(&route_to_evict);
}
- };
- let mut clients = match self.clients.lock() {
- Ok(clients) => clients,
- Err(error) => panic!("route-aware client cache lock should not be poisoned: {error}"),
- };
- if let Some(existing_client) = clients.get(&route) {
- return Ok((
- route,
- existing_client.clone(),
- SelectedTlsBackend::TransportDefault,
- ));
- }
- if clients.len() >= MAX_CACHED_ROUTES
- && let Some(route_to_evict) = clients.keys().next().cloned()
- {
- clients.remove(&route_to_evict);
- }
- clients.insert(route.clone(), client.clone());
+ clients.insert(build_route, client.clone());
+ Ok::<_, BuildRouteAwareHttpClientError>(client)
+ })
+ .await
+ .map_err(RouteAwareClientPoolError::BuildTask)??;
Ok((route, client, SelectedTlsBackend::TransportDefault))
}
diff --git a/codex-rs/http-client/src/route_aware_client_pool_tests.rs b/codex-rs/http-client/src/route_aware_client_pool_tests.rs
index af1a0cdcd1..711bc6b0e5 100644
--- a/codex-rs/http-client/src/route_aware_client_pool_tests.rs
+++ b/codex-rs/http-client/src/route_aware_client_pool_tests.rs
@@ -12,6 +12,7 @@ use std::time::Duration;
use std::time::Instant;
use bytes::Bytes;
+use futures::FutureExt;
use futures::stream;
use http::HeaderValue;
use pretty_assertions::assert_eq;
@@ -710,6 +711,75 @@ async fn request_timeout_covers_route_selection() {
assert_eq!(resolver_calls.load(Ordering::SeqCst), 1);
}
+#[test]
+fn managed_request_timeout_covers_queued_transport_construction() {
+ let runtime = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .max_blocking_threads(/*val*/ 1)
+ .build()
+ .unwrap();
+ runtime.block_on(async {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ listener.set_nonblocking(/*nonblocking*/ true).unwrap();
+ let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+ let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
+ let blocker = tokio::task::spawn_blocking(move || {
+ started_tx.send(()).unwrap();
+ let _ = release_rx.recv();
+ });
+ started_rx.await.unwrap();
+
+ let controller = crate::NetworkPolicyController::default();
+ let policy = controller.policy();
+ controller.publish(policy.revision(), crate::DestinationPolicy::Unrestricted);
+ let pool = RouteAwareClientPool::new(
+ HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault).with_network_policy(policy),
+ ClientRouteClass::Api,
+ );
+ let url = format!("http://{}/", listener.local_addr().unwrap());
+ let requests = (0..8).map(|_| {
+ pool.get(&url)
+ .timeout(Duration::from_millis(/*millis*/ 200))
+ .send()
+ });
+ for result in futures::future::join_all(requests).await {
+ assert!(result.unwrap_err().is_timeout());
+ }
+ assert!(pool.client_build.try_lock().is_err());
+ assert!(pool.clients.lock().unwrap().is_empty());
+ assert_eq!(
+ listener.accept().unwrap_err().kind(),
+ io::ErrorKind::WouldBlock
+ );
+ drop(release_tx);
+ blocker.await.unwrap();
+
+ let build_finished =
+ tokio::time::timeout(Duration::from_secs(/*secs*/ 5), pool.client_build.lock())
+ .await
+ .unwrap();
+ assert!(
+ pool.clients
+ .lock()
+ .unwrap()
+ .contains_key(&OutboundProxyRoute::TransportDefault)
+ );
+ let (_, _, backend) = pool
+ .client_for_url_with_resolver(&url, |_| async {
+ Ok(OutboundProxyRoute::TransportDefault)
+ })
+ .now_or_never()
+ .unwrap()
+ .unwrap();
+ assert_eq!(backend, SelectedTlsBackend::TransportDefault);
+ drop(build_finished);
+ assert_eq!(
+ listener.accept().unwrap_err().kind(),
+ io::ErrorKind::WouldBlock
+ );
+ });
+}
+
#[tokio::test]
async fn request_timeout_is_shared_across_redirect_hops() {
let (address, server) = spawn_response_server(vec![
diff --git a/codex-rs/login/src/auth/default_client.rs b/codex-rs/login/src/auth/default_client.rs
index 461f658d18..ae509a4a73 100644
--- a/codex-rs/login/src/auth/default_client.rs
+++ b/codex-rs/login/src/auth/default_client.rs
@@ -250,6 +250,15 @@ pub fn create_client_for_route(
ClientRedirectPolicy::Default => default_http_client_builder(),
ClientRedirectPolicy::Reject => default_http_client_builder().without_redirects(),
};
+ create_client_for_route_with_builder(http_client_factory, request_url, route_class, builder)
+}
+
+fn create_client_for_route_with_builder(
+ http_client_factory: &HttpClientFactory,
+ request_url: &str,
+ route_class: ClientRouteClass,
+ builder: HttpClientBuilder,
+) -> Result<HttpClient, BuildRouteAwareHttpClientError> {
if http_client_factory.network_policy().is_managed() {
let mut pool = codex_http_client::RouteAwareClientPool::with_builder(
http_client_factory.clone(),
@@ -295,6 +304,39 @@ pub async fn create_client_for_route_async(
request_url: String,
route_class: ClientRouteClass,
redirect_policy: ClientRedirectPolicy,
+) -> std::io::Result<HttpClient> {
+ create_client_for_route_async_with_builder(
+ http_client_factory,
+ request_url,
+ route_class,
+ move || match redirect_policy {
+ ClientRedirectPolicy::Default => default_http_client_builder(),
+ ClientRedirectPolicy::Reject => default_http_client_builder().without_redirects(),
+ },
+ )
+ .await
+}
+
+/// Builds a routed default client without request URL or response-header diagnostics.
+pub async fn create_client_for_route_without_request_logging_async(
+ http_client_factory: HttpClientFactory,
+ request_url: String,
+ route_class: ClientRouteClass,
+) -> std::io::Result<HttpClient> {
+ create_client_for_route_async_with_builder(
+ http_client_factory,
+ request_url,
+ route_class,
+ || default_http_client_builder().without_request_logging(),
+ )
+ .await
+}
+
+async fn create_client_for_route_async_with_builder(
+ http_client_factory: HttpClientFactory,
+ request_url: String,
+ route_class: ClientRouteClass,
+ builder: impl FnOnce() -> HttpClientBuilder + Send + 'static,
) -> std::io::Result<HttpClient> {
let permit = ROUTE_AWARE_CLIENT_BUILD_PERMIT
.acquire()
@@ -302,11 +344,11 @@ pub async fn create_client_for_route_async(
.map_err(std::io::Error::other)?;
tokio::task::spawn_blocking(move || {
let _permit = permit;
- create_client_for_route(
+ create_client_for_route_with_builder(
&http_client_factory,
&request_url,
route_class,
- redirect_policy,
+ builder(),
)
.map_err(std::io::Error::from)
})