Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Refresh rejected proxy credentials and back off failed renewals (#55299)

ee54f99017f23a
diff --git a/codex-rs/responses-api-proxy/src/auth.rs b/codex-rs/responses-api-proxy/src/auth.rs
index 1b61b5c34f..87b8a531c8 100644
--- a/codex-rs/responses-api-proxy/src/auth.rs
+++ b/codex-rs/responses-api-proxy/src/auth.rs
@@ -1,11 +1,14 @@
//! Authenticates proxy requests using a locked API key or renewable OpenAI credentials.
-//! Serializes exchanges and never reuses expired credentials on renewal errors.
+//! Federated exchanges are serialized, fail closed on renewal errors, and never replay requests.
//! Token clients never follow redirects or emit credential-bearing URL diagnostics.
use std::fmt;
use std::future::Future;
use std::pin::Pin;
+use std::sync::Arc;
use std::sync::Mutex;
+use std::sync::atomic::AtomicBool;
+use std::sync::atomic::Ordering;
use std::time::Duration;
use std::time::Instant;
use std::time::SystemTime;
@@ -37,6 +40,8 @@ const MAX_RESPONSE_BYTES: usize = 64 * 1024;
const MAX_EXCHANGE_REQUEST_BYTES: usize = 64 * 1024;
pub(crate) const TOKEN_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
const MAX_FEDERATED_TOKEN_LIFETIME: Duration = Duration::from_secs(3600);
+const INITIAL_RENEWAL_BACKOFF: Duration = Duration::from_secs(1);
+const MAX_RENEWAL_BACKOFF: Duration = Duration::from_secs(30);
/// Supplies a fresh JWT whenever an OpenAI credential needs to be issued.
/// Acquisition stays asynchronous so callers can compose it with the exchange.
@@ -94,10 +99,21 @@ impl fmt::Debug for Auth {
}
}
-/// The sensitive header used for an individual request.
+/// The sensitive header and, for federation, the rejection state of this issuance.
+/// In-flight requests share only their own issuance's flag, never a replacement's.
#[derive(Clone, Debug)]
pub(crate) struct AuthHeader {
pub(crate) value: HeaderValue,
+ rejected: Option<Arc<AtomicBool>>,
+}
+
+impl AuthHeader {
+ pub(crate) fn invalidate(&self) {
+ if let Some(rejected) = &self.rejected {
+ // This flag carries no other state; credentials remain mutex-protected.
+ rejected.store(true, Ordering::Relaxed);
+ }
+ }
}
impl Auth {
@@ -123,6 +139,7 @@ impl Auth {
provider,
service_account,
cached: None,
+ renewal_failure: None,
})),
})
}
@@ -146,12 +163,15 @@ impl Auth {
// Keep the existing locked, static API key allocation.
let mut header = HeaderValue::from_static(value);
header.set_sensitive(true);
- Ok(AuthHeader { value: header })
+ Ok(AuthHeader {
+ value: header,
+ rejected: None,
+ })
}
Self::Federated { state } => state
.lock()
.map_err(|_| anyhow!("workload identity credential cache unavailable"))?
- .header(),
+ .header(Instant::now),
}
}
}
@@ -164,6 +184,7 @@ pub(crate) struct FederatedAuth {
provider: String,
service_account: String,
cached: Option<CachedToken>,
+ renewal_failure: Option<RenewalFailure>,
}
struct CachedToken {
@@ -172,22 +193,60 @@ struct CachedToken {
expires_at: SystemTime,
}
+struct RenewalFailure {
+ retry_at: Instant,
+ delay: Duration,
+}
+
impl FederatedAuth {
- fn header(&mut self) -> Result<AuthHeader> {
+ fn header(&mut self, clock: impl Fn() -> Instant) -> Result<AuthHeader> {
+ let now = clock();
if let Some(cached) = &self.cached
- && Instant::now() < cached.renew_at
+ && now < cached.renew_at
// Monotonic time can stop while a host is suspended.
&& SystemTime::now() < cached.expires_at
+ && cached
+ .header
+ .rejected
+ .as_ref()
+ .is_none_or(|rejected| !rejected.load(Ordering::Relaxed))
{
return Ok(cached.header.clone());
}
- // Discard the old credential before a fallible exchange. Never reuse it
- // when acquisition fails.
+ if let Some(failure) = &self.renewal_failure
+ && now < failure.retry_at
+ {
+ // Keep neither the failed token nor an error that could hold source
+ // credentials. The initiating request reports the original safe error.
+ return Err(anyhow!(
+ "workload identity renewal temporarily unavailable; retry later"
+ ));
+ }
+ // One exchange at a time (under Auth's mutex), including during startup.
+ // On failure, fail closed instead of reusing an expired/revoked token.
self.cached = None;
- let cached = self.exchange()?;
- let header = cached.header.clone();
- self.cached = Some(cached);
- Ok(header)
+ match self.exchange() {
+ Ok(cached) => {
+ let header = cached.header.clone();
+ self.cached = Some(cached);
+ self.renewal_failure = None;
+ Ok(header)
+ }
+ Err(err) => {
+ // Waiters share this failure instead of each retrying the token
+ // endpoints. Later requests may retry after a bounded delay.
+ let delay = self
+ .renewal_failure
+ .as_ref()
+ .map(|failure| (failure.delay * 2).min(MAX_RENEWAL_BACKOFF))
+ .unwrap_or(INITIAL_RENEWAL_BACKOFF);
+ self.renewal_failure = Some(RenewalFailure {
+ retry_at: clock() + delay,
+ delay,
+ });
+ Err(err)
+ }
+ }
}
fn exchange(&mut self) -> Result<CachedToken> {
@@ -272,7 +331,10 @@ impl FederatedAuth {
.checked_add(remaining - skew)
.ok_or_else(|| anyhow!("OpenAI returned an invalid token lifetime"))?;
Ok(CachedToken {
- header: AuthHeader { value: header },
+ header: AuthHeader {
+ value: header,
+ rejected: Some(Arc::new(AtomicBool::new(/*v*/ false))),
+ },
renew_at,
expires_at,
})
diff --git a/codex-rs/responses-api-proxy/src/auth_tests.rs b/codex-rs/responses-api-proxy/src/auth_tests.rs
index b9df94a72b..e45d12dc55 100644
--- a/codex-rs/responses-api-proxy/src/auth_tests.rs
+++ b/codex-rs/responses-api-proxy/src/auth_tests.rs
@@ -2,12 +2,15 @@
use super::*;
use pretty_assertions::assert_eq;
+use reqwest::blocking::Client;
use serde_json::json;
use std::sync::Arc;
+use std::sync::Barrier;
use std::sync::mpsc;
use std::thread;
use tiny_http::Header;
use tiny_http::Server;
+use tiny_http::StatusCode;
pub(crate) fn direct_token_client() -> TokenClient {
TokenClient(TokenClient::builder().build_direct().unwrap())
@@ -96,6 +99,22 @@ fn recv_exchange(server: &Server, subject: &str) -> tiny_http::Request {
req
}
+fn recv_response_request(
+ server: &Server,
+ expected_token: &str,
+ expected_body: &str,
+) -> tiny_http::Request {
+ let mut req = recv(server);
+ assert_eq!(req.url(), "/v1/responses");
+ assert_eq!(req.method(), &tiny_http::Method::Post);
+ assert!(req.headers().iter().any(|h| h.field.equiv("Authorization")
+ && h.value.as_str() == format!("Bearer {expected_token}")));
+ let mut body = String::new();
+ req.as_reader().read_to_string(&mut body).unwrap();
+ assert_eq!(body, expected_body);
+ req
+}
+
fn serve_exchange(server: &Server, subject: &str, token: &str, expires_in: u64, expires_at: u64) {
respond_exchange(
recv_exchange(server, subject),
@@ -373,6 +392,50 @@ fn absolute_expiry_forces_renewal_and_discards_cached_credential_on_failure() {
worker.join().unwrap();
}
+#[test]
+fn rejected_renewal_discards_old_credential_and_does_not_log_or_follow_redirect() {
+ let (server, url) = server();
+ let target = url.join("/credential-leak").unwrap();
+ let worker = thread::spawn(move || {
+ serve_exchange(
+ &server,
+ "issuer.subject.1",
+ "original.token",
+ /*expires_in*/ 300,
+ epoch_seconds() + 300,
+ );
+ recv_exchange(&server, "issuer.subject.2")
+ .respond(
+ tiny_http::Response::from_string("issuer.subject.2 original.token")
+ .with_status_code(StatusCode(302))
+ .with_header(Header::from_bytes("Location", target.as_str()).unwrap()),
+ )
+ .unwrap();
+ assert!(
+ server
+ .recv_timeout(Duration::from_millis(200))
+ .unwrap()
+ .is_none()
+ );
+ });
+ let auth = auth_at(&url);
+ assert_eq!(
+ auth.header().unwrap().value.to_str().unwrap(),
+ "Bearer original.token"
+ );
+ let Auth::Federated { ref state, .. } = auth else {
+ unreachable!()
+ };
+ state.lock().unwrap().cached.as_mut().unwrap().renew_at = Instant::now();
+ let error = format!("{:#}", auth.header().unwrap_err());
+ assert!(error.contains("302"), "{error}");
+ for secret in ["issuer.subject.2", "original.token", url.as_str()] {
+ assert!(!error.contains(secret), "{error}");
+ }
+ assert!(state.lock().unwrap().cached.is_none());
+ worker.join().unwrap();
+}
+
#[test]
fn invalid_subjects_are_rejected_before_exchange() {
struct InvalidSource(String);
@@ -394,3 +457,396 @@ fn invalid_subjects_are_rejected_before_exchange() {
);
}
}
+
+#[test]
+fn forwards_401_without_replay_then_reacquires_but_keeps_token_on_403() {
+ let (upstream, upstream_url) = server();
+ let auth = auth_at(&upstream_url);
+ let (proxy, proxy_url) = server();
+ let upstream_worker = thread::spawn(move || {
+ serve_exchange(
+ &upstream,
+ "issuer.subject.1",
+ "openai.first.jwt",
+ /*expires_in*/ 300,
+ epoch_seconds() + 300,
+ );
+ recv_response_request(&upstream, "openai.first.jwt", "first")
+ .respond(
+ tiny_http::Response::from_string("rejected")
+ .with_status_code(StatusCode(401))
+ .with_header(Header::from_bytes("WWW-Authenticate", "Bearer").unwrap()),
+ )
+ .unwrap();
+ // No Responses replay or speculative exchange: the caller submits a
+ // distinct POST. A permission failure on that POST must not evict auth.
+ serve_exchange(
+ &upstream,
+ "issuer.subject.2",
+ "openai.second.jwt",
+ /*expires_in*/ 300,
+ epoch_seconds() + 300,
+ );
+ recv_response_request(&upstream, "openai.second.jwt", "second")
+ .respond(
+ tiny_http::Response::from_string("forbidden").with_status_code(StatusCode(403)),
+ )
+ .unwrap();
+ recv_response_request(&upstream, "openai.second.jwt", "third")
+ .respond(tiny_http::Response::from_string("success"))
+ .unwrap();
+ });
+ let proxy_worker = thread::spawn(move || {
+ let client = Client::builder()
+ .no_proxy()
+ .timeout(Duration::from_secs(5))
+ .redirect(auth.redirect_policy())
+ .build()
+ .unwrap();
+ let config = crate::ForwardConfig {
+ host_header: HeaderValue::from_str(upstream_url.authority()).unwrap(),
+ upstream_url: upstream_url.join("/v1/responses").unwrap(),
+ };
+ for _ in 0..3 {
+ crate::forward_request(
+ &client,
+ &auth,
+ &config,
+ /*dump_dir*/ None,
+ recv(&proxy),
+ )
+ .unwrap();
+ }
+ });
+ let client = Client::builder()
+ .no_proxy()
+ .timeout(Duration::from_secs(5))
+ .redirect(reqwest::redirect::Policy::none())
+ .build()
+ .unwrap();
+ let response_url = proxy_url.join("/v1/responses").unwrap();
+ let response = client
+ .post(response_url.clone())
+ .body("first")
+ .send()
+ .unwrap();
+ assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED);
+ assert_eq!(
+ response.headers().get("www-authenticate").unwrap(),
+ "Bearer"
+ );
+ assert_eq!(response.text().unwrap(), "rejected");
+ let response = client
+ .post(response_url.clone())
+ .body("second")
+ .send()
+ .unwrap();
+ assert_eq!(response.status(), reqwest::StatusCode::FORBIDDEN);
+ assert_eq!(response.text().unwrap(), "forbidden");
+ let response = client.post(response_url).body("third").send().unwrap();
+ assert_eq!(response.status(), reqwest::StatusCode::OK);
+ assert_eq!(response.text().unwrap(), "success");
+ proxy_worker.join().unwrap();
+ upstream_worker.join().unwrap();
+}
+
+#[test]
+fn late_rejection_does_not_invalidate_a_newer_credential() {
+ let (server, url) = server();
+ let worker = thread::spawn(move || {
+ serve_exchange(
+ &server,
+ "issuer.subject.1",
+ "openai.first.jwt",
+ /*expires_in*/ 300,
+ epoch_seconds() + 300,
+ );
+ serve_exchange(
+ &server,
+ "issuer.subject.2",
+ "openai.second.jwt",
+ /*expires_in*/ 300,
+ epoch_seconds() + 300,
+ );
+ });
+ let auth = auth_at(&url);
+ let original = auth.header().unwrap();
+ original.invalidate();
+ let renewed = auth.header().unwrap();
+ assert_eq!(renewed.value.to_str().unwrap(), "Bearer openai.second.jwt");
+ worker.join().unwrap();
+ // Model an in-flight request made with the old token completing after
+ // the refresh. The server is closed, so an accidental third exchange fails.
+ original.invalidate();
+ assert_eq!(auth.header().unwrap().value, renewed.value);
+}
+
+#[test]
+fn forwards_401_before_an_inflight_renewal_finishes() {
+ let (upstream, upstream_url) = server();
+ let auth = Arc::new(auth_at(&upstream_url));
+ let acquiring = Arc::clone(&auth);
+ let first = thread::spawn(move || acquiring.header().unwrap());
+ serve_exchange(
+ &upstream,
+ "issuer.subject.1",
+ "openai.first.jwt",
+ /*expires_in*/ 300,
+ epoch_seconds() + 300,
+ );
+ let original = first.join().unwrap();
+ let (proxy, proxy_url) = server();
+ let forwarding = Arc::clone(&auth);
+ let config = crate::ForwardConfig {
+ host_header: HeaderValue::from_str(upstream_url.authority()).unwrap(),
+ upstream_url: upstream_url.join("/v1/responses").unwrap(),
+ };
+ let proxy_worker = thread::spawn(move || {
+ let client = Client::builder()
+ .no_proxy()
+ .redirect(forwarding.redirect_policy())
+ .build()
+ .unwrap();
+ crate::forward_request(
+ &client,
+ &forwarding,
+ &config,
+ /*dump_dir*/ None,
+ recv(&proxy),
+ )
+ .unwrap();
+ });
+ let (sent, received) = mpsc::sync_channel(1);
+ let caller = thread::spawn(move || {
+ let response = Client::builder()
+ .no_proxy()
+ .build()
+ .unwrap()
+ .post(proxy_url.join("/v1/responses").unwrap())
+ .body("first")
+ .send()
+ .unwrap();
+ sent.send((response.status(), response.text().unwrap()))
+ .unwrap();
+ });
+ // The request is already at upstream with the first credential. Hold its
+ // response until a concurrent rejection has started a new exchange.
+ let response_request = recv_response_request(&upstream, "openai.first.jwt", "first");
+ original.invalidate();
+ let acquiring = Arc::clone(&auth);
+ let renewal = thread::spawn(move || acquiring.header().unwrap());
+ let exchange = recv_exchange(&upstream, "issuer.subject.2");
+ response_request
+ .respond(tiny_http::Response::from_string("rejected").with_status_code(StatusCode(401)))
+ .unwrap();
+ // This must arrive before we let the exchange finish and release the mutex.
+ // Always release it before asserting so a regression cannot strand workers.
+ let response = received.recv_timeout(Duration::from_secs(5));
+ respond_exchange(
+ exchange,
+ "openai.second.jwt",
+ /*expires_in*/ 300,
+ epoch_seconds() + 300,
+ );
+ let renewed = renewal.join().unwrap();
+ caller.join().unwrap();
+ proxy_worker.join().unwrap();
+ assert_eq!(
+ response.unwrap(),
+ (reqwest::StatusCode::UNAUTHORIZED, "rejected".to_string())
+ );
+ assert_eq!(auth.header().unwrap().value, renewed.value);
+}
+
+fn header_at(auth: &Auth, now: Instant) -> Result<AuthHeader> {
+ let Auth::Federated { state } = auth else {
+ unreachable!()
+ };
+ state.lock().unwrap().header(|| now)
+}
+
+// Advance to the recorded retry deadline without waiting on real time.
+fn elapse_backoff(auth: &Auth, now: &mut Instant) -> Duration {
+ let Auth::Federated { state } = auth else {
+ unreachable!()
+ };
+ let state = state.lock().unwrap();
+ assert!(state.cached.is_none());
+ let failure = state.renewal_failure.as_ref().unwrap();
+ *now = failure.retry_at;
+ failure.delay
+}
+
+#[test]
+fn renewal_backoff_starts_after_a_slow_exchange_fails() {
+ let (server, url) = server();
+ let started = Instant::now();
+ let failed = started + Duration::from_secs(10);
+ let clock = Arc::new(Mutex::new(started));
+ let upstream_clock = Arc::clone(&clock);
+ let worker = thread::spawn(move || {
+ serve_exchange(
+ &server,
+ "issuer.subject.1",
+ "openai.first.jwt",
+ /*expires_in*/ 300,
+ epoch_seconds() + 300,
+ );
+ let exchange = recv_exchange(&server, "issuer.subject.2");
+ // Simulate time spent waiting on the failed renewal, longer than its cooldown.
+ *upstream_clock.lock().unwrap() = failed;
+ exchange
+ .respond(
+ tiny_http::Response::from_string("temporarily unavailable")
+ .with_status_code(StatusCode(503)),
+ )
+ .unwrap();
+ });
+ let auth = auth_at(&url);
+ auth.header().unwrap().invalidate();
+ let Auth::Federated { state } = &auth else {
+ unreachable!()
+ };
+ let error = state
+ .lock()
+ .unwrap()
+ .header(|| *clock.lock().unwrap())
+ .unwrap_err();
+ worker.join().unwrap();
+ assert_eq!(
+ error.to_string(),
+ "OpenAI workload identity token request rejected (HTTP 503)"
+ );
+ let retry_at = failed + Duration::from_secs(1);
+ assert_eq!(
+ state
+ .lock()
+ .unwrap()
+ .renewal_failure
+ .as_ref()
+ .unwrap()
+ .retry_at,
+ retry_at
+ );
+ assert_eq!(
+ header_at(&auth, retry_at - Duration::from_nanos(1))
+ .unwrap_err()
+ .to_string(),
+ "workload identity renewal temporarily unavailable; retry later"
+ );
+}
+
+#[test]
+fn concurrent_renewal_failure_backs_off_caps_and_resets_after_success() {
+ let (server, url) = server();
+ let worker = thread::spawn(move || {
+ serve_exchange(
+ &server,
+ "issuer.subject.1",
+ "openai.first.jwt",
+ /*expires_in*/ 300,
+ epoch_seconds() + 300,
+ );
+ for subject in 2..=8 {
+ recv_exchange(&server, &format!("issuer.subject.{subject}"))
+ .respond(
+ tiny_http::Response::from_string("private-issuer-token openai.first.jwt")
+ .with_status_code(StatusCode(503)),
+ )
+ .unwrap();
+ }
+ serve_exchange(
+ &server,
+ "issuer.subject.9",
+ "openai.recovered.jwt",
+ /*expires_in*/ 300,
+ epoch_seconds() + 300,
+ );
+ recv_exchange(&server, "issuer.subject.10")
+ .respond(
+ tiny_http::Response::from_string("private-issuer-token")
+ .with_status_code(StatusCode(503)),
+ )
+ .unwrap();
+ });
+ let auth = Arc::new(auth_at(&url));
+ let original = auth.header().unwrap();
+ let Auth::Federated { state, .. } = auth.as_ref() else {
+ unreachable!()
+ };
+ let mut now = Instant::now();
+ state.lock().unwrap().cached.as_mut().unwrap().renew_at = now;
+ let barrier = Arc::new(Barrier::new(8));
+ // Start every participant before joining any of them.
+ let mut callers = Vec::with_capacity(8);
+ for _ in 0..8 {
+ let auth = Arc::clone(&auth);
+ let barrier = Arc::clone(&barrier);
+ callers.push(thread::spawn(move || {
+ barrier.wait();
+ format!("{:#}", header_at(&auth, now).unwrap_err())
+ }));
+ }
+ let errors: Vec<_> = callers
+ .into_iter()
+ .map(|caller| caller.join().unwrap())
+ .collect();
+ assert_eq!(errors.iter().filter(|e| e.contains("503")).count(), 1);
+ assert_eq!(
+ errors.iter().filter(|e| e.contains("retry later")).count(),
+ 7
+ );
+ for error in &errors {
+ for secret in ["private-issuer-token", "openai.first.jwt", url.as_str()] {
+ assert!(!error.contains(secret), "{error}");
+ }
+ }
+ // An older 401 arriving during backoff cannot trigger another renewal.
+ original.invalidate();
+ assert!(
+ header_at(&auth, now)
+ .unwrap_err()
+ .to_string()
+ .contains("retry later")
+ );
+ for seconds in [1, 2, 4, 8, 16, 30] {
+ assert_eq!(
+ elapse_backoff(&auth, &mut now),
+ Duration::from_secs(seconds)
+ );
+ assert!(
+ header_at(&auth, now)
+ .unwrap_err()
+ .to_string()
+ .contains("503")
+ );
+ assert!(
+ header_at(&auth, now)
+ .unwrap_err()
+ .to_string()
+ .contains("retry later")
+ );
+ }
+ assert_eq!(elapse_backoff(&auth, &mut now), Duration::from_secs(30));
+ let renewed = header_at(&auth, now).unwrap();
+ assert_eq!(
+ renewed.value.to_str().unwrap(),
+ "Bearer openai.recovered.jwt"
+ );
+ assert!(state.lock().unwrap().renewal_failure.is_none());
+ renewed.invalidate();
+ assert!(
+ header_at(&auth, now)
+ .unwrap_err()
+ .to_string()
+ .contains("503")
+ );
+ assert!(
+ header_at(&auth, now)
+ .unwrap_err()
+ .to_string()
+ .contains("retry later")
+ );
+ assert_eq!(elapse_backoff(&auth, &mut now), Duration::from_secs(1));
+ worker.join().unwrap();
+}
diff --git a/codex-rs/responses-api-proxy/src/lib.rs b/codex-rs/responses-api-proxy/src/lib.rs
index 5cc423d8e3..83288eeb48 100644
--- a/codex-rs/responses-api-proxy/src/lib.rs
+++ b/codex-rs/responses-api-proxy/src/lib.rs
@@ -226,7 +226,7 @@ fn forward_request(
return Err(err);
}
};
- headers.insert(AUTHORIZATION, auth_header.value);
+ headers.insert(AUTHORIZATION, auth_header.value.clone());
headers.insert(HOST, config.host_header.clone());
@@ -242,6 +242,11 @@ fn forward_request(
// implements `Read`, so we can use it directly as the body of the
// `tiny_http::Response`.
let status = upstream_resp.status();
+ if status == reqwest::StatusCode::UNAUTHORIZED {
+ // Do not replay a Responses POST. A subsequent request can obtain a
+ // fresh credential, but a late 401 must not evict one renewed in-flight.
+ auth_header.invalidate();
+ }
let mut response_headers = Vec::new();
for (name, value) in upstream_resp.headers().iter() {
// Skip headers that tiny_http manages itself.