Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

keep auth refresh flight through credential commit (#3417)

kt5504a55c0dcc
diff --git a/.changes/fix-auth-refresh-flight-commit.md b/.changes/fix-auth-refresh-flight-commit.md
new file mode 100644
index 0000000000..eb50a0f3b1
--- /dev/null
+++ b/.changes/fix-auth-refresh-flight-commit.md
@@ -0,0 +1 @@
+- Concurrent OAuth credential lookups now keep the refresh flight reserved until the refreshed credential is committed and reloaded, preventing a second lookup from reusing the expired refresh token between fetch and persistence.
diff --git a/crates/pa-core/src/auth/manager.rs b/crates/pa-core/src/auth/manager.rs
index d741ce429b..1cb1135b13 100644
--- a/crates/pa-core/src/auth/manager.rs
+++ b/crates/pa-core/src/auth/manager.rs
@@ -41,20 +41,23 @@ fn now_epoch_ms() -> i64 {
.map_or(i64::MAX, |d| d.as_millis() as i64)
}
-/// One OAuth refresh in flight per provider: the token fetch runs outside every
-/// lock, so TS's single-threaded single-flight needs its own gate.
-fn refresh_flight(provider: &str) -> std::sync::MutexGuard<'static, ()> {
+/// Provider gate for the whole refresh, including credential commit and reload.
+fn refresh_flight_lock(provider: &str) -> &'static std::sync::Mutex<()> {
static FLIGHTS: std::sync::OnceLock<
std::sync::Mutex<std::collections::HashMap<String, &'static std::sync::Mutex<()>>>,
> = std::sync::OnceLock::new();
let registry = FLIGHTS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()));
- let lock = {
+ {
let mut registry = registry.lock().expect("auth refresh-flight registry");
- *registry
+ registry
.entry(provider.to_string())
.or_insert_with(|| Box::leak(Box::new(std::sync::Mutex::new(()))))
- };
- lock.lock()
+ }
+}
+
+fn refresh_flight(provider: &str) -> std::sync::MutexGuard<'static, ()> {
+ refresh_flight_lock(provider)
+ .lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
diff --git a/crates/pa-core/src/auth/manager/lookup.rs b/crates/pa-core/src/auth/manager/lookup.rs
index 9e6f3157ac..c410dc8ca5 100644
--- a/crates/pa-core/src/auth/manager/lookup.rs
+++ b/crates/pa-core/src/auth/manager/lookup.rs
@@ -146,8 +146,8 @@ impl AuthStorage {
/// and write). The phases:
///
/// 1. LOAD: the document through the read arm (no document lock).
- /// 2. FETCH: the token call outside every lock, behind [`refresh_flight`]'s
- /// single-flight gate; the expiry is re-checked under the gate so a second
+ /// 2. FETCH: the token call outside the document lock, behind [`refresh_flight`]'s
+ /// single-flight gate, held through WRITE and reload; expiry is re-checked so a second
/// fetch never wastes a single-use refresh token.
/// 3. WRITE: the locked read-modify-write, holding the lock only for the re-read,
/// insert, and atomic write. A peer that refreshed meanwhile keeps its fresher
@@ -181,27 +181,25 @@ impl AuthStorage {
self.reload();
return Some(credential);
}
- // FETCH: outside every lock, one flight per provider.
- let fetched = {
- let _flight = refresh_flight(provider_id);
- // The gate may have just released a flight that wrote a fresh
- // credential; re-check before spending a refresh token.
- let content = self.storage.read().unwrap_or_default();
- if let Some(credential) = parse_storage_data(content.as_deref())
- .ok()
- .and_then(|data| data.credential(provider_id))
- .filter(|credential| {
- matches!(
- credential,
- AuthCredential::Oauth { expires, .. } if now_epoch_ms() < *expires
- )
- })
- {
- self.reload();
- return Some(credential);
- }
- self.oauth.refresh(provider_id, &data)
- };
+ // FETCH: outside the document lock, one flight through commit and reload.
+ let _flight = refresh_flight(provider_id);
+ // The gate may have just released a flight that wrote a fresh
+ // credential; re-check before spending a refresh token.
+ let content = self.storage.read().unwrap_or_default();
+ if let Some(credential) = parse_storage_data(content.as_deref())
+ .ok()
+ .and_then(|data| data.credential(provider_id))
+ .filter(|credential| {
+ matches!(
+ credential,
+ AuthCredential::Oauth { expires, .. } if now_epoch_ms() < *expires
+ )
+ })
+ {
+ self.reload();
+ return Some(credential);
+ }
+ let fetched = self.oauth.refresh(provider_id, &data);
let Some(new_credential) = fetched else {
// Refresh failed: keep credentials for a later retry; a peer
// may have refreshed meanwhile, so reload before failing.
diff --git a/crates/pa-core/src/auth/manager/tests.rs b/crates/pa-core/src/auth/manager/tests.rs
index 4ddebe0ca4..903087c51c 100644
--- a/crates/pa-core/src/auth/manager/tests.rs
+++ b/crates/pa-core/src/auth/manager/tests.rs
@@ -659,6 +659,57 @@ fn the_token_fetch_holds_no_document_lock_and_a_peer_write_keeps_its_fresher_cre
);
}
+/// Probe the persistence boundary itself: main releases the flight before this write.
+#[test]
+fn the_refresh_flight_stays_held_through_credential_commit() {
+ struct CommitProbe {
+ provider: &'static str,
+ storage: crate::auth::storage::InMemoryAuthStorageBackend,
+ flight_held: std::sync::Mutex<Vec<bool>>,
+ }
+
+ impl AuthStorageBackend for CommitProbe {
+ fn read(&self) -> anyhow::Result<Option<String>> {
+ self.storage.read()
+ }
+
+ fn with_lock(
+ &self,
+ update: &mut dyn FnMut(Option<String>) -> anyhow::Result<((), Option<String>)>,
+ ) -> anyhow::Result<()> {
+ self.flight_held
+ .lock()
+ .unwrap()
+ .push(refresh_flight_lock(self.provider).try_lock().is_err());
+ self.storage.with_lock(update)
+ }
+ }
+
+ let provider = "x-flight-commit-probe";
+ let backend = Arc::new(CommitProbe {
+ provider,
+ storage: crate::auth::storage::InMemoryAuthStorageBackend::default(),
+ flight_held: std::sync::Mutex::new(Vec::new()),
+ });
+ let mut data = AuthStorageData::default();
+ data.insert(provider, &expired_oauth("old-access"));
+ let seed = serde_json::to_string(&data.0).unwrap();
+ backend
+ .storage
+ .with_lock(&mut |_| Ok(((), Some(seed.clone()))))
+ .unwrap();
+ let oauth = Arc::new(CountingOAuth {
+ calls: std::sync::atomic::AtomicUsize::new(0),
+ delay_ms: 0,
+ });
+ let mut auth = AuthStorage::from_storage(backend.clone(), oauth);
+ assert_eq!(
+ auth.get_api_key(provider).as_deref(),
+ Some("fetched-access")
+ );
+ assert_eq!(*backend.flight_held.lock().unwrap(), vec![true]);
+}
+
#[test]
fn a_second_refresh_joins_the_first_flight_instead_of_fetching_again() {
// Two resolutions race: the flight gate serializes them, and the