Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(auth): single-flight token refresh across processes and keep sessions alive (#4633)

ddupontbc3a0f074b6f
diff --git a/libs/cua/crates/cua-auth/Cargo.toml b/libs/cua/crates/cua-auth/Cargo.toml
index e44d1bb313..e38a176a05 100644
--- a/libs/cua/crates/cua-auth/Cargo.toml
+++ b/libs/cua/crates/cua-auth/Cargo.toml
@@ -19,7 +19,7 @@ rand.workspace = true
sha2.workspace = true
url.workspace = true
rustls.workspace = true
-tokio = { workspace = true, features = ["net", "time", "sync", "io-util", "macros"] }
+tokio = { workspace = true, features = ["net", "time", "sync", "io-util", "macros", "rt"] }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "json"] }
chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
diff --git a/libs/cua/crates/cua-auth/src/lib.rs b/libs/cua/crates/cua-auth/src/lib.rs
index a2c21ff465..2a565bd433 100644
--- a/libs/cua/crates/cua-auth/src/lib.rs
+++ b/libs/cua/crates/cua-auth/src/lib.rs
@@ -29,6 +29,7 @@
use serde::{Deserialize, Serialize};
use std::{
path::{Path, PathBuf},
+ sync::Arc,
time::Duration,
};
@@ -374,6 +375,23 @@ impl Store {
let _ = std::fs::remove_file(self.marker_path());
}
+ /// The lock file that serialises token refreshes across processes using
+ /// this store (no token material).
+ fn refresh_lock_path(&self) -> PathBuf {
+ match self {
+ Store::File(p) => {
+ let name = p
+ .file_name()
+ .map(|n| n.to_string_lossy().into_owned())
+ .unwrap_or_else(|| "credentials".into());
+ p.with_file_name(format!(".{name}.refresh.lock"))
+ }
+ Store::TestKeychain(d) => d.join("refresh.lock"),
+ #[cfg(any(target_os = "macos", target_os = "windows"))]
+ Store::Keyring => cua_home().join("auth-refresh.lock"),
+ }
+ }
+
/// Human description.
pub fn describe(&self) -> String {
match self {
@@ -953,9 +971,11 @@ impl Oidc {
.await?;
if status != 200 {
let msg = describe(&v, "your session expired; run 'cua auth login' again");
- // 4xx from the token endpoint means the grant is dead; anything
- // else may be transient.
- return Err(if (400..500).contains(&status) {
+ // Only a refusal of the grant itself (`invalid_grant`, or a 401)
+ // means the session is dead. Throttling, timeouts and other 4xx
+ // or 5xx answers are transient and must never sign the user out.
+ let dead = status == 401 || (status == 400 && v["error"] == "invalid_grant");
+ return Err(if dead {
Error::Unauthenticated(msg)
} else {
Error::Http(msg)
@@ -1411,11 +1431,61 @@ async fn await_callback(listener: &tokio::net::TcpListener, state: &str) -> Resu
// ---------------------------------------------------------------- session
+/// Longest a refresh waits for another process's refresh to finish.
+const REFRESH_LOCK_WAIT: Duration = Duration::from_secs(100);
+/// How often the background keep-alive checks the session.
+const KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(120);
+/// The keep-alive refreshes when the access token expires within this.
+const KEEP_ALIVE_WINDOW: Duration = Duration::from_secs(300);
+
+/// An exclusive advisory file lock held while refreshing, shared by every
+/// process that uses the same store. Released on drop (or process exit).
+struct RefreshLock(#[allow(dead_code)] std::fs::File);
+
+impl RefreshLock {
+ async fn acquire(store: &Store) -> Result<Self> {
+ let path = store.refresh_lock_path();
+ tokio::task::spawn_blocking(move || {
+ cua_home::guard_write(&path).map_err(store_err)?;
+ if let Some(d) = path.parent() {
+ std::fs::create_dir_all(d).map_err(store_err)?;
+ }
+ let f = std::fs::OpenOptions::new()
+ .create(true)
+ .truncate(false)
+ .write(true)
+ .open(&path)
+ .map_err(store_err)?;
+ let deadline = std::time::Instant::now() + REFRESH_LOCK_WAIT;
+ loop {
+ match f.try_lock() {
+ Ok(()) => return Ok(Self(f)),
+ Err(std::fs::TryLockError::WouldBlock) => {
+ if std::time::Instant::now() >= deadline {
+ // Transient: the other refresh is still running.
+ return Err(Error::Http(
+ "another Cua process is refreshing the session; try again".into(),
+ ));
+ }
+ std::thread::sleep(Duration::from_millis(25));
+ }
+ Err(std::fs::TryLockError::Error(e)) => return Err(store_err(e)),
+ }
+ }
+ })
+ .await
+ .map_err(store_err)?
+ }
+}
+
/// The shared, refreshing user session.
pub struct Session {
oidc: Oidc,
store: Store,
cached: tokio::sync::Mutex<Option<Credentials>>,
+ /// A refresh succeeded but the store refused the write: the cached copy
+ /// holds the only valid (rotated) refresh token and must be saved again.
+ unsaved: std::sync::atomic::AtomicBool,
}
impl Session {
@@ -1425,6 +1495,7 @@ impl Session {
oidc,
store,
cached: tokio::sync::Mutex::new(None),
+ unsaved: std::sync::atomic::AtomicBool::new(false),
}
}
@@ -1453,7 +1524,16 @@ impl Session {
/// Who is signed in (no network).
pub fn identity(&self) -> Option<Identity> {
- self.store.load().ok().flatten().map(|c| c.identity())
+ match self.store.load() {
+ Ok(c) => c.map(|c| c.identity()),
+ // A store that cannot be read right now (locked keychain after
+ // sleep) is not a sign-out: report the last identity seen.
+ Err(_) => self
+ .cached
+ .try_lock()
+ .ok()
+ .and_then(|g| g.as_ref().map(|c| c.identity())),
+ }
}
/// Starts a sign-in.
@@ -1469,23 +1549,60 @@ impl Session {
Ok(id)
}
- /// A valid access token, refreshed (and persisted) when it expires
- /// within a minute or `force`. `Error::NotLoggedIn` without a session;
- /// a refresh the provider refuses clears the session.
- pub async fn access_token(&self, force: bool) -> Result<String> {
- let mut guard = self.cached.lock().await;
- // The store is the source of truth: another process may have signed
- // out, or refreshed and rotated the tokens. A store that cannot be
- // read right now falls back to the last good copy.
- let stored = match self.store.load() {
- Ok(Some(c)) => c,
+ /// Reads the store (the source of truth: another process may have
+ /// signed out, or refreshed and rotated the tokens). A store that cannot
+ /// be read right now (locked keychain, I/O error) falls back to the last
+ /// good copy and is never read as signed out.
+ fn read_stored(&self, guard: &mut Option<Credentials>) -> Result<Credentials> {
+ use std::sync::atomic::Ordering::SeqCst;
+ if self.unsaved.load(SeqCst)
+ && let Some(c) = guard.clone()
+ {
+ // The store still holds the superseded refresh token.
+ if self.store.save(&c).is_ok() {
+ self.unsaved.store(false, SeqCst);
+ }
+ return Ok(c);
+ }
+ match self.store.load() {
+ Ok(Some(c)) => Ok(c),
Ok(None) => {
*guard = None;
- return Err(Error::NotLoggedIn);
+ Err(Error::NotLoggedIn)
}
- Err(Error::Store(e)) => guard.clone().ok_or(Error::Store(e))?,
- Err(e) => return Err(e),
- };
+ Err(Error::Store(e)) => guard.clone().ok_or(Error::Store(e)),
+ Err(e) => Err(e),
+ }
+ }
+
+ /// Saves rotated credentials, retrying; if the store keeps refusing, the
+ /// cached copy is kept and saved again on the next call.
+ async fn persist(&self, c: &Credentials) {
+ for attempt in 0..4u64 {
+ if self.store.save(c).is_ok() {
+ self.unsaved
+ .store(false, std::sync::atomic::Ordering::SeqCst);
+ return;
+ }
+ tokio::time::sleep(Duration::from_millis(100 * (attempt + 1))).await;
+ }
+ self.unsaved
+ .store(true, std::sync::atomic::Ordering::SeqCst);
+ }
+
+ /// A valid access token, refreshed (and persisted) when it expires
+ /// within [`REFRESH_SKEW_SECS`] or `force`. `Error::NotLoggedIn` without
+ /// a session. Only a refresh the provider refuses as a dead grant clears
+ /// the session; network and store failures are returned as such.
+ ///
+ /// Refreshing is single-flight across processes: the app, the daemon and
+ /// the CLI share one store, and a rotated refresh token that a second
+ /// process then presents makes the provider revoke the whole session. A
+ /// file lock serialises refreshes and the store is re-read under it, so
+ /// the second process uses the first one's result instead.
+ pub async fn access_token(&self, force: bool) -> Result<String> {
+ let mut guard = self.cached.lock().await;
+ let stored = self.read_stored(&mut guard)?;
let rotated = guard
.as_ref()
.is_some_and(|c| c.access_token != stored.access_token);
@@ -1494,24 +1611,31 @@ impl Session {
*guard = Some(stored);
return Ok(t);
}
- match self.oidc.refresh(&stored).await {
+ let _lock = RefreshLock::acquire(&self.store).await?;
+ // Another process may have refreshed while we waited for the lock.
+ let current = self.read_stored(&mut guard)?;
+ if current.access_token != stored.access_token && !current.needs_refresh() {
+ let t = current.access_token.clone();
+ *guard = Some(current);
+ return Ok(t);
+ }
+ match self.oidc.refresh(&current).await {
Ok(n) => {
- self.store.save(&n)?;
+ // Persist before anything else: the old refresh token is
+ // already spent.
+ self.persist(&n).await;
let t = n.access_token.clone();
*guard = Some(n);
Ok(t)
}
Err(Error::Unauthenticated(m)) => {
- // Another process may have refreshed (and rotated the
- // refresh token) while ours was in flight, so the provider
- // refused the one we sent. Its fresh session is in the store
- // now: use it rather than signing the user out. Only a store
- // that still holds the refused session is cleared.
+ // The grant is dead. Clear only a store that still holds the
+ // refused session; one a sign-in just replaced is kept.
match self.store.load() {
- Ok(Some(current)) if current != stored => {
- if !current.needs_refresh() {
- let t = current.access_token.clone();
- *guard = Some(current);
+ Ok(Some(now)) if now != current => {
+ if !now.needs_refresh() {
+ let t = now.access_token.clone();
+ *guard = Some(now);
return Ok(t);
}
*guard = None;
@@ -1522,9 +1646,11 @@ impl Session {
let _ = self.store.clear();
Err(Error::Unauthenticated(m))
}
- _ => {
+ // Unreadable store: not proof the session is gone.
+ Err(_) => Err(Error::Http(m)),
+ Ok(None) => {
*guard = None;
- Err(Error::Unauthenticated(m))
+ Err(Error::NotLoggedIn)
}
}
}
@@ -1532,6 +1658,50 @@ impl Session {
}
}
+ /// Refreshes now when the access token expires within `window`, so a
+ /// session that is not otherwise used stays alive. Errors are transient
+ /// or already signed-out; callers ignore them.
+ pub async fn refresh_if_due(&self, window: Duration) -> Result<()> {
+ let due = match self.store.load() {
+ Ok(Some(c)) => c.expires().map_or(true, |e| {
+ e <= chrono::Utc::now() + chrono::Duration::from_std(window).unwrap_or_default()
+ }),
+ Ok(None) => return Ok(()),
+ Err(_) => return Ok(()),
+ };
+ if due {
+ self.access_token(true).await.map(|_| ())
+ } else {
+ Ok(())
+ }
+ }
+
+ /// Keeps the session fresh in the background (proactive refresh, ahead of
+ /// expiry and of the provider's idle timeout) until the session is
+ /// dropped. A no-op outside a tokio runtime.
+ pub fn spawn_keep_alive(self: &Arc<Self>) {
+ let Ok(rt) = tokio::runtime::Handle::try_current() else {
+ return;
+ };
+ let weak = Arc::downgrade(self);
+ rt.spawn(async move {
+ loop {
+ tokio::time::sleep(KEEP_ALIVE_INTERVAL).await;
+ let Some(s) = weak.upgrade() else { return };
+ let _ = s.refresh_if_due(KEEP_ALIVE_WINDOW).await;
+ }
+ });
+ }
+
+ /// Whether the stored session holds an offline refresh token (granted
+ /// `offline_access`), which outlives the provider's SSO session limits.
+ /// `None` when unknown (nothing stored or no scope recorded).
+ pub fn is_offline(&self) -> Option<bool> {
+ let c = self.store.load().ok().flatten()?;
+ let scope = c.scope?;
+ Some(scope.split_whitespace().any(|s| s == "offline_access"))
+ }
+
/// Revokes (best effort) and clears the session. Returns whether a
/// session existed and whether the issuer confirmed revocation.
pub async fn logout(&self) -> Result<(bool, bool)> {
diff --git a/libs/cua/crates/cua-auth/tests/two_process.rs b/libs/cua/crates/cua-auth/tests/two_process.rs
new file mode 100644
index 0000000000..4e166ed6c3
--- /dev/null
+++ b/libs/cua/crates/cua-auth/tests/two_process.rs
@@ -0,0 +1,234 @@
+//! Refresh across processes: the app, the daemon and the CLI share one
+//! credential store. With refresh-token rotation and reuse detection, two of
+//! them refreshing at once with the same token revokes the session. The
+//! fake issuer here is strict that way: a refresh token works once, and
+//! presenting a spent one revokes everything.
+//!
+//! The children are this test binary re-run (`CUA_TWO_PROCESS_CHILD`), file
+//! store in a temp dir, never the OS keychain or ~/.cua.
+
+use cua_auth::{Credentials, Oidc, Session, Store};
+use std::io::{Read, Write};
+use std::net::TcpListener;
+use std::sync::{Arc, Mutex};
+
+#[derive(Default)]
+struct Issuer {
+ /// Refresh tokens already spent.
+ spent: Vec<String>,
+ current: u32,
+ refreshes: u32,
+ reuse_detected: bool,
+ /// Answer refreshes with this status instead (transient failures).
+ fail_with: Option<u16>,
+}
+
+fn serve(st: Arc<Mutex<Issuer>>) -> String {
+ let l = TcpListener::bind("127.0.0.1:0").unwrap();
+ let base = format!("http://{}", l.local_addr().unwrap());
+ let b = base.clone();
+ std::thread::spawn(move || {
+ for s in l.incoming() {
+ let Ok(mut s) = s else { continue };
+ let (st, base) = (st.clone(), b.clone());
+ std::thread::spawn(move || {
+ let mut buf = Vec::new();
+ let mut tmp = [0u8; 4096];
+ let body = loop {
+ let n = s.read(&mut tmp).unwrap_or(0);
+ if n == 0 {
+ return;
+ }
+ buf.extend_from_slice(&tmp[..n]);
+ if let Some(i) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
+ let head = String::from_utf8_lossy(&buf[..i]).to_lowercase();
+ let len: usize = head
+ .lines()
+ .find_map(|l| l.strip_prefix("content-length:"))
+ .and_then(|v| v.trim().parse().ok())
+ .unwrap_or(0);
+ if buf.len() >= i + 4 + len {
+ break String::from_utf8_lossy(&buf[i + 4..i + 4 + len]).to_string();
+ }
+ }
+ };
+ let line = String::from_utf8_lossy(&buf)
+ .lines()
+ .next()
+ .unwrap()
+ .to_string();
+ let (status, out) = if line.contains("openid-configuration") {
+ (
+ 200,
+ serde_json::json!({ "token_endpoint": format!("{base}/token") }),
+ )
+ } else {
+ let rt = url::form_urlencoded::parse(body.as_bytes())
+ .find(|(k, _)| k == "refresh_token")
+ .map(|(_, v)| v.into_owned())
+ .unwrap_or_default();
+ let mut g = st.lock().unwrap();
+ std::thread::sleep(std::time::Duration::from_millis(150));
+ if let Some(code) = g.fail_with {
+ (
+ code,
+ serde_json::json!({ "error": "temporarily_unavailable" }),
+ )
+ } else if g.reuse_detected || g.spent.contains(&rt) {
+ g.reuse_detected = true;
+ (
+ 400,
+ serde_json::json!({ "error": "invalid_grant", "error_description": "Token reuse detected" }),
+ )
+ } else {
+ g.spent.push(rt);
+ g.current += 1;
+ g.refreshes += 1;
+ let n = g.current;
+ (
+ 200,
+ serde_json::json!({ "access_token": format!("at-{n}"),
+ "refresh_token": format!("rt-{n}"), "expires_in": 900,
+ "scope": "openid profile offline_access" }),
+ )
+ }
+ };
+ let text = out.to_string();
+ let _ = write!(
+ s,
+ "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{text}",
+ text.len()
+ );
+ });
+ }
+ });
+ base
+}
+
+fn old_creds() -> Credentials {
+ Credentials {
+ access_token: "at-0".into(),
+ refresh_token: Some("rt-0".into()),
+ expires_at: "2000-01-01T00:00:00Z".into(),
+ token_type: "Bearer".into(),
+ scope: Some("openid offline_access".into()),
+ id_token: None,
+ }
+}
+
+/// Child role: refresh through the shared store and report.
+#[test]
+fn child_process() {
+ let (Ok(url), Ok(path)) = (
+ std::env::var("CUA_TWO_PROCESS_ISSUER"),
+ std::env::var("CUA_TWO_PROCESS_STORE"),
+ ) else {
+ return;
+ };
+ let rt = tokio::runtime::Runtime::new().unwrap();
+ let s = Session::new(Oidc::new(url, "cua-cli"), Store::File(path.into()));
+ let r = rt.block_on(s.access_token(true));
+ println!(
+ "CHILD_RESULT {}",
+ r.map_or_else(|e| format!("ERR {e}"), |t| format!("OK {t}"))
+ );
+}
+
+#[test]
+fn concurrent_processes_refresh_once_and_nobody_is_signed_out() {
+ let st = Arc::new(Mutex::new(Issuer::default()));
+ let url = serve(st.clone());
+ let d = tempfile::tempdir().unwrap();
+ let path = d.path().join("credentials.json");
+ Store::File(path.clone()).save(&old_creds()).unwrap();
+
+ let exe = std::env::current_exe().unwrap();
+ let kids: Vec<_> = (0..6)
+ .map(|_| {
+ std::process::Command::new(&exe)
+ .args([
+ "child_process",
+ "--exact",
+ "--nocapture",
+ "--test-threads=1",
+ ])
+ .env("CUA_TWO_PROCESS_ISSUER", &url)
+ .env("CUA_TWO_PROCESS_STORE", &path)
+ .stdout(std::process::Stdio::piped())
+ .spawn()
+ .unwrap()
+ })
+ .collect();
+ for k in kids {
+ let out = k.wait_with_output().unwrap();
+ let text = String::from_utf8_lossy(&out.stdout);
+ assert!(text.contains("CHILD_RESULT OK"), "child failed: {text}");
+ }
+ let g = st.lock().unwrap();
+ assert!(!g.reuse_detected, "a spent refresh token was presented");
+ assert_eq!(g.refreshes, 1, "exactly one process refreshes");
+ let stored = Store::File(path).load().unwrap().unwrap();
+ assert_eq!(stored.refresh_token.as_deref(), Some("rt-1"));
+}
+
+#[tokio::test]
+async fn transient_refresh_failures_keep_the_session() {
+ for code in [429u16, 500, 503] {
+ let st = Arc::new(Mutex::new(Issuer {
+ fail_with: Some(code),
+ ..Default::default()
+ }));
+ let url = serve(st);
+ let d = tempfile::tempdir().unwrap();
+ let path = d.path().join("credentials.json");
+ let store = Store::File(path.clone());
+ store.save(&old_creds()).unwrap();
+ let s = Session::new(Oidc::new(url, "cua-cli"), Store::File(path));
+ let e = s.access_token(false).await.unwrap_err();
+ assert!(
+ !matches!(
+ e,
+ cua_auth::Error::Unauthenticated(_) | cua_auth::Error::NotLoggedIn
+ ),
+ "{code}: {e}"
+ );
+ assert_eq!(
+ store.load().unwrap(),
+ Some(old_creds()),
+ "{code} wiped the session"
+ );
+ }
+}
+
+#[tokio::test]
+async fn an_unreadable_store_is_not_signed_out() {
+ // A directory where the credentials file should be: reads fail with an
+ // I/O error (like a locked keychain), which is not "no session".
+ let st = Arc::new(Mutex::new(Issuer::default()));
+ let url = serve(st.clone());
+ let d = tempfile::tempdir().unwrap();
+ let path = d.path().join("credentials.json");
+ std::fs::create_dir(&path).unwrap();
+ let s = Session::new(Oidc::new(url, "cua-cli"), Store::File(path.clone()));
+ let e = s.access_token(false).await.unwrap_err();
+ assert!(matches!(e, cua_auth::Error::Store(_)), "{e}");
+ assert!(path.is_dir());
+ assert_eq!(st.lock().unwrap().refreshes, 0);
+}
+
+#[tokio::test]
+async fn a_dead_grant_still_signs_out() {
+ let st = Arc::new(Mutex::new(Issuer::default()));
+ st.lock().unwrap().reuse_detected = true;
+ let url = serve(st);
+ let d = tempfile::tempdir().unwrap();
+ let path = d.path().join("credentials.json");
+ let store = Store::File(path.clone());
+ store.save(&old_creds()).unwrap();
+ let s = Session::new(Oidc::new(url, "cua-cli"), Store::File(path));
+ assert!(matches!(
+ s.access_token(false).await,
+ Err(cua_auth::Error::Unauthenticated(_))
+ ));
+ assert!(store.load().unwrap().is_none());
+}
diff --git a/libs/cua/crates/cua-sdk/src/native/auth.rs b/libs/cua/crates/cua-sdk/src/native/auth.rs
index 49618d1d6d..3c241b61d1 100644
--- a/libs/cua/crates/cua-sdk/src/native/auth.rs
+++ b/libs/cua/crates/cua-sdk/src/native/auth.rs
@@ -201,9 +201,11 @@ impl Auth {
/// On an explicit session (Rust hosts and tests: a file store in a
/// temporary directory and a fake issuer).
pub fn with_session(session: cua_auth::Session) -> Arc<Self> {
- Arc::new(Auth {
- session: Arc::new(session),
- })
+ let session = Arc::new(session);
+ // Refresh ahead of expiry in long-lived hosts (the app, the daemon)
+ // instead of only when a call finds the token expired.
+ session.spawn_keep_alive();
+ Arc::new(Auth { session })
}
/// The underlying session (Rust hosts: a Fleet token provider).