Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(auth): single-flight token refresh across processes and keep sessions alive (#4633)
ddupontbc3a0f074b6f
diff --git a/libs/cua/crates/cua-auth/Cargo.toml b/libs/cua/crates/cua-auth/Cargo.tomlindex e44d1bb313..e38a176a05 100644--- a/libs/cua/crates/cua-auth/Cargo.toml+++ b/libs/cua/crates/cua-auth/Cargo.toml@@ -19,7 +19,7 @@ rand.workspace = truesha2.workspace = trueurl.workspace = truerustls.workspace = true-tokio = { workspace = true, features = ["net", "time", "sync", "io-util", "macros"] }+tokio = { workspace = true, features = ["net", "time", "sync", "io-util", "macros", "rt"] }reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "json"] }chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }diff --git a/libs/cua/crates/cua-auth/src/lib.rs b/libs/cua/crates/cua-auth/src/lib.rsindex a2c21ff465..2a565bd433 100644--- a/libs/cua/crates/cua-auth/src/lib.rs+++ b/libs/cua/crates/cua-auth/src/lib.rs@@ -29,6 +29,7 @@use serde::{Deserialize, Serialize};use std::{path::{Path, PathBuf},+ sync::Arc,time::Duration,};@@ -374,6 +375,23 @@ impl Store {let _ = std::fs::remove_file(self.marker_path());}+ /// The lock file that serialises token refreshes across processes using+ /// this store (no token material).+ fn refresh_lock_path(&self) -> PathBuf {+ match self {+ Store::File(p) => {+ let name = p+ .file_name()+ .map(|n| n.to_string_lossy().into_owned())+ .unwrap_or_else(|| "credentials".into());+ p.with_file_name(format!(".{name}.refresh.lock"))+ }+ Store::TestKeychain(d) => d.join("refresh.lock"),+ #[cfg(any(target_os = "macos", target_os = "windows"))]+ Store::Keyring => cua_home().join("auth-refresh.lock"),+ }+ }+/// Human description.pub fn describe(&self) -> String {match self {@@ -953,9 +971,11 @@ impl Oidc {.await?;if status != 200 {let msg = describe(&v, "your session expired; run 'cua auth login' again");- // 4xx from the token endpoint means the grant is dead; anything- // else may be transient.- return Err(if (400..500).contains(&status) {+ // Only a refusal of the grant itself (`invalid_grant`, or a 401)+ // means the session is dead. Throttling, timeouts and other 4xx+ // or 5xx answers are transient and must never sign the user out.+ let dead = status == 401 || (status == 400 && v["error"] == "invalid_grant");+ return Err(if dead {Error::Unauthenticated(msg)} else {Error::Http(msg)@@ -1411,11 +1431,61 @@ async fn await_callback(listener: &tokio::net::TcpListener, state: &str) -> Resu// ---------------------------------------------------------------- session+/// Longest a refresh waits for another process's refresh to finish.+const REFRESH_LOCK_WAIT: Duration = Duration::from_secs(100);+/// How often the background keep-alive checks the session.+const KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(120);+/// The keep-alive refreshes when the access token expires within this.+const KEEP_ALIVE_WINDOW: Duration = Duration::from_secs(300);++/// An exclusive advisory file lock held while refreshing, shared by every+/// process that uses the same store. Released on drop (or process exit).+struct RefreshLock(#[allow(dead_code)] std::fs::File);++impl RefreshLock {+ async fn acquire(store: &Store) -> Result<Self> {+ let path = store.refresh_lock_path();+ tokio::task::spawn_blocking(move || {+ cua_home::guard_write(&path).map_err(store_err)?;+ if let Some(d) = path.parent() {+ std::fs::create_dir_all(d).map_err(store_err)?;+ }+ let f = std::fs::OpenOptions::new()+ .create(true)+ .truncate(false)+ .write(true)+ .open(&path)+ .map_err(store_err)?;+ let deadline = std::time::Instant::now() + REFRESH_LOCK_WAIT;+ loop {+ match f.try_lock() {+ Ok(()) => return Ok(Self(f)),+ Err(std::fs::TryLockError::WouldBlock) => {+ if std::time::Instant::now() >= deadline {+ // Transient: the other refresh is still running.+ return Err(Error::Http(+ "another Cua process is refreshing the session; try again".into(),+ ));+ }+ std::thread::sleep(Duration::from_millis(25));+ }+ Err(std::fs::TryLockError::Error(e)) => return Err(store_err(e)),+ }+ }+ })+ .await+ .map_err(store_err)?+ }+}+/// The shared, refreshing user session.pub struct Session {oidc: Oidc,store: Store,cached: tokio::sync::Mutex<Option<Credentials>>,+ /// A refresh succeeded but the store refused the write: the cached copy+ /// holds the only valid (rotated) refresh token and must be saved again.+ unsaved: std::sync::atomic::AtomicBool,}impl Session {@@ -1425,6 +1495,7 @@ impl Session {oidc,store,cached: tokio::sync::Mutex::new(None),+ unsaved: std::sync::atomic::AtomicBool::new(false),}}@@ -1453,7 +1524,16 @@ impl Session {/// Who is signed in (no network).pub fn identity(&self) -> Option<Identity> {- self.store.load().ok().flatten().map(|c| c.identity())+ match self.store.load() {+ Ok(c) => c.map(|c| c.identity()),+ // A store that cannot be read right now (locked keychain after+ // sleep) is not a sign-out: report the last identity seen.+ Err(_) => self+ .cached+ .try_lock()+ .ok()+ .and_then(|g| g.as_ref().map(|c| c.identity())),+ }}/// Starts a sign-in.@@ -1469,23 +1549,60 @@ impl Session {Ok(id)}- /// A valid access token, refreshed (and persisted) when it expires- /// within a minute or `force`. `Error::NotLoggedIn` without a session;- /// a refresh the provider refuses clears the session.- pub async fn access_token(&self, force: bool) -> Result<String> {- let mut guard = self.cached.lock().await;- // The store is the source of truth: another process may have signed- // out, or refreshed and rotated the tokens. A store that cannot be- // read right now falls back to the last good copy.- let stored = match self.store.load() {- Ok(Some(c)) => c,+ /// Reads the store (the source of truth: another process may have+ /// signed out, or refreshed and rotated the tokens). A store that cannot+ /// be read right now (locked keychain, I/O error) falls back to the last+ /// good copy and is never read as signed out.+ fn read_stored(&self, guard: &mut Option<Credentials>) -> Result<Credentials> {+ use std::sync::atomic::Ordering::SeqCst;+ if self.unsaved.load(SeqCst)+ && let Some(c) = guard.clone()+ {+ // The store still holds the superseded refresh token.+ if self.store.save(&c).is_ok() {+ self.unsaved.store(false, SeqCst);+ }+ return Ok(c);+ }+ match self.store.load() {+ Ok(Some(c)) => Ok(c),Ok(None) => {*guard = None;- return Err(Error::NotLoggedIn);+ Err(Error::NotLoggedIn)}- Err(Error::Store(e)) => guard.clone().ok_or(Error::Store(e))?,- Err(e) => return Err(e),- };+ Err(Error::Store(e)) => guard.clone().ok_or(Error::Store(e)),+ Err(e) => Err(e),+ }+ }++ /// Saves rotated credentials, retrying; if the store keeps refusing, the+ /// cached copy is kept and saved again on the next call.+ async fn persist(&self, c: &Credentials) {+ for attempt in 0..4u64 {+ if self.store.save(c).is_ok() {+ self.unsaved+ .store(false, std::sync::atomic::Ordering::SeqCst);+ return;+ }+ tokio::time::sleep(Duration::from_millis(100 * (attempt + 1))).await;+ }+ self.unsaved+ .store(true, std::sync::atomic::Ordering::SeqCst);+ }++ /// A valid access token, refreshed (and persisted) when it expires+ /// within [`REFRESH_SKEW_SECS`] or `force`. `Error::NotLoggedIn` without+ /// a session. Only a refresh the provider refuses as a dead grant clears+ /// the session; network and store failures are returned as such.+ ///+ /// Refreshing is single-flight across processes: the app, the daemon and+ /// the CLI share one store, and a rotated refresh token that a second+ /// process then presents makes the provider revoke the whole session. A+ /// file lock serialises refreshes and the store is re-read under it, so+ /// the second process uses the first one's result instead.+ pub async fn access_token(&self, force: bool) -> Result<String> {+ let mut guard = self.cached.lock().await;+ let stored = self.read_stored(&mut guard)?;let rotated = guard.as_ref().is_some_and(|c| c.access_token != stored.access_token);@@ -1494,24 +1611,31 @@ impl Session {*guard = Some(stored);return Ok(t);}- match self.oidc.refresh(&stored).await {+ let _lock = RefreshLock::acquire(&self.store).await?;+ // Another process may have refreshed while we waited for the lock.+ let current = self.read_stored(&mut guard)?;+ if current.access_token != stored.access_token && !current.needs_refresh() {+ let t = current.access_token.clone();+ *guard = Some(current);+ return Ok(t);+ }+ match self.oidc.refresh(¤t).await {Ok(n) => {- self.store.save(&n)?;+ // Persist before anything else: the old refresh token is+ // already spent.+ self.persist(&n).await;let t = n.access_token.clone();*guard = Some(n);Ok(t)}Err(Error::Unauthenticated(m)) => {- // Another process may have refreshed (and rotated the- // refresh token) while ours was in flight, so the provider- // refused the one we sent. Its fresh session is in the store- // now: use it rather than signing the user out. Only a store- // that still holds the refused session is cleared.+ // The grant is dead. Clear only a store that still holds the+ // refused session; one a sign-in just replaced is kept.match self.store.load() {- Ok(Some(current)) if current != stored => {- if !current.needs_refresh() {- let t = current.access_token.clone();- *guard = Some(current);+ Ok(Some(now)) if now != current => {+ if !now.needs_refresh() {+ let t = now.access_token.clone();+ *guard = Some(now);return Ok(t);}*guard = None;@@ -1522,9 +1646,11 @@ impl Session {let _ = self.store.clear();Err(Error::Unauthenticated(m))}- _ => {+ // Unreadable store: not proof the session is gone.+ Err(_) => Err(Error::Http(m)),+ Ok(None) => {*guard = None;- Err(Error::Unauthenticated(m))+ Err(Error::NotLoggedIn)}}}@@ -1532,6 +1658,50 @@ impl Session {}}+ /// Refreshes now when the access token expires within `window`, so a+ /// session that is not otherwise used stays alive. Errors are transient+ /// or already signed-out; callers ignore them.+ pub async fn refresh_if_due(&self, window: Duration) -> Result<()> {+ let due = match self.store.load() {+ Ok(Some(c)) => c.expires().map_or(true, |e| {+ e <= chrono::Utc::now() + chrono::Duration::from_std(window).unwrap_or_default()+ }),+ Ok(None) => return Ok(()),+ Err(_) => return Ok(()),+ };+ if due {+ self.access_token(true).await.map(|_| ())+ } else {+ Ok(())+ }+ }++ /// Keeps the session fresh in the background (proactive refresh, ahead of+ /// expiry and of the provider's idle timeout) until the session is+ /// dropped. A no-op outside a tokio runtime.+ pub fn spawn_keep_alive(self: &Arc<Self>) {+ let Ok(rt) = tokio::runtime::Handle::try_current() else {+ return;+ };+ let weak = Arc::downgrade(self);+ rt.spawn(async move {+ loop {+ tokio::time::sleep(KEEP_ALIVE_INTERVAL).await;+ let Some(s) = weak.upgrade() else { return };+ let _ = s.refresh_if_due(KEEP_ALIVE_WINDOW).await;+ }+ });+ }++ /// Whether the stored session holds an offline refresh token (granted+ /// `offline_access`), which outlives the provider's SSO session limits.+ /// `None` when unknown (nothing stored or no scope recorded).+ pub fn is_offline(&self) -> Option<bool> {+ let c = self.store.load().ok().flatten()?;+ let scope = c.scope?;+ Some(scope.split_whitespace().any(|s| s == "offline_access"))+ }+/// Revokes (best effort) and clears the session. Returns whether a/// session existed and whether the issuer confirmed revocation.pub async fn logout(&self) -> Result<(bool, bool)> {diff --git a/libs/cua/crates/cua-auth/tests/two_process.rs b/libs/cua/crates/cua-auth/tests/two_process.rsnew file mode 100644index 0000000000..4e166ed6c3--- /dev/null+++ b/libs/cua/crates/cua-auth/tests/two_process.rs@@ -0,0 +1,234 @@+//! Refresh across processes: the app, the daemon and the CLI share one+//! credential store. With refresh-token rotation and reuse detection, two of+//! them refreshing at once with the same token revokes the session. The+//! fake issuer here is strict that way: a refresh token works once, and+//! presenting a spent one revokes everything.+//!+//! The children are this test binary re-run (`CUA_TWO_PROCESS_CHILD`), file+//! store in a temp dir, never the OS keychain or ~/.cua.++use cua_auth::{Credentials, Oidc, Session, Store};+use std::io::{Read, Write};+use std::net::TcpListener;+use std::sync::{Arc, Mutex};++#[derive(Default)]+struct Issuer {+ /// Refresh tokens already spent.+ spent: Vec<String>,+ current: u32,+ refreshes: u32,+ reuse_detected: bool,+ /// Answer refreshes with this status instead (transient failures).+ fail_with: Option<u16>,+}++fn serve(st: Arc<Mutex<Issuer>>) -> String {+ let l = TcpListener::bind("127.0.0.1:0").unwrap();+ let base = format!("http://{}", l.local_addr().unwrap());+ let b = base.clone();+ std::thread::spawn(move || {+ for s in l.incoming() {+ let Ok(mut s) = s else { continue };+ let (st, base) = (st.clone(), b.clone());+ std::thread::spawn(move || {+ let mut buf = Vec::new();+ let mut tmp = [0u8; 4096];+ let body = loop {+ let n = s.read(&mut tmp).unwrap_or(0);+ if n == 0 {+ return;+ }+ buf.extend_from_slice(&tmp[..n]);+ if let Some(i) = buf.windows(4).position(|w| w == b"\r\n\r\n") {+ let head = String::from_utf8_lossy(&buf[..i]).to_lowercase();+ let len: usize = head+ .lines()+ .find_map(|l| l.strip_prefix("content-length:"))+ .and_then(|v| v.trim().parse().ok())+ .unwrap_or(0);+ if buf.len() >= i + 4 + len {+ break String::from_utf8_lossy(&buf[i + 4..i + 4 + len]).to_string();+ }+ }+ };+ let line = String::from_utf8_lossy(&buf)+ .lines()+ .next()+ .unwrap()+ .to_string();+ let (status, out) = if line.contains("openid-configuration") {+ (+ 200,+ serde_json::json!({ "token_endpoint": format!("{base}/token") }),+ )+ } else {+ let rt = url::form_urlencoded::parse(body.as_bytes())+ .find(|(k, _)| k == "refresh_token")+ .map(|(_, v)| v.into_owned())+ .unwrap_or_default();+ let mut g = st.lock().unwrap();+ std::thread::sleep(std::time::Duration::from_millis(150));+ if let Some(code) = g.fail_with {+ (+ code,+ serde_json::json!({ "error": "temporarily_unavailable" }),+ )+ } else if g.reuse_detected || g.spent.contains(&rt) {+ g.reuse_detected = true;+ (+ 400,+ serde_json::json!({ "error": "invalid_grant", "error_description": "Token reuse detected" }),+ )+ } else {+ g.spent.push(rt);+ g.current += 1;+ g.refreshes += 1;+ let n = g.current;+ (+ 200,+ serde_json::json!({ "access_token": format!("at-{n}"),+ "refresh_token": format!("rt-{n}"), "expires_in": 900,+ "scope": "openid profile offline_access" }),+ )+ }+ };+ let text = out.to_string();+ let _ = write!(+ s,+ "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{text}",+ text.len()+ );+ });+ }+ });+ base+}++fn old_creds() -> Credentials {+ Credentials {+ access_token: "at-0".into(),+ refresh_token: Some("rt-0".into()),+ expires_at: "2000-01-01T00:00:00Z".into(),+ token_type: "Bearer".into(),+ scope: Some("openid offline_access".into()),+ id_token: None,+ }+}++/// Child role: refresh through the shared store and report.+#[test]+fn child_process() {+ let (Ok(url), Ok(path)) = (+ std::env::var("CUA_TWO_PROCESS_ISSUER"),+ std::env::var("CUA_TWO_PROCESS_STORE"),+ ) else {+ return;+ };+ let rt = tokio::runtime::Runtime::new().unwrap();+ let s = Session::new(Oidc::new(url, "cua-cli"), Store::File(path.into()));+ let r = rt.block_on(s.access_token(true));+ println!(+ "CHILD_RESULT {}",+ r.map_or_else(|e| format!("ERR {e}"), |t| format!("OK {t}"))+ );+}++#[test]+fn concurrent_processes_refresh_once_and_nobody_is_signed_out() {+ let st = Arc::new(Mutex::new(Issuer::default()));+ let url = serve(st.clone());+ let d = tempfile::tempdir().unwrap();+ let path = d.path().join("credentials.json");+ Store::File(path.clone()).save(&old_creds()).unwrap();++ let exe = std::env::current_exe().unwrap();+ let kids: Vec<_> = (0..6)+ .map(|_| {+ std::process::Command::new(&exe)+ .args([+ "child_process",+ "--exact",+ "--nocapture",+ "--test-threads=1",+ ])+ .env("CUA_TWO_PROCESS_ISSUER", &url)+ .env("CUA_TWO_PROCESS_STORE", &path)+ .stdout(std::process::Stdio::piped())+ .spawn()+ .unwrap()+ })+ .collect();+ for k in kids {+ let out = k.wait_with_output().unwrap();+ let text = String::from_utf8_lossy(&out.stdout);+ assert!(text.contains("CHILD_RESULT OK"), "child failed: {text}");+ }+ let g = st.lock().unwrap();+ assert!(!g.reuse_detected, "a spent refresh token was presented");+ assert_eq!(g.refreshes, 1, "exactly one process refreshes");+ let stored = Store::File(path).load().unwrap().unwrap();+ assert_eq!(stored.refresh_token.as_deref(), Some("rt-1"));+}++#[tokio::test]+async fn transient_refresh_failures_keep_the_session() {+ for code in [429u16, 500, 503] {+ let st = Arc::new(Mutex::new(Issuer {+ fail_with: Some(code),+ ..Default::default()+ }));+ let url = serve(st);+ let d = tempfile::tempdir().unwrap();+ let path = d.path().join("credentials.json");+ let store = Store::File(path.clone());+ store.save(&old_creds()).unwrap();+ let s = Session::new(Oidc::new(url, "cua-cli"), Store::File(path));+ let e = s.access_token(false).await.unwrap_err();+ assert!(+ !matches!(+ e,+ cua_auth::Error::Unauthenticated(_) | cua_auth::Error::NotLoggedIn+ ),+ "{code}: {e}"+ );+ assert_eq!(+ store.load().unwrap(),+ Some(old_creds()),+ "{code} wiped the session"+ );+ }+}++#[tokio::test]+async fn an_unreadable_store_is_not_signed_out() {+ // A directory where the credentials file should be: reads fail with an+ // I/O error (like a locked keychain), which is not "no session".+ let st = Arc::new(Mutex::new(Issuer::default()));+ let url = serve(st.clone());+ let d = tempfile::tempdir().unwrap();+ let path = d.path().join("credentials.json");+ std::fs::create_dir(&path).unwrap();+ let s = Session::new(Oidc::new(url, "cua-cli"), Store::File(path.clone()));+ let e = s.access_token(false).await.unwrap_err();+ assert!(matches!(e, cua_auth::Error::Store(_)), "{e}");+ assert!(path.is_dir());+ assert_eq!(st.lock().unwrap().refreshes, 0);+}++#[tokio::test]+async fn a_dead_grant_still_signs_out() {+ let st = Arc::new(Mutex::new(Issuer::default()));+ st.lock().unwrap().reuse_detected = true;+ let url = serve(st);+ let d = tempfile::tempdir().unwrap();+ let path = d.path().join("credentials.json");+ let store = Store::File(path.clone());+ store.save(&old_creds()).unwrap();+ let s = Session::new(Oidc::new(url, "cua-cli"), Store::File(path));+ assert!(matches!(+ s.access_token(false).await,+ Err(cua_auth::Error::Unauthenticated(_))+ ));+ assert!(store.load().unwrap().is_none());+}diff --git a/libs/cua/crates/cua-sdk/src/native/auth.rs b/libs/cua/crates/cua-sdk/src/native/auth.rsindex 49618d1d6d..3c241b61d1 100644--- a/libs/cua/crates/cua-sdk/src/native/auth.rs+++ b/libs/cua/crates/cua-sdk/src/native/auth.rs@@ -201,9 +201,11 @@ impl Auth {/// On an explicit session (Rust hosts and tests: a file store in a/// temporary directory and a fake issuer).pub fn with_session(session: cua_auth::Session) -> Arc<Self> {- Arc::new(Auth {- session: Arc::new(session),- })+ let session = Arc::new(session);+ // Refresh ahead of expiry in long-lived hosts (the app, the daemon)+ // instead of only when a call finds the token expired.+ session.spawn_keep_alive();+ Arc::new(Auth { session })}/// The underlying session (Rust hosts: a Fleet token provider).