Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-relay): mint a new service token on a 401 and refresh at half-life (CUA-2959) (#6348)
cua-agents[bot]eed1e7830867
diff --git a/libs/cua-spacesd/crates/cua-relay/src/auth_log.rs b/libs/cua-spacesd/crates/cua-relay/src/auth_log.rsindex cfcccc89f2..5be15466e2 100644--- a/libs/cua-spacesd/crates/cua-relay/src/auth_log.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/auth_log.rs@@ -22,6 +22,8 @@//! Tokens, token fragments, emails, account and user ids, IP addresses,//! machine ids and device names are never logged.+use std::sync::atomic::{AtomicU64, Ordering};+use axum::http::{header, HeaderMap};/// The `tracing` target of the 401 log lines.@@ -77,6 +79,34 @@ pub enum Reason {}impl Reason {+ /// Every reason, in [`Self::index`] order.+ pub const ALL: [Reason; 20] = [+ Self::MissingToken,+ Self::Malformed,+ Self::UnsupportedAlgorithm,+ Self::UnknownSigningKey,+ Self::JwksUnavailable,+ Self::BadSignature,+ Self::WrongIssuer,+ Self::WrongAudience,+ Self::Expired,+ Self::NotYetValid,+ Self::MissingClaim,+ Self::UnknownMachineToken,+ Self::InvalidRelayToken,+ Self::InvalidAdminToken,+ Self::DeviceProofStale,+ Self::DeviceProofBadSignature,+ Self::DeviceProofReplayed,+ Self::SpacesdCredentialMissing,+ Self::InvalidPairingCode,+ Self::TicketRevoked,+ ];++ fn index(self) -> usize {+ Self::ALL.iter().position(|r| *r == self).unwrap_or(0)+ }+/// The category as logged.pub fn as_str(self) -> &'static str {match self {@@ -120,6 +150,19 @@ pub enum Route {}impl Route {+ /// Every route, in [`Self::index`] order.+ pub const ALL: [Route; 5] = [+ Self::Connect,+ Self::Admin,+ Self::Machines,+ Self::Devices,+ Self::Proxy,+ ];++ fn index(self) -> usize {+ Self::ALL.iter().position(|r| *r == self).unwrap_or(0)+ }+/// The class as logged.pub fn as_str(self) -> &'static str {match self {@@ -270,6 +313,7 @@ pub fn client_kind(headers: &HeaderMap) -> &'static str {/// Logs one 401.pub fn rejected(headers: &HeaderMap, route: Route, reason: Reason, facts: Option<&TokenFacts>) {+ REJECTIONS[route.index()][reason.index()].fetch_add(1, Ordering::Relaxed);let facts = facts.cloned().unwrap_or_default();tracing::info!(target: TARGET,@@ -283,6 +327,35 @@ pub fn rejected(headers: &HeaderMap, route: Route, reason: Reason, facts: Option);}+/// 401s answered, by route and reason (process lifetime).+static REJECTIONS: [[AtomicU64; Reason::ALL.len()]; Route::ALL.len()] =+ [const { [const { AtomicU64::new(0) }; Reason::ALL.len()] }; Route::ALL.len()];++/// `cua_relay_auth_rejections_total{route,reason}` in Prometheus text+/// format, one line per pair seen at least once.+///+/// These are the relay refusing its own callers, kept apart from+/// `cua_relay_org_api_errors_total` (the relay's calls to the Spaces API).+/// Most are expected: `unknown_machine_token` and `ticket_revoked` are a+/// deleted, rotated or revoked machine or device still reconnecting,+/// `expired` a client that has not refreshed yet.+pub fn metrics() -> String {+ let mut out = String::from("# TYPE cua_relay_auth_rejections_total counter\n");+ for route in Route::ALL {+ for reason in Reason::ALL {+ let n = REJECTIONS[route.index()][reason.index()].load(Ordering::Relaxed);+ if n > 0 {+ out.push_str(&format!(+ "cua_relay_auth_rejections_total{{route=\"{}\",reason=\"{}\"}} {n}\n",+ route.as_str(),+ reason.as_str()+ ));+ }+ }+ }+ out+}+/// Logs one `cua share` web event (`web_login_ok`, `web_login_failed`,/// `web_session_issued`, `web_session_refused`, `public_rate_limited`, …)/// with its coarse `detail`. Like every line here it carries no token,@@ -306,6 +379,44 @@ mod tests {)}+ #[test]+ fn every_reason_and_route_has_its_own_slot() {+ let reasons: std::collections::HashSet<_> =+ Reason::ALL.iter().map(|r| r.as_str()).collect();+ assert_eq!(reasons.len(), Reason::ALL.len());+ for (i, r) in Reason::ALL.iter().enumerate() {+ assert_eq!(r.index(), i);+ }+ for (i, r) in Route::ALL.iter().enumerate() {+ assert_eq!(r.index(), i);+ }+ }++ #[test]+ fn revoked_machines_are_counted_apart_by_reason() {+ let before = REJECTIONS[Route::Connect.index()][Reason::UnknownMachineToken.index()]+ .load(Ordering::Relaxed);+ rejected(+ &HeaderMap::new(),+ Route::Connect,+ Reason::UnknownMachineToken,+ None,+ );+ rejected(+ &HeaderMap::new(),+ Route::Connect,+ Reason::UnknownMachineToken,+ None,+ );+ let m = metrics();+ let want = format!(+ "cua_relay_auth_rejections_total{{route=\"connect\",reason=\"unknown_machine_token\"}} {}\n",+ before + 2+ );+ assert!(m.contains(&want), "{m}");+ assert!(!m.contains("org_api_errors"), "{m}");+ }+#[test]fn facts_keep_only_public_issuer_and_audience() {let audiences = vec!["cua-relay".to_owned()];diff --git a/libs/cua-spacesd/crates/cua-relay/src/org_access.rs b/libs/cua-spacesd/crates/cua-relay/src/org_access.rsindex ce967e60a8..6ca64dd679 100644--- a/libs/cua-spacesd/crates/cua-relay/src/org_access.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/org_access.rs@@ -85,6 +85,25 @@ pub const DEFAULT_STALE_WINDOW: Duration = Duration::from_secs(15 * 60);const MAX_ENTRIES: usize = 50_000;/// How often the clean-up of outside shares runs.pub const SWEEP_EVERY: Duration = Duration::from_secs(24 * 3600);+/// Start of the error for a 401 the Spaces API gave a freshly minted+/// service token: a configuration problem, never worth asking again.+pub const TOKEN_REFUSED: &str = "Spaces API refused the relay's service token";++/// How long a service token is used before a new one is minted: half its+/// lifetime (`expires_in`, 60 s when absent), at least one second.+///+/// The relay cannot see the issuer's or the API's clock, only the+/// lifetime. Keycloak stamps `exp` from its own clock and the API checks it+/// against its own (30 s leeway), so a token used until shortly before its+/// nominal end is refused whenever the issuer runs behind the API by more+/// than the remaining margin plus the leeway (CUA-2959: a node 60 s slow,+/// refusals in the last second of every 270 s token). Half the lifetime+/// (150 s for the relay's 300 s tokens) leaves room for minutes of skew,+/// the usual OAuth client practice; a refusal past that is still+/// recovered by [`OrgAccess::send_authed`].+pub fn token_refresh_after(expires_in: Option<u64>) -> Duration {+ Duration::from_millis(expires_in.unwrap_or(60).saturating_mul(500).max(1000))+}/// The clock cache expiry reads (see [`crate::clock::Clock`]).pub use crate::clock::Clock;@@ -245,6 +264,13 @@ pub struct OrgCounters {/// Answers served from the last-known-good cache during an outage.pub stale_served: AtomicU64,pub membership_pushes: AtomicU64,+ /// Service tokens minted at the token endpoint.+ pub token_fetches: AtomicU64,+ /// Mints that failed (token endpoint down, secret refused).+ pub token_errors: AtomicU64,+ /// 401s to a cached service token, answered by minting a new one and+ /// asking again (not API errors; see [`OrgAccess::send_authed`]).+ pub token_refused: AtomicU64,}/// The organization access path: API client plus caches.@@ -362,42 +388,107 @@ impl OrgAccess {None}- /// The relay's service token (client credentials), cached until shortly- /// before it expires. `None` without a token endpoint or secret (a test- /// API without auth).+ /// The relay's service token (client credentials), cached and minted+ /// again at half its lifetime (see [`token_refresh_after`]). `None`+ /// without a token endpoint or secret (a test API without auth).async fn service_token(&self) -> Result<Option<String>, String> {+ self.service_token_replacing(None).await+ }++ /// Like [`Self::service_token`], but when the cached token is+ /// `refused` (the API just answered 401 to it), mints a new one. Callers+ /// that were refused together mint once: whoever comes second finds a+ /// different, fresh token already cached and uses it.+ async fn service_token_replacing(+ &self,+ refused: Option<&str>,+ ) -> Result<Option<String>, String> {let (Some(url), Some(secret)) = (&self.config.token_url, &self.config.client_secret) else {return Ok(None);};let mut cached = self.token.lock().await;- if let Some((token, until)) = cached.as_ref() {- if Instant::now() < *until {+ if let Some((token, refresh_at)) = cached.as_ref() {+ if Instant::now() < *refresh_at && refused != Some(token.as_str()) {return Ok(Some(token.clone()));}}- let reply = self- .http- .post(url)- .form(&[- ("grant_type", "client_credentials"),- ("client_id", self.config.client_id.as_str()),- ("client_secret", secret.as_str()),- ])- .send()- .await- .map_err(|e| format!("service token: {}", error_chain(&e.without_url())))?;- if !reply.status().is_success() {- return Err(format!("service token: HTTP {}", reply.status()));+ *cached = None;+ self.counters.token_fetches.fetch_add(1, Ordering::Relaxed);+ let minted = async {+ let reply = self+ .http+ .post(url)+ .form(&[+ ("grant_type", "client_credentials"),+ ("client_id", self.config.client_id.as_str()),+ ("client_secret", secret.as_str()),+ ])+ .send()+ .await+ .map_err(|e| format!("service token: {}", error_chain(&e.without_url())))?;+ if !reply.status().is_success() {+ return Err(format!("service token: HTTP {}", reply.status()));+ }+ reply+ .json::<TokenReply>()+ .await+ .map_err(|e| format!("service token: {}", e.without_url()))}- let t: TokenReply = reply- .json()- .await- .map_err(|e| format!("service token: {e}"))?;- let life = Duration::from_secs(t.expires_in.unwrap_or(60).saturating_sub(30).max(5));- *cached = Some((t.access_token.clone(), Instant::now() + life));+ .await;+ let t = minted.inspect_err(|_| {+ self.counters.token_errors.fetch_add(1, Ordering::Relaxed);+ })?;+ *cached = Some((+ t.access_token.clone(),+ Instant::now() + token_refresh_after(t.expires_in),+ ));Ok(Some(t.access_token))}+ /// Sends the request `build` makes with the service token. A 401 means+ /// the API refused that token: it is minted again and the request sent+ /// once more, so a token the API considers expired before the relay+ /// does (the issuer's clock behind the API's, CUA-2959) costs one extra+ /// round trip, not an error. Only a 401 to a freshly minted token is+ /// returned as such ([`TOKEN_REFUSED`]): that is a configuration+ /// problem (client secret, `SPACES_CP_SERVICE_CLIENTS`), not something+ /// to retry.+ async fn send_authed(+ &self,+ build: impl Fn(&reqwest::Client) -> reqwest::RequestBuilder,+ ) -> Result<reqwest::Response, String> {+ let mut token = self.service_token().await?;+ let mut refreshed = false;+ loop {+ let mut req = build(&self.http);+ if let Some(t) = &token {+ req = req.bearer_auth(t);+ }+ self.counters.api_calls.fetch_add(1, Ordering::Relaxed);+ let r = req.send().await.map_err(|e| {+ self.counters.api_errors.fetch_add(1, Ordering::Relaxed);+ format!("Spaces API: {}", error_chain(&e.without_url()))+ })?;+ if r.status() != reqwest::StatusCode::UNAUTHORIZED {+ return Ok(r);+ }+ let Some(used) = token.as_deref() else {+ // No service token configured: nothing to refresh.+ self.counters.api_errors.fetch_add(1, Ordering::Relaxed);+ return Err(format!(+ "{TOKEN_REFUSED} (HTTP 401, no service token configured)"+ ));+ };+ if refreshed {+ self.counters.api_errors.fetch_add(1, Ordering::Relaxed);+ return Err(format!("{TOKEN_REFUSED} (HTTP 401 to a fresh token)"));+ }+ self.counters.token_refused.fetch_add(1, Ordering::Relaxed);+ token = self.service_token_replacing(Some(used)).await?;+ refreshed = true;+ }+ }+pub(crate) async fn get(&self,path: &str,@@ -408,22 +499,17 @@ impl OrgAccess {.api_url.as_deref().ok_or_else(|| "no Spaces API configured".to_owned())?;- let token = self.service_token().await?;- let mut req = self- .http- .get(format!("{}{path}", base.trim_end_matches('/')));- if let Some(t) = token {- req = req.bearer_auth(t);- }- if let Some(e) = etag {- req = req.header(reqwest::header::IF_NONE_MATCH, e);- }- self.counters.api_calls.fetch_add(1, Ordering::Relaxed);- let r = req.send().await.map_err(|e| {- self.counters.api_errors.fetch_add(1, Ordering::Relaxed);- format!("Spaces API: {}", error_chain(&e.without_url()))- })?;- if r.status().is_server_error() || r.status() == reqwest::StatusCode::UNAUTHORIZED {+ let url = format!("{}{path}", base.trim_end_matches('/'));+ let r = self+ .send_authed(|http| {+ let req = http.get(&url);+ match etag {+ Some(e) => req.header(reqwest::header::IF_NONE_MATCH, e),+ None => req,+ }+ })+ .await?;+ if r.status().is_server_error() {self.counters.api_errors.fetch_add(1, Ordering::Relaxed);return Err(format!("Spaces API: HTTP {}", r.status()));}@@ -446,19 +532,8 @@ impl OrgAccess {.api_url.as_deref().ok_or_else(|| "no Spaces API configured".to_owned())?;- let token = self.service_token().await?;- let mut req = self- .http- .post(format!("{}{path}", base.trim_end_matches('/')))- .json(body);- if let Some(t) = token {- req = req.bearer_auth(t);- }- self.counters.api_calls.fetch_add(1, Ordering::Relaxed);- let r = req.send().await.map_err(|e| {- self.counters.api_errors.fetch_add(1, Ordering::Relaxed);- format!("Spaces API: {}", e.without_url())- })?;+ let url = format!("{}{path}", base.trim_end_matches('/'));+ let r = self.send_authed(|http| http.post(&url).json(body)).await?;if !r.status().is_success() {self.counters.api_errors.fetch_add(1, Ordering::Relaxed);}@@ -477,21 +552,10 @@ impl OrgAccess {.api_url.as_deref().ok_or_else(|| "no Spaces API configured".to_owned())?;- let token = self.service_token().await?;- let mut req = self- .http- .post(format!("{}{path}", base.trim_end_matches('/')))- .json(body);- if let Some(t) = token {- req = req.bearer_auth(t);- }- self.counters.api_calls.fetch_add(1, Ordering::Relaxed);- let r = req.send().await.map_err(|e| {- self.counters.api_errors.fetch_add(1, Ordering::Relaxed);- format!("Spaces API: {}", e.without_url())- })?;+ let url = format!("{}{path}", base.trim_end_matches('/'));+ let r = self.send_authed(|http| http.post(&url).json(body)).await?;let status = r.status();- if status.is_server_error() || status == reqwest::StatusCode::UNAUTHORIZED {+ if status.is_server_error() {self.counters.api_errors.fetch_add(1, Ordering::Relaxed);return Err(format!("Spaces API: HTTP {status}"));}@@ -1012,7 +1076,10 @@ impl OrgAccess {# TYPE cua_relay_org_stale_served_total counter\ncua_relay_org_stale_served_total {}\n\# TYPE cua_relay_teams_shadow_checks_total counter\ncua_relay_teams_shadow_checks_total {}\n\# TYPE cua_relay_teams_shadow_mismatches_total counter\ncua_relay_teams_shadow_mismatches_total {}\n\- # TYPE cua_relay_org_membership_pushes_total counter\ncua_relay_org_membership_pushes_total {}\n",+ # TYPE cua_relay_org_membership_pushes_total counter\ncua_relay_org_membership_pushes_total {}\n\+ # TYPE cua_relay_org_service_token_fetches_total counter\ncua_relay_org_service_token_fetches_total {}\n\+ # TYPE cua_relay_org_service_token_errors_total counter\ncua_relay_org_service_token_errors_total {}\n\+ # TYPE cua_relay_org_service_token_refused_total counter\ncua_relay_org_service_token_refused_total {}\n",self.config.source.as_str(),get(&c.api_calls),get(&c.api_errors),@@ -1022,6 +1089,9 @@ impl OrgAccess {get(&c.shadow_checks),get(&c.shadow_mismatches),get(&c.membership_pushes),+ get(&c.token_fetches),+ get(&c.token_errors),+ get(&c.token_refused),)}}@@ -1062,6 +1132,7 @@ fn error_chain(e: &dyn std::error::Error) -> String {#[cfg(test)]mod tests {use super::*;+ use std::sync::Arc;#[test]fn parses_sources() {@@ -1078,6 +1149,141 @@ mod tests {assert!(machine_ids(&serde_json::json!({})).is_empty());}+ #[test]+ fn service_tokens_are_replaced_at_half_their_lifetime() {+ assert_eq!(token_refresh_after(Some(300)), Duration::from_secs(150));+ assert_eq!(token_refresh_after(None), Duration::from_secs(30));+ assert_eq!(token_refresh_after(Some(0)), Duration::from_secs(1));+ // The CUA-2959 case: Keycloak 60 s behind the API, 30 s leeway. The+ // token must be replaced before the API (true clock) sees it as+ // expired: at 300 - 60 + 30 = 270 s of real age.+ assert!(token_refresh_after(Some(300)) < Duration::from_secs(270 - 60));+ }++ /// A fake token endpoint minting `t1`, `t2`, ... and a fake Spaces API+ /// answering `/internal/v1/entitlements/x` 200 only to the tokens in+ /// `accepted` (401 otherwise). Returns (base URL, mints, API hits).+ async fn fake_auth_api(+ accepted: &'static [&'static str],+ ) -> (String, Arc<AtomicU64>, Arc<AtomicU64>) {+ let mints = Arc::new(AtomicU64::new(0));+ let hits = Arc::new(AtomicU64::new(0));+ let (m, h) = (mints.clone(), hits.clone());+ let app = axum::Router::new()+ .route(+ "/token",+ axum::routing::post(move || {+ let n = m.fetch_add(1, Ordering::Relaxed) + 1;+ async move {+ axum::Json(serde_json::json!({+ "access_token": format!("t{n}"), "expires_in": 300+ }))+ }+ }),+ )+ .route(+ "/internal/v1/entitlements/x",+ axum::routing::get(move |headers: axum::http::HeaderMap| {+ h.fetch_add(1, Ordering::Relaxed);+ async move {+ let auth = headers+ .get("authorization")+ .and_then(|v| v.to_str().ok())+ .unwrap_or("");+ if accepted.iter().any(|t| auth == format!("Bearer {t}")) {+ (axum::http::StatusCode::OK, r#"{"plan":"pro"}"#)+ } else {+ (+ axum::http::StatusCode::UNAUTHORIZED,+ r#"{"error":"unauthorized"}"#,+ )+ }+ }+ }),+ );+ let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let url = format!("http://{}", l.local_addr().unwrap());+ tokio::spawn(async move { axum::serve(l, app).await.unwrap() });+ (url, mints, hits)+ }++ fn authed_org(url: &str) -> OrgAccess {+ OrgAccess::new(OrgAccessConfig {+ api_url: Some(url.to_owned()),+ token_url: Some(format!("{url}/token")),+ client_id: "cua-relay-svc".into(),+ client_secret: Some("test-secret".into()),+ ..Default::default()+ })+ }++ #[tokio::test]+ async fn a_refused_cached_token_is_replaced_and_not_an_api_error() {+ // The API refuses t1 (it thinks it expired), accepts t2.+ let (url, mints, hits) = fake_auth_api(&["t2"]).await;+ let org = authed_org(&url);+ let r = org.get("/internal/v1/entitlements/x", None).await.unwrap();+ assert_eq!(r.status(), reqwest::StatusCode::OK);+ assert_eq!(mints.load(Ordering::Relaxed), 2);+ assert_eq!(hits.load(Ordering::Relaxed), 2);+ assert_eq!(org.counters.api_errors.load(Ordering::Relaxed), 0);+ assert_eq!(org.counters.token_refused.load(Ordering::Relaxed), 1);+ // The new token is cached: the next call mints nothing.+ org.get("/internal/v1/entitlements/x", None).await.unwrap();+ assert_eq!(mints.load(Ordering::Relaxed), 2);+ let m = org.metrics();+ assert!(m.contains("cua_relay_org_api_errors_total 0\n"), "{m}");+ assert!(+ m.contains("cua_relay_org_service_token_refused_total 1\n"),+ "{m}"+ );+ assert!(+ m.contains("cua_relay_org_service_token_fetches_total 2\n"),+ "{m}"+ );+ }++ #[tokio::test]+ async fn a_refused_fresh_token_is_one_api_error_and_final() {+ // A misconfigured client: every token is refused.+ let (url, mints, hits) = fake_auth_api(&[]).await;+ let org = Arc::new(authed_org(&url));+ let err = org+ .get("/internal/v1/entitlements/x", None)+ .await+ .unwrap_err();+ assert!(err.starts_with(TOKEN_REFUSED), "{err}");+ // One retry with a fresh token, never more.+ assert_eq!(mints.load(Ordering::Relaxed), 2);+ assert_eq!(hits.load(Ordering::Relaxed), 2);+ assert_eq!(org.counters.api_errors.load(Ordering::Relaxed), 1);++ // The plan lookup does not ask again on top (no 6x fan-out).+ let plans = crate::plan_source::EntitlementPlans::new(org.clone(), None);+ hits.store(0, Ordering::Relaxed);+ plans.warm("x", Duration::from_secs(5)).await;+ assert_eq!(hits.load(Ordering::Relaxed), 2);+ }++ #[tokio::test]+ async fn concurrent_refusals_mint_one_new_token() {+ let (url, mints, _) = fake_auth_api(&["t2"]).await;+ let org = Arc::new(authed_org(&url));+ // Cache t1 first.+ let _ = org.service_token().await.unwrap();+ let calls: Vec<_> = (0..4)+ .map(|_| {+ let org = org.clone();+ tokio::spawn(async move { org.get("/internal/v1/entitlements/x", None).await })+ })+ .collect();+ for c in calls {+ assert!(c.await.unwrap().is_ok());+ }+ assert_eq!(mints.load(Ordering::Relaxed), 2);+ assert_eq!(org.counters.api_errors.load(Ordering::Relaxed), 0);+ }+#[tokio::test]async fn no_api_fails_closed() {let org = OrgAccess::new(OrgAccessConfig {diff --git a/libs/cua-spacesd/crates/cua-relay/src/plan_source.rs b/libs/cua-spacesd/crates/cua-relay/src/plan_source.rsindex ad0f0c6fca..045e40c4a5 100644--- a/libs/cua-spacesd/crates/cua-relay/src/plan_source.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/plan_source.rs@@ -27,7 +27,7 @@ use std::time::{Duration, Instant};use serde::Deserialize;use crate::meter::{Assignment, Membership, PlanSource, TeamPlan, DEFAULT_PLAN};-use crate::org_access::OrgAccess;+use crate::org_access::{OrgAccess, TOKEN_REFUSED};/// Default lifetime of a cached plan.pub const DEFAULT_TTL: Duration = Duration::from_secs(60);@@ -462,8 +462,11 @@ impl Inner {}/// [`Self::fetch`], asked again with a short backoff after a transport- /// error, a 5xx or a refused service token. The permit is held per try,- /// never while waiting, so a retry cannot starve other users' lookups.+ /// error or a 5xx. A 401 is not retried here: the client already minted+ /// a new service token and asked again once ([`OrgAccess`]'s+ /// `send_authed`), so one still refused is a configuration problem. The+ /// permit is held per try, never while waiting, so a retry cannot+ /// starve other users' lookups.async fn fetch_with_retry(&self, user: &str) -> Result<Assignment, String> {let mut wait = FETCH_BACKOFF;let mut last = Err("no lookup made".to_owned());@@ -472,9 +475,11 @@ impl Inner {let _permit = self.permits.acquire().await;self.fetch(user).await};- // Only a transport error, a 5xx or a refused token is worth asking- // again; a 4xx (429, 403) or an unreadable answer is final.- let final_answer = matches!(&last, Err(e) if e.starts_with(ANSWERED));+ // Only a transport error or a 5xx is worth asking again; a 4xx+ // (429, 403, a refused fresh token) or an unreadable answer is+ // final.+ let final_answer = matches!(&last,+ Err(e) if e.starts_with(ANSWERED) || e.starts_with(TOKEN_REFUSED));if last.is_ok() || final_answer || attempt == FETCH_ATTEMPTS || !self.org.has_api() {break;}diff --git a/libs/cua-spacesd/crates/cua-relay/src/server.rs b/libs/cua-spacesd/crates/cua-relay/src/server.rsindex 29cf369e1c..f849d1fd34 100644--- a/libs/cua-spacesd/crates/cua-relay/src/server.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/server.rs@@ -1440,6 +1440,7 @@ async fn metrics(State(relay): State<Relay>) -> Response {.unwrap_or_default();text.push_str(&relay.conns.render());text.push_str(&relay.mesh.metrics());+ text.push_str(&crate::auth_log::metrics());#[cfg(feature = "hosted")]{text.push_str(&relay.org.metrics());