Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(spaces): retry 'too many requests' and say so in plain words (CUA-2275) (#5929)
cua-agents[bot]c421b1caa948
diff --git a/apps/cua-spaces-desktop/src/model/host-failure.ts b/apps/cua-spaces-desktop/src/model/host-failure.tsindex c4b54f3a57..7fc30dcb94 100644--- a/apps/cua-spaces-desktop/src/model/host-failure.ts+++ b/apps/cua-spaces-desktop/src/model/host-failure.ts@@ -9,7 +9,7 @@// that runs the host service.import type { PresentedFailure } from "../bridge/host";-export type HostFailureKind = "download" | "network" | "guiSession" | "signedOut" | "service" | "other";+export type HostFailureKind = "download" | "network" | "throttled" | "guiSession" | "signedOut" | "service" | "other";export interface HostSetupFailure extends PresentedFailure {kind: HostFailureKind;@@ -27,8 +27,22 @@ export function thisComputer(platform: NodeJS.Platform): string {const SYSTEM: Partial<Record<NodeJS.Platform, string>> = { darwin: "macOS", win32: "Windows", linux: "Linux" };+/**+ * Seconds to wait when the raw error is the server telling us to slow down+ * (the host's own sentence, or a bare HTTP 429 "Too Many Requests" from an+ * older host): a number, null for "unknown", undefined when it is not that.+ * Only the generic throttle: a 429 that says why (a limit, a quota) is not.+ */+export function throttledWait(raw: string): number | null | undefined {+ const own = /too many attempts right now\. wait (\d+) seconds?/i.exec(raw);+ if (own) return Number(own[1]);+ if (/\bhttp 429\b[:\s]*too many requests\s*$/i.test(raw.trim())) return null;+ return undefined;+}+/** Which kind of failure the raw setup error is. */export function classify(raw: string): HostFailureKind {+ if (throttledWait(raw) !== undefined) return "throttled";const s = raw.toLowerCase();const any = (needles: string[]) => needles.some((n) => s.includes(n));const word = (w: string) => new RegExp(`\\b${w}\\b`).test(s);@@ -66,7 +80,7 @@ export function classify(raw: string): HostFailureKind {}/** The title and message for a kind. */-export function failureWords(kind: HostFailureKind, platform: NodeJS.Platform = "darwin"): [string, string] {+export function failureWords(kind: HostFailureKind, platform: NodeJS.Platform = "darwin", raw = ""): [string, string] {const here = thisComputer(platform);const system = SYSTEM[platform] ?? "The system";const remote = platform === "darwin" ? "Screen Sharing" : "Remote Desktop";@@ -79,6 +93,13 @@ export function failureWords(kind: HostFailureKind, platform: NodeJS.Platform =];case "network":return ["Couldn’t connect", "Cua Spaces couldn’t reach the internet. Check your connection and try again."];+ case "throttled": {+ const secs = throttledWait(raw);+ return [+ "Too many attempts right now",+ secs ? `Trying again in ${secs} ${secs === 1 ? "second" : "seconds"}.` : "Wait a few seconds, then try again.",+ ];+ }case "guiSession":return [`Sign in to ${here} first`,@@ -102,6 +123,6 @@ export function failureWords(kind: HostFailureKind, platform: NodeJS.Platform =/** The failure for a raw error (its kind read from the words, or given: the account refused, whatever its words). */export function presentHostFailure(raw: string, platform: NodeJS.Platform = "darwin", kind: HostFailureKind = classify(raw)): HostSetupFailure {- const [title, message] = failureWords(kind, platform);+ const [title, message] = failureWords(kind, platform, raw);return { kind, title, message, details: raw, actionLabel: kind === "signedOut" ? SIGN_IN_LABEL : RETRY_LABEL };}diff --git a/apps/cua-spaces-desktop/test/host-failure.test.ts b/apps/cua-spaces-desktop/test/host-failure.test.tsindex 5b5c127f4a..c650440368 100644--- a/apps/cua-spaces-desktop/test/host-failure.test.ts+++ b/apps/cua-spaces-desktop/test/host-failure.test.ts@@ -40,6 +40,29 @@ describe("host setup failures", () => {expect(f.message).not.toContain("http");});+ it("a throttled setup says how long to wait, in plain words", () => {+ const raw = "http: Too many attempts right now. Wait 3 seconds, then try again.";+ const f = presentHostFailure(raw);+ expect(f.kind).toBe("throttled");+ expect(f.title).toBe("Too many attempts right now");+ expect(f.message).toBe("Trying again in 3 seconds.");+ expect(f.details).toBe(raw);+ expect(f.actionLabel).toBe("Retry");+ expect(presentHostFailure("Too many attempts right now. Wait 1 second, then try again.").message).toBe("Trying again in 1 second.");+ });++ it("a raw HTTP 429 from an older host is the same, without a number and without the raw words", () => {+ const f = presentHostFailure("http: relay: HTTP 429: Too Many Requests");+ expect(f.kind).toBe("throttled");+ expect(f.message).toBe("Wait a few seconds, then try again.");+ expect(f.message).not.toMatch(/429|http|relay/i);+ });++ it.each(["http: relay: HTTP 429: computer limit for this token reached", "HTTP 429: at most 5 unused pairing codes", "You're using 5 of 5 live sessions on Pro. Close one or upgrade to Teams."])(+ "a limit the relay explains is not a throttle: %s",+ (raw) => expect(classify(raw)).toBe("other"),+ );+it.each(["launchctl bootstrap gui/501: Bootstrap failed: 125: Domain does not support specified action (no Aqua session)", "no GUI session: sign in at the console"])("no one at the screen: %s",(raw) => {diff --git a/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Models/HostSetupFailure.swift b/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Models/HostSetupFailure.swiftindex d813cb69ca..8241143871 100644--- a/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Models/HostSetupFailure.swift+++ b/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Models/HostSetupFailure.swift@@ -12,6 +12,8 @@ public struct HostSetupFailure: Equatable, Sendable {case download/// No network, a timeout, or a server that could not be reached.case network+ /// The server is throttling this Mac (HTTP 429): too many attempts.+ case throttled/// No one is signed in at this Mac's screen (launchd's Aqua session).case guiSession/// The Cua account is signed out or its session expired.@@ -36,7 +38,7 @@ public struct HostSetupFailure: Equatable, Sendable {/// A failure whose kind the error itself says (the account refused,/// whatever its words), with the raw error as details.public static func presenting(_ raw: String, as kind: Kind) -> HostSetupFailure {- let (title, message) = words(kind)+ let (title, message) = words(kind, raw: raw)return HostSetupFailure(kind: kind, title: title, message: message, details: raw)}@@ -47,11 +49,42 @@ public struct HostSetupFailure: Equatable, Sendable {/// Maps the raw setup error to what the sheet shows.public static func presenting(_ raw: String) -> HostSetupFailure {let kind = classify(raw)- let (title, message) = words(kind)+ let (title, message) = words(kind, raw: raw)return HostSetupFailure(kind: kind, title: title, message: message, details: raw)}+ /// Seconds to wait when the raw error is the server telling this Mac to+ /// slow down: the host's own sentence, or a bare HTTP 429 "Too Many+ /// Requests" from an older host (`.some(nil)`, no number). `nil` when it+ /// is not that. A 429 that says why (a limit, a quota) is not a throttle.+ static func throttledWait(_ raw: String) -> Int?? {+ let own = #"too many attempts right now\. wait (\d+) seconds?"#+ if let r = raw.range(of: own, options: [.regularExpression, .caseInsensitive]) {+ let digits = raw[r].filter(\.isNumber)+ return .some(Int(digits))+ }+ let bare = #"\bhttp 429\b[:\s]*too many requests\s*$"#+ if raw.trimmingCharacters(in: .whitespacesAndNewlines)+ .range(of: bare, options: [.regularExpression, .caseInsensitive]) != nil {+ return .some(nil)+ }+ return nil+ }++ /// Seconds before the app tries again by itself, when the server is+ /// throttling this Mac and said how long to wait.+ public var throttleSeconds: Int? {+ kind == .throttled ? (Self.throttledWait(details) ?? nil) : nil+ }++ /// The line once the one automatic retry is used up.+ static func manualWait(_ f: HostSetupFailure) -> String {+ guard let s = f.throttleSeconds else { return f.message }+ return "Wait \(s) \(s == 1 ? "second" : "seconds"), then try again."+ }+static func classify(_ raw: String) -> Kind {+ if throttledWait(raw) != nil { return .throttled }let s = raw.lowercased()func any(_ needles: [String]) -> Bool { needles.contains { s.contains($0) } }func word(_ pattern: String) -> Bool {@@ -84,7 +117,7 @@ public struct HostSetupFailure: Equatable, Sendable {return .other}- static func words(_ kind: Kind) -> (String, String) {+ static func words(_ kind: Kind, raw: String = "") -> (String, String) {switch kind {case .download:return ("Couldn\u{2019}t download the Cua host service",@@ -93,6 +126,11 @@ public struct HostSetupFailure: Equatable, Sendable {case .network:return ("Couldn\u{2019}t connect","Cua Spaces couldn\u{2019}t reach the internet. Check your connection and try again.")+ case .throttled:+ let seconds = throttledWait(raw) ?? nil+ let wait = seconds.map { "Trying again in \($0) \($0 == 1 ? "second" : "seconds")." }+ ?? "Wait a few seconds, then try again."+ return ("Too many attempts right now", wait)case .guiSession:return ("Sign in to this Mac first","Setting up access needs someone signed in at this Mac\u{2019}s own screen, not only over "diff --git a/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Views/ThisMachineView.swift b/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Views/ThisMachineView.swiftindex 2c229c79fe..da5ecc6d4b 100644--- a/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Views/ThisMachineView.swift+++ b/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Views/ThisMachineView.swift@@ -435,6 +435,10 @@ struct HostSetupFailureView: View {let retrying: Boollet retry: () -> Void@State private var showDetails = false+ /// Seconds left before a throttled setup tries again by itself.+ @State private var remaining: Int?+ /// One automatic retry per setup: a second throttle waits for the user.+ @State private var autoRetried = falsevar body: some View {VStack(alignment: .leading, spacing: 6) {@@ -455,10 +459,11 @@ struct HostSetupFailureView: View {.disabled(retrying).accessibilityIdentifier("host-setup-retry")}- Text(failure.message)+ Text(remaining.map { "Trying again in \($0) \($0 == 1 ? "second" : "seconds")." } ?? (autoRetried ? HostSetupFailure.manualWait(failure) : failure.message)).font(.callout).foregroundStyle(.secondary).fixedSize(horizontal: false, vertical: true)+ .accessibilityIdentifier("host-setup-failure-message")DisclosureGroup(HostSetupFailure.detailsLabel, isExpanded: $showDetails) {HStack(alignment: .top, spacing: 8) {// A long error scrolls instead of pushing the page.@@ -487,5 +492,20 @@ struct HostSetupFailureView: View {}.accessibilityElement(children: .contain).accessibilityIdentifier("host-setup-failure")+ .task(id: failure) {+ // A throttled setup counts down, then tries again once.+ guard !autoRetried, var left = failure.throttleSeconds else { remaining = nil; return }+ while left > 0 {+ remaining = left+ try? await Task.sleep(for: .seconds(1))+ if Task.isCancelled { return }+ left -= 1+ }+ remaining = nil+ if !retrying {+ autoRetried = true+ retry()+ }+ }}}diff --git a/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/HostSetupFailureTests.swift b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/HostSetupFailureTests.swiftindex 42628ff7fe..e008869c77 100644--- a/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/HostSetupFailureTests.swift+++ b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/HostSetupFailureTests.swift@@ -23,6 +23,31 @@ struct HostSetupFailureTests {#expect(f.details == Self.missingRelease, "details stay raw")}+ @Test func aThrottledSetupSaysHowLongToWaitInPlainWords() {+ let raw = "http: Too many attempts right now. Wait 3 seconds, then try again."+ let f = HostSetupFailure.presenting(raw)+ #expect(f.kind == .throttled)+ #expect(f.title == "Too many attempts right now")+ #expect(f.message == "Trying again in 3 seconds.")+ #expect(f.details == raw)+ #expect(f.actionLabel == "Retry")+ }++ @Test func aRawHTTP429FromAnOlderHostIsTheSameWithoutANumber() {+ let f = HostSetupFailure.presenting("http: relay: HTTP 429: Too Many Requests")+ #expect(f.kind == .throttled)+ #expect(f.message == "Wait a few seconds, then try again.")+ #expect(!f.message.contains("429"))+ }++ @Test(arguments: [+ "http: relay: HTTP 429: computer limit for this token reached",+ "HTTP 429: at most 5 unused pairing codes",+ ])+ func aLimitTheRelayExplainsIsNotAThrottle(_ raw: String) {+ #expect(HostSetupFailure.presenting(raw).kind == .other)+ }+@Test(arguments: ["cua-spacesd: no published release for this platform","download: cua-spacesd v0.2.0 is not available",diff --git a/apps/cua-spaces-web/src/bridge/__tests__/friendly-errors.test.ts b/apps/cua-spaces-web/src/bridge/__tests__/friendly-errors.test.tsindex 5c7d5b386b..f977db0d93 100644--- a/apps/cua-spaces-web/src/bridge/__tests__/friendly-errors.test.ts+++ b/apps/cua-spaces-web/src/bridge/__tests__/friendly-errors.test.ts@@ -3,7 +3,7 @@import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";-import { limitsFor, plainJoinError, plainPairingError, stripInternalPrefixes } from "../friendly-errors";+import { throttleSeconds, limitsFor, plainJoinError, plainThrottleError, plainPairingError, stripInternalPrefixes } from "../friendly-errors";let warn: ReturnType<typeof vi.spyOn>;beforeEach(() => {@@ -30,6 +30,25 @@ describe("plain host errors", () => {expect(plainJoinError(new Error("cua host setup timed out"))).toMatch(/took too long/);});+ it("joining and pairing say how long to wait when the server is throttling, never the raw 429", () => {+ const own = "http: Too many attempts right now. Wait 4 seconds, then try again.";+ expect(plainJoinError(new Error(own))).toBe("Too many attempts right now. Wait 4 seconds, then try again.");+ expect(plainPairingError(new Error(own))).toBe("Too many attempts right now. Wait 4 seconds, then try again.");+ const old = new Error("http: relay: HTTP 429: Too Many Requests");+ expect(plainJoinError(old)).toBe("Too many attempts right now. Wait a few seconds, then try again.");+ expect(plainPairingError(old)).not.toMatch(/429|relay/i);+ });++ it("reads the wait to count down from", () => {+ expect(throttleSeconds("http: Too many attempts right now. Wait 12 seconds, then try again.")).toBe(12);+ expect(throttleSeconds("http: relay: HTTP 429: Too Many Requests")).toBeNull();+ });++ it("a 429 the relay explains is not a throttle and keeps its reason", () => {+ expect(plainThrottleError("relay: HTTP 429: computer limit for this token reached")).toBeNull();+ expect(plainPairingError(new Error("relay: HTTP 429: at most 5 unused pairing codes"))).toContain("at most 5 unused pairing codes");+ });+it("pairing: an unregistered computer is told to set up first", () => {const text = plainPairingError(new Error("not found: relay: no such device"));expect(text).toBe("This computer isn't set up for access yet. Set it up first, then add computers.");diff --git a/apps/cua-spaces-web/src/bridge/__tests__/hosts.test.ts b/apps/cua-spaces-web/src/bridge/__tests__/hosts.test.tsindex 292af31a27..dfc2dc8c8f 100644--- a/apps/cua-spaces-web/src/bridge/__tests__/hosts.test.ts+++ b/apps/cua-spaces-web/src/bridge/__tests__/hosts.test.ts@@ -797,6 +797,12 @@ describe("webkit adapter", () => {expect(hostFailure(e)).toEqual({ title: "Sign in to Cua", message: presented.message, details: presented.details, actionLabel: "Sign In" });// A plain error stays plain.expect(hostFailure(new HostError("nope", "failed"))).toEqual({ message: "nope" });+ // A throttled host that only says the error: plain words, raw kept.+ expect(hostFailure(new Error("http: Too many attempts right now. Wait 3 seconds, then try again."))).toEqual({+ title: "Too many attempts right now",+ message: "Trying again in 3 seconds.",+ details: "http: Too many attempts right now. Wait 3 seconds, then try again.",+ });} finally {vi.useRealTimers();}diff --git a/apps/cua-spaces-web/src/bridge/friendly-errors.ts b/apps/cua-spaces-web/src/bridge/friendly-errors.tsindex 8a569172b5..eebe0ed03f 100644--- a/apps/cua-spaces-web/src/bridge/friendly-errors.ts+++ b/apps/cua-spaces-web/src/bridge/friendly-errors.ts@@ -21,10 +21,31 @@ function rawText(e: unknown): string {return e instanceof Error ? e.message : String(e);}+/**+ * The plain line for the server telling us to slow down: the host's own+ * sentence ("Too many attempts right now. Wait 3 seconds, then try again.")+ * or a bare HTTP 429 "Too Many Requests" from an older host. A 429 that says+ * why (a limit, a quota) is not this; null for anything else.+ */+export function plainThrottleError(raw: string): string | null {+ const own = /too many attempts right now\. wait (\d+) seconds?/i.exec(raw);+ if (own) return `Too many attempts right now. Wait ${own[1]} ${own[1] === "1" ? "second" : "seconds"}, then try again.`;+ if (/\bhttp 429\b[:\s]*too many requests\s*$/i.test(raw.trim())) return "Too many attempts right now. Wait a few seconds, then try again.";+ return null;+}++/** Seconds a throttled host asked to wait (its own sentence), else null. */+export function throttleSeconds(raw: string): number | null {+ const own = /too many attempts right now\. wait (\d+) seconds?/i.exec(raw);+ return own ? Number(own[1]) : null;+}+/** What "Join with a code" says for a failure. */export function plainJoinError(e: unknown): string {const raw = rawText(e);console.warn("[Cua Spaces] join with a code failed:", raw);+ const throttled = plainThrottleError(raw);+ if (throttled) return throttled;if (/pairing code|unauthenticated|not valid|expired|revoked|already used|invalid code/i.test(raw)) {return "That code didn't work. It may be used, expired or mistyped. Make a new one in Add a computer.";}@@ -38,6 +59,8 @@ export function plainJoinError(e: unknown): string {export function plainPairingError(e: unknown): string {const raw = rawText(e);console.warn("[Cua Spaces] making a pairing link failed:", raw);+ const throttled = plainThrottleError(raw);+ if (throttled) return throttled;if (/no such device|not enrolled|not registered|unknown device/i.test(raw)) {return "This computer isn't set up for access yet. Set it up first, then add computers.";}diff --git a/apps/cua-spaces-web/src/bridge/this-machine.ts b/apps/cua-spaces-web/src/bridge/this-machine.tsindex 44e1031f5f..9fbf9863b2 100644--- a/apps/cua-spaces-web/src/bridge/this-machine.ts+++ b/apps/cua-spaces-web/src/bridge/this-machine.ts@@ -13,6 +13,7 @@ import { useContext, useSyncExternalStore } from "react";import { HostError, type DataAdapter } from "./adapter";import { BridgeContext } from "./BridgeProvider";import type { CoreClient } from "./core";+import { plainThrottleError } from "./friendly-errors";import type { HostActionId, HostFormView as HostFormViewBase } from "./contracts/host";import type { HostFormAction, HostFormState, HostRunAction, HostSetupRequest } from "./ops/host-setup";@@ -83,7 +84,10 @@ const IDLE: ThisMachineState = { form: null, busy: false, error: null, setupErroexport function hostFailure(e: unknown): HostFailure {const message = e instanceof Error ? e.message : String(e);const p = e instanceof HostError ? e.presented : undefined;- return p ? { title: p.title, message, details: p.details, actionLabel: p.actionLabel } : { message };+ if (p) return { title: p.title, message, details: p.details, actionLabel: p.actionLabel };+ // A host that only says the error: the server's "slow down" in plain words.+ const throttled = plainThrottleError(message);+ return throttled ? { title: "Too many attempts right now", message: throttled.replace(/^Too many attempts right now\. Wait (\d+) (seconds?), then try again\./, "Trying again in $1 $2.").replace("Wait a few seconds, then try again.", "Trying again in a few seconds."), details: message } : { message };}export class ThisMachineStore {diff --git a/apps/cua-spaces-web/src/components/this-machine.tsx b/apps/cua-spaces-web/src/components/this-machine.tsxindex dcfdc06970..cead446d6a 100644--- a/apps/cua-spaces-web/src/components/this-machine.tsx+++ b/apps/cua-spaces-web/src/components/this-machine.tsx@@ -2,7 +2,7 @@// Copyright (c) 2026 Cua AI, Inc.import { CheckIcon, ChevronDownIcon, ChevronRightIcon, CopyIcon, LoaderCircleIcon, TriangleAlertIcon } from "lucide-react";-import { useState, type ReactNode } from "react";+import { useEffect, useRef, useState, type ReactNode } from "react";import {useSession,@@ -13,6 +13,7 @@ import {type HostPanelView,type HostSetupFormView,} from "@/bridge";+import { throttleSeconds } from "@/bridge/friendly-errors";import { Button } from "@/components/ui/button";import { Checkbox } from "@/components/ui/checkbox";import { Dialog, DialogPopup, DialogTitle } from "@/components/ui/dialog";@@ -326,6 +327,30 @@ export function FailureView({ failure, retrying, onRetry }: { failure: HostFailuconst [open, setOpen] = useState(false);const [copied, setCopied] = useState(false);const details = failure.details && failure.details !== failure.message ? failure.details : null;+ // One automatic retry per setup: a second throttle waits for the user.+ const autoRetried = useRef(false);+ const wait = autoRetried.current ? null : throttleSeconds(failure.details ?? failure.message);+ const [left, setLeft] = useState<number | null>(wait);+ // A throttled setup counts down live, then tries again once.+ useEffect(() => {+ if (wait === null) return setLeft(null);+ let n = wait;+ setLeft(n);+ const timer = setInterval(() => {+ n -= 1;+ if (n > 0) return setLeft(n);+ clearInterval(timer);+ setLeft(null);+ if (!retrying) {+ autoRetried.current = true;+ onRetry();+ }+ }, 1000);+ return () => clearInterval(timer);+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [failure]);+ const manual = autoRetried.current ? throttleSeconds(failure.details ?? failure.message) : null;+ const message = manual ? `Wait ${manual} ${manual === 1 ? "second" : "seconds"}, then try again.` : left === null ? failure.message : `Trying again in ${left} ${left === 1 ? "second" : "seconds"}.`;return (<div data-host-failure className="text-[13px]"><div className="flex items-center gap-2">@@ -339,7 +364,7 @@ export function FailureView({ failure, retrying, onRetry }: { failure: HostFailu</div>{failure.title ? (<p data-host-failure-message className="mt-1 text-xs text-muted-foreground">- {failure.message}+ {message}</p>) : null}{details ? (diff --git a/apps/cua-spaces/src-tauri/Cargo.lock b/apps/cua-spaces/src-tauri/Cargo.lockindex 91003b66a1..075ca2558d 100644--- a/apps/cua-spaces/src-tauri/Cargo.lock+++ b/apps/cua-spaces/src-tauri/Cargo.lock@@ -1896,6 +1896,7 @@ dependencies = ["cua-home","flate2","hex",+ "httpdate","libc","rand 0.9.5","reqwest 0.12.28",diff --git a/examples/openkoalabot-example-tauri/src-tauri/Cargo.lock b/examples/openkoalabot-example-tauri/src-tauri/Cargo.lockindex 740444b25d..2e765653fb 100644--- a/examples/openkoalabot-example-tauri/src-tauri/Cargo.lock+++ b/examples/openkoalabot-example-tauri/src-tauri/Cargo.lock@@ -1886,6 +1886,7 @@ dependencies = ["cua-home","flate2","hex",+ "httpdate","libc","rand 0.9.5","reqwest 0.12.28",diff --git a/examples/streaming/rust/Cargo.lock b/examples/streaming/rust/Cargo.lockindex 44e9add7ac..5f793a48ab 100644--- a/examples/streaming/rust/Cargo.lock+++ b/examples/streaming/rust/Cargo.lock@@ -1516,6 +1516,7 @@ dependencies = ["cua-home","flate2","hex",+ "httpdate","libc","rand 0.9.5","reqwest 0.12.28",diff --git a/libs/cua-spacesd/Cargo.lock b/libs/cua-spacesd/Cargo.lockindex 30e8494582..49c567402e 100644--- a/libs/cua-spacesd/Cargo.lock+++ b/libs/cua-spacesd/Cargo.lock@@ -1568,6 +1568,7 @@ dependencies = ["cua-home","flate2","hex",+ "httpdate","libc","rand 0.9.5","reqwest",diff --git a/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock b/libs/cua-spacesd/tests/spaces-e2e/Cargo.lockindex 324a36350e..9d3878c26c 100644--- a/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock+++ b/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock@@ -2365,6 +2365,7 @@ dependencies = ["cua-home","flate2","hex",+ "httpdate","libc","rand 0.9.5","reqwest 0.12.28",diff --git a/libs/cua/Cargo.lock b/libs/cua/Cargo.lockindex ca13a47d1c..2769e9c887 100644--- a/libs/cua/Cargo.lock+++ b/libs/cua/Cargo.lock@@ -2687,6 +2687,7 @@ dependencies = ["cua-home","flate2","hex",+ "httpdate","libc","rand 0.9.5","reqwest 0.12.28",diff --git a/libs/cua/crates/cua-cli/src/host.rs b/libs/cua/crates/cua-cli/src/host.rsindex cb6450e8d0..a0be422364 100644--- a/libs/cua/crates/cua-cli/src/host.rs+++ b/libs/cua/crates/cua-cli/src/host.rs@@ -1520,7 +1520,7 @@ fn host_error(e: cua_host::Error, m: String) -> CuaError {E::Unauthenticated(_) => CuaError::Unauthenticated(m),E::PermissionDenied(_) => CuaError::PermissionDenied(m),E::NotFound(_) => CuaError::NotFound(m),- E::Relay(_) | E::Download(_) => CuaError::Http(m),+ E::Relay(_) | E::Download(_) | E::Throttled { .. } => CuaError::Http(m),// The plain sentence, then where to upgrade.E::PlanLimit(_) => {let url = cua_home::plan_limit::PlanLimit::parse(&m)diff --git a/libs/cua/crates/cua-cli/src/usage_cmd.rs b/libs/cua/crates/cua-cli/src/usage_cmd.rsindex 1ab4e458f0..14de09a87f 100644--- a/libs/cua/crates/cua-cli/src/usage_cmd.rs+++ b/libs/cua/crates/cua-cli/src/usage_cmd.rs@@ -258,4 +258,20 @@ mod tests {);assert!(m.contains("/buy/teams?from=plan_limit"), "{m}");}++ #[test]+ fn a_throttled_error_says_how_long_to_wait_in_plain_words() {+ let e = crate::host::host_err(cua_host::Error::Throttled {+ retry_after_secs: 3,+ });+ let m = e.to_string();+ assert!(+ m.contains("Too many attempts right now. Wait 3 seconds, then try again."),+ "{m}"+ );+ assert!(+ !m.contains("429") && !m.contains("Too Many Requests"),+ "{m}"+ );+ }}diff --git a/libs/cua/crates/cua-host/Cargo.toml b/libs/cua/crates/cua-host/Cargo.tomlindex 88dc5bf6d9..62085a399b 100644--- a/libs/cua/crates/cua-host/Cargo.toml+++ b/libs/cua/crates/cua-host/Cargo.toml@@ -27,6 +27,7 @@ cua-auth.workspace = truecua-home.workspace = trueflate2 = "1"hex.workspace = true+httpdate = "1"rand.workspace = truereqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "json"] }# Client device key (P-256, the curve the Secure Enclave uses) and its@@ -50,3 +51,4 @@ windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_System_[dev-dependencies]axum = { workspace = true, features = ["json"] }tempfile.workspace = true+tokio = { workspace = true, features = ["test-util"] }diff --git a/libs/cua/crates/cua-host/src/lib.rs b/libs/cua/crates/cua-host/src/lib.rsindex 29d6c61496..3ea40a7d84 100644--- a/libs/cua/crates/cua-host/src/lib.rs+++ b/libs/cua/crates/cua-host/src/lib.rs@@ -157,6 +157,14 @@ pub enum Error {/// ([`cua_home::plan_limit::PlanLimit::sentence`]); never retried.#[error("{0}")]PlanLimit(String),+ /// The proxy in front of the relay throttled this client (HTTP 429, bare+ /// `Too Many Requests`) and kept doing so through every retry. The+ /// request never reached the relay.+ #[error("Too many attempts right now. Wait {retry_after_secs} seconds, then try again.")]+ Throttled {+ /// How long the proxy asked to wait (`Retry-After`), at least 1.+ retry_after_secs: u64,+ },/// Driver download failed.#[error("download: {0}")]Download(String),@@ -182,7 +190,7 @@ impl Error {Error::PermissionDenied(m) if m.starts_with("relay: ") => Some(403),Error::NotFound(m) if m.starts_with("relay: ") => Some(404),Error::Conflict(m) if m.starts_with("relay: ") => Some(409),- Error::PlanLimit(_) => Some(429),+ Error::PlanLimit(_) | Error::Throttled { .. } => Some(429),Error::Relay(m) | Error::Download(m) => status_in(m),_ => None,}@@ -195,6 +203,7 @@ impl Error {/// missing machine or an answer that could not be read.pub fn is_transient(&self) -> bool {match self {+ Error::Throttled { .. } => true,Error::Relay(m) => match status_in(m) {Some(code) => matches!(code, 429 | 502 | 503 | 504),None => !m.starts_with("bad relay response"),diff --git a/libs/cua/crates/cua-host/src/relay.rs b/libs/cua/crates/cua-host/src/relay.rsindex 033d0f9cd0..95fde591f9 100644--- a/libs/cua/crates/cua-host/src/relay.rs+++ b/libs/cua/crates/cua-host/src/relay.rs@@ -484,6 +484,45 @@ pub struct DeviceSession {pub expires_at: u64,}+/// Most times one request is sent while the proxy throttles this client.+const THROTTLE_ATTEMPTS: u32 = 8;+/// Most time spent waiting between those attempts.+const THROTTLE_BUDGET: Duration = Duration::from_secs(60);++/// How long to wait before sending again after the `attempt`th throttled+/// answer: the proxy's `Retry-After` (`retry_after_secs`) or a capped+/// exponential backoff (1 s, 2 s, 4 s, 8 s), whichever is longer, plus up to+/// a quarter of that as jitter so many clients do not return together.+fn throttle_wait(retry_after_secs: u64, attempt: u32) -> Duration {+ let backoff = 1u64 << attempt.saturating_sub(1).min(3);+ // Clamped before jitter so a absurd `Retry-After` cannot overflow+ // `Duration`; anything past the budget makes the caller give up anyway.+ let secs = retry_after_secs+ .max(backoff)+ .min(THROTTLE_BUDGET.as_secs() + 1);+ let base = Duration::from_secs(secs);+ base + base.mul_f64(rand::random::<f64>() * 0.25)+}++/// The proxy's own 429 has the generic body `Too Many Requests` (or none).+fn is_proxy_throttle(body: &str) -> bool {+ let b = body.trim();+ b.is_empty() || b.eq_ignore_ascii_case("too many requests")+}++/// `Retry-After` as a wait: whole seconds or an HTTP date.+fn retry_after(h: &reqwest::header::HeaderMap) -> Option<Duration> {+ let v = h.get(reqwest::header::RETRY_AFTER)?.to_str().ok()?.trim();+ if let Ok(secs) = v.parse::<u64>() {+ return Some(Duration::from_secs(secs));+ }+ let at = httpdate::parse_http_date(v).ok()?;+ Some(+ at.duration_since(std::time::SystemTime::now())+ .unwrap_or_default(),+ )+}+/// Times a machine registration is tried again after a transient failure.const REGISTER_RETRIES: u32 = 3;/// The first wait before trying again (doubling each time).@@ -508,6 +547,27 @@ pub struct RelayClient {base: String,pub(crate) http: reqwest::Client,device_session: Option<String>,+ sleeper: Sleeper,+}++/// Waits between throttled attempts. Unit tests of this crate record the+/// waits instead of sleeping them.+#[derive(Clone, Debug, Default)]+struct Sleeper {+ #[cfg(test)]+ waits: std::sync::Arc<std::sync::Mutex<Vec<Duration>>>,+}++impl Sleeper {+ #[cfg(not(test))]+ async fn sleep(&self, d: Duration) {+ tokio::time::sleep(d).await;+ }++ #[cfg(test)]+ async fn sleep(&self, d: Duration) {+ self.waits.lock().unwrap().push(d);+ }}impl RelayClient {@@ -528,6 +588,7 @@ impl RelayClient {base,http,device_session: None,+ sleeper: Sleeper::default(),})}@@ -625,17 +686,59 @@ impl RelayClient {self.send_once(req, bearer).await.map_err(|(e, _)| e)}- /// One request. On failure, also whether the relay certainly did not- /// act on it and it may be sent again: the connection never opened, or- /// the relay (or the proxy in front of it) answered 429, 502 or 503.+ /// One request, sent again while the proxy in front of the relay is+ /// throttling this client (see [`throttle_wait`]). On failure, also+ /// whether the relay certainly did not act on it and it may be sent+ /// again: the connection never opened, or the relay (or the proxy in+ /// front of it) answered 502 or 503. A throttled request never reached+ /// the relay, so sending it again is safe for any method.async fn send_once<T: serde::de::DeserializeOwned>(&self,req: reqwest::RequestBuilder,bearer: &str,) -> std::result::Result<T, (Error, bool)> {- let mut req = req.bearer_auth(bearer);- if let Some(session) = &self.device_session {- req = req.header(DEVICE_SESSION_HEADER, session);+ self.send_with(req, Some(bearer)).await+ }++ async fn send_with<T: serde::de::DeserializeOwned>(+ &self,+ req: reqwest::RequestBuilder,+ bearer: Option<&str>,+ ) -> std::result::Result<T, (Error, bool)> {+ let mut waited = Duration::ZERO;+ let mut attempt = 0u32;+ let mut req = req;+ loop {+ // A body that cannot be copied (a stream) is sent once.+ let next = req.try_clone();+ let result = self.send_attempt(req, bearer).await;+ let (Err((Error::Throttled { retry_after_secs }, _)), Some(next)) = (&result, next)+ else {+ return result;+ };+ attempt += 1;+ let wait = throttle_wait(*retry_after_secs, attempt);+ if attempt >= THROTTLE_ATTEMPTS || waited + wait > THROTTLE_BUDGET {+ return result;+ }+ tracing::debug!(attempt, ?wait, "relay is throttling this client, retrying");+ self.sleeper.sleep(wait).await;+ waited += wait;+ req = next;+ }+ }++ async fn send_attempt<T: serde::de::DeserializeOwned>(+ &self,+ req: reqwest::RequestBuilder,+ bearer: Option<&str>,+ ) -> std::result::Result<T, (Error, bool)> {+ let mut req = req;+ if let Some(b) = bearer {+ req = req.bearer_auth(b);+ if let Some(session) = &self.device_session {+ req = req.header(DEVICE_SESSION_HEADER, session);+ }}let resp = req.send()@@ -643,6 +746,7 @@ impl RelayClient {.map_err(|e| (Error::Relay(format!("{}: {e}", self.base)), e.is_connect()))?;let status = resp.status();let limit = plan_limit(resp.headers());+ let retry_after = retry_after(resp.headers());let body = resp.text().await@@ -653,7 +757,23 @@ impl RelayClient {if let (429, Some(l)) = (status.as_u16(), limit) {return Err((Error::PlanLimit(l.sentence()), false));}- let retry = matches!(status.as_u16(), 429 | 502 | 503);+ if status.as_u16() == 429 {+ // Only the proxy's bare answer is a throttle. The relay's+ // own 429s (quotas, pairing-code caps) say why in the body+ // and sending them again gets the same answer.+ if is_proxy_throttle(&body) {+ // An hour at most, so the message and any countdown stay sane.+ let secs = retry_after.map_or(1, |d| d.as_secs().clamp(1, 3600));+ return Err((+ Error::Throttled {+ retry_after_secs: secs,+ },+ false,+ ));+ }+ return Err((http_error(429, &body), false));+ }+ let retry = matches!(status.as_u16(), 502 | 503);return Err((http_error(status.as_u16(), &body), retry));}let body = if body.trim().is_empty() {@@ -702,35 +822,17 @@ impl RelayClient {/// `GET /v1/info` (no auth).pub async fn info(&self) -> Result<RelayInfo> {- let resp = self- .http- .get(self.url("/v1/info"))- .send()+ self.send_with(self.http.get(self.url("/v1/info")), None).await- .map_err(|e| Error::Relay(format!("{}: {e}", self.base)))?;- let status = resp.status();- let body = resp.text().await.unwrap_or_default();- if !status.is_success() {- return Err(http_error(status.as_u16(), &body));- }- serde_json::from_str(&body).map_err(|e| Error::Relay(format!("bad relay response: {e}")))+ .map_err(|(e, _)| e)}/// `GET /.well-known/jwks.json` (no auth): the relay's assertion/// signing keys, as a registration returns them (a machine pins them).pub async fn jwks(&self) -> Result<serde_json::Value> {- let resp = self- .http- .get(self.url("/.well-known/jwks.json"))- .send()+ self.send_with(self.http.get(self.url("/.well-known/jwks.json")), None).await- .map_err(|e| Error::Relay(format!("{}: {e}", self.base)))?;- let status = resp.status();- let body = resp.text().await.unwrap_or_default();- if !status.is_success() {- return Err(http_error(status.as_u16(), &body));- }- serde_json::from_str(&body).map_err(|e| Error::Relay(format!("bad relay response: {e}")))+ .map_err(|(e, _)| e)}/// Registers (or re-registers, rotating the machine token) a machine@@ -1158,4 +1260,212 @@ mod tests {assert!(!e.is_transient());assert_eq!(hits.load(Ordering::SeqCst), 1);}++ /// A relay that answers the first `throttled` calls to `/v1/machines`+ /// with the proxy's bare 429 (`retry_after` as the header), then `[]`.+ async fn throttling_relay(+ throttled: usize,+ retry_after: &'static str,+ body: &'static str,+ ) -> (RelayClient, std::sync::Arc<std::sync::atomic::AtomicUsize>) {+ use std::sync::Arc;+ use std::sync::atomic::{AtomicUsize, Ordering};+ let hits = Arc::new(AtomicUsize::new(0));+ let counter = hits.clone();+ let app = axum::Router::new().route(+ "/v1/machines",+ axum::routing::get(move || {+ let n = counter.fetch_add(1, Ordering::SeqCst);+ async move {+ if n < throttled {+ (+ axum::http::StatusCode::TOO_MANY_REQUESTS,+ [("retry-after", retry_after)],+ body,+ )+ .into_response()+ } else {+ (axum::http::StatusCode::OK, "[]").into_response()+ }+ }+ }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ (RelayClient::new(&format!("http://{addr}")).unwrap(), hits)+ }++ use axum::response::IntoResponse;++ #[tokio::test]+ async fn proxy_throttling_is_retried_until_it_succeeds() {+ let (client, hits) = throttling_relay(2, "1", "Too Many Requests").await;+ let machines = client.machines("tok").await.unwrap();+ assert!(machines.is_empty());+ assert_eq!(hits.load(std::sync::atomic::Ordering::SeqCst), 3);+ // Two waits: Retry-After (1 s) and the backoff (2 s), plus jitter.+ let waits = client.sleeper.waits.lock().unwrap().clone();+ assert_eq!(waits.len(), 2);+ assert!(waits[0] >= Duration::from_secs(1) && waits[0] <= Duration::from_millis(1250));+ assert!(waits[1] >= Duration::from_secs(2) && waits[1] <= Duration::from_millis(2500));+ }++ #[tokio::test]+ async fn retry_after_is_honoured_in_seconds_and_as_a_date() {+ let (client, _) = throttling_relay(1, "7", "").await;+ client.machines("tok").await.unwrap();+ assert!(client.sleeper.waits.lock().unwrap()[0] >= Duration::from_secs(7));++ let date = httpdate::fmt_http_date(std::time::SystemTime::now() + Duration::from_secs(9));+ let (client, _) = throttling_relay(1, Box::leak(date.into_boxed_str()), "").await;+ client.machines("tok").await.unwrap();+ let wait = client.sleeper.waits.lock().unwrap()[0];+ assert!(wait >= Duration::from_secs(7), "{wait:?}");+ }++ #[tokio::test]+ async fn it_gives_up_after_the_cap_with_a_plain_message() {+ let (client, hits) = throttling_relay(usize::MAX, "2", "Too Many Requests").await;+ let e = client.machines("tok").await.unwrap_err();+ assert!(+ matches!(+ e,+ Error::Throttled {+ retry_after_secs: 2+ }+ ),+ "{e:?}"+ );+ assert_eq!(+ e.to_string(),+ "Too many attempts right now. Wait 2 seconds, then try again."+ );+ assert_eq!(e.http_status(), Some(429));+ assert!(e.is_transient());+ // Waits of about 2, 2, 4, 8, 8, ... s stop at the 60 s cap.+ let total: Duration = client.sleeper.waits.lock().unwrap().iter().sum();+ assert!(total <= Duration::from_secs(60), "{total:?}");+ assert!(total >= Duration::from_secs(30), "{total:?}");+ let tries = hits.load(std::sync::atomic::Ordering::SeqCst);+ assert!((4..=8).contains(&tries), "{tries}");+ assert_eq!(client.sleeper.waits.lock().unwrap().len(), tries - 1);+ }++ #[tokio::test]+ async fn a_post_is_retried_too_because_a_throttled_call_never_arrived() {+ use std::sync::Arc;+ use std::sync::atomic::{AtomicUsize, Ordering};+ let hits = Arc::new(AtomicUsize::new(0));+ let counter = hits.clone();+ let app = axum::Router::new().route(+ "/v1/devices",+ axum::routing::post(move |body: String| {+ let n = counter.fetch_add(1, Ordering::SeqCst);+ async move {+ if n == 0 {+ (+ axum::http::StatusCode::TOO_MANY_REQUESTS,+ "Too Many Requests",+ )+ .into_response()+ } else {+ assert!(body.contains("\"k\""), "{body}");+ (+ axum::http::StatusCode::OK,+ r#"{"device_id":"d","user_id":"u"}"#,+ )+ .into_response()+ }+ }+ }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ let client = RelayClient::new(&format!("http://{addr}")).unwrap();+ let r = client+ .send_json::<_, serde_json::Value>(+ reqwest::Method::POST,+ "/v1/devices",+ Some(&serde_json::json!({"k": 1})),+ "tok",+ )+ .await;+ assert!(r.is_ok(), "{r:?}");+ assert_eq!(hits.load(Ordering::SeqCst), 2);+ }++ #[test]+ fn the_backoff_doubles_to_a_cap_and_never_undercuts_retry_after() {+ for (retry_after, attempt, floor) in [+ (1, 1, 1),+ (1, 2, 2),+ (1, 3, 4),+ (1, 4, 8),+ (1, 9, 8),+ (30, 1, 30),+ ] {+ let w = throttle_wait(retry_after, attempt);+ let floor = Duration::from_secs(floor);+ assert!(+ w >= floor && w <= floor.mul_f64(1.25),+ "{retry_after} {attempt} {w:?}"+ );+ }+ }++ #[test]+ fn a_huge_retry_after_is_clamped_and_never_panics() {+ let w = throttle_wait(u64::MAX, 1);+ assert!(w > THROTTLE_BUDGET && w <= THROTTLE_BUDGET * 2, "{w:?}");+ }++ #[tokio::test]+ async fn a_huge_retry_after_gives_up_at_once_with_a_sane_wait() {+ let (client, hits) = throttling_relay(usize::MAX, "18446744073709551615", "").await;+ let e = client.machines("tok").await.unwrap_err();+ assert!(+ matches!(+ e,+ Error::Throttled {+ retry_after_secs: 3600+ }+ ),+ "{e:?}"+ );+ assert_eq!(hits.load(std::sync::atomic::Ordering::SeqCst), 1);+ assert!(client.sleeper.waits.lock().unwrap().is_empty());+ }++ #[test]+ fn retry_after_parses_seconds_dates_and_junk() {+ let h = |v: &str| {+ let mut m = reqwest::header::HeaderMap::new();+ m.insert(reqwest::header::RETRY_AFTER, v.parse().unwrap());+ m+ };+ assert_eq!(retry_after(&h("5")), Some(Duration::from_secs(5)));+ assert_eq!(retry_after(&h(" 12 ")), Some(Duration::from_secs(12)));+ assert_eq!(+ retry_after(&h("Wed, 21 Oct 2015 07:28:00 GMT")),+ Some(Duration::ZERO)+ );+ assert_eq!(retry_after(&h("soon")), None);+ assert_eq!(retry_after(&reqwest::header::HeaderMap::new()), None);+ }++ #[tokio::test]+ async fn the_relays_own_429s_are_not_retried() {+ for body in [+ "computer limit for this token reached",+ "at most 5 unused pairing codes",+ ] {+ let (client, hits) = throttling_relay(usize::MAX, "1", body).await;+ let e = client.machines("tok").await.unwrap_err();+ assert!(matches!(e, Error::Relay(_)), "{e:?}");+ assert!(e.to_string().contains(body), "{e}");+ assert_eq!(hits.load(std::sync::atomic::Ordering::SeqCst), 1);+ }+ }}diff --git a/libs/cua/crates/cua-host/src/testing.rs b/libs/cua/crates/cua-host/src/testing.rsindex 38e35404be..eebcb1ac12 100644--- a/libs/cua/crates/cua-host/src/testing.rs+++ b/libs/cua/crates/cua-host/src/testing.rs@@ -301,6 +301,10 @@ async fn register(st.register_calls += 1;if let Some(code) = st.fail_register.pop_front() {let status = StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);+ if code == 429 {+ // The proxy's bare throttle, which the client waits out.+ return (status, "Too Many Requests").into_response();+ }return err(status, "the fake relay failed this registration");}let id = body["id"].as_str().unwrap_or_default().to_string();diff --git a/libs/cua/crates/cua-sdk/src/native/host.rs b/libs/cua/crates/cua-sdk/src/native/host.rsindex 52d642005f..bb51817ea2 100644--- a/libs/cua/crates/cua-sdk/src/native/host.rs+++ b/libs/cua/crates/cua-sdk/src/native/host.rs@@ -16,7 +16,7 @@ impl From<cua_host::Error> for CuaError {E::Unauthenticated(_) => CuaError::Unauthenticated(m),E::PermissionDenied(_) => CuaError::PermissionDenied(m),E::NotFound(_) => CuaError::NotFound(m),- E::Relay(_) | E::Download(_) => CuaError::Http(m),+ E::Relay(_) | E::Download(_) | E::Throttled { .. } => CuaError::Http(m),// The plain sentence, then where to upgrade, as the CLI says it;// the Python and TypeScript SDKs read both back as `PlanLimitError`.E::PlanLimit(_) => {diff --git a/tests/e2e/cua-sdk/rust/Cargo.lock b/tests/e2e/cua-sdk/rust/Cargo.lockindex 515f7f130e..5283f3ae1f 100644--- a/tests/e2e/cua-sdk/rust/Cargo.lock+++ b/tests/e2e/cua-sdk/rust/Cargo.lock@@ -1516,6 +1516,7 @@ dependencies = ["cua-home","flate2","hex",+ "httpdate","libc","rand 0.9.5","reqwest 0.12.28",diff --git a/tests/e2e/local-runtimes/Cargo.lock b/tests/e2e/local-runtimes/Cargo.lockindex 0f44b0bc97..091ea633c6 100644--- a/tests/e2e/local-runtimes/Cargo.lock+++ b/tests/e2e/local-runtimes/Cargo.lock@@ -1312,6 +1312,7 @@ dependencies = ["cua-home","flate2","hex",+ "httpdate","libc","rand 0.9.5","reqwest 0.12.28",