Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(macos): offer retry when stream provider creation fails (#4807)
injaneity81540962dbfb
diff --git a/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Views/SpaceDetailView.swift b/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Views/SpaceDetailView.swiftindex 89416e2e26..763ea9d7c1 100644--- a/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Views/SpaceDetailView.swift+++ b/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Views/SpaceDetailView.swift@@ -26,6 +26,7 @@ struct SpaceDetailView: View {/// Connect (or Try again) was pressed: the stream opens even with/// "Connect to the desktop automatically" off.@State private var connectRequested = false+ @State private var providerFailed = false@Environment(\.openWindow) private var openWindowprivate let copy = appSpaceDetailCopy()@@ -34,10 +35,12 @@ struct SpaceDetailView: View {private var off: Bool { space.power.map { $0.off || $0.turningOn == false } ?? false }/// What the live parts (stream, polling) restart on.private var live: String { "\(space.id) \(deleting) \(off)" }+ /// Provider creation can fail before a stream session exists.+ private var providerPhase: AppStreamPhase { providerFailed ? .failed : .noSession }var body: some View {let detail = model.detail(space)- let wantsStream = model.cover(detail, requested: connectRequested, stream: .noSession).openStream+ let wantsStream = model.cover(detail, requested: connectRequested, stream: providerPhase).openStreamScrollViewReader { scroller inForm {if let note = detail.desktopNote {@@ -169,7 +172,7 @@ struct SpaceDetailView: View {}} else {StreamPhaseReader(session: detail.canStream ? session : nil) { phase in- let cover = model.cover(detail, requested: connectRequested, stream: phase)+ let cover = model.cover(detail, requested: connectRequested, stream: session == nil ? providerPhase : phase)ZStack {if let session, detail.canStream {SpaceScreenView(session: session, isInteractive: true, showsControls: false)@@ -192,6 +195,7 @@ struct SpaceDetailView: View {/// failed one again.private func press(_ cover: AppDesktopCover) {connectRequested = true+ providerFailed = falseguard cover.kind == .status, let session else { return }Task {await session.stop()@@ -312,7 +316,7 @@ struct SpaceDetailView: View {private func connect(_ detail: AppSpaceDetail) async {// The core decides: auto-connect on, or Connect pressed.guard session == nil,- model.cover(detail, requested: connectRequested, stream: .noSession).openStream else { return }+ model.cover(detail, requested: connectRequested, stream: providerPhase).openStream else { return }do {let provider = try await model.backend.streamProvider(id: space.id)guard !Task.isCancelled else { return }@@ -329,6 +333,7 @@ struct SpaceDetailView: View {}} catch {guard !Task.isCancelled else { return }+ providerFailed = truemodel.show(error: "Could not open \(space.name): \(LiveSpacesBackend.words(error))")}}@@ -338,6 +343,7 @@ struct SpaceDetailView: View {private func disconnect() {keepLastFrame()connectRequested = false+ providerFailed = falseagents = nilpips?.popInAll()if let session { Task { await session.stop() } }diff --git a/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/ProviderFailureTests.swift b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/ProviderFailureTests.swiftnew file mode 100644index 0000000000..aeda5ce594--- /dev/null+++ b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/ProviderFailureTests.swift@@ -0,0 +1,198 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++import CuaSDK+import CuaSpacesFFI+import CuaSpacesStreaming+@testable import CuaSpacesMacKit+import Foundation+import AppKit+import Testing+import SwiftUI+import Vision++/// Scripted provider creation and media failures; no host is contacted.+private final class ProviderFailureBackend: SpacesBackend, @unchecked Sendable {+ var result: Result<[AppSpaceRow], Error> = .success(FixtureSpacesBackend.sample)+ var failures = 2+ let provider = RefusingMediaProvider()+ private let lock = NSLock()+ private var ids: [String] = []+ var requestedIds: [String] { lock.withLock { ids } }+ func rows() async throws -> [AppSpaceRow] { try result.get() }+ func create(_ args: AppCreateSpaceArgs, createId: String, progress: @escaping @Sendable (SpaceCreateProgress) -> Void) async throws -> String { fatalError("unexpected create") }+ func cancelCreate(createId: String) async throws {}+ func add(url: String, token: String?, name: String?) async throws {}+ func remove(id: String, removeOnly: Bool) async throws {}+ func setPower(id: String, on: Bool) async throws {}+ func streamProvider(id: String) async throws -> SpaceStreamSourceProviding {+ let attempt = lock.withLock { ids.append(id); return ids.count }+ if attempt <= failures { throw CuaError.Internal(message: "scripted provider failure") }+ return provider+ }+ func localBackends() async -> [String]? { nil }+ func localStorage() async -> LocalStorage? { nil }+ func cloudAvailable() async -> Bool { false }+ func teleportContext(id: String) async throws -> (CuaSpacesFFI.Teleport, CuaSDK.Space)? { nil }+ func teleportHandle() -> CuaSpacesFFI.Teleport? { nil }+ func sendFiles(id: String, paths: [String]) async throws -> [AppSentFileInfo] { [] }+ func agentRuns(id: String) async throws -> [AppSpaceAgentRun] { [] }+}++private final class RefusingMediaProvider: SpaceStreamSourceProviding, @unchecked Sendable {+ private let lock = NSLock()+ private var opens = 0+ var attempts: Int { lock.withLock { opens } }+ func availableWindows() async throws -> [StreamWindow] { [] }+ func openSession(_ source: StreamSource, frames: FrameSink, audio: AudioSink?) async throws -> SpaceStreamSession {+ lock.withLock { opens += 1 }+ throw StreamError.noStream("scripted media failure")+ }+ func joinPresence(name: String, color: String?) async throws -> SpacePresence {+ throw StreamError.noStream("no presence in fixture")+ }+}++@MainActor @Suite("Provider creation recovery", .serialized)+struct ProviderFailureTests {+ @MainActor private struct Fixture {+ let backend: ProviderFailureBackend+ let model: AppModel+ let devices: FixtureDevices+ let space: AppSpace+ let host: NSView+ let window: NSWindow+ let directory: URL++ func close() {+ window.contentView = nil+ window.close()+ try? FileManager.default.removeItem(at: directory)+ }++ func text() throws -> [VNRecognizedTextObservation] {+ host.layoutSubtreeIfNeeded()+ window.display()+ let bitmap = try #require(host.bitmapImageRepForCachingDisplay(in: host.bounds))+ host.cacheDisplay(in: host.bounds, to: bitmap)+ let request = VNRecognizeTextRequest()+ request.recognitionLanguages = ["en-US"]+ request.recognitionLevel = .accurate+ request.usesLanguageCorrection = false+ try VNImageRequestHandler(cgImage: #require(bitmap.cgImage), options: [:]).perform([request])+ return request.results ?? []+ }++ func press(_ label: String) throws {+ let observations = try text()+ let match = try #require(observations.first { $0.topCandidates(1).first?.string == label },+ "the actual detail renders \(label)")+ let box = match.boundingBox+ let local = NSPoint(x: box.midX * host.bounds.width,+ y: (host.isFlipped ? 1 - box.midY : box.midY) * host.bounds.height)+ let point = host.convert(local, to: nil)+ let key = NSApp.keyWindow+ // AppKit dispatch within this process to this window only: no+ // system event posting, Accessibility grant, or real desktop.+ for kind in [NSEvent.EventType.leftMouseDown, .leftMouseUp] {+ let event = try #require(NSEvent.mouseEvent(with: kind, location: point, modifierFlags: [],+ timestamp: ProcessInfo.processInfo.systemUptime, windowNumber: window.windowNumber,+ context: nil, eventNumber: 0, clickCount: 1, pressure: 1))+ try #require(event.window === window)+ NSApp.sendEvent(event)+ }+ #expect(NSApp.keyWindow === key)+ }++ func settle() async throws { try await Task.sleep(for: .milliseconds(350)) }+ func expectProviders(_ count: Int) async throws {+ for _ in 0..<80 where backend.requestedIds.count < count {+ try await Task.sleep(for: .milliseconds(50))+ }+ try await settle()+ #expect(backend.requestedIds == Array(repeating: space.id, count: count))+ }+ func approve(_ approved: Bool) async throws {+ devices.current.devices[0].state = approved ? "enrolled" : "pending"+ await model.devices.refresh()+ try await settle()+ }+ }++ private func open(autoConnect: Bool, failures: Int = 2) async throws -> Fixture {+ _ = NSApplication.shared+ NSApp.finishLaunching()+ let backend = ProviderFailureBackend()+ backend.failures = failures+ var row = FixtureSpacesBackend.sample[0]+ row.id = "relay:studio"+ row.provider = "relay"+ backend.result = .success([row])+ let devices = FixtureDevices(snapshot: FixtureDevices.sample(now: UInt64(Date().timeIntervalSince1970)))+ let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)+ let model = AppModel(backend: backend, keyvault: KeyvaultModel(client: nil),+ onboarding: OnboardingModel(statePath: nil), settingsPath: directory.appendingPathComponent("settings.json").path,+ telemetry: FixtureTelemetry(), devices: devices, presence: FixturePresence())+ model.settings.autoConnect = autoConnect+ await model.refresh()+ model.devices.signedIn = true+ await model.devices.refresh()+ let space = try #require(model.spaces.first { $0.id == row.id })+ #expect(model.detail(space).access == nil)+ let host = NSHostingView(rootView: SpaceDetailView(model: model, space: space).frame(width: 800, height: 800))+ let window = NSWindow(contentRect: NSRect(x: -20000, y: -20000, width: 800, height: 800),+ styleMask: [.borderless], backing: .buffered, defer: false)+ window.isReleasedWhenClosed = false+ window.contentView = host+ let key = NSApp.keyWindow+ window.orderBack(nil)+ #expect(NSApp.keyWindow === key)+ try #require(NSScreen.screens.allSatisfy { !$0.frame.intersects(window.frame) })+ try #require(window.windowNumber > 0, "requires a WindowServer-backed test window")+ host.layoutSubtreeIfNeeded()+ return Fixture(backend: backend, model: model, devices: devices, space: space,+ host: host, window: window, directory: directory)+ }++ @Test(arguments: [false, true])+ func retriesProviderCreationThenAttemptsMedia(autoConnect: Bool) async throws {+ let f = try await open(autoConnect: autoConnect)+ defer { f.close() }+ if !autoConnect {+ try await f.settle()+ #expect(f.backend.requestedIds.isEmpty)+ try f.press("Connect")+ }+ try await f.expectProviders(1)+ #expect(f.model.banner?.contains("scripted provider failure") == true)+ await f.model.refresh()+ try await f.settle()+ #expect(f.backend.requestedIds == [f.space.id], "polling does not retry a failure")+ for attempt in 2...3 {+ try f.press("Try again")+ try await f.expectProviders(attempt)+ }+ for _ in 0..<80 where f.backend.provider.attempts < 1 {+ try await Task.sleep(for: .milliseconds(50))+ }+ #expect(f.backend.provider.attempts == 1, "provider creation recovered and attempted to open media")+ try await f.settle()+ await f.model.refresh()+ try await f.settle()+ #expect(f.backend.provider.attempts == 1)+ #expect(f.backend.requestedIds == Array(repeating: f.space.id, count: 3))+ }++ @Test func approvalAndPollingDoNotRetryAnExistingFailure() async throws {+ let f = try await open(autoConnect: true, failures: 1)+ defer { f.close() }+ try await f.expectProviders(1)+ try await f.approve(false)+ #expect(f.model.detail(f.space).access != nil)+ try await f.approve(true)+ #expect(f.model.detail(f.space).access == nil)+ #expect(f.backend.requestedIds == [f.space.id])+ try f.press("Try again")+ try await f.expectProviders(2)+ }+}