Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(spaces-macos): send Command chords to the Space while the viewer has the keyboard (#4848)
ddupontf2b32eb2d854
diff --git a/apps/cua-spaces-macos/README.md b/apps/cua-spaces-macos/README.mdindex c561af4e6b..094f2ed3a9 100644--- a/apps/cua-spaces-macos/README.md+++ b/apps/cua-spaces-macos/README.md@@ -17,6 +17,15 @@ render and do platform glue.| Launch at login | Settings → General: `SMAppService.mainApp`, showing what macOS reports (on, off, waiting for approval with a button to Login Items, not found); the first run's Done page ticks it. A login launch opens without the main window, and the app's daemon (host Spaces, persistent agents, Cua Volume) starts with it || Notch panel | The "N Spaces" tab, Space tiles with live thumbnails, and the Teleport prompt; hover opens it after 300 ms, a dragged window or file drops on a tile. Window drags need Accessibility; the panel says so with an Open Settings button |+Keyboard in the live desktop (main window and floating viewer): once you+click the stream it has the keyboard, and Command chords go to the Space+instead of the app's shortcuts (Super on a Linux guest, Command on macOS).+Press and release Control+Option, or click outside the stream, to give+Command shortcuts back to the Mac. Shortcuts macOS reserves (⌘Tab, ⌘Space,+⌘⇧3/4/5, Mission Control) stay on the Mac: capturing them would need an+event tap and the Accessibility permission. Policy: `KeyCapture` in+`libs/spaces-app-swift`.+## One core, two shells```diff --git a/libs/spaces-app-swift/Sources/CuaSpacesStreaming/KeyCapture.swift b/libs/spaces-app-swift/Sources/CuaSpacesStreaming/KeyCapture.swiftnew file mode 100644index 0000000000..2480617515--- /dev/null+++ b/libs/spaces-app-swift/Sources/CuaSpacesStreaming/KeyCapture.swift@@ -0,0 +1,70 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++#if canImport(AppKit)+import AppKit++/// Where a key event goes while an interactive stream view may own the+/// keyboard.+public enum KeyRoute: Equatable, Sendable {+ /// AppKit as usual: the responder chain, then the app's menus.+ case app+ /// To the Space, before the app's menus and shortcuts see it.+ case guest+ /// The release chord: the stream hands the keyboard back to the Mac.+ case release+}++/// The keyboard capture policy of an interactive stream view.+///+/// While the stream owns keyboard focus (it is its window's first responder,+/// which a click on it makes it), every Command chord goes to the Space+/// instead of the Cua Spaces menus: the Space sends Command to a macOS guest+/// and Super to a Linux one (the Windows key on Windows), so ⌘2 is Super+2+/// in Hyprland. This holds in the main window and in the floating viewer.+///+/// Pressing and releasing Control+Option, with no other key in between,+/// hands the keyboard back (Command shortcuts reach the Mac again) until the+/// stream is clicked again; so does clicking outside the stream. Control++/// Option chords such as ⌃⌥T still reach the Space.+///+/// Shortcuts macOS reserves for itself (⌘Tab, ⌘Space, ⌘⇧3/4/5, ⌃ arrows,+/// Mission Control and other system shortcuts) never reach the app, so they+/// stay on the Mac: capturing them needs an event tap and the Accessibility+/// permission, which the viewer does not ask for.+public struct KeyCapture: Equatable, Sendable {+ /// Control+Option, pressed and released alone.+ public static let releaseChord: NSEvent.ModifierFlags = [.control, .option]++ /// The release chord is held with no other key typed since.+ private var releaseArmed = false++ public init() {}++ /// The route for one key event. `focused`: the event is for the window+ /// whose first responder is the interactive stream.+ public mutating func route(_ type: NSEvent.EventType, flags: NSEvent.ModifierFlags,+ focused: Bool) -> KeyRoute {+ let held = flags.intersection([.command, .shift, .option, .control])+ guard focused else {+ releaseArmed = false+ return .app+ }+ switch type {+ case .keyDown:+ releaseArmed = false+ return held.contains(.command) ? .guest : .app+ case .flagsChanged:+ if held == Self.releaseChord {+ releaseArmed = true+ return .app+ }+ let released = releaseArmed && held.isEmpty+ if !held.isSubset(of: Self.releaseChord) || held.isEmpty { releaseArmed = false }+ return released ? .release : .app+ default:+ return .app+ }+ }+}+#endifdiff --git a/libs/spaces-app-swift/Sources/CuaSpacesStreaming/LiveStreamView.swift b/libs/spaces-app-swift/Sources/CuaSpacesStreaming/LiveStreamView.swiftindex 51048aed67..e7c2e3f424 100644--- a/libs/spaces-app-swift/Sources/CuaSpacesStreaming/LiveStreamView.swift+++ b/libs/spaces-app-swift/Sources/CuaSpacesStreaming/LiveStreamView.swift@@ -86,6 +86,10 @@ public final class LiveStreamInputView: NSView {/// button-up still lands somewhere the user pointed at — and never at a/// sentinel.private var lastInFramePoint: CGPoint?+ /// Claims Command chords for the Space before the app's menus see them+ /// (``KeyCapture``); installed while the view is in a window.+ private var keyMonitor: Any?+ private var keyCapture = KeyCapture()/// The single source of the view↔surface mapping.public var geometry: StreamGeometry {@@ -295,6 +299,43 @@ public final class LiveStreamInputView: NSView {// MARK: - Keyboard+ public override func viewDidMoveToWindow() {+ super.viewDidMoveToWindow()+ if window == nil {+ if let keyMonitor { NSEvent.removeMonitor(keyMonitor) }+ keyMonitor = nil+ } else if keyMonitor == nil {+ // A local monitor runs before AppKit dispatches the event, so it+ // wins over the main menu and SwiftUI shortcuts, in a+ // non-activating panel too.+ keyMonitor = NSEvent.addLocalMonitorForEvents(matching: [.keyDown, .flagsChanged]) { [weak self] event in+ MainActor.assumeIsolated { self?.capture(event) ?? false } ? nil : event+ }+ }+ }++ /// Whether the event was claimed for the Space (``KeyCapture``).+ func capture(_ event: NSEvent) -> Bool {+ let focused = isInteractive && window != nil && event.window === window && window?.firstResponder === self+ switch keyCapture.route(event.type, flags: event.modifierFlags, focused: focused) {+ case .app:+ return false+ case .guest:+ sendChord(event)+ return true+ case .release:+ window?.makeFirstResponder(nil)+ return false+ }+ }++ /// AppKit does not reliably send the key-up of a Command chord to the+ /// view, and a press with no release is a stuck key on the Space: send+ /// the chord whole. A late real key-up is a harmless repeat.+ private func sendChord(_ event: NSEvent) {+ onInput?(encoder.keyEvents(for: event, down: true) + encoder.keyEvents(for: event, down: false))+ }+public override func keyDown(with event: NSEvent) {guard isInteractive else { return super.keyDown(with: event) }onInput?(encoder.keyEvents(for: event, down: true))@@ -306,13 +347,11 @@ public final class LiveStreamInputView: NSView {}/// Swallow the command chords AppKit would otherwise eat, so ⌘C in the- /// stream reaches the Space rather than the host app.+ /// stream reaches the Space rather than the host app (the key monitor+ /// usually claims them first).public override func performKeyEquivalent(with event: NSEvent) -> Bool {guard isInteractive, window?.firstResponder === self else { return false }- // AppKit does not reliably send the key-up of a Command chord to the- // view, and a press with no release is a stuck key on the Space:- // send the chord whole. A late real key-up is a harmless repeat.- onInput?(encoder.keyEvents(for: event, down: true) + encoder.keyEvents(for: event, down: false))+ sendChord(event)return true}}diff --git a/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/KeyCaptureTests.swift b/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/KeyCaptureTests.swiftnew file mode 100644index 0000000000..25e333b4d0--- /dev/null+++ b/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/KeyCaptureTests.swift@@ -0,0 +1,43 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++#if canImport(AppKit)+import AppKit+import Testing+@testable import CuaSpacesStreaming++/// Command chords typed while the interactive stream has the keyboard go to+/// the Space (Super on a Linux guest), not to the app's menus (#4609).+@Suite @MainActor struct KeyCaptureTests {++ @Test func commandChordsGoToTheGuestWhileTheStreamHasFocus() {+ var c = KeyCapture()+ #expect(c.route(.keyDown, flags: .command, focused: true) == .guest, "⌘2 → Super+2")+ #expect(c.route(.keyDown, flags: [.command, .shift], focused: true) == .guest)+ #expect(c.route(.keyDown, flags: [.command, .control], focused: true) == .guest)+ // Plain keys and Control chords take the view's own keyDown.+ #expect(c.route(.keyDown, flags: [], focused: true) == .app)+ #expect(c.route(.keyDown, flags: .control, focused: true) == .app)+ // Without the keyboard the app keeps its shortcuts.+ #expect(c.route(.keyDown, flags: .command, focused: false) == .app)+ }++ /// Control+Option pressed and released alone hands the keyboard back;+ /// a Control+Option chord (⌃⌥T) does not.+ @Test func controlOptionAloneReleasesTheKeyboard() {+ var c = KeyCapture()+ #expect(c.route(.flagsChanged, flags: .control, focused: true) == .app)+ #expect(c.route(.flagsChanged, flags: [.control, .option], focused: true) == .app)+ #expect(c.route(.flagsChanged, flags: .option, focused: true) == .app)+ #expect(c.route(.flagsChanged, flags: [], focused: true) == .release)++ #expect(c.route(.flagsChanged, flags: [.control, .option], focused: true) == .app)+ #expect(c.route(.keyDown, flags: [.control, .option], focused: true) == .app, "⌃⌥T goes to the view")+ #expect(c.route(.flagsChanged, flags: [], focused: true) == .app, "a chord, not the release")++ #expect(c.route(.flagsChanged, flags: [.control, .option], focused: true) == .app)+ #expect(c.route(.flagsChanged, flags: [.control, .option, .shift], focused: true) == .app)+ #expect(c.route(.flagsChanged, flags: [], focused: true) == .app, "another modifier joined")+ }+}+#endifdiff --git a/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/ViewerInputTests.swift b/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/ViewerInputTests.swiftindex 91ce02e086..c3709a1153 100644--- a/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/ViewerInputTests.swift+++ b/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/ViewerInputTests.swift@@ -278,5 +278,45 @@ private func pointerEvents(in text: String) -> [[String: Any]] {#expect(provider.policies.last == "allow_activation")await session.stop()}++ /// A Command chord typed while the stream has the keyboard goes to the+ /// Space (Super on a Linux guest) before the app's own shortcuts: here a+ /// SwiftUI ⌘N in the same window, which used to take it (#4609).+ @Test func commandChordReachesTheSpaceBeforeTheAppsShortcuts() throws {+ _ = NSApplication.shared+ var shortcutFired = false+ let session = LiveStreamSession(provider: OfflineStreamSourceProvider())+ let root = VStack {+ Button("New Space") { shortcutFired = true }.keyboardShortcut("n", modifiers: .command)+ LiveStreamView(session: session, isInteractive: true).frame(width: 400, height: 300)+ }+ let window = NSWindow(contentRect: CGRect(x: 0, y: 0, width: 500, height: 400),+ styleMask: [.titled], backing: .buffered, defer: false)+ window.isReleasedWhenClosed = false+ let host = NSHostingView(rootView: root)+ host.frame = CGRect(x: 0, y: 0, width: 500, height: 400)+ window.contentView = host+ host.layoutSubtreeIfNeeded()+ defer { window.contentView = nil; window.close() }++ func find(_ v: NSView) -> LiveStreamInputView? {+ if let s = v as? LiveStreamInputView { return s }+ for sub in v.subviews { if let f = find(sub) { return f } }+ return nil+ }+ let stream = try #require(find(host))+ var keys: [String] = []+ stream.onInput = { events in+ for case let .key(key, down, modifiers, _) in events {+ keys.append("\(modifiers.map(\.rawValue).joined(separator: "+"))+\(key) \(down ? "down" : "up")")+ }+ }+ #expect(window.makeFirstResponder(stream))+ NSApp.sendEvent(NSEvent.keyEvent(with: .keyDown, location: .zero, modifierFlags: .command, timestamp: 0,+ windowNumber: window.windowNumber, context: nil, characters: "n",+ charactersIgnoringModifiers: "n", isARepeat: false, keyCode: 45)!)+ #expect(keys == ["command+n down", "command+n up"], "the chord reached the Space")+ #expect(!shortcutFired, "the app's ⌘N did not run")+ }}#endif