Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(spaces-macos): send Command chords to the Space while the viewer has the keyboard (#4848)

ddupontf2b32eb2d854
diff --git a/apps/cua-spaces-macos/README.md b/apps/cua-spaces-macos/README.md
index c561af4e6b..094f2ed3a9 100644
--- a/apps/cua-spaces-macos/README.md
+++ b/apps/cua-spaces-macos/README.md
@@ -17,6 +17,15 @@ render and do platform glue.
| Launch at login | Settings → General: `SMAppService.mainApp`, showing what macOS reports (on, off, waiting for approval with a button to Login Items, not found); the first run's Done page ticks it. A login launch opens without the main window, and the app's daemon (host Spaces, persistent agents, Cua Volume) starts with it |
| Notch panel | The "N Spaces" tab, Space tiles with live thumbnails, and the Teleport prompt; hover opens it after 300 ms, a dragged window or file drops on a tile. Window drags need Accessibility; the panel says so with an Open Settings button |
+Keyboard in the live desktop (main window and floating viewer): once you
+click the stream it has the keyboard, and Command chords go to the Space
+instead of the app's shortcuts (Super on a Linux guest, Command on macOS).
+Press and release Control+Option, or click outside the stream, to give
+Command shortcuts back to the Mac. Shortcuts macOS reserves (⌘Tab, ⌘Space,
+⌘⇧3/4/5, Mission Control) stay on the Mac: capturing them would need an
+event tap and the Accessibility permission. Policy: `KeyCapture` in
+`libs/spaces-app-swift`.
+
## One core, two shells
```
diff --git a/libs/spaces-app-swift/Sources/CuaSpacesStreaming/KeyCapture.swift b/libs/spaces-app-swift/Sources/CuaSpacesStreaming/KeyCapture.swift
new file mode 100644
index 0000000000..2480617515
--- /dev/null
+++ b/libs/spaces-app-swift/Sources/CuaSpacesStreaming/KeyCapture.swift
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+#if canImport(AppKit)
+import AppKit
+
+/// Where a key event goes while an interactive stream view may own the
+/// keyboard.
+public enum KeyRoute: Equatable, Sendable {
+ /// AppKit as usual: the responder chain, then the app's menus.
+ case app
+ /// To the Space, before the app's menus and shortcuts see it.
+ case guest
+ /// The release chord: the stream hands the keyboard back to the Mac.
+ case release
+}
+
+/// The keyboard capture policy of an interactive stream view.
+///
+/// While the stream owns keyboard focus (it is its window's first responder,
+/// which a click on it makes it), every Command chord goes to the Space
+/// instead of the Cua Spaces menus: the Space sends Command to a macOS guest
+/// and Super to a Linux one (the Windows key on Windows), so ⌘2 is Super+2
+/// in Hyprland. This holds in the main window and in the floating viewer.
+///
+/// Pressing and releasing Control+Option, with no other key in between,
+/// hands the keyboard back (Command shortcuts reach the Mac again) until the
+/// stream is clicked again; so does clicking outside the stream. Control+
+/// Option chords such as ⌃⌥T still reach the Space.
+///
+/// Shortcuts macOS reserves for itself (⌘Tab, ⌘Space, ⌘⇧3/4/5, ⌃ arrows,
+/// Mission Control and other system shortcuts) never reach the app, so they
+/// stay on the Mac: capturing them needs an event tap and the Accessibility
+/// permission, which the viewer does not ask for.
+public struct KeyCapture: Equatable, Sendable {
+ /// Control+Option, pressed and released alone.
+ public static let releaseChord: NSEvent.ModifierFlags = [.control, .option]
+
+ /// The release chord is held with no other key typed since.
+ private var releaseArmed = false
+
+ public init() {}
+
+ /// The route for one key event. `focused`: the event is for the window
+ /// whose first responder is the interactive stream.
+ public mutating func route(_ type: NSEvent.EventType, flags: NSEvent.ModifierFlags,
+ focused: Bool) -> KeyRoute {
+ let held = flags.intersection([.command, .shift, .option, .control])
+ guard focused else {
+ releaseArmed = false
+ return .app
+ }
+ switch type {
+ case .keyDown:
+ releaseArmed = false
+ return held.contains(.command) ? .guest : .app
+ case .flagsChanged:
+ if held == Self.releaseChord {
+ releaseArmed = true
+ return .app
+ }
+ let released = releaseArmed && held.isEmpty
+ if !held.isSubset(of: Self.releaseChord) || held.isEmpty { releaseArmed = false }
+ return released ? .release : .app
+ default:
+ return .app
+ }
+ }
+}
+#endif
diff --git a/libs/spaces-app-swift/Sources/CuaSpacesStreaming/LiveStreamView.swift b/libs/spaces-app-swift/Sources/CuaSpacesStreaming/LiveStreamView.swift
index 51048aed67..e7c2e3f424 100644
--- a/libs/spaces-app-swift/Sources/CuaSpacesStreaming/LiveStreamView.swift
+++ b/libs/spaces-app-swift/Sources/CuaSpacesStreaming/LiveStreamView.swift
@@ -86,6 +86,10 @@ public final class LiveStreamInputView: NSView {
/// button-up still lands somewhere the user pointed at — and never at a
/// sentinel.
private var lastInFramePoint: CGPoint?
+ /// Claims Command chords for the Space before the app's menus see them
+ /// (``KeyCapture``); installed while the view is in a window.
+ private var keyMonitor: Any?
+ private var keyCapture = KeyCapture()
/// The single source of the view↔surface mapping.
public var geometry: StreamGeometry {
@@ -295,6 +299,43 @@ public final class LiveStreamInputView: NSView {
// MARK: - Keyboard
+ public override func viewDidMoveToWindow() {
+ super.viewDidMoveToWindow()
+ if window == nil {
+ if let keyMonitor { NSEvent.removeMonitor(keyMonitor) }
+ keyMonitor = nil
+ } else if keyMonitor == nil {
+ // A local monitor runs before AppKit dispatches the event, so it
+ // wins over the main menu and SwiftUI shortcuts, in a
+ // non-activating panel too.
+ keyMonitor = NSEvent.addLocalMonitorForEvents(matching: [.keyDown, .flagsChanged]) { [weak self] event in
+ MainActor.assumeIsolated { self?.capture(event) ?? false } ? nil : event
+ }
+ }
+ }
+
+ /// Whether the event was claimed for the Space (``KeyCapture``).
+ func capture(_ event: NSEvent) -> Bool {
+ let focused = isInteractive && window != nil && event.window === window && window?.firstResponder === self
+ switch keyCapture.route(event.type, flags: event.modifierFlags, focused: focused) {
+ case .app:
+ return false
+ case .guest:
+ sendChord(event)
+ return true
+ case .release:
+ window?.makeFirstResponder(nil)
+ return false
+ }
+ }
+
+ /// AppKit does not reliably send the key-up of a Command chord to the
+ /// view, and a press with no release is a stuck key on the Space: send
+ /// the chord whole. A late real key-up is a harmless repeat.
+ private func sendChord(_ event: NSEvent) {
+ onInput?(encoder.keyEvents(for: event, down: true) + encoder.keyEvents(for: event, down: false))
+ }
+
public override func keyDown(with event: NSEvent) {
guard isInteractive else { return super.keyDown(with: event) }
onInput?(encoder.keyEvents(for: event, down: true))
@@ -306,13 +347,11 @@ public final class LiveStreamInputView: NSView {
}
/// Swallow the command chords AppKit would otherwise eat, so ⌘C in the
- /// stream reaches the Space rather than the host app.
+ /// stream reaches the Space rather than the host app (the key monitor
+ /// usually claims them first).
public override func performKeyEquivalent(with event: NSEvent) -> Bool {
guard isInteractive, window?.firstResponder === self else { return false }
- // AppKit does not reliably send the key-up of a Command chord to the
- // view, and a press with no release is a stuck key on the Space:
- // send the chord whole. A late real key-up is a harmless repeat.
- onInput?(encoder.keyEvents(for: event, down: true) + encoder.keyEvents(for: event, down: false))
+ sendChord(event)
return true
}
}
diff --git a/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/KeyCaptureTests.swift b/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/KeyCaptureTests.swift
new file mode 100644
index 0000000000..25e333b4d0
--- /dev/null
+++ b/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/KeyCaptureTests.swift
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+#if canImport(AppKit)
+import AppKit
+import Testing
+@testable import CuaSpacesStreaming
+
+/// Command chords typed while the interactive stream has the keyboard go to
+/// the Space (Super on a Linux guest), not to the app's menus (#4609).
+@Suite @MainActor struct KeyCaptureTests {
+
+ @Test func commandChordsGoToTheGuestWhileTheStreamHasFocus() {
+ var c = KeyCapture()
+ #expect(c.route(.keyDown, flags: .command, focused: true) == .guest, "⌘2 → Super+2")
+ #expect(c.route(.keyDown, flags: [.command, .shift], focused: true) == .guest)
+ #expect(c.route(.keyDown, flags: [.command, .control], focused: true) == .guest)
+ // Plain keys and Control chords take the view's own keyDown.
+ #expect(c.route(.keyDown, flags: [], focused: true) == .app)
+ #expect(c.route(.keyDown, flags: .control, focused: true) == .app)
+ // Without the keyboard the app keeps its shortcuts.
+ #expect(c.route(.keyDown, flags: .command, focused: false) == .app)
+ }
+
+ /// Control+Option pressed and released alone hands the keyboard back;
+ /// a Control+Option chord (⌃⌥T) does not.
+ @Test func controlOptionAloneReleasesTheKeyboard() {
+ var c = KeyCapture()
+ #expect(c.route(.flagsChanged, flags: .control, focused: true) == .app)
+ #expect(c.route(.flagsChanged, flags: [.control, .option], focused: true) == .app)
+ #expect(c.route(.flagsChanged, flags: .option, focused: true) == .app)
+ #expect(c.route(.flagsChanged, flags: [], focused: true) == .release)
+
+ #expect(c.route(.flagsChanged, flags: [.control, .option], focused: true) == .app)
+ #expect(c.route(.keyDown, flags: [.control, .option], focused: true) == .app, "⌃⌥T goes to the view")
+ #expect(c.route(.flagsChanged, flags: [], focused: true) == .app, "a chord, not the release")
+
+ #expect(c.route(.flagsChanged, flags: [.control, .option], focused: true) == .app)
+ #expect(c.route(.flagsChanged, flags: [.control, .option, .shift], focused: true) == .app)
+ #expect(c.route(.flagsChanged, flags: [], focused: true) == .app, "another modifier joined")
+ }
+}
+#endif
diff --git a/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/ViewerInputTests.swift b/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/ViewerInputTests.swift
index 91ce02e086..c3709a1153 100644
--- a/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/ViewerInputTests.swift
+++ b/libs/spaces-app-swift/Tests/CuaSpacesStreamingTests/ViewerInputTests.swift
@@ -278,5 +278,45 @@ private func pointerEvents(in text: String) -> [[String: Any]] {
#expect(provider.policies.last == "allow_activation")
await session.stop()
}
+
+ /// A Command chord typed while the stream has the keyboard goes to the
+ /// Space (Super on a Linux guest) before the app's own shortcuts: here a
+ /// SwiftUI ⌘N in the same window, which used to take it (#4609).
+ @Test func commandChordReachesTheSpaceBeforeTheAppsShortcuts() throws {
+ _ = NSApplication.shared
+ var shortcutFired = false
+ let session = LiveStreamSession(provider: OfflineStreamSourceProvider())
+ let root = VStack {
+ Button("New Space") { shortcutFired = true }.keyboardShortcut("n", modifiers: .command)
+ LiveStreamView(session: session, isInteractive: true).frame(width: 400, height: 300)
+ }
+ let window = NSWindow(contentRect: CGRect(x: 0, y: 0, width: 500, height: 400),
+ styleMask: [.titled], backing: .buffered, defer: false)
+ window.isReleasedWhenClosed = false
+ let host = NSHostingView(rootView: root)
+ host.frame = CGRect(x: 0, y: 0, width: 500, height: 400)
+ window.contentView = host
+ host.layoutSubtreeIfNeeded()
+ defer { window.contentView = nil; window.close() }
+
+ func find(_ v: NSView) -> LiveStreamInputView? {
+ if let s = v as? LiveStreamInputView { return s }
+ for sub in v.subviews { if let f = find(sub) { return f } }
+ return nil
+ }
+ let stream = try #require(find(host))
+ var keys: [String] = []
+ stream.onInput = { events in
+ for case let .key(key, down, modifiers, _) in events {
+ keys.append("\(modifiers.map(\.rawValue).joined(separator: "+"))+\(key) \(down ? "down" : "up")")
+ }
+ }
+ #expect(window.makeFirstResponder(stream))
+ NSApp.sendEvent(NSEvent.keyEvent(with: .keyDown, location: .zero, modifierFlags: .command, timestamp: 0,
+ windowNumber: window.windowNumber, context: nil, characters: "n",
+ charactersIgnoringModifiers: "n", isARepeat: false, keyCode: 45)!)
+ #expect(keys == ["command+n down", "command+n up"], "the chord reached the Space")
+ #expect(!shortcutFired, "the app's ⌘N did not run")
+ }
}
#endif