Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

11.1 KB
Raw Blame
// SPDX-License-Identifier: FSL-1.1-MIT
// Copyright (c) 2026 Cua AI, Inc.

#if canImport(AppKit)
import AppKit
import Combine
import SwiftUI

/// The picture-in-picture pop-out.
///
/// A floating always-on-top panel carrying the **same** `LiveStreamSession` as
/// the in-app view. Popping out does not open a second RCDP session, does not
/// reconnect, and does not reset the decoder: the controller only moves which
/// view is hosting a session that stays live throughout. Popping back in is the
/// same move in reverse.
///
/// This also sidesteps the failure this stack has seen twice — a second stream
/// window silently evicting the first. One session, two observers, no second
/// claim on the target.
///
/// A panel can also own a session of its own (``popOut(provider:source:interactive:)``):
/// a single Space window, streamed on its own target, stopped when the panel
/// closes. ``StreamPiPSet`` keeps one panel per source.
@MainActor
public final class StreamPiPController: NSObject, ObservableObject, NSWindowDelegate {
    @Published public private(set) var isOpen = false

    /// What the panel shows. `.desktop` for the shared-session pop-out.
    public private(set) var source: StreamSource = .desktop

    /// Whether the panel owns its session (``popOut(provider:source:interactive:)``)
    /// rather than borrowing the app's.
    public private(set) var ownsSession = false

    private var panel: NSPanel?
    private weak var sharedSession: LiveStreamSession?
    private var ownedSession: LiveStreamSession?
    private var sizeWatch: AnyCancellable?

    /// The session on screen in the panel, shared or owned.
    public var session: LiveStreamSession? { ownedSession ?? sharedSession }

    /// Pop the stream out into a floating panel.
    public func popOut(session: LiveStreamSession, interactive: Bool = true) {
        if let panel {
            if Self.presentsPanels { panel.makeKeyAndOrderFront(nil) }
            isOpen = true
            session.isPoppedOut = true
            return
        }
        self.sharedSession = session
        self.source = session.source
        self.ownsSession = false
        present(session, title: session.title.isEmpty ? "Space" : session.title, interactive: interactive)
    }

    /// Pop out one source on a session of its own: a single window of the
    /// Space (or its desktop) streamed independently of the app's session.
    /// The session starts now and stops when the panel closes.
    public func popOut(provider: SpaceStreamSourceProviding, source: StreamSource,
                       interactive: Bool = true) {
        if let panel {
            if Self.presentsPanels { panel.makeKeyAndOrderFront(nil) }
            isOpen = true
            return
        }
        let session = LiveStreamSession(provider: provider)
        self.ownedSession = session
        self.source = source
        self.ownsSession = true
        present(session, title: source.pipTitle, interactive: interactive)
        Task { await session.select(source) }
    }

    private func present(_ session: LiveStreamSession, title: String, interactive: Bool) {
        let aspect = Self.aspect(of: session.surfaceSize)
        let width: CGFloat = Self.defaultWidth
        let contentRect = NSRect(x: 0, y: 0, width: width, height: (width / aspect).rounded())

        let panel = NSPanel(contentRect: contentRect,
                            styleMask: [.titled, .closable, .resizable, .utilityWindow, .nonactivatingPanel],
                            backing: .buffered,
                            defer: false)
        panel.title = title
        panel.isFloatingPanel = true
        // Always on top, and present on top of full-screen apps too — a PiP that
        // disappears behind the thing you are watching is not a PiP.
        panel.level = .floating
        panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary]
        panel.hidesOnDeactivate = false
        panel.isMovableByWindowBackground = true
        panel.backgroundColor = .black
        panel.isReleasedWhenClosed = false
        panel.delegate = self
        // Keep the panel's shape locked to the stream so the letterbox stays
        // empty and the click mapping keeps a 1:1 aspect.
        panel.contentAspectRatio = NSSize(width: aspect, height: 1)

        let host = NSHostingView(rootView: PiPStreamContent(session: session, isInteractive: interactive))
        host.autoresizingMask = [.width, .height]
        panel.contentView = host
        panel.center()
        if Self.presentsPanels { panel.makeKeyAndOrderFront(nil) }

        // A window's size is only known once its first frame decodes: follow
        // it, so a tall window gets a tall panel.
        sizeWatch = session.$surfaceSize
            .removeDuplicates()
            .sink { [weak panel] size in
                guard let panel, size.width > 0, size.height > 0 else { return }
                let a = Self.aspect(of: size)
                panel.contentAspectRatio = NSSize(width: a, height: 1)
                let w = panel.contentLayoutRect.width
                panel.setContentSize(NSSize(width: w, height: (w / a).rounded()))
            }

        self.panel = panel
        isOpen = true
        session.isPoppedOut = true
    }

    /// Pop back in. A shared session keeps streaming; an owned one stops.
    public func popIn() {
        let panel = self.panel
        self.panel = nil
        panel?.delegate = nil
        panel?.orderOut(nil)
        panel?.contentView = nil
        panel?.close()
        finish()
    }

    public func toggle(session: LiveStreamSession) {
        if isOpen { popIn() } else { popOut(session: session) }
    }

    public func windowWillClose(_ notification: Notification) {
        // Closing the panel is "pop back in", never "stop the app's stream".
        panel = nil
        finish()
    }

    private func finish() {
        sizeWatch = nil
        isOpen = false
        sharedSession?.isPoppedOut = false
        sharedSession = nil
        if let owned = ownedSession {
            ownedSession = nil
            owned.isPoppedOut = false
            Task { await owned.stop() }
        }
    }

    /// The panel's window, for window-only captures and tests.
    public var window: NSWindow? { panel }

    static let defaultWidth: CGFloat = 480
    /// Off in tests: build the panel, never put it on screen.
    static var presentsPanels = true

    static func aspect(of size: CGSize) -> CGFloat {
        size.width > 0 && size.height > 0 ? size.width / size.height : 16.0 / 10.0
    }
}

/// Several pop-outs at once, one per source: the desktop and any number of
/// single windows, each in its own floating panel.
///
/// The desktop can borrow the app's session (``toggle(_:sharing:)``), so
/// popping it out does not open a second stream; every window gets a session
/// of its own that stops when its panel closes.
@MainActor
public final class StreamPiPSet: ObservableObject {
    /// Keys (``StreamSource/pipKey``) of the panels that are open.
    @Published public private(set) var openKeys: Set<String> = []

    private let provider: SpaceStreamSourceProviding
    private var controllers: [String: StreamPiPController] = [:]
    private var watches: [String: AnyCancellable] = [:]

    public init(provider: SpaceStreamSourceProviding) {
        self.provider = provider
    }

    public func isOpen(_ source: StreamSource) -> Bool { openKeys.contains(source.pipKey) }

    /// The controller for an open source.
    public func controller(for source: StreamSource) -> StreamPiPController? {
        controllers[source.pipKey]
    }

    /// Pop `source` out. With `sharing`, the panel shows that session (the
    /// app's own desktop stream) instead of opening another.
    public func popOut(_ source: StreamSource, sharing session: LiveStreamSession? = nil,
                       interactive: Bool = true) {
        let key = source.pipKey
        let controller = controllers[key] ?? StreamPiPController()
        controllers[key] = controller
        watches[key] = controller.$isOpen.dropFirst().sink { [weak self] open in
            guard let self, !open else { return }
            self.openKeys.remove(key)
            self.controllers[key] = nil
            self.watches[key] = nil
        }
        if let session {
            controller.popOut(session: session, interactive: interactive)
        } else {
            controller.popOut(provider: provider, source: source, interactive: interactive)
        }
        openKeys.insert(key)
    }

    public func popIn(_ source: StreamSource) {
        controllers[source.pipKey]?.popIn()
    }

    public func toggle(_ source: StreamSource, sharing session: LiveStreamSession? = nil) {
        if isOpen(source) { popIn(source) } else { popOut(source, sharing: session) }
    }

    /// Close every panel (app shutdown).
    public func popInAll() {
        for c in Array(controllers.values) { c.popIn() }
    }
}

public extension StreamSource {
    /// One panel per desktop or window handle, whatever its epoch or size.
    var pipKey: String {
        switch self {
        case .desktop: return "desktop"
        case let .window(w): return "window:\(w.id)"
        }
    }

    /// The pop-out's title: the window's name, or "Desktop".
    var pipTitle: String {
        switch self {
        case .desktop: return "Desktop"
        case let .window(w): return w.title.isEmpty ? w.app : w.title
        }
    }
}

/// The panel's contents: the live stream plus a minimal chrome that does not
/// steal space from the picture.
private struct PiPStreamContent: View {
    @ObservedObject public var session: LiveStreamSession
    public var isInteractive: Bool

    public var body: some View {
        ZStack(alignment: .topLeading) {
            LiveStreamView(session: session, isInteractive: isInteractive,
                           showsCursorOverlay: session.presenceView == nil)
            PresenceOverlay(session: session)
            if !session.status.isLive {
                StreamStatusBadge(status: session.status)
                    .padding(8)
            }
        }
        .background(Color.black)
    }
}

/// Small status chip used by both the in-app view and the PiP.
public struct StreamStatusBadge: View {
    public var status: LiveStreamSession.Status

    public var body: some View {
        HStack(spacing: 6) {
            Circle().fill(color).frame(width: 7, height: 7)
            Text(text).font(.system(size: 11, weight: .medium))
        }
        .padding(.horizontal, 8)
        .padding(.vertical, 4)
        .background(.black.opacity(0.65), in: Capsule())
        .foregroundStyle(.white)
    }

    private var color: Color {
        switch status {
        case .streaming: return .green
        case .connecting: return .yellow
        case .suspended: return .orange
        case .failed: return .red
        case .idle: return .gray
        }
    }

    private var text: String {
        switch status {
        case .idle: return "Idle"
        case .connecting: return "Connecting…"
        case .streaming: return "Live"
        case let .suspended(reason): return "Paused: \(reason)"
        case let .failed(reason): return "Failed: \(reason)"
        }
    }
}
#endif