Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

5.1 KB
Raw Blame
import AppKit
import AVFoundation
import Cua
import Foundation

/// Window mode: an AppKit window showing the decoded frames and speaker
/// playback through AVAudioEngine. The scenario runs on a background task and
/// terminates the app when it is done.
enum WindowMode {
    static func run(cfg: Config) {
        let app = NSApplication.shared
        app.setActivationPolicy(.regular)
        let presenter = WindowPresenter()
        let delegate = AppDelegate(presenter: presenter)
        app.delegate = delegate
        Task.detached {
            var code: Int32 = 1
            do { code = try await CuaStreamingMain.runScenario(cfg: cfg, presenter: presenter) } catch {
                fputs("error: \(error)\n", stderr)
            }
            presenter.stopAudio()
            DispatchQueue.main.async { exit(code) }
        }
        withExtendedLifetime(delegate) { app.run() }
    }
}

final class AppDelegate: NSObject, NSApplicationDelegate {
    let presenter: WindowPresenter
    init(presenter: WindowPresenter) { self.presenter = presenter }

    func applicationDidFinishLaunching(_ notification: Notification) {
        presenter.makeWindow()
        NSApp.activate(ignoringOtherApps: true)
    }

    func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
}

/// Renders BGRA frames into a layer and plays PCM. Frame callbacks arrive
/// on the SDK's delivery thread: at most one frame is in flight to the main
/// thread (newer frames replace a pending one), audio is scheduled directly.
final class WindowPresenter: StreamPresenter, @unchecked Sendable {
    private var window: NSWindow?
    private var view: NSView?
    private let lock = NSLock()
    private var pending: CGImage?
    private var scheduled = false

    private let engine = AVAudioEngine()
    private let player = AVAudioPlayerNode()
    private var format: AVAudioFormat?
    private let audioQueue = DispatchQueue(label: "cua.streaming.audio")

    func makeWindow() {
        let w = NSWindow(
            contentRect: NSRect(x: 0, y: 0, width: 1024, height: 640),
            styleMask: [.titled, .closable, .resizable, .miniaturizable],
            backing: .buffered, defer: false)
        w.title = "cua streaming (Swift)"
        let v = NSView()
        v.wantsLayer = true
        v.layer?.contentsGravity = .resizeAspect
        v.layer?.backgroundColor = NSColor.black.cgColor
        w.contentView = v
        w.center()
        w.makeKeyAndOrderFront(nil)
        window = w
        view = v
    }

    func present(frame: DecodedVideoFrame) {
        guard let image = Self.image(frame) else { return }
        lock.lock()
        pending = image
        let needSchedule = !scheduled
        scheduled = true
        lock.unlock()
        guard needSchedule else { return }
        DispatchQueue.main.async { [self] in
            lock.lock()
            let img = pending
            pending = nil
            scheduled = false
            lock.unlock()
            view?.layer?.contents = img
        }
    }

    func play(audio: PcmAudio) {
        audioQueue.async { [self] in
            let ch = AVAudioChannelCount(max(1, audio.channels))
            if format == nil || format!.sampleRate != Double(audio.sampleRate) || format!.channelCount != ch {
                reconfigure(sampleRate: Double(audio.sampleRate), channels: ch)
            }
            guard let format else { return }
            let n = audio.samples.count / Int(ch)
            guard n > 0, let buf = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(n)),
                  let planes = buf.floatChannelData
            else { return }
            buf.frameLength = AVAudioFrameCount(n)
            for i in 0..<n {
                for c in 0..<Int(ch) { planes[c][i] = Float(audio.samples[i * Int(ch) + c]) / 32768 }
            }
            player.scheduleBuffer(buf)
        }
    }

    func stopAudio() {
        audioQueue.sync {
            player.stop()
            engine.stop()
        }
    }

    private func reconfigure(sampleRate: Double, channels: AVAudioChannelCount) {
        player.stop()
        engine.stop()
        if player.engine == nil { engine.attach(player) }
        guard let f = AVAudioFormat(
            commonFormat: .pcmFormatFloat32, sampleRate: sampleRate, channels: channels, interleaved: false)
        else { return }
        engine.connect(player, to: engine.mainMixerNode, format: f)
        do {
            try engine.start()
            player.play()
            format = f
        } catch {
            fputs("audio: \(error)\n", stderr)
            format = nil
        }
    }

    private static func image(_ f: DecodedVideoFrame) -> CGImage? {
        guard let provider = CGDataProvider(data: f.data as CFData) else { return nil }
        return CGImage(
            width: Int(f.width), height: Int(f.height), bitsPerComponent: 8, bitsPerPixel: 32,
            bytesPerRow: Int(f.stride), space: CGColorSpaceCreateDeviceRGB(),
            bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.noneSkipFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue),
            provider: provider, decode: nil, shouldInterpolate: false, intent: .defaultIntent)
    }
}