Skip to content

๐Ÿ‘พ Open Computer Use โ€“ Open-Source Alternative to Codex Computer Use

15.2 KB
Raw Blame
import AppKit
import CoreGraphics
import Foundation
import XCTest
@testable import OpenComputerUseKit

@MainActor
final class SkyKeyboardLiveTests: XCTestCase {
    private struct WindowRecord {
        let id: CGWindowID
        let pid: pid_t
        let bounds: CGRect
        let name: String
    }

    func testCoveredChromeReceivesSkyKeyTypingWithoutForegroundSideEffects() throws {
        guard ProcessInfo.processInfo.environment["OPEN_COMPUTER_USE_RUN_SKY_KEY_LIVE_TEST"] == "1" else {
            throw XCTSkip("Set OPEN_COMPUTER_USE_RUN_SKY_KEY_LIVE_TEST=1 to run the isolated Chrome live test")
        }
        let spi = SkyLightSPI.shared
        guard spi.capability.isAvailable else {
            throw XCTSkip("SkyLight SPI unavailable: \(spi.capability.unavailableReason)")
        }

        let chromeURL = URL(fileURLWithPath: "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome")
        guard FileManager.default.isExecutableFile(atPath: chromeURL.path) else {
            throw XCTSkip("Google Chrome is not installed at the standard path")
        }
        let originalFrontApp = NSWorkspace.shared.frontmostApplication
        defer {
            if let originalFrontApp {
                _ = originalFrontApp.activate(options: [.activateAllWindows])
            }
        }

        let coverExecutable = Self.packageRoot
            .appendingPathComponent(".build/debug/OpenComputerUseFixture")
        guard FileManager.default.isExecutableFile(atPath: coverExecutable.path) else {
            throw XCTSkip("Build OpenComputerUseFixture before running the live test")
        }
        let coverProbe = try launch(executable: coverExecutable)
        defer {
            stop(coverProbe)
        }
        let coverProbeWindow = try waitForWindow(
            pid: coverProbe.processIdentifier,
            nameContaining: "OpenComputerUseFixture"
        )

        let testRoot = Self.packageRoot.appendingPathComponent(".build", isDirectory: true)
            .appendingPathComponent("ocu-sky-key-live-\(UUID().uuidString)", isDirectory: true)
        let profileURL = testRoot.appendingPathComponent("chrome-profile", isDirectory: true)
        let pageURL = testRoot.appendingPathComponent("index.html")
        try FileManager.default.createDirectory(at: profileURL, withIntermediateDirectories: true)
        try Self.liveTestHTML.write(to: pageURL, atomically: false, encoding: .utf8)
        defer {
            do {
                try FileManager.default.removeItem(at: testRoot)
            } catch {
                print("sky_key live test cleanup warning: \(error)")
            }
        }

        let targetWidth = Int(min(640, coverProbeWindow.bounds.width - 80))
        let targetHeight = Int(min(480, coverProbeWindow.bounds.height - 80))
        let targetX = Int(coverProbeWindow.bounds.minX + 40)
        let targetY = Int(coverProbeWindow.bounds.minY + 40)

        let chrome = Process()
        chrome.executableURL = chromeURL
        chrome.arguments = [
            "--user-data-dir=\(profileURL.path)",
            "--no-first-run",
            "--no-default-browser-check",
            "--disable-background-networking",
            "--disable-component-update",
            "--window-position=\(targetX),\(targetY)",
            "--window-size=\(targetWidth),\(targetHeight)",
            "--app=\(pageURL.absoluteString)",
        ]
        chrome.standardOutput = FileHandle.nullDevice
        chrome.standardError = FileHandle.nullDevice
        try chrome.run()
        defer {
            stop(chrome)
        }

        let readyWindow = try waitForWindow(
            pid: chrome.processIdentifier,
            nameContaining: "ocu-sky-key-ready"
        )
        print("sky_key live test: target window=\(readyWindow.id) owner pid=\(readyWindow.pid)")
        stop(coverProbe)
        let cover = try launch(executable: coverExecutable)
        defer {
            stop(cover)
        }
        let coverWindow = try waitForWindow(
            pid: cover.processIdentifier,
            nameContaining: "OpenComputerUseFixture"
        )
        try waitUntil(timeout: 5, failure: "Isolated Chrome remained the frontmost app") {
            guard let frontPID = NSWorkspace.shared.frontmostApplication?.processIdentifier else {
                return false
            }
            return frontPID != readyWindow.pid
        }
        try waitUntil(timeout: 5, failure: "The cover window did not move above isolated Chrome") {
            let ordered = windows()
            guard
                let currentCoverIndex = ordered.firstIndex(where: { $0.id == coverWindow.id }),
                let currentTargetIndex = ordered.firstIndex(where: { $0.id == readyWindow.id })
            else {
                return false
            }
            return currentCoverIndex < currentTargetIndex
        }

        let orderedBefore = windows()
        guard
            let coveredWindow = orderedBefore.first(where: { $0.id == readyWindow.id }),
            let freshCoverWindow = orderedBefore.first(where: { $0.id == coverWindow.id }),
            let coverIndex = orderedBefore.firstIndex(where: { $0.id == coverWindow.id }),
            let targetIndex = orderedBefore.firstIndex(where: { $0.id == readyWindow.id })
        else {
            return XCTFail("Could not re-read the controlled Chrome and cover windows")
        }
        XCTAssertTrue(
            freshCoverWindow.bounds.contains(coveredWindow.bounds),
            "The controlled Chrome window must be fully covered before sky_key"
        )
        XCTAssertLessThan(coverIndex, targetIndex, "The cover window must be above Chrome in z-order")
        try FixtureBridge.post(
            FixtureCommand(kind: "click", identifier: "fixture-input")
        )
        let foregroundStateBefore = try waitForFixtureState(
            pid: cover.processIdentifier,
            failure: "The foreground fixture did not keep its text field focused"
        ) { state in
            state.isActive == true
                && state.isKeyWindow == true
                && state.focusedIdentifier == "fixture-input"
        }

        let frontPIDBefore = NSWorkspace.shared.frontmostApplication?.processIdentifier
        let cursorBefore = CGEvent(source: nil)?.location
        let target = SkyKeyboardTarget(windowID: coveredWindow.id, pid: coveredWindow.pid)

        try SkyKeyboardDispatcher.typeText(target: target, text: "hello", spi: spi)
        let typedWindow = try waitForWindow(pid: readyWindow.pid, nameContaining: "ocu-sky-key-value-hello|")
        XCTAssertTrue(
            typedWindow.name.hasPrefix("ocu-sky-key-value-hello|"),
            "sky_key typing must land in the covered Chrome input exactly once"
        )

        try SkyKeyboardDispatcher.pressKey(target: target, key: "backspace", spi: spi)
        try SkyKeyboardDispatcher.pressKey(target: target, key: "shift+a", spi: spi)
        _ = try waitForWindow(pid: readyWindow.pid, nameContaining: "ocu-sky-key-value-hellA|")

        // Menu-bar key equivalents go through the target's Edit menu.
        try SkyKeyboardDispatcher.pressKey(target: target, key: "cmd+a", spi: spi)
        try SkyKeyboardDispatcher.typeText(target: target, text: "Ok!", spi: spi)
        let editedWindow = try waitForWindow(pid: readyWindow.pid, nameContaining: "ocu-sky-key-value-Ok!|")
        RunLoop.current.run(until: Date().addingTimeInterval(0.5))
        let finalWindow = try XCTUnwrap(windows().first(where: { $0.id == editedWindow.id }))
        XCTAssertEqual(
            finalWindow.name,
            "ocu-sky-key-value-Ok!|unfocused",
            "sky_key must deliver keys, key-binding chords and menu chords, then release Chrome's synthetic key state"
        )

        // Control: the plain CGEvent.postToPid keyboard path that `auto` uses.
        // Chromium receives the key events but refuses text insertion while its
        // window is not key. Observed, not asserted, so a future Chromium change
        // cannot fail the sky_key regression.
        try InputSimulation.typeText("plain", pid: coveredWindow.pid)
        RunLoop.current.run(until: Date().addingTimeInterval(0.5))
        let controlTitle = windows().first(where: { $0.id == editedWindow.id })?.name ?? ""
        print("sky_key live test: public postToPid control title after release=\(controlTitle)")

        XCTAssertEqual(
            NSWorkspace.shared.frontmostApplication?.processIdentifier,
            frontPIDBefore,
            "sky_key must not change the frontmost app"
        )
        let foregroundStateAfter = try waitForFixtureState(
            pid: cover.processIdentifier,
            failure: "The foreground fixture state was unavailable after sky_key"
        ) { _ in true }
        XCTAssertEqual(foregroundStateAfter.isActive, true, "sky_key must keep the foreground app active")
        XCTAssertEqual(foregroundStateAfter.isKeyWindow, true, "sky_key must keep the foreground window key")
        XCTAssertEqual(
            foregroundStateAfter.focusedIdentifier,
            foregroundStateBefore.focusedIdentifier,
            "sky_key must preserve the foreground first responder"
        )
        XCTAssertEqual(
            foregroundStateAfter.activationLossCount,
            foregroundStateBefore.activationLossCount,
            "sky_key must not transiently resign the foreground app"
        )
        XCTAssertEqual(
            foregroundStateAfter.keyWindowLossCount,
            foregroundStateBefore.keyWindowLossCount,
            "sky_key must not transiently resign the foreground key window"
        )
        XCTAssertEqual(
            foregroundStateAfter.elements.first(where: { $0.identifier == "fixture-input" })?.value,
            foregroundStateBefore.elements.first(where: { $0.identifier == "fixture-input" })?.value,
            "sky_key keystrokes must not leak into the foreground text field"
        )

        if let cursorBefore, let cursorAfter = CGEvent(source: nil)?.location {
            print("sky_key live test: cursor before=\(cursorBefore) after=\(cursorAfter)")
            XCTAssertLessThan(hypot(cursorAfter.x - cursorBefore.x, cursorAfter.y - cursorBefore.y), 0.5, "sky_key must not move the real pointer (a user moving the mouse during the run also trips this)")
        }

        let orderedAfter = windows()
        guard
            let coverIndexAfter = orderedAfter.firstIndex(where: { $0.id == coverWindow.id }),
            let targetIndexAfter = orderedAfter.firstIndex(where: { $0.id == editedWindow.id })
        else {
            return XCTFail("Could not verify final window z-order")
        }
        XCTAssertLessThan(coverIndexAfter, targetIndexAfter, "sky_key must not raise the Chrome window")
    }

    private func waitForWindow(
        pid: pid_t? = nil,
        nameContaining marker: String,
        timeout: TimeInterval = 10
    ) throws -> WindowRecord {
        var match: WindowRecord?
        do {
            try waitUntil(timeout: timeout, failure: "Timed out waiting for Chrome window title containing \(marker)") {
                match = windows().first(where: { window in
                    (pid == nil || window.pid == pid) && window.name.contains(marker)
                })
                return match != nil
            }
        } catch {
            let observed = windows()
                .filter { pid == nil || $0.pid == pid }
                .map { "\($0.id):\($0.name)" }
                .joined(separator: ", ")
            throw ComputerUseError.message(
                "Timed out waiting for Chrome window title containing \(marker); observed [\(observed)]"
            )
        }
        return try XCTUnwrap(match)
    }

    private func waitUntil(
        timeout: TimeInterval,
        failure: String,
        condition: () -> Bool
    ) throws {
        let deadline = Date().addingTimeInterval(timeout)
        while Date() < deadline {
            if condition() {
                return
            }
            RunLoop.current.run(until: Date().addingTimeInterval(0.05))
        }
        throw ComputerUseError.message(failure)
    }

    private func waitForFixtureState(
        pid: pid_t,
        failure: String,
        condition: (FixtureAppState) -> Bool
    ) throws -> FixtureAppState {
        var match: FixtureAppState?
        try waitUntil(timeout: 5, failure: failure) {
            guard
                let state = try? FixtureBridge.readState(),
                state.processIdentifier == pid,
                condition(state)
            else {
                return false
            }
            match = state
            return true
        }
        return try XCTUnwrap(match)
    }

    private func stop(_ process: Process) {
        if process.isRunning {
            process.terminate()
            let deadline = Date().addingTimeInterval(5)
            while process.isRunning, Date() < deadline {
                RunLoop.current.run(until: Date().addingTimeInterval(0.05))
            }
        }
        RunLoop.current.run(until: Date().addingTimeInterval(0.25))
    }

    private func launch(executable: URL) throws -> Process {
        let process = Process()
        process.executableURL = executable
        process.standardOutput = FileHandle.nullDevice
        process.standardError = FileHandle.nullDevice
        try process.run()
        return process
    }

    private func windows() -> [WindowRecord] {
        let rawWindows = CGWindowListCopyWindowInfo(
            [.optionOnScreenOnly, .excludeDesktopElements],
            kCGNullWindowID
        ) as? [[String: Any]] ?? []

        return rawWindows.compactMap { info in
            guard
                let number = info[kCGWindowNumber as String] as? NSNumber,
                let ownerPID = info[kCGWindowOwnerPID as String] as? NSNumber,
                let layer = info[kCGWindowLayer as String] as? NSNumber,
                layer.intValue == 0,
                let boundsDictionary = info[kCGWindowBounds as String] as? [String: Any],
                let bounds = CGRect(dictionaryRepresentation: boundsDictionary as CFDictionary),
                bounds.width > 0,
                bounds.height > 0
            else {
                return nil
            }

            return WindowRecord(
                id: number.uint32Value,
                pid: ownerPID.int32Value,
                bounds: bounds,
                name: info[kCGWindowName as String] as? String ?? ""
            )
        }
    }

    private static let liveTestHTML = #"""
    <!doctype html>
    <meta charset="utf-8">
    <title>ocu-sky-key-ready</title>
    <style>
      html, body { width: 100%; height: 100%; margin: 0; }
      input { width: 90%; font: 24px system-ui; margin: 24px; }
    </style>
    <input id="target" autofocus placeholder="sky_key probe">
    <script>
      const input = document.querySelector('#target');
      input.focus();
      const update = () => {
        document.title = `ocu-sky-key-value-${input.value}|${document.hasFocus() ? 'focused' : 'unfocused'}`;
      };
      input.addEventListener('input', update);
      window.addEventListener('focus', update);
      window.addEventListener('blur', update);
    </script>
    """#

    private static let packageRoot: URL = {
        var url = URL(fileURLWithPath: #filePath)
        for _ in 0..<5 {
            url.deleteLastPathComponent()
        }
        return url
    }()
}