Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

21.4 KB
Raw Blame
# Run the canonical Rust desktop matrix in an active Windows user session.
# Scenario definitions and assertions stay in the Rust integration test.
param(
    [switch]$NoBuild,
    [switch]$RequireGui
)

Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"

$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Definition
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..\..")).Path
$driverRoot = Join-Path $repoRoot "libs\cua-driver"
$rustRoot = Join-Path $driverRoot "rust"
$suite = if ([string]::IsNullOrWhiteSpace($env:CUA_E2E_INTERNAL_LANE)) { "all" } else { $env:CUA_E2E_INTERNAL_LANE }
if ($suite -notin @("shared", "native", "capture", "all")) {
    throw "Unsupported internal lane: $suite"
}
$artifactDir = Join-Path $repoRoot "artifacts\cua-driver\windows"
New-Item -ItemType Directory -Force $artifactDir | Out-Null
$recordingRoot = Join-Path $artifactDir "recordings"
Remove-Item -Path $recordingRoot -Recurse -Force -ErrorAction SilentlyContinue
New-Item -ItemType Directory -Force $recordingRoot | Out-Null
$resultsPath = Join-Path $artifactDir "results.jsonl"
$casesPath = Join-Path $artifactDir "cases.jsonl"
$environmentPath = Join-Path $artifactDir "environment.jsonl"
$summaryPath = Join-Path $artifactDir "summary.md"
foreach ($path in @($casesPath, $environmentPath, $resultsPath)) {
    New-Item -ItemType File -Force $path | Out-Null
    Clear-Content $path
}
Remove-Item -Force -ErrorAction SilentlyContinue $summaryPath
$env:CUA_E2E_DECLARATIONS_FILE = $casesPath
$env:CUA_E2E_ENVIRONMENT_FILE = $environmentPath
$env:CUA_E2E_RESULTS_FILE = $resultsPath
$env:CUA_E2E_RECORDINGS_ROOT = $recordingRoot

$ffmpeg = Get-Command ffmpeg.exe -ErrorAction SilentlyContinue
$ffprobe = Get-Command ffprobe.exe -ErrorAction SilentlyContinue
if ($null -eq $ffmpeg -or $null -eq $ffprobe) {
    throw "FFmpeg and ffprobe are required for E2E trajectory videos"
}

& (Join-Path $scriptDir "verify-user-session.ps1")
if ($RequireGui) { $env:CUA_REQUIRE_GUI = "1" }

# Harness hosts named by CUA_E2E_HARNESS_FILTER (the Rust test reads the same
# variable and keeps only those hosts); empty means every host. The fixture
# target "webview" is the host "webview2".
function Test-HarnessWanted([string]$Name) {
    $selected = @(([string]$env:CUA_E2E_HARNESS_FILTER).Split(",") | ForEach-Object { $_.Trim() } | Where-Object { $_ })
    if ($selected.Count -eq 0) { return $true }
    if ($Name -eq "webview") { $Name = "webview2" }
    return $selected -contains $Name
}

$env:CUA_TEST_WORKSPACE_ROOT = $rustRoot
$env:CUA_TEST_DRIVER_BIN = Join-Path $rustRoot "target\release\cua-driver.exe"
$env:CUA_TEST_APPS_ROOT = Join-Path $rustRoot "test-apps"
$env:CUA_TEST_REQUIRE_FIXTURES = "1"
$env:CUA_TEST_DRIVER_STDERR = "1"
$env:CUA_E2E_FORBID_SKIPS = "1"
# This matrix runs in a disposable, authenticated desktop session. This
# testkit-only switch gives spawned behavior daemons explicit unrestricted
# authorization without leaking product authorization variables into the
# SDK/runtime tests executed by this runner.
$env:CUA_E2E_UNRESTRICTED_GUI = "1"
$sourceMarker = Join-Path $repoRoot ".cua-e2e-source-sha"
if (Test-Path $sourceMarker) {
    $env:CUA_E2E_SOURCE_MARKER = $sourceMarker
    if ([string]::IsNullOrWhiteSpace($env:CUA_E2E_SOURCE_SHA)) {
        $env:CUA_E2E_SOURCE_SHA = (Get-Content -Raw $sourceMarker).Trim()
    }
}
if (-not [string]::IsNullOrWhiteSpace($env:CUA_E2E_SOURCE_SHA)) {
    $env:CUA_DRIVER_SOURCE_SHA = $env:CUA_E2E_SOURCE_SHA
}
Remove-Item Env:CUA_E2E_EXPECTED_MIN_CELLS -ErrorAction SilentlyContinue
if (($suite -in @("shared", "all")) -and
    [string]::IsNullOrWhiteSpace($env:CUA_E2E_CELL_FILTER) -and
    [string]::IsNullOrWhiteSpace($env:CUA_E2E_HARNESS_FILTER)) {
    $env:CUA_E2E_EXPECTED_MIN_CELLS = "80"
}

if (-not $NoBuild) {
    $previousPreference = $ErrorActionPreference
    $ErrorActionPreference = "Continue"
    try {
        & cargo build --release -p cua-driver --manifest-path (Join-Path $rustRoot "Cargo.toml")
        $buildExit = $LASTEXITCODE
    } finally {
        $ErrorActionPreference = $previousPreference
    }
    if ($buildExit -ne 0) { throw "Rust driver build failed" }
    $fixtureTargets = switch ($suite) {
        # The PR quick lane sets CUA_E2E_HARNESS_FILTER=electron: build only the
        # harnesses it names, like the Linux runner does. Empty builds them all.
        "shared" { @(@("electron", "tauri", "webview") | Where-Object { Test-HarnessWanted $_ }) }
        "native" { @("wpf", "winui3", "webview", "electron") }
        "capture" { @("wpf", "electron") }
        default { @("wpf", "winui3", "webview", "electron", "tauri") }
    }
    & (Join-Path $scriptDir "build-harnesses.ps1") -Targets $fixtureTargets
}

if (-not (Test-Path $env:CUA_TEST_DRIVER_BIN)) {
    throw "Driver binary not found: $($env:CUA_TEST_DRIVER_BIN)"
}
$requiredFixtures = @()
$requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-electron\CuaTestHarness.Electron.exe"
if (($suite -in @("shared", "all")) -and (Test-HarnessWanted "tauri")) {
    $requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-tauri\CuaTestHarness.Tauri.exe"
}
if ($suite -in @("native", "capture", "all")) {
    $requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-wpf\CuaTestHarness.Wpf.exe"
}
if (($suite -in @("shared", "native", "all")) -and (Test-HarnessWanted "webview")) {
    $requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-webview\CuaTestHarness.WebView.exe"
}
if ($suite -in @("native", "all")) {
    $requiredFixtures += @(
        (Join-Path $env:CUA_TEST_APPS_ROOT "harness-winui3\CuaTestHarness.WinUI3.exe")
    )
}
foreach ($fixture in $requiredFixtures) {
    if (-not (Test-Path $fixture)) { throw "Required fixture was not built: $fixture" }
}

function Invoke-E2eReport {
    Push-Location $rustRoot
    try {
        $previousPreference = $ErrorActionPreference
        $ErrorActionPreference = "Continue"
        try {
            $reportOutput = @(& cargo run -p cua-driver-testkit --bin cua-e2e-report -- `
                --declarations $casesPath `
                --environment $environmentPath `
                --results $resultsPath `
                --artifact-root $artifactDir `
                --require-video `
                --output $summaryPath 2>&1) | ForEach-Object {
                    if ($_ -is [System.Management.Automation.ErrorRecord]) {
                        $_.Exception.Message
                    } else {
                        $_.ToString()
                    }
                }
            $exitCode = $LASTEXITCODE
        } finally {
            $ErrorActionPreference = $previousPreference
        }
        $reportOutput | Out-Host
        return $exitCode
    } finally {
        Pop-Location
    }
}

Write-Host "[PREFLIGHT] Windows desktop, fixture, UIA, capture, and video" -ForegroundColor Yellow
Push-Location $rustRoot
try {
    $preflightLog = Join-Path $artifactDir "environment-preflight.log"
    # Windows PowerShell 5.1 wraps native stderr as ErrorRecord objects. With
    # the script-wide Stop preference, normal Cargo progress would terminate
    # the lane before LASTEXITCODE can be inspected. Relax only around the
    # captured native process; the real exit code remains authoritative.
    $previousPreference = $ErrorActionPreference
    $ErrorActionPreference = "Continue"
    try {
        $preflightOutput = @(& cargo test -p cua-driver-e2e --test e2e_environment_preflight_test -- `
            --ignored --exact canonical_e2e_environment_is_ready --nocapture --test-threads=1 2>&1) | `
            ForEach-Object {
                if ($_ -is [System.Management.Automation.ErrorRecord]) {
                    $_.Exception.Message
                } else {
                    $_.ToString()
                }
            }
        $preflightExit = $LASTEXITCODE
    } finally {
        $ErrorActionPreference = $previousPreference
    }
    $preflightOutput | Tee-Object -FilePath $preflightLog
} finally {
    Pop-Location
}
if ($preflightExit -ne 0) {
    Invoke-E2eReport | Out-Null
    throw "Windows E2E environment preflight failed"
}

function Invoke-CargoTestAttempt {
    param([string[]]$Arguments, [string]$LogPath)
    Push-Location $rustRoot
    try {
        $previousPreference = $ErrorActionPreference
        $ErrorActionPreference = "Continue"
        try {
            $output = @(& cargo @Arguments 2>&1) | ForEach-Object {
                if ($_ -is [System.Management.Automation.ErrorRecord]) {
                    $_.Exception.Message
                } else {
                    $_.ToString()
                }
            }
            $exitCode = $LASTEXITCODE
        } finally {
            $ErrorActionPreference = $previousPreference
        }
        $output | Tee-Object -FilePath $LogPath | Out-Host
        return $exitCode
    } finally {
        Pop-Location
    }
}

# With CUA_E2E_RETRY_ONCE=1 a failed case is rerun exactly once. A pass on the
# rerun is "flaky": it is recorded in flaky-cases.jsonl and does not fail the
# gate. Two failures are a real failure. Mirrors scripts/ci/lib/e2e-retry-once.sh.
function Invoke-CargoTest {
    param([string]$Name, [string[]]$Arguments)
    Write-Host "[RUN] $Name" -ForegroundColor Yellow
    $logPath = Join-Path $artifactDir ("$($Name -replace '[^A-Za-z0-9_.-]', '-').log")
    $retry = $env:CUA_E2E_RETRY_ONCE -eq "1"
    if ($retry) {
        $resultsLines = @(Get-Content $resultsPath).Count
        $caseLines = @(Get-Content $casesPath).Count
        $beforeDirs = @(Get-ChildItem -Path $recordingRoot -Directory | ForEach-Object { $_.FullName })
    }
    $exitCode = Invoke-CargoTestAttempt -Arguments $Arguments -LogPath $logPath
    if ($exitCode -ne 0 -and $retry) {
        Write-Host "[RETRY] $Name failed (exit $exitCode); rerunning once" -ForegroundColor Yellow
        $quarantine = Join-Path $artifactDir ("retried-attempts\" + ($Name -replace '[^A-Za-z0-9_.-]', '-'))
        New-Item -ItemType Directory -Force (Join-Path $quarantine "recordings") | Out-Null
        Copy-Item $logPath (Join-Path $quarantine "attempt1.log")
        $resultRows = @(Get-Content $resultsPath)
        $caseRows = @(Get-Content $casesPath)
        $resultRows | Select-Object -Skip $resultsLines | Set-Content (Join-Path $quarantine "results.jsonl")
        $caseRows | Select-Object -Skip $caseLines | Set-Content (Join-Path $quarantine "cases.jsonl")
        Set-Content -Path $resultsPath -Value ($resultRows | Select-Object -First $resultsLines)
        Set-Content -Path $casesPath -Value ($caseRows | Select-Object -First $caseLines)
        Get-ChildItem -Path $recordingRoot -Directory | Where-Object { $beforeDirs -notcontains $_.FullName } |
            ForEach-Object { Move-Item $_.FullName (Join-Path $quarantine "recordings") }
        $secondExit = Invoke-CargoTestAttempt -Arguments $Arguments -LogPath $logPath
        if ($secondExit -eq 0) {
            Write-Host "[FLAKY] $Name failed once (exit $exitCode) then passed; not failing the gate"
            $record = [ordered]@{ case = $Name; platform = "windows"; first_attempt_exit = $exitCode; second_attempt = "passed" }
            Add-Content -Path (Join-Path $artifactDir "flaky-cases.jsonl") -Value ($record | ConvertTo-Json -Compress)
            return
        }
        Write-Host "[FAIL] $Name failed twice (exit $exitCode, then $secondExit); real failure"
        $exitCode = $secondExit
    }
    if ($exitCode -ne 0) {
        $script:FailureCount++
    }
}

function Invoke-ComputerHistoryGate {
    $historyHome = Join-Path $artifactDir "history-product-home"
    $historyBinDir = Join-Path $artifactDir "history-product-bin"
    $historyLocalAppData = Join-Path $artifactDir "history-local-app-data"
    $historyPipeName = "cua-driver-local-history-$PID"
    $historyPipe = "\\.\pipe\$historyPipeName"
    $env:CUA_DRIVER_LOCAL_HOME = $historyHome
    $env:CUA_DRIVER_LOCAL_INSTALL_DIR = $historyBinDir
    $env:CUA_E2E_HISTORY_DAEMON_SOCKET = $historyPipe
    $env:LOCALAPPDATA = $historyLocalAppData

    Write-Host "[HISTORY] Installing the exact candidate into an isolated local namespace" -ForegroundColor Yellow
    $installLog = Join-Path $artifactDir "history-install-local.log"
    $previousPreference = $ErrorActionPreference
    $ErrorActionPreference = "Continue"
    try {
        $installOutput = @(& (Join-Path $driverRoot "scripts\install-local.ps1") `
            -NoAutoStart -NoPathUpdate 2>&1) | ForEach-Object {
                if ($_ -is [System.Management.Automation.ErrorRecord]) {
                    $_.Exception.Message
                } else {
                    $_.ToString()
                }
            }
        $installExit = $LASTEXITCODE
    } finally {
        $ErrorActionPreference = $previousPreference
    }
    $installOutput | Tee-Object -FilePath $installLog
    if ($installExit -ne 0) {
        $script:FailureCount++
        return
    }

    $installed = Join-Path $historyHome "packages\current\cua-driver-local.exe"
    $env:CUA_E2E_INSTALLED_DRIVER_BIN = $installed
    if (-not (Test-Path -LiteralPath $installed)) {
        Write-Host "[HISTORY FAIL] Installed driver is missing: $installed" -ForegroundColor Red
        $script:FailureCount++
        return
    }

    $previousPreference = $ErrorActionPreference
    $ErrorActionPreference = "Continue"
    try {
        & $installed history purge-offline --yes 2>&1 |
            Set-Content (Join-Path $artifactDir "history-purge-preflight.log")
        $purgeExit = $LASTEXITCODE
    } finally {
        $ErrorActionPreference = $previousPreference
    }
    if ($purgeExit -ne 0) {
        Write-Host "[HISTORY FAIL] Installed driver could not establish an empty encrypted store" -ForegroundColor Red
        $script:FailureCount++
        return
    }

    $daemonLog = Join-Path $artifactDir "history-daemon.log"
    $daemonErr = Join-Path $artifactDir "history-daemon.err.log"
    $daemon = Start-Process -FilePath $installed -ArgumentList @(
        "serve",
        "--socket", $historyPipe,
        "--permission-mode", "unrestricted",
        "--dangerously-bypass-approvals"
    ) -RedirectStandardOutput $daemonLog -RedirectStandardError $daemonErr -PassThru

    $ready = $false
    for ($attempt = 0; $attempt -lt 150; $attempt++) {
        if ($daemon.HasExited) { break }
        try {
            $probe = [System.IO.Pipes.NamedPipeClientStream]::new(
                ".",
                $historyPipeName,
                [System.IO.Pipes.PipeDirection]::InOut
            )
            $probe.Connect(100)
            $probe.Dispose()
            $ready = $true
            break
        } catch {
            Start-Sleep -Milliseconds 100
        }
    }
    if (-not $ready) {
        Write-Host "[HISTORY FAIL] Installed daemon did not become ready" -ForegroundColor Red
        $script:FailureCount++
        if (-not $daemon.HasExited) { Stop-Process -Id $daemon.Id -Force }
        return
    }

    Invoke-CargoTest "computer-history-encrypted-lifecycle" @(
        "test", "-p", "cua-driver-e2e", "--test", "computer_history_cross_platform_test", "--",
        "--ignored", "--exact", "encrypted_history_survives_restart_and_cryptographically_purges",
        "--nocapture", "--test-threads=1"
    )

    $previousPreference = $ErrorActionPreference
    $ErrorActionPreference = "Continue"
    try {
        & $installed stop --socket $historyPipe 2>&1 | Add-Content $daemonLog
    } finally {
        $ErrorActionPreference = $previousPreference
    }
    if (-not $daemon.HasExited) {
        $daemon.WaitForExit(5000) | Out-Null
    }
    if (-not $daemon.HasExited) { Stop-Process -Id $daemon.Id -Force }
}

function Test-E2eRecordings {
    $failureCount = 0
    $errors = @(Get-ChildItem -Path $recordingRoot -Filter "recording-error.txt" -Recurse -ErrorAction SilentlyContinue)
    foreach ($errorFile in $errors) {
        Write-Host "[VIDEO FAIL] $($errorFile.FullName)" -ForegroundColor Red
        Get-Content $errorFile.FullName | Write-Host
        $failureCount++
    }

    $videos = @(Get-ChildItem -Path $recordingRoot -Filter "recording.mp4" -Recurse -ErrorAction SilentlyContinue)
    if ($videos.Count -eq 0) {
        Write-Host "[VIDEO FAIL] No E2E trajectory videos were produced" -ForegroundColor Red
        return ($failureCount + 1)
    }
    foreach ($video in $videos) {
        & $ffprobe.Source -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 $video.FullName | Out-Null
        if ($LASTEXITCODE -ne 0 -or $video.Length -eq 0) {
            Write-Host "[VIDEO FAIL] Unplayable trajectory: $($video.FullName)" -ForegroundColor Red
            $failureCount++
        } else {
            Write-Host "[VIDEO PASS] $($video.FullName)" -ForegroundColor Green
        }
    }

    $ownedVideos = @{}
    Get-Content $resultsPath | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | ForEach-Object {
        $result = $_ | ConvertFrom-Json
        if ($null -ne $result.evidence.video) {
            $ownedVideos[$result.evidence.video.Replace("\", "/")] = $true
        }
    }
    foreach ($video in $videos) {
        $artifactPrefix = ([System.IO.Path]::GetFullPath($artifactDir)).TrimEnd("\") + "\"
        $videoPath = [System.IO.Path]::GetFullPath($video.FullName)
        if (-not $videoPath.StartsWith($artifactPrefix, [System.StringComparison]::OrdinalIgnoreCase)) {
            Write-Host "[VIDEO FAIL] Trajectory escaped artifact root: $videoPath" -ForegroundColor Red
            $failureCount++
            continue
        }
        $relative = $videoPath.Substring($artifactPrefix.Length).Replace("\", "/")
        if ($relative -like "recordings/environment-preflight-*/recording.mp4") {
            continue
        }
        if (-not $ownedVideos.ContainsKey($relative)) {
            Write-Host "[VIDEO FAIL] Orphan trajectory has no typed result row: $relative" -ForegroundColor Red
            $failureCount++
        }
    }
    return $failureCount
}

$script:FailureCount = 0

if ($suite -in @("shared", "all")) {
    Invoke-CargoTest "protected permission prompt socket" @(
        "test", "-p", "cua-driver", "--test", "permission_prompt_authorization_test", "--",
        "--test-threads=1"
    )
    Invoke-CargoTest "SDK runtime contract" @(
        "test", "-p", "cua-driver-sdk", "--lib", "--", "--test-threads=1"
    )
    Invoke-CargoTest "SDK runtime configuration" @(
        "test", "-p", "cua-driver-sdk", "--test", "runtime_configuration", "--",
        "--test-threads=1"
    )
    Invoke-CargoTest "private worker lifecycle" @(
        "test", "-p", "cua-driver", "--test", "private_worker_test", "--",
        "--test-threads=1"
    )
    Invoke-CargoTest "shared behavior matrix" @(
        "test", "-p", "cua-driver-e2e", "--test", "cross_platform_behavior_test", "--",
        "--ignored", "--exact", "shared_web_action_matrix_is_state_verified",
        "--nocapture", "--test-threads=1"
    )
    Invoke-CargoTest "embedded browser routes" @(
        "test", "-p", "cua-driver-e2e", "--test", "cross_platform_behavior_test", "--",
        "--ignored", "--exact", "embedded_browser_routes_are_exact_or_refused",
        "--nocapture", "--test-threads=1"
    )
}

if ($suite -in @("native", "all")) {
    Invoke-CargoTest "Windows installed-app discovery deadline" @(
        "test", "-p", "cua-driver", "--test", "protocol_tools_call_test", "--",
        "--ignored", "--exact", "tools_call_list_apps", "--nocapture", "--test-threads=1"
    )
    Invoke-CargoTest "Agent cursor showcase" @(
        "test", "-p", "cua-driver-e2e", "--test", "agent_cursor_showcase_test", "--",
        "--ignored", "--nocapture", "--test-threads=1"
    )
    Invoke-CargoTest "Windows native harnesses" @(
        "test", "-p", "cua-driver-e2e", "--test", "harness_wpf_test", "--",
        "--ignored", "--nocapture", "--test-threads=1"
    )
    Invoke-CargoTest "WinUI3 harnesses" @(
        "test", "-p", "cua-driver-e2e", "--test", "harness_winui3_test", "--",
        "--ignored", "--nocapture", "--test-threads=1"
    )
    Invoke-CargoTest "Windows web harnesses" @(
        "test", "-p", "cua-driver-e2e", "--test", "harness_web_test", "--",
        "--ignored", "--nocapture", "--test-threads=1"
    )
    Invoke-CargoTest "Windows minimized launch" @(
        "test", "-p", "cua-driver-e2e", "--test", "launch_windows_test", "--",
        "--ignored", "--nocapture", "--test-threads=1"
    )
    Invoke-CargoTest "Windows agent cursor" @(
        "test", "-p", "cua-driver-e2e", "--test", "agent_cursor_windows_test", "--",
        "--ignored", "--nocapture", "--test-threads=1"
    )
}

if ($suite -in @("capture", "all")) {
    Invoke-CargoTest "capture contract" @(
        "test", "-p", "cua-driver-e2e", "--test", "capture_contract_test", "--",
        "--ignored", "--nocapture", "--test-threads=1"
    )
    Invoke-CargoTest "Windows desktop scope" @(
        "test", "-p", "cua-driver-e2e", "--test", "desktop_scope_windows_test", "--",
        "--ignored", "--nocapture", "--test-threads=1"
    )
    # Includes the published-catalog row: it downloads the pinned
    # cua-perception release and needs a Python with Tk (py -3 or python).
    Invoke-CargoTest "perception capture loop" @(
        "test", "-p", "cua-driver-e2e", "--test", "perception_capture_loop_test", "--",
        "--ignored", "--nocapture", "--test-threads=1"
    )
}

if ($suite -in @("shared", "all")) {
    Invoke-ComputerHistoryGate
}

$script:FailureCount += (Test-E2eRecordings)

$reportExit = Invoke-E2eReport
if ($reportExit -ne 0) {
    Write-Host "Windows E2E result validation failed" -ForegroundColor Red
    $script:FailureCount++
}

if ($script:FailureCount -ne 0) {
    throw "Windows Rust e2e suite had $($script:FailureCount) failing lane(s)"
}

Write-Host "Windows Rust e2e matrix completed: $suite" -ForegroundColor Green