Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
# Run the canonical Rust desktop matrix in an active Windows user session.
# Scenario definitions and assertions stay in the Rust integration test.
param(
[switch]$NoBuild,
[switch]$RequireGui
)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Definition
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..\..")).Path
$driverRoot = Join-Path $repoRoot "libs\cua-driver"
$rustRoot = Join-Path $driverRoot "rust"
$suite = if ([string]::IsNullOrWhiteSpace($env:CUA_E2E_INTERNAL_LANE)) { "all" } else { $env:CUA_E2E_INTERNAL_LANE }
if ($suite -notin @("shared", "native", "capture", "all")) {
throw "Unsupported internal lane: $suite"
}
$artifactDir = Join-Path $repoRoot "artifacts\cua-driver\windows"
New-Item -ItemType Directory -Force $artifactDir | Out-Null
$recordingRoot = Join-Path $artifactDir "recordings"
Remove-Item -Path $recordingRoot -Recurse -Force -ErrorAction SilentlyContinue
New-Item -ItemType Directory -Force $recordingRoot | Out-Null
$resultsPath = Join-Path $artifactDir "results.jsonl"
$casesPath = Join-Path $artifactDir "cases.jsonl"
$environmentPath = Join-Path $artifactDir "environment.jsonl"
$summaryPath = Join-Path $artifactDir "summary.md"
foreach ($path in @($casesPath, $environmentPath, $resultsPath)) {
New-Item -ItemType File -Force $path | Out-Null
Clear-Content $path
}
Remove-Item -Force -ErrorAction SilentlyContinue $summaryPath
$env:CUA_E2E_DECLARATIONS_FILE = $casesPath
$env:CUA_E2E_ENVIRONMENT_FILE = $environmentPath
$env:CUA_E2E_RESULTS_FILE = $resultsPath
$env:CUA_E2E_RECORDINGS_ROOT = $recordingRoot
$ffmpeg = Get-Command ffmpeg.exe -ErrorAction SilentlyContinue
$ffprobe = Get-Command ffprobe.exe -ErrorAction SilentlyContinue
if ($null -eq $ffmpeg -or $null -eq $ffprobe) {
throw "FFmpeg and ffprobe are required for E2E trajectory videos"
}
& (Join-Path $scriptDir "verify-user-session.ps1")
if ($RequireGui) { $env:CUA_REQUIRE_GUI = "1" }
# Harness hosts named by CUA_E2E_HARNESS_FILTER (the Rust test reads the same
# variable and keeps only those hosts); empty means every host. The fixture
# target "webview" is the host "webview2".
function Test-HarnessWanted([string]$Name) {
$selected = @(([string]$env:CUA_E2E_HARNESS_FILTER).Split(",") | ForEach-Object { $_.Trim() } | Where-Object { $_ })
if ($selected.Count -eq 0) { return $true }
if ($Name -eq "webview") { $Name = "webview2" }
return $selected -contains $Name
}
$env:CUA_TEST_WORKSPACE_ROOT = $rustRoot
$env:CUA_TEST_DRIVER_BIN = Join-Path $rustRoot "target\release\cua-driver.exe"
$env:CUA_TEST_APPS_ROOT = Join-Path $rustRoot "test-apps"
$env:CUA_TEST_REQUIRE_FIXTURES = "1"
$env:CUA_TEST_DRIVER_STDERR = "1"
$env:CUA_E2E_FORBID_SKIPS = "1"
# This matrix runs in a disposable, authenticated desktop session. This
# testkit-only switch gives spawned behavior daemons explicit unrestricted
# authorization without leaking product authorization variables into the
# SDK/runtime tests executed by this runner.
$env:CUA_E2E_UNRESTRICTED_GUI = "1"
$sourceMarker = Join-Path $repoRoot ".cua-e2e-source-sha"
if (Test-Path $sourceMarker) {
$env:CUA_E2E_SOURCE_MARKER = $sourceMarker
if ([string]::IsNullOrWhiteSpace($env:CUA_E2E_SOURCE_SHA)) {
$env:CUA_E2E_SOURCE_SHA = (Get-Content -Raw $sourceMarker).Trim()
}
}
if (-not [string]::IsNullOrWhiteSpace($env:CUA_E2E_SOURCE_SHA)) {
$env:CUA_DRIVER_SOURCE_SHA = $env:CUA_E2E_SOURCE_SHA
}
Remove-Item Env:CUA_E2E_EXPECTED_MIN_CELLS -ErrorAction SilentlyContinue
if (($suite -in @("shared", "all")) -and
[string]::IsNullOrWhiteSpace($env:CUA_E2E_CELL_FILTER) -and
[string]::IsNullOrWhiteSpace($env:CUA_E2E_HARNESS_FILTER)) {
$env:CUA_E2E_EXPECTED_MIN_CELLS = "80"
}
if (-not $NoBuild) {
$previousPreference = $ErrorActionPreference
$ErrorActionPreference = "Continue"
try {
& cargo build --release -p cua-driver --manifest-path (Join-Path $rustRoot "Cargo.toml")
$buildExit = $LASTEXITCODE
} finally {
$ErrorActionPreference = $previousPreference
}
if ($buildExit -ne 0) { throw "Rust driver build failed" }
$fixtureTargets = switch ($suite) {
# The PR quick lane sets CUA_E2E_HARNESS_FILTER=electron: build only the
# harnesses it names, like the Linux runner does. Empty builds them all.
"shared" { @(@("electron", "tauri", "webview") | Where-Object { Test-HarnessWanted $_ }) }
"native" { @("wpf", "winui3", "webview", "electron") }
"capture" { @("wpf", "electron") }
default { @("wpf", "winui3", "webview", "electron", "tauri") }
}
& (Join-Path $scriptDir "build-harnesses.ps1") -Targets $fixtureTargets
}
if (-not (Test-Path $env:CUA_TEST_DRIVER_BIN)) {
throw "Driver binary not found: $($env:CUA_TEST_DRIVER_BIN)"
}
$requiredFixtures = @()
$requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-electron\CuaTestHarness.Electron.exe"
if (($suite -in @("shared", "all")) -and (Test-HarnessWanted "tauri")) {
$requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-tauri\CuaTestHarness.Tauri.exe"
}
if ($suite -in @("native", "capture", "all")) {
$requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-wpf\CuaTestHarness.Wpf.exe"
}
if (($suite -in @("shared", "native", "all")) -and (Test-HarnessWanted "webview")) {
$requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-webview\CuaTestHarness.WebView.exe"
}
if ($suite -in @("native", "all")) {
$requiredFixtures += @(
(Join-Path $env:CUA_TEST_APPS_ROOT "harness-winui3\CuaTestHarness.WinUI3.exe")
)
}
foreach ($fixture in $requiredFixtures) {
if (-not (Test-Path $fixture)) { throw "Required fixture was not built: $fixture" }
}
function Invoke-E2eReport {
Push-Location $rustRoot
try {
$previousPreference = $ErrorActionPreference
$ErrorActionPreference = "Continue"
try {
$reportOutput = @(& cargo run -p cua-driver-testkit --bin cua-e2e-report -- `
--declarations $casesPath `
--environment $environmentPath `
--results $resultsPath `
--artifact-root $artifactDir `
--require-video `
--output $summaryPath 2>&1) | ForEach-Object {
if ($_ -is [System.Management.Automation.ErrorRecord]) {
$_.Exception.Message
} else {
$_.ToString()
}
}
$exitCode = $LASTEXITCODE
} finally {
$ErrorActionPreference = $previousPreference
}
$reportOutput | Out-Host
return $exitCode
} finally {
Pop-Location
}
}
Write-Host "[PREFLIGHT] Windows desktop, fixture, UIA, capture, and video" -ForegroundColor Yellow
Push-Location $rustRoot
try {
$preflightLog = Join-Path $artifactDir "environment-preflight.log"
# Windows PowerShell 5.1 wraps native stderr as ErrorRecord objects. With
# the script-wide Stop preference, normal Cargo progress would terminate
# the lane before LASTEXITCODE can be inspected. Relax only around the
# captured native process; the real exit code remains authoritative.
$previousPreference = $ErrorActionPreference
$ErrorActionPreference = "Continue"
try {
$preflightOutput = @(& cargo test -p cua-driver-e2e --test e2e_environment_preflight_test -- `
--ignored --exact canonical_e2e_environment_is_ready --nocapture --test-threads=1 2>&1) | `
ForEach-Object {
if ($_ -is [System.Management.Automation.ErrorRecord]) {
$_.Exception.Message
} else {
$_.ToString()
}
}
$preflightExit = $LASTEXITCODE
} finally {
$ErrorActionPreference = $previousPreference
}
$preflightOutput | Tee-Object -FilePath $preflightLog
} finally {
Pop-Location
}
if ($preflightExit -ne 0) {
Invoke-E2eReport | Out-Null
throw "Windows E2E environment preflight failed"
}
function Invoke-CargoTestAttempt {
param([string[]]$Arguments, [string]$LogPath)
Push-Location $rustRoot
try {
$previousPreference = $ErrorActionPreference
$ErrorActionPreference = "Continue"
try {
$output = @(& cargo @Arguments 2>&1) | ForEach-Object {
if ($_ -is [System.Management.Automation.ErrorRecord]) {
$_.Exception.Message
} else {
$_.ToString()
}
}
$exitCode = $LASTEXITCODE
} finally {
$ErrorActionPreference = $previousPreference
}
$output | Tee-Object -FilePath $LogPath | Out-Host
return $exitCode
} finally {
Pop-Location
}
}
# With CUA_E2E_RETRY_ONCE=1 a failed case is rerun exactly once. A pass on the
# rerun is "flaky": it is recorded in flaky-cases.jsonl and does not fail the
# gate. Two failures are a real failure. Mirrors scripts/ci/lib/e2e-retry-once.sh.
function Invoke-CargoTest {
param([string]$Name, [string[]]$Arguments)
Write-Host "[RUN] $Name" -ForegroundColor Yellow
$logPath = Join-Path $artifactDir ("$($Name -replace '[^A-Za-z0-9_.-]', '-').log")
$retry = $env:CUA_E2E_RETRY_ONCE -eq "1"
if ($retry) {
$resultsLines = @(Get-Content $resultsPath).Count
$caseLines = @(Get-Content $casesPath).Count
$beforeDirs = @(Get-ChildItem -Path $recordingRoot -Directory | ForEach-Object { $_.FullName })
}
$exitCode = Invoke-CargoTestAttempt -Arguments $Arguments -LogPath $logPath
if ($exitCode -ne 0 -and $retry) {
Write-Host "[RETRY] $Name failed (exit $exitCode); rerunning once" -ForegroundColor Yellow
$quarantine = Join-Path $artifactDir ("retried-attempts\" + ($Name -replace '[^A-Za-z0-9_.-]', '-'))
New-Item -ItemType Directory -Force (Join-Path $quarantine "recordings") | Out-Null
Copy-Item $logPath (Join-Path $quarantine "attempt1.log")
$resultRows = @(Get-Content $resultsPath)
$caseRows = @(Get-Content $casesPath)
$resultRows | Select-Object -Skip $resultsLines | Set-Content (Join-Path $quarantine "results.jsonl")
$caseRows | Select-Object -Skip $caseLines | Set-Content (Join-Path $quarantine "cases.jsonl")
Set-Content -Path $resultsPath -Value ($resultRows | Select-Object -First $resultsLines)
Set-Content -Path $casesPath -Value ($caseRows | Select-Object -First $caseLines)
Get-ChildItem -Path $recordingRoot -Directory | Where-Object { $beforeDirs -notcontains $_.FullName } |
ForEach-Object { Move-Item $_.FullName (Join-Path $quarantine "recordings") }
$secondExit = Invoke-CargoTestAttempt -Arguments $Arguments -LogPath $logPath
if ($secondExit -eq 0) {
Write-Host "[FLAKY] $Name failed once (exit $exitCode) then passed; not failing the gate"
$record = [ordered]@{ case = $Name; platform = "windows"; first_attempt_exit = $exitCode; second_attempt = "passed" }
Add-Content -Path (Join-Path $artifactDir "flaky-cases.jsonl") -Value ($record | ConvertTo-Json -Compress)
return
}
Write-Host "[FAIL] $Name failed twice (exit $exitCode, then $secondExit); real failure"
$exitCode = $secondExit
}
if ($exitCode -ne 0) {
$script:FailureCount++
}
}
function Invoke-ComputerHistoryGate {
$historyHome = Join-Path $artifactDir "history-product-home"
$historyBinDir = Join-Path $artifactDir "history-product-bin"
$historyLocalAppData = Join-Path $artifactDir "history-local-app-data"
$historyPipeName = "cua-driver-local-history-$PID"
$historyPipe = "\\.\pipe\$historyPipeName"
$env:CUA_DRIVER_LOCAL_HOME = $historyHome
$env:CUA_DRIVER_LOCAL_INSTALL_DIR = $historyBinDir
$env:CUA_E2E_HISTORY_DAEMON_SOCKET = $historyPipe
$env:LOCALAPPDATA = $historyLocalAppData
Write-Host "[HISTORY] Installing the exact candidate into an isolated local namespace" -ForegroundColor Yellow
$installLog = Join-Path $artifactDir "history-install-local.log"
$previousPreference = $ErrorActionPreference
$ErrorActionPreference = "Continue"
try {
$installOutput = @(& (Join-Path $driverRoot "scripts\install-local.ps1") `
-NoAutoStart -NoPathUpdate 2>&1) | ForEach-Object {
if ($_ -is [System.Management.Automation.ErrorRecord]) {
$_.Exception.Message
} else {
$_.ToString()
}
}
$installExit = $LASTEXITCODE
} finally {
$ErrorActionPreference = $previousPreference
}
$installOutput | Tee-Object -FilePath $installLog
if ($installExit -ne 0) {
$script:FailureCount++
return
}
$installed = Join-Path $historyHome "packages\current\cua-driver-local.exe"
$env:CUA_E2E_INSTALLED_DRIVER_BIN = $installed
if (-not (Test-Path -LiteralPath $installed)) {
Write-Host "[HISTORY FAIL] Installed driver is missing: $installed" -ForegroundColor Red
$script:FailureCount++
return
}
$previousPreference = $ErrorActionPreference
$ErrorActionPreference = "Continue"
try {
& $installed history purge-offline --yes 2>&1 |
Set-Content (Join-Path $artifactDir "history-purge-preflight.log")
$purgeExit = $LASTEXITCODE
} finally {
$ErrorActionPreference = $previousPreference
}
if ($purgeExit -ne 0) {
Write-Host "[HISTORY FAIL] Installed driver could not establish an empty encrypted store" -ForegroundColor Red
$script:FailureCount++
return
}
$daemonLog = Join-Path $artifactDir "history-daemon.log"
$daemonErr = Join-Path $artifactDir "history-daemon.err.log"
$daemon = Start-Process -FilePath $installed -ArgumentList @(
"serve",
"--socket", $historyPipe,
"--permission-mode", "unrestricted",
"--dangerously-bypass-approvals"
) -RedirectStandardOutput $daemonLog -RedirectStandardError $daemonErr -PassThru
$ready = $false
for ($attempt = 0; $attempt -lt 150; $attempt++) {
if ($daemon.HasExited) { break }
try {
$probe = [System.IO.Pipes.NamedPipeClientStream]::new(
".",
$historyPipeName,
[System.IO.Pipes.PipeDirection]::InOut
)
$probe.Connect(100)
$probe.Dispose()
$ready = $true
break
} catch {
Start-Sleep -Milliseconds 100
}
}
if (-not $ready) {
Write-Host "[HISTORY FAIL] Installed daemon did not become ready" -ForegroundColor Red
$script:FailureCount++
if (-not $daemon.HasExited) { Stop-Process -Id $daemon.Id -Force }
return
}
Invoke-CargoTest "computer-history-encrypted-lifecycle" @(
"test", "-p", "cua-driver-e2e", "--test", "computer_history_cross_platform_test", "--",
"--ignored", "--exact", "encrypted_history_survives_restart_and_cryptographically_purges",
"--nocapture", "--test-threads=1"
)
$previousPreference = $ErrorActionPreference
$ErrorActionPreference = "Continue"
try {
& $installed stop --socket $historyPipe 2>&1 | Add-Content $daemonLog
} finally {
$ErrorActionPreference = $previousPreference
}
if (-not $daemon.HasExited) {
$daemon.WaitForExit(5000) | Out-Null
}
if (-not $daemon.HasExited) { Stop-Process -Id $daemon.Id -Force }
}
function Test-E2eRecordings {
$failureCount = 0
$errors = @(Get-ChildItem -Path $recordingRoot -Filter "recording-error.txt" -Recurse -ErrorAction SilentlyContinue)
foreach ($errorFile in $errors) {
Write-Host "[VIDEO FAIL] $($errorFile.FullName)" -ForegroundColor Red
Get-Content $errorFile.FullName | Write-Host
$failureCount++
}
$videos = @(Get-ChildItem -Path $recordingRoot -Filter "recording.mp4" -Recurse -ErrorAction SilentlyContinue)
if ($videos.Count -eq 0) {
Write-Host "[VIDEO FAIL] No E2E trajectory videos were produced" -ForegroundColor Red
return ($failureCount + 1)
}
foreach ($video in $videos) {
& $ffprobe.Source -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 $video.FullName | Out-Null
if ($LASTEXITCODE -ne 0 -or $video.Length -eq 0) {
Write-Host "[VIDEO FAIL] Unplayable trajectory: $($video.FullName)" -ForegroundColor Red
$failureCount++
} else {
Write-Host "[VIDEO PASS] $($video.FullName)" -ForegroundColor Green
}
}
$ownedVideos = @{}
Get-Content $resultsPath | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | ForEach-Object {
$result = $_ | ConvertFrom-Json
if ($null -ne $result.evidence.video) {
$ownedVideos[$result.evidence.video.Replace("\", "/")] = $true
}
}
foreach ($video in $videos) {
$artifactPrefix = ([System.IO.Path]::GetFullPath($artifactDir)).TrimEnd("\") + "\"
$videoPath = [System.IO.Path]::GetFullPath($video.FullName)
if (-not $videoPath.StartsWith($artifactPrefix, [System.StringComparison]::OrdinalIgnoreCase)) {
Write-Host "[VIDEO FAIL] Trajectory escaped artifact root: $videoPath" -ForegroundColor Red
$failureCount++
continue
}
$relative = $videoPath.Substring($artifactPrefix.Length).Replace("\", "/")
if ($relative -like "recordings/environment-preflight-*/recording.mp4") {
continue
}
if (-not $ownedVideos.ContainsKey($relative)) {
Write-Host "[VIDEO FAIL] Orphan trajectory has no typed result row: $relative" -ForegroundColor Red
$failureCount++
}
}
return $failureCount
}
$script:FailureCount = 0
if ($suite -in @("shared", "all")) {
Invoke-CargoTest "protected permission prompt socket" @(
"test", "-p", "cua-driver", "--test", "permission_prompt_authorization_test", "--",
"--test-threads=1"
)
Invoke-CargoTest "SDK runtime contract" @(
"test", "-p", "cua-driver-sdk", "--lib", "--", "--test-threads=1"
)
Invoke-CargoTest "SDK runtime configuration" @(
"test", "-p", "cua-driver-sdk", "--test", "runtime_configuration", "--",
"--test-threads=1"
)
Invoke-CargoTest "private worker lifecycle" @(
"test", "-p", "cua-driver", "--test", "private_worker_test", "--",
"--test-threads=1"
)
Invoke-CargoTest "shared behavior matrix" @(
"test", "-p", "cua-driver-e2e", "--test", "cross_platform_behavior_test", "--",
"--ignored", "--exact", "shared_web_action_matrix_is_state_verified",
"--nocapture", "--test-threads=1"
)
Invoke-CargoTest "embedded browser routes" @(
"test", "-p", "cua-driver-e2e", "--test", "cross_platform_behavior_test", "--",
"--ignored", "--exact", "embedded_browser_routes_are_exact_or_refused",
"--nocapture", "--test-threads=1"
)
}
if ($suite -in @("native", "all")) {
Invoke-CargoTest "Windows installed-app discovery deadline" @(
"test", "-p", "cua-driver", "--test", "protocol_tools_call_test", "--",
"--ignored", "--exact", "tools_call_list_apps", "--nocapture", "--test-threads=1"
)
Invoke-CargoTest "Agent cursor showcase" @(
"test", "-p", "cua-driver-e2e", "--test", "agent_cursor_showcase_test", "--",
"--ignored", "--nocapture", "--test-threads=1"
)
Invoke-CargoTest "Windows native harnesses" @(
"test", "-p", "cua-driver-e2e", "--test", "harness_wpf_test", "--",
"--ignored", "--nocapture", "--test-threads=1"
)
Invoke-CargoTest "WinUI3 harnesses" @(
"test", "-p", "cua-driver-e2e", "--test", "harness_winui3_test", "--",
"--ignored", "--nocapture", "--test-threads=1"
)
Invoke-CargoTest "Windows web harnesses" @(
"test", "-p", "cua-driver-e2e", "--test", "harness_web_test", "--",
"--ignored", "--nocapture", "--test-threads=1"
)
Invoke-CargoTest "Windows minimized launch" @(
"test", "-p", "cua-driver-e2e", "--test", "launch_windows_test", "--",
"--ignored", "--nocapture", "--test-threads=1"
)
Invoke-CargoTest "Windows agent cursor" @(
"test", "-p", "cua-driver-e2e", "--test", "agent_cursor_windows_test", "--",
"--ignored", "--nocapture", "--test-threads=1"
)
}
if ($suite -in @("capture", "all")) {
Invoke-CargoTest "capture contract" @(
"test", "-p", "cua-driver-e2e", "--test", "capture_contract_test", "--",
"--ignored", "--nocapture", "--test-threads=1"
)
Invoke-CargoTest "Windows desktop scope" @(
"test", "-p", "cua-driver-e2e", "--test", "desktop_scope_windows_test", "--",
"--ignored", "--nocapture", "--test-threads=1"
)
# Includes the published-catalog row: it downloads the pinned
# cua-perception release and needs a Python with Tk (py -3 or python).
Invoke-CargoTest "perception capture loop" @(
"test", "-p", "cua-driver-e2e", "--test", "perception_capture_loop_test", "--",
"--ignored", "--nocapture", "--test-threads=1"
)
}
if ($suite -in @("shared", "all")) {
Invoke-ComputerHistoryGate
}
$script:FailureCount += (Test-E2eRecordings)
$reportExit = Invoke-E2eReport
if ($reportExit -ne 0) {
Write-Host "Windows E2E result validation failed" -ForegroundColor Red
$script:FailureCount++
}
if ($script:FailureCount -ne 0) {
throw "Windows Rust e2e suite had $($script:FailureCount) failing lane(s)"
}
Write-Host "Windows Rust e2e matrix completed: $suite" -ForegroundColor Green