Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci(cua-driver): publish desktop e2e matrix results

Francesco Bonaccid563dfdce9ac
diff --git a/.github/workflows/e2e-rust-linux.yml b/.github/workflows/e2e-rust-linux.yml
index 4214ccb2ad..847b787511 100644
--- a/.github/workflows/e2e-rust-linux.yml
+++ b/.github/workflows/e2e-rust-linux.yml
@@ -18,8 +18,9 @@ permissions:
contents: read
jobs:
- e2e:
- name: Linux interactive Rust suite
+ shared:
+ if: inputs.suite == 'shared' || inputs.suite == 'all'
+ name: "Linux / shared Electron"
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
@@ -36,15 +37,78 @@ jobs:
libspa-0.2-dev libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev \
libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \
libayatana-appindicator3-dev librsvg2-dev
- - name: Run canonical Rust matrix
+ - name: Run shared Rust behavior matrix
run: |
xvfb-run -a --server-args="-screen 0 1920x1080x24" \
dbus-run-session -- bash -lc \
- "scripts/ci/linux/run-rust-e2e.sh --suite '${{ inputs.suite }}'"
- - name: Upload logs
+ "scripts/ci/linux/run-rust-e2e.sh --suite shared"
+ - name: Upload shared results
if: always()
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
with:
- name: rust-linux-e2e-artifacts
+ name: rust-linux-shared
path: artifacts/cua-driver/linux
if-no-files-found: ignore
+
+ modality:
+ if: inputs.suite == 'modality' || inputs.suite == 'all'
+ name: "Linux / modality and desktop scope"
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ ref: ${{ inputs.ref }}
+ - uses: dtolnay/rust-toolchain@stable
+ - name: Install GUI and Rust build dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends \
+ xvfb dbus-x11 at-spi2-core openbox python3-gi gir1.2-gtk-3.0 \
+ libgtk-3-dev clang pkg-config libdbus-1-dev libpipewire-0.3-dev \
+ libspa-0.2-dev libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev \
+ libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \
+ libayatana-appindicator3-dev librsvg2-dev
+ - name: Run modality Rust matrix
+ run: |
+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \
+ dbus-run-session -- bash -lc \
+ "scripts/ci/linux/run-rust-e2e.sh --suite modality"
+ - name: Upload modality results
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: rust-linux-modality
+ path: artifacts/cua-driver/linux
+ if-no-files-found: ignore
+
+ summary:
+ if: always()
+ needs: [shared, modality]
+ name: "Linux / matrix summary"
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - name: Download lane results
+ uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+ with:
+ path: artifacts
+ - name: Publish matrix summary
+ shell: bash
+ run: |
+ {
+ echo "# CUA Rust Linux E2E matrix"
+ echo
+ echo "The lane jobs above are independent; a failure in one lane does not hide the others."
+ echo
+ } >> "$GITHUB_STEP_SUMMARY"
+ found=0
+ while IFS= read -r summary; do
+ found=1
+ echo "## $(basename "$(dirname "$summary")")" >> "$GITHUB_STEP_SUMMARY"
+ cat "$summary" >> "$GITHUB_STEP_SUMMARY"
+ echo >> "$GITHUB_STEP_SUMMARY"
+ done < <(find artifacts -type f -name summary.md -print | sort)
+ if [[ "$found" == 0 ]]; then
+ echo "No lane summary artifact was produced." >> "$GITHUB_STEP_SUMMARY"
+ fi
diff --git a/.github/workflows/e2e-rust-windows.yml b/.github/workflows/e2e-rust-windows.yml
index 61930b074c..08242ee5a1 100644
--- a/.github/workflows/e2e-rust-windows.yml
+++ b/.github/workflows/e2e-rust-windows.yml
@@ -22,25 +22,81 @@ permissions:
contents: read
jobs:
- e2e:
- name: Windows interactive Rust suite
+ shared:
+ if: inputs.suite == 'shared' || inputs.suite == 'all'
+ name: "Windows / shared Electron + Tauri"
runs-on: ${{ inputs.runner }}
timeout-minutes: 90
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
ref: ${{ inputs.ref }}
- - name: Run from the interactive user session
+ - name: Run shared Rust behavior matrix
shell: pwsh
- run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite ${{ inputs.suite }} -RequireGui
+ run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite shared -RequireGui
- name: Collect logs
if: always()
shell: pwsh
run: .\scripts\ci\windows\collect-artifacts.ps1
- - name: Upload logs
+ - name: Upload shared results
if: always()
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
with:
- name: rust-windows-e2e-artifacts
+ name: rust-windows-shared
path: artifacts/cua-driver/windows
if-no-files-found: ignore
+
+ native:
+ if: inputs.suite == 'native' || inputs.suite == 'all'
+ name: "Windows / native WPF + WebView2"
+ runs-on: ${{ inputs.runner }}
+ timeout-minutes: 90
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ ref: ${{ inputs.ref }}
+ - name: Run native Rust harnesses
+ shell: pwsh
+ run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite native -RequireGui
+ - name: Collect logs
+ if: always()
+ shell: pwsh
+ run: .\scripts\ci\windows\collect-artifacts.ps1
+ - name: Upload native results
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: rust-windows-native
+ path: artifacts/cua-driver/windows
+ if-no-files-found: ignore
+
+ summary:
+ if: always()
+ needs: [shared, native]
+ name: "Windows / matrix summary"
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - name: Download lane results
+ uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+ with:
+ path: artifacts
+ - name: Publish matrix summary
+ shell: bash
+ run: |
+ {
+ echo "# CUA Rust Windows E2E matrix"
+ echo
+ echo "The lane jobs above are independent; a failure in one lane does not hide the others."
+ echo
+ } >> "$GITHUB_STEP_SUMMARY"
+ found=0
+ while IFS= read -r summary; do
+ found=1
+ echo "## $(basename "$(dirname "$summary")")" >> "$GITHUB_STEP_SUMMARY"
+ cat "$summary" >> "$GITHUB_STEP_SUMMARY"
+ echo >> "$GITHUB_STEP_SUMMARY"
+ done < <(find artifacts -type f -name summary.md -print | sort)
+ if [[ "$found" == 0 ]]; then
+ echo "No lane summary artifact was produced." >> "$GITHUB_STEP_SUMMARY"
+ fi
diff --git a/.gitignore b/.gitignore
index f1f7a42c6c..9b06bbf0ad 100644
--- a/.gitignore
+++ b/.gitignore
@@ -72,6 +72,7 @@ docs/_build/
# PyBuilder
.pybuilder/
target/
+artifacts/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
diff --git a/libs/cua-driver/docs/e2e-ci-reporting.md b/libs/cua-driver/docs/e2e-ci-reporting.md
new file mode 100644
index 0000000000..5895482893
--- /dev/null
+++ b/libs/cua-driver/docs/e2e-ci-reporting.md
@@ -0,0 +1,64 @@
+# Rust E2E CI Reporting
+
+The Rust desktop harness is the behavioral source of truth. GitHub Actions only
+selects the lane and publishes its results.
+
+## Workflow UI
+
+The Linux and Windows manual workflows split the expensive suites into
+independent jobs. The workflow summary job collects their artifacts and writes
+one Markdown table to the GitHub Actions run summary.
+
+This means a failure in the shared Electron/Tauri lane does not prevent the
+native or modality lane from running. The workflow still fails when a required
+lane fails.
+
+## Result files
+
+When `CUA_E2E_RESULTS_FILE` is set, Rust shared behavior tests append one JSON
+object per host and scenario:
+
+```json
+{
+ "schema": "cua-e2e-result/v1",
+ "platform": "windows",
+ "host": "tauri",
+ "scenario": "keyboard",
+ "status": "FAIL",
+ "message": "application state did not reach key_state=enter",
+ "duration_ms": 24360
+}
+```
+
+When `CUA_E2E_SUMMARY_FILE` is set, the same test writes a Markdown row for the
+GitHub summary. Native and modality scripts add lane-level rows to the same
+files and retain their full logs alongside them.
+
+Statuses are deliberately explicit:
+
+- `PASS`: the external application-state oracle passed.
+- `FAIL`: the driver response or external oracle failed.
+- `SKIP`: the fixture was unavailable and was not required by the invocation.
+
+The CI scripts set `CUA_TEST_REQUIRE_FIXTURES=1`, so missing required fixtures
+are reported as failures rather than silently becoming skips.
+
+## Running locally
+
+Windows:
+
+```powershell
+.\scripts\ci\windows\run-rust-e2e.ps1 -Suite shared -RequireGui
+.\scripts\ci\windows\run-rust-e2e.ps1 -Suite native -RequireGui
+```
+
+Linux:
+
+```bash
+xvfb-run -a --server-args="-screen 0 1920x1080x24" \
+ dbus-run-session -- bash -lc \
+ "scripts/ci/linux/run-rust-e2e.sh --suite shared"
+```
+
+The generated files are under `artifacts/cua-driver/<platform>/` and should
+not be committed.
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
index eef8d280c8..9132ecaad0 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
@@ -15,6 +15,10 @@
#![cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))]
+use std::any::Any;
+use std::fs::OpenOptions;
+use std::io::Write;
+use std::panic::{self, AssertUnwindSafe};
use std::path::PathBuf;
use std::process::{Command, Stdio};
use std::thread;
@@ -105,6 +109,106 @@ fn host_specs() -> Vec<HostSpec> {
hosts
}
+fn panic_message(payload: &Box<dyn Any + Send>) -> String {
+ payload
+ .downcast_ref::<String>()
+ .cloned()
+ .or_else(|| {
+ payload
+ .downcast_ref::<&str>()
+ .map(|message| (*message).to_owned())
+ })
+ .unwrap_or_else(|| "test panicked without a string payload".to_owned())
+}
+
+fn escape_markdown(value: &str) -> String {
+ value
+ .replace('|', "\\|")
+ .replace('\r', "")
+ .replace('\n', " ")
+}
+
+fn append_result_line(path: &str, line: &str) {
+ let Some(parent) = std::path::Path::new(path).parent() else {
+ return;
+ };
+ let _ = std::fs::create_dir_all(parent);
+ if let Ok(mut file) = OpenOptions::new().create(true).append(true).open(path) {
+ let _ = writeln!(file, "{line}");
+ }
+}
+
+fn record_result(scenario: &str, host: &str, status: &str, message: &str, duration: Duration) {
+ let message = escape_markdown(message);
+ if let Ok(path) = std::env::var("CUA_E2E_RESULTS_FILE") {
+ let line = serde_json::json!({
+ "schema": "cua-e2e-result/v1",
+ "platform": std::env::consts::OS,
+ "host": host,
+ "scenario": scenario,
+ "status": status,
+ "message": message,
+ "duration_ms": duration.as_millis(),
+ });
+ append_result_line(&path, &line.to_string());
+ }
+ if let Ok(path) = std::env::var("CUA_E2E_SUMMARY_FILE") {
+ append_result_line(
+ &path,
+ &format!(
+ "| {} | {} | {} | {} | {} ms | {} |",
+ std::env::consts::OS,
+ escape_markdown(host),
+ escape_markdown(scenario),
+ status,
+ duration.as_millis(),
+ if message.is_empty() { "-" } else { &message },
+ ),
+ );
+ }
+}
+
+fn run_host_case<F>(scenario: &str, spec: &HostSpec, test: F) -> Option<Box<dyn Any + Send>>
+where
+ F: FnOnce(Fixture),
+{
+ let started = Instant::now();
+ let outcome = panic::catch_unwind(AssertUnwindSafe(|| {
+ let Some(fixture) = launch_host(spec) else {
+ return false;
+ };
+ test(fixture);
+ true
+ }));
+ match outcome {
+ Ok(true) => {
+ record_result(scenario, spec.name, "PASS", "", started.elapsed());
+ None
+ }
+ Ok(false) => {
+ record_result(
+ scenario,
+ spec.name,
+ "SKIP",
+ "fixture unavailable",
+ started.elapsed(),
+ );
+ None
+ }
+ Err(payload) => {
+ let message = panic_message(&payload);
+ record_result(scenario, spec.name, "FAIL", &message, started.elapsed());
+ Some(payload)
+ }
+ }
+}
+
+fn resume_first_failure(failure: Option<Box<dyn Any + Send>>) {
+ if let Some(payload) = failure {
+ panic::resume_unwind(payload);
+ }
+}
+
fn spawn_driver() -> Option<McpDriver> {
#[cfg(target_os = "macos")]
{
@@ -142,7 +246,10 @@ fn launch_host(spec: &HostSpec) -> Option<Fixture> {
let Some(mut driver) = spawn_driver() else {
if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() {
- panic!("cua-driver could not be started for the required {} fixture", spec.name);
+ panic!(
+ "cua-driver could not be started for the required {} fixture",
+ spec.name
+ );
}
return None;
};
@@ -338,299 +445,319 @@ fn click_ax(fixture: &mut Fixture, id: &str) {
#[test]
#[ignore]
fn shared_web_calculator_ax_route_is_state_verified() {
+ let mut failure = None;
for spec in host_specs() {
- let Some(mut fixture) = launch_host(&spec) else {
- continue;
- };
- click_ax(&mut fixture, "calc-1");
- click_ax(&mut fixture, "calc-2");
- click_ax(&mut fixture, "calc-plus");
- click_ax(&mut fixture, "calc-4");
- click_ax(&mut fixture, "calc-equals");
- assert_tree_contains(&mut fixture, "display=16");
- println!("✅ {} calculator AX route", fixture.name);
- }
-}
-
-#[test]
-#[ignore]
-fn shared_web_keyboard_routes_are_state_verified() {
- for spec in host_specs() {
- let Some(mut fixture) = launch_host(&spec) else {
- continue;
- };
- let pre = snapshot(&mut fixture);
- let input = require_element(&pre, "keyboard-input");
- let (origin_x, origin_y) = window_origin(&fixture, &pre);
- let scale = screenshot_scale(&pre);
- let (screen_x, screen_y) = element_center(&pre, input);
- let focus = fixture.driver.call(
- "click",
- serde_json::json!({
- "pid": fixture.pid as i64,
- "window_id": fixture.wid,
- "x": (screen_x - origin_x) * scale,
- "y": (screen_y - origin_y) * scale,
- "delivery_mode": "background"
- }),
- );
- assert!(
- !focus.is_error(),
- "{}: keyboard input focus click failed: {}",
- fixture.name,
- focus.text()
- );
- thread::sleep(Duration::from_millis(150));
- let enter = fixture.driver.call(
- "press_key",
- serde_json::json!({
- "pid": fixture.pid as i64,
- "window_id": fixture.wid,
- "element_index": input,
- "key": "return",
- "delivery_mode": "background"
- }),
- );
- assert!(
- !enter.is_error(),
- "{}: return failed: {}",
- fixture.name,
- enter.text()
- );
- assert_tree_contains(&mut fixture, "key_state=enter");
-
- let hotkey = fixture.driver.call(
- "hotkey",
- serde_json::json!({
- "pid": fixture.pid as i64,
- "window_id": fixture.wid,
- "keys": ["ctrl", "shift", "7"],
- "x": (screen_x - origin_x) * scale,
- "y": (screen_y - origin_y) * scale,
- "delivery_mode": "background"
- }),
- );
- #[cfg(target_os = "windows")]
- {
- if hotkey.is_error()
- || hotkey
- .text()
- .contains("Background delivery is not available")
- {
- println!(
- "✅ {} keyboard AX route: background hotkey refused honestly",
- fixture.name
- );
- continue;
- }
- }
- #[cfg(target_os = "linux")]
- {
- if hotkey.is_error()
- && hotkey.structured()["code"].as_str() == Some("background_unavailable")
- {
- println!(
- "✅ {} keyboard AX route: background hotkey refused honestly",
- fixture.name
- );
- continue;
- }
+ let result = run_host_case("calculator_ax", &spec, |mut fixture| {
+ click_ax(&mut fixture, "calc-1");
+ click_ax(&mut fixture, "calc-2");
+ click_ax(&mut fixture, "calc-plus");
+ click_ax(&mut fixture, "calc-4");
+ click_ax(&mut fixture, "calc-equals");
+ assert_tree_contains(&mut fixture, "display=16");
+ println!("✅ {} calculator AX route", fixture.name);
+ });
+ if failure.is_none() {
+ failure = result;
}
- assert!(
- !hotkey.is_error(),
- "{}: Ctrl+Shift+7 failed: {}",
- fixture.name,
- hotkey.text()
- );
- assert_tree_contains(&mut fixture, "key_state=hotkey");
- println!("✅ {} keyboard AX routes", fixture.name);
}
+ resume_first_failure(failure);
}
#[test]
#[ignore]
-fn shared_web_calculator_pixel_route_is_state_verified() {
+fn shared_web_keyboard_routes_are_state_verified() {
+ let mut failure = None;
for spec in host_specs() {
- let Some(mut fixture) = launch_host(&spec) else {
- continue;
- };
- for id in ["calc-1", "calc-2", "calc-plus", "calc-4", "calc-equals"] {
+ let result = run_host_case("keyboard", &spec, |mut fixture| {
let pre = snapshot(&mut fixture);
- let origin = window_origin(&fixture, &pre);
+ let input = require_element(&pre, "keyboard-input");
+ let (origin_x, origin_y) = window_origin(&fixture, &pre);
let scale = screenshot_scale(&pre);
- let index = require_element(&pre, id);
- let (screen_x, screen_y) = element_center(&pre, index);
- let response = fixture.driver.call(
+ let (screen_x, screen_y) = element_center(&pre, input);
+ let focus = fixture.driver.call(
"click",
serde_json::json!({
"pid": fixture.pid as i64,
"window_id": fixture.wid,
- "x": (screen_x - origin.0) * scale,
- "y": (screen_y - origin.1) * scale,
+ "x": (screen_x - origin_x) * scale,
+ "y": (screen_y - origin_y) * scale,
"delivery_mode": "background"
}),
);
assert!(
- !response.is_error(),
- "{}: pixel click {id} failed: {}",
+ !focus.is_error(),
+ "{}: keyboard input focus click failed: {}",
fixture.name,
- response.text()
+ focus.text()
+ );
+ thread::sleep(Duration::from_millis(150));
+ let enter = fixture.driver.call(
+ "press_key",
+ serde_json::json!({
+ "pid": fixture.pid as i64,
+ "window_id": fixture.wid,
+ "element_index": input,
+ "key": "return",
+ "delivery_mode": "background"
+ }),
+ );
+ assert!(
+ !enter.is_error(),
+ "{}: return failed: {}",
+ fixture.name,
+ enter.text()
+ );
+ assert_tree_contains(&mut fixture, "key_state=enter");
+
+ let hotkey = fixture.driver.call(
+ "hotkey",
+ serde_json::json!({
+ "pid": fixture.pid as i64,
+ "window_id": fixture.wid,
+ "keys": ["ctrl", "shift", "7"],
+ "x": (screen_x - origin_x) * scale,
+ "y": (screen_y - origin_y) * scale,
+ "delivery_mode": "background"
+ }),
);
+ #[cfg(target_os = "windows")]
+ {
+ if hotkey.is_error()
+ || hotkey
+ .text()
+ .contains("Background delivery is not available")
+ {
+ println!(
+ "✅ {} keyboard AX route: background hotkey refused honestly",
+ fixture.name
+ );
+ return;
+ }
+ }
+ #[cfg(target_os = "linux")]
+ {
+ if hotkey.is_error()
+ && hotkey.structured()["code"].as_str() == Some("background_unavailable")
+ {
+ println!(
+ "✅ {} keyboard AX route: background hotkey refused honestly",
+ fixture.name
+ );
+ return;
+ }
+ }
+ assert!(
+ !hotkey.is_error(),
+ "{}: Ctrl+Shift+7 failed: {}",
+ fixture.name,
+ hotkey.text()
+ );
+ assert_tree_contains(&mut fixture, "key_state=hotkey");
+ println!("✅ {} keyboard AX routes", fixture.name);
+ });
+ if failure.is_none() {
+ failure = result;
}
- assert_tree_contains(&mut fixture, "display=16");
- println!("✅ {} calculator pixel route", fixture.name);
}
+ resume_first_failure(failure);
}
#[test]
#[ignore]
-fn shared_web_editor_and_scroll_are_state_verified() {
+fn shared_web_calculator_pixel_route_is_state_verified() {
+ let mut failure = None;
for spec in host_specs() {
- let Some(mut fixture) = launch_host(&spec) else {
- continue;
- };
- let pre = snapshot(&mut fixture);
- let editor = require_element(&pre, "editor-document");
- let text = fixture.driver.call(
- "type_text",
- serde_json::json!({
- "pid": fixture.pid as i64,
- "window_id": fixture.wid,
- "element_index": editor,
- "text": "CUA saved this note.",
- "delivery_mode": "background"
- }),
- );
- assert!(
- !text.is_error(),
- "{}: editor type_text failed: {}",
- fixture.name,
- text.text()
- );
- click_ax(&mut fixture, "editor-save");
- assert_tree_contains(&mut fixture, "editor_status=saved:");
-
- let scroll = require_element(&snapshot(&mut fixture), "scroll-tall");
- let response = fixture.driver.call(
- "scroll",
- serde_json::json!({
- "pid": fixture.pid as i64,
- "window_id": fixture.wid,
- "element_index": scroll,
- "direction": "down",
- "by": "page",
- "amount": 4,
- "delivery_mode": "background"
- }),
- );
- #[cfg(target_os = "windows")]
- {
- if response.is_error()
- || response
- .text()
- .contains("Background delivery is not available")
- {
- println!(
- "✅ {} editor + scroll: background scroll refused honestly",
- fixture.name
+ let result = run_host_case("calculator_pixel", &spec, |mut fixture| {
+ for id in ["calc-1", "calc-2", "calc-plus", "calc-4", "calc-equals"] {
+ let pre = snapshot(&mut fixture);
+ let origin = window_origin(&fixture, &pre);
+ let scale = screenshot_scale(&pre);
+ let index = require_element(&pre, id);
+ let (screen_x, screen_y) = element_center(&pre, index);
+ let response = fixture.driver.call(
+ "click",
+ serde_json::json!({
+ "pid": fixture.pid as i64,
+ "window_id": fixture.wid,
+ "x": (screen_x - origin.0) * scale,
+ "y": (screen_y - origin.1) * scale,
+ "delivery_mode": "background"
+ }),
+ );
+ assert!(
+ !response.is_error(),
+ "{}: pixel click {id} failed: {}",
+ fixture.name,
+ response.text()
);
- continue;
}
+ assert_tree_contains(&mut fixture, "display=16");
+ println!("✅ {} calculator pixel route", fixture.name);
+ });
+ if failure.is_none() {
+ failure = result;
}
- assert!(
- !response.is_error(),
- "{}: scroll failed: {}",
- fixture.name,
- response.text()
- );
- let post = snapshot(&mut fixture);
- let offset = post
- .tree_text()
- .lines()
- .find_map(|line| line.split("scroll_offset=").nth(1))
- .and_then(|tail| tail.split(|ch: char| !ch.is_ascii_digit()).next())
- .and_then(|value| value.parse::<u64>().ok())
- .unwrap_or(0);
- assert!(
- offset > 0,
- "{}: successful background scroll did not advance the external scroll oracle: {}",
- fixture.name,
- post.tree_text()
- );
- println!("✅ {} editor + scroll state oracles", fixture.name);
}
+ resume_first_failure(failure);
}
#[test]
#[ignore]
-fn shared_web_child_window_and_drag_have_external_oracles() {
+fn shared_web_editor_and_scroll_are_state_verified() {
+ let mut failure = None;
for spec in host_specs() {
- let Some(mut fixture) = launch_host(&spec) else {
- continue;
- };
- let pre = snapshot(&mut fixture);
- let (window_x, window_y) = window_origin(&fixture, &pre);
- let scale = screenshot_scale(&pre);
- let source = require_element(&pre, "drag-source");
- let frame = pre.structured()["elements"]
- .as_array()
- .and_then(|elements| {
- elements
- .iter()
- .find(|element| element["element_index"].as_u64() == Some(source))
- })
- .and_then(|element| element["frame"].as_object())
- .unwrap_or_else(|| panic!("{}: drag-source has no frame", fixture.name));
- let x = (frame["x"].as_f64().unwrap_or(0.0) - window_x
- + frame["w"].as_f64().unwrap_or(0.0) / 2.0)
- * scale;
- let y = (frame["y"].as_f64().unwrap_or(0.0) - window_y
- + frame["h"].as_f64().unwrap_or(0.0) / 2.0)
- * scale;
- let target_index = require_element(&pre, "drop-target");
- let target = pre.structured()["elements"]
- .as_array()
- .and_then(|elements| {
- elements
- .iter()
- .find(|element| element["element_index"].as_u64() == Some(target_index))
- })
- .and_then(|element| element["frame"].as_object())
- .unwrap_or_else(|| panic!("{}: drop-target has no frame", fixture.name));
- let tx = (target["x"].as_f64().unwrap_or(0.0) - window_x
- + target["w"].as_f64().unwrap_or(0.0) / 2.0)
- * scale;
- let ty = (target["y"].as_f64().unwrap_or(0.0) - window_y
- + target["h"].as_f64().unwrap_or(0.0) / 2.0)
- * scale;
- #[cfg(not(target_os = "macos"))]
- let _ = (x, y, tx, ty);
- #[cfg(target_os = "macos")]
- {
- let drag = fixture.driver.call(
- "drag",
+ let result = run_host_case("editor_scroll", &spec, |mut fixture| {
+ let pre = snapshot(&mut fixture);
+ let editor = require_element(&pre, "editor-document");
+ let text = fixture.driver.call(
+ "type_text",
serde_json::json!({
"pid": fixture.pid as i64,
"window_id": fixture.wid,
- "from_x": x,
- "from_y": y,
- "to_x": tx,
- "to_y": ty,
- "duration_ms": 500,
- "delivery_mode": "foreground"
+ "element_index": editor,
+ "text": "CUA saved this note.",
+ "delivery_mode": "background"
}),
);
assert!(
- !drag.is_error(),
- "{}: drag failed: {}",
+ !text.is_error(),
+ "{}: editor type_text failed: {}",
+ fixture.name,
+ text.text()
+ );
+ click_ax(&mut fixture, "editor-save");
+ assert_tree_contains(&mut fixture, "editor_status=saved:");
+
+ let scroll = require_element(&snapshot(&mut fixture), "scroll-tall");
+ let response = fixture.driver.call(
+ "scroll",
+ serde_json::json!({
+ "pid": fixture.pid as i64,
+ "window_id": fixture.wid,
+ "element_index": scroll,
+ "direction": "down",
+ "by": "page",
+ "amount": 4,
+ "delivery_mode": "background"
+ }),
+ );
+ #[cfg(target_os = "windows")]
+ {
+ if response.is_error()
+ || response
+ .text()
+ .contains("Background delivery is not available")
+ {
+ println!(
+ "✅ {} editor + scroll: background scroll refused honestly",
+ fixture.name
+ );
+ return;
+ }
+ }
+ assert!(
+ !response.is_error(),
+ "{}: scroll failed: {}",
+ fixture.name,
+ response.text()
+ );
+ let post = snapshot(&mut fixture);
+ let offset = post
+ .tree_text()
+ .lines()
+ .find_map(|line| line.split("scroll_offset=").nth(1))
+ .and_then(|tail| tail.split(|ch: char| !ch.is_ascii_digit()).next())
+ .and_then(|value| value.parse::<u64>().ok())
+ .unwrap_or(0);
+ assert!(
+ offset > 0,
+ "{}: successful background scroll did not advance the external scroll oracle: {}",
fixture.name,
- drag.text()
+ post.tree_text()
);
- assert_tree_contains(&mut fixture, "drag_status=dropped");
+ println!("✅ {} editor + scroll state oracles", fixture.name);
+ });
+ if failure.is_none() {
+ failure = result;
}
+ }
+ resume_first_failure(failure);
+}
- click_ax(&mut fixture, "btn-open-child-window");
- assert_tree_contains(&mut fixture, "child_windows=1");
- println!("✅ {} child-window + drag state oracles", fixture.name);
+#[test]
+#[ignore]
+fn shared_web_child_window_and_drag_have_external_oracles() {
+ let mut failure = None;
+ for spec in host_specs() {
+ let result = run_host_case("child_window_drag", &spec, |mut fixture| {
+ let pre = snapshot(&mut fixture);
+ let (window_x, window_y) = window_origin(&fixture, &pre);
+ let scale = screenshot_scale(&pre);
+ let source = require_element(&pre, "drag-source");
+ let frame = pre.structured()["elements"]
+ .as_array()
+ .and_then(|elements| {
+ elements
+ .iter()
+ .find(|element| element["element_index"].as_u64() == Some(source))
+ })
+ .and_then(|element| element["frame"].as_object())
+ .unwrap_or_else(|| panic!("{}: drag-source has no frame", fixture.name));
+ let x = (frame["x"].as_f64().unwrap_or(0.0) - window_x
+ + frame["w"].as_f64().unwrap_or(0.0) / 2.0)
+ * scale;
+ let y = (frame["y"].as_f64().unwrap_or(0.0) - window_y
+ + frame["h"].as_f64().unwrap_or(0.0) / 2.0)
+ * scale;
+ let target_index = require_element(&pre, "drop-target");
+ let target = pre.structured()["elements"]
+ .as_array()
+ .and_then(|elements| {
+ elements
+ .iter()
+ .find(|element| element["element_index"].as_u64() == Some(target_index))
+ })
+ .and_then(|element| element["frame"].as_object())
+ .unwrap_or_else(|| panic!("{}: drop-target has no frame", fixture.name));
+ let tx = (target["x"].as_f64().unwrap_or(0.0) - window_x
+ + target["w"].as_f64().unwrap_or(0.0) / 2.0)
+ * scale;
+ let ty = (target["y"].as_f64().unwrap_or(0.0) - window_y
+ + target["h"].as_f64().unwrap_or(0.0) / 2.0)
+ * scale;
+ #[cfg(not(target_os = "macos"))]
+ let _ = (x, y, tx, ty);
+ #[cfg(target_os = "macos")]
+ {
+ let drag = fixture.driver.call(
+ "drag",
+ serde_json::json!({
+ "pid": fixture.pid as i64,
+ "window_id": fixture.wid,
+ "from_x": x,
+ "from_y": y,
+ "to_x": tx,
+ "to_y": ty,
+ "duration_ms": 500,
+ "delivery_mode": "foreground"
+ }),
+ );
+ assert!(
+ !drag.is_error(),
+ "{}: drag failed: {}",
+ fixture.name,
+ drag.text()
+ );
+ assert_tree_contains(&mut fixture, "drag_status=dropped");
+ }
+
+ click_ax(&mut fixture, "btn-open-child-window");
+ assert_tree_contains(&mut fixture, "child_windows=1");
+ println!("✅ {} child-window + drag state oracles", fixture.name);
+ });
+ if failure.is_none() {
+ failure = result;
+ }
}
+ resume_first_failure(failure);
}
diff --git a/scripts/ci/linux/run-rust-e2e.sh b/scripts/ci/linux/run-rust-e2e.sh
index d90c021fa5..bce7bd14f8 100755
--- a/scripts/ci/linux/run-rust-e2e.sh
+++ b/scripts/ci/linux/run-rust-e2e.sh
@@ -35,6 +35,17 @@ case "$SUITE" in
esac
mkdir -p "${REPO_ROOT}/artifacts/cua-driver/linux"
+RESULTS_FILE="${REPO_ROOT}/artifacts/cua-driver/linux/results.jsonl"
+SUMMARY_FILE="${REPO_ROOT}/artifacts/cua-driver/linux/summary.md"
+: > "${RESULTS_FILE}"
+cat > "${SUMMARY_FILE}" <<'EOF'
+# CUA Rust Linux E2E matrix
+
+| Platform | Host/lane | Scenario | Status | Duration | Details |
+| --- | --- | --- | --- | --- | --- |
+EOF
+export CUA_E2E_RESULTS_FILE="${RESULTS_FILE}"
+export CUA_E2E_SUMMARY_FILE="${SUMMARY_FILE}"
export CUA_TEST_WORKSPACE_ROOT="${RUST_ROOT}"
export CUA_TEST_DRIVER_BIN="${RUST_ROOT}/target/release/cua-driver"
export CUA_TEST_APPS_ROOT="${RUST_ROOT}/test-apps"
@@ -55,11 +66,28 @@ if [[ ! -x "${CUA_TEST_APPS_ROOT}/harness-electron/CuaTestHarness.Electron" ]];
exit 1
fi
+FAILURE_COUNT=0
+
run_test() {
local name="$1"
shift
echo "[RUN] ${name}"
+ set +e
(cd "${RUST_ROOT}" && "$@") 2>&1 | tee "${REPO_ROOT}/artifacts/cua-driver/linux/${name}.log"
+ local exit_code=${PIPESTATUS[0]}
+ set -e
+ local status="PASS"
+ if [[ "${exit_code}" != 0 ]]; then
+ status="FAIL"
+ FAILURE_COUNT=$((FAILURE_COUNT + 1))
+ fi
+ local details="-"
+ if [[ "${exit_code}" != 0 ]]; then
+ details="exit code ${exit_code}"
+ fi
+ printf '{"schema":"cua-e2e-result/v1","platform":"linux","host":"lane","scenario":"%s","status":"%s","message":"%s"}\n' \
+ "${name}" "${status}" "${details}" >> "${RESULTS_FILE}"
+ printf '| Linux | lane | %s | %s | n/a | %s |\n' "${name}" "${status}" "${details}" >> "${SUMMARY_FILE}"
}
if [[ "${SUITE}" == shared || "${SUITE}" == all ]]; then
@@ -77,4 +105,9 @@ if [[ "${SUITE}" == modality || "${SUITE}" == all ]]; then
--ignored --nocapture --test-threads=1
fi
+if [[ "${FAILURE_COUNT}" != 0 ]]; then
+ echo "Linux Rust e2e suite had ${FAILURE_COUNT} failing lane(s)" >&2
+ exit 1
+fi
+
echo "Linux Rust e2e suite completed: ${SUITE}"
diff --git a/scripts/ci/windows/run-rust-e2e.ps1 b/scripts/ci/windows/run-rust-e2e.ps1
index dc77c89e32..2c092bf64f 100644
--- a/scripts/ci/windows/run-rust-e2e.ps1
+++ b/scripts/ci/windows/run-rust-e2e.ps1
@@ -16,6 +16,17 @@ $driverRoot = Join-Path $repoRoot "libs\cua-driver"
$rustRoot = Join-Path $driverRoot "rust"
$artifactDir = Join-Path $repoRoot "artifacts\cua-driver\windows"
New-Item -ItemType Directory -Force $artifactDir | Out-Null
+$resultsPath = Join-Path $artifactDir "results.jsonl"
+$summaryPath = Join-Path $artifactDir "summary.md"
+@(
+ "# CUA Rust Windows E2E matrix",
+ "",
+ "| Platform | Host/lane | Scenario | Status | Duration | Details |",
+ "| --- | --- | --- | --- | --- | --- |"
+) | Set-Content -Path $summaryPath
+Remove-Item -Force -ErrorAction SilentlyContinue $resultsPath
+$env:CUA_E2E_RESULTS_FILE = $resultsPath
+$env:CUA_E2E_SUMMARY_FILE = $summaryPath
& (Join-Path $scriptDir "verify-user-session.ps1")
if ($RequireGui) { $env:CUA_REQUIRE_GUI = "1" }
@@ -47,16 +58,32 @@ function Invoke-CargoTest {
Write-Host "[RUN] $Name" -ForegroundColor Yellow
Push-Location $rustRoot
try {
- $logPath = Join-Path $artifactDir ("$($Name -replace '[^A-Za-z0-9_.-]', '-')}.log")
+ $logPath = Join-Path $artifactDir ("$($Name -replace '[^A-Za-z0-9_.-]', '-').log")
$output = & cargo @Arguments 2>&1
$exitCode = $LASTEXITCODE
$output | Tee-Object -FilePath $logPath
- if ($exitCode -ne 0) { throw "$Name failed with exit code $exitCode" }
+ $status = if ($exitCode -eq 0) { "PASS" } else { "FAIL" }
+ $record = [ordered]@{
+ schema = "cua-e2e-result/v1"
+ platform = "windows"
+ host = "lane"
+ scenario = $Name
+ status = $status
+ message = if ($exitCode -eq 0) { "" } else { "exit code $exitCode" }
+ } | ConvertTo-Json -Compress
+ Add-Content -Path $resultsPath -Value $record
+ $details = if ($exitCode -eq 0) { "-" } else { "exit code $exitCode" }
+ Add-Content -Path $summaryPath -Value "| Windows | lane | $Name | $status | n/a | $details |"
+ if ($exitCode -ne 0) {
+ $script:FailureCount++
+ }
} finally {
Pop-Location
}
}
+$script:FailureCount = 0
+
if ($Suite -in @("shared", "all")) {
Invoke-CargoTest "shared behavior matrix" @(
"test", "-p", "cua-driver", "--test", "cross_platform_behavior_test", "--",
@@ -75,4 +102,8 @@ if ($Suite -in @("native", "all")) {
)
}
+if ($script:FailureCount -ne 0) {
+ throw "Windows Rust e2e suite had $($script:FailureCount) failing lane(s)"
+}
+
Write-Host "Windows Rust e2e suite completed: $Suite" -ForegroundColor Green