Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(docs): wait for the window manager before focusing the guide terminals (#5965)
injaneitydc8bae76a856
diff --git a/docs/content/docs/cua-sdk/guides/desktop.mdx b/docs/content/docs/cua-sdk/guides/desktop.mdxindex e5d97ad3f2..d156363a5c 100644--- a/docs/content/docs/cua-sdk/guides/desktop.mdx+++ b/docs/content/docs/cua-sdk/guides/desktop.mdx@@ -24,6 +24,8 @@ Python, TypeScript and Rust, see [Use Cua in your own harness](/cua-sdk/guides/o+The window manager can start after the sandbox is ready, so the example waits until the terminal really has focus before it types.+<Tabs groupId="language" persist items={['Python', 'TypeScript', 'CLI']}><Tab value="Python">@@ -31,14 +33,23 @@ Python, TypeScript and Rust, see [Use Cua in your own harness](/cua-sdk/guides/ofrom pathlib import Pathfrom cua_sandbox import Image, Sandbox-FOCUS_TERMINAL = ("export DISPLAY=:1; setsid -f xfce4-terminal -T demo >/dev/null 2>&1; "- "xdotool search --sync --onlyvisible --name demo windowactivate --sync")+FOCUS_TERMINAL = (+ "export DISPLAY=:1; setsid -f xfce4-terminal -T demo >/dev/null 2>&1; "+ "end=$(($(date +%s) + 20)); "+ "while [ $(date +%s) -lt $end ]; do "+ "w=$(xdotool search --onlyvisible --limit 1 --name ^demo$) && "+ "xprop -id $w WM_STATE | grep -q window.state && "+ "timeout 5 xdotool windowactivate --sync $w && exit 0; "+ "sleep 0.2; done; "+ "echo demo terminal never got keyboard focus >&2; exit 1"+)async with Sandbox.ephemeral(Image.linux()) as sb:width, height = await sb.get_dimensions()await sb.mouse.click(width // 2, height // 2)# Keys go to the focused window, so open one.- assert (await sb.shell.run(FOCUS_TERMINAL)).success+ focus = await sb.shell.run(FOCUS_TERMINAL, timeout=40)+ assert focus.success, f"terminal focus failed ({focus.returncode}): {focus.stderr}"await sb.keyboard.type("hello")await sb.keyboard.keypress(["ctrl", "a"])Path("desk.png").write_bytes(await sb.screenshot())@@ -49,11 +60,17 @@ async with Sandbox.ephemeral(Image.linux()) as sb:```ts test="container" id="desktop-typescript"import { writeFile } from 'node:fs/promises';-import { embedded, SandboxCreateOptions, SpacesdCommand } from '@trycua/cua';+import { embedded, SandboxCreateOptions } from '@trycua/cua';const FOCUS_TERMINAL ='export DISPLAY=:1; setsid -f xfce4-terminal -T demo >/dev/null 2>&1; ' +- 'xdotool search --sync --onlyvisible --name demo windowactivate --sync';+ 'end=$(($(date +%s) + 20)); ' ++ 'while [ $(date +%s) -lt $end ]; do ' ++ 'w=$(xdotool search --onlyvisible --limit 1 --name ^demo$) && ' ++ 'xprop -id $w WM_STATE | grep -q window.state && ' ++ 'timeout 5 xdotool windowactivate --sync $w && exit 0; ' ++ 'sleep 0.2; done; ' ++ 'echo demo terminal never got keyboard focus >&2; exit 1';const sb = await embedded().sandboxes().create(SandboxCreateOptions.create({@@ -65,7 +82,12 @@ try {const shot = await env.screenshot(undefined);await env.click(Math.floor(shot.width / 2), Math.floor(shot.height / 2));// Keys go to the focused window, so open one.- await env.run(SpacesdCommand.create({ program: 'sh', args: ['-c', FOCUS_TERMINAL] }));+ const focus = await env.sh(FOCUS_TERMINAL, 40_000);+ if (!focus.exit.success) {+ throw new Error(+ `terminal focus failed (${JSON.stringify(focus.exit)}): ${new TextDecoder().decode(focus.stderr)}`+ );+ }await env.typeText('hello');await env.hotkey(['ctrl', 'a']);await writeFile('desk.png', Buffer.from((await env.screenshot(undefined)).image));diff --git a/docs/content/docs/cua-sdk/guides/own-harness.mdx b/docs/content/docs/cua-sdk/guides/own-harness.mdxindex 41b1d49931..bbf7789d54 100644--- a/docs/content/docs/cua-sdk/guides/own-harness.mdx+++ b/docs/content/docs/cua-sdk/guides/own-harness.mdx@@ -40,7 +40,13 @@ from cua_sandbox import Image, Sandbox# Open a terminal titled "harness" and focus it, so typed keys land there.OPEN_TERMINAL = ("export DISPLAY=:1; setsid -f xfce4-terminal -T harness >/dev/null 2>&1; "- "xdotool search --sync --onlyvisible --name harness windowactivate --sync"+ "end=$(($(date +%s) + 20)); "+ "while [ $(date +%s) -lt $end ]; do "+ "w=$(xdotool search --onlyvisible --limit 1 --name ^harness$) && "+ "xprop -id $w WM_STATE | grep -q window.state && "+ "timeout 5 xdotool windowactivate --sync $w && exit 0; "+ "sleep 0.2; done; "+ "echo harness terminal never got keyboard focus >&2; exit 1")@@ -58,7 +64,8 @@ async def main():# Click, then type into a terminal.width, height = await sb.get_dimensions()await sb.mouse.click(width // 2, height // 2)- await sb.shell.run(OPEN_TERMINAL)+ opened = await sb.shell.run(OPEN_TERMINAL, timeout=40)+ assert opened.success, f"terminal focus failed ({opened.returncode}): {opened.stderr}"await sb.keyboard.type("echo typed by my harness")await sb.keyboard.keypress("enter")await asyncio.sleep(1) # let the terminal draw@@ -95,7 +102,13 @@ import { embedded, SandboxCreateOptions } from '@trycua/cua';// Open a terminal titled "harness" and focus it, so typed keys land there.const OPEN_TERMINAL ='export DISPLAY=:1; setsid -f xfce4-terminal -T harness >/dev/null 2>&1; ' +- 'xdotool search --sync --onlyvisible --name harness windowactivate --sync';+ 'end=$(($(date +%s) + 20)); ' ++ 'while [ $(date +%s) -lt $end ]; do ' ++ 'w=$(xdotool search --onlyvisible --limit 1 --name ^harness$) && ' ++ 'xprop -id $w WM_STATE | grep -q window.state && ' ++ 'timeout 5 xdotool windowactivate --sync $w && exit 0; ' ++ 'sleep 0.2; done; ' ++ 'echo harness terminal never got keyboard focus >&2; exit 1';const text = (bytes: ArrayBuffer) => new TextDecoder().decode(bytes);// A Linux desktop on this machine.@@ -115,7 +128,12 @@ try {// Click, then type into a terminal.const { width, height } = await env.screenshot(undefined);await env.click(Math.floor(width / 2), Math.floor(height / 2));- await env.sh(OPEN_TERMINAL, undefined);+ const opened = await env.sh(OPEN_TERMINAL, 40_000);+ if (!opened.exit.success) {+ throw new Error(+ `terminal focus failed (${JSON.stringify(opened.exit)}): ${text(opened.stderr)}`+ );+ }await env.typeText('echo typed by my harness');await env.press('enter');await new Promise((resolve) => setTimeout(resolve, 1000)); // let the terminal draw@@ -159,7 +177,13 @@ use serde_json::{json, Value};// Open a terminal titled "harness" and focus it, so typed keys land there.const OPEN_TERMINAL: &str = "export DISPLAY=:1; setsid -f xfce4-terminal -T harness >/dev/null 2>&1; \- xdotool search --sync --onlyvisible --name harness windowactivate --sync";+ end=$(($(date +%s) + 20)); \+ while [ $(date +%s) -lt $end ]; do \+ w=$(xdotool search --onlyvisible --limit 1 --name ^harness$) && \+ xprop -id $w WM_STATE | grep -q window.state && \+ timeout 5 xdotool windowactivate --sync $w && exit 0; \+ sleep 0.2; done; \+ echo harness terminal never got keyboard focus >&2; exit 1";#[tokio::main]async fn main() -> Result<(), Box<dyn std::error::Error>> {@@ -195,7 +219,15 @@ async fn drive(sb: &cua_sdk::Sandbox) -> Result<(), Box<dyn std::error::Error>>let shot = env.screenshot(None).await?;let (width, height) = (shot.width, shot.height);env.click(f64::from(width / 2), f64::from(height / 2)).await?;- env.sh(OPEN_TERMINAL.into(), None).await?;+ let opened = env.sh(OPEN_TERMINAL.into(), Some(40_000)).await?;+ if !opened.exit.success {+ return Err(format!(+ "terminal focus failed ({:?}): {}",+ opened.exit,+ String::from_utf8_lossy(&opened.stderr)+ )+ .into());+ }env.type_text("echo typed by my harness".into()).await?;env.press("enter".into()).await?;tokio::time::sleep(std::time::Duration::from_secs(1)).await; // let the terminal draw