Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(docs): wait for the window manager before focusing the guide terminals (#5965)

injaneitydc8bae76a856
diff --git a/docs/content/docs/cua-sdk/guides/desktop.mdx b/docs/content/docs/cua-sdk/guides/desktop.mdx
index e5d97ad3f2..d156363a5c 100644
--- a/docs/content/docs/cua-sdk/guides/desktop.mdx
+++ b/docs/content/docs/cua-sdk/guides/desktop.mdx
@@ -24,6 +24,8 @@ Python, TypeScript and Rust, see [Use Cua in your own harness](/cua-sdk/guides/o
![A terminal on a local Linux sandbox desktop, opened from the dock and typed into through the HTML5 viewer](/img/cua-sdk/viewer-terminal.png)
+The window manager can start after the sandbox is ready, so the example waits until the terminal really has focus before it types.
+
<Tabs groupId="language" persist items={['Python', 'TypeScript', 'CLI']}>
<Tab value="Python">
@@ -31,14 +33,23 @@ Python, TypeScript and Rust, see [Use Cua in your own harness](/cua-sdk/guides/o
from pathlib import Path
from cua_sandbox import Image, Sandbox
-FOCUS_TERMINAL = ("export DISPLAY=:1; setsid -f xfce4-terminal -T demo >/dev/null 2>&1; "
- "xdotool search --sync --onlyvisible --name demo windowactivate --sync")
+FOCUS_TERMINAL = (
+ "export DISPLAY=:1; setsid -f xfce4-terminal -T demo >/dev/null 2>&1; "
+ "end=$(($(date +%s) + 20)); "
+ "while [ $(date +%s) -lt $end ]; do "
+ "w=$(xdotool search --onlyvisible --limit 1 --name ^demo$) && "
+ "xprop -id $w WM_STATE | grep -q window.state && "
+ "timeout 5 xdotool windowactivate --sync $w && exit 0; "
+ "sleep 0.2; done; "
+ "echo demo terminal never got keyboard focus >&2; exit 1"
+)
async with Sandbox.ephemeral(Image.linux()) as sb:
width, height = await sb.get_dimensions()
await sb.mouse.click(width // 2, height // 2)
# Keys go to the focused window, so open one.
- assert (await sb.shell.run(FOCUS_TERMINAL)).success
+ focus = await sb.shell.run(FOCUS_TERMINAL, timeout=40)
+ assert focus.success, f"terminal focus failed ({focus.returncode}): {focus.stderr}"
await sb.keyboard.type("hello")
await sb.keyboard.keypress(["ctrl", "a"])
Path("desk.png").write_bytes(await sb.screenshot())
@@ -49,11 +60,17 @@ async with Sandbox.ephemeral(Image.linux()) as sb:
```ts test="container" id="desktop-typescript"
import { writeFile } from 'node:fs/promises';
-import { embedded, SandboxCreateOptions, SpacesdCommand } from '@trycua/cua';
+import { embedded, SandboxCreateOptions } from '@trycua/cua';
const FOCUS_TERMINAL =
'export DISPLAY=:1; setsid -f xfce4-terminal -T demo >/dev/null 2>&1; ' +
- 'xdotool search --sync --onlyvisible --name demo windowactivate --sync';
+ 'end=$(($(date +%s) + 20)); ' +
+ 'while [ $(date +%s) -lt $end ]; do ' +
+ 'w=$(xdotool search --onlyvisible --limit 1 --name ^demo$) && ' +
+ 'xprop -id $w WM_STATE | grep -q window.state && ' +
+ 'timeout 5 xdotool windowactivate --sync $w && exit 0; ' +
+ 'sleep 0.2; done; ' +
+ 'echo demo terminal never got keyboard focus >&2; exit 1';
const sb = await embedded().sandboxes().create(
SandboxCreateOptions.create({
@@ -65,7 +82,12 @@ try {
const shot = await env.screenshot(undefined);
await env.click(Math.floor(shot.width / 2), Math.floor(shot.height / 2));
// Keys go to the focused window, so open one.
- await env.run(SpacesdCommand.create({ program: 'sh', args: ['-c', FOCUS_TERMINAL] }));
+ const focus = await env.sh(FOCUS_TERMINAL, 40_000);
+ if (!focus.exit.success) {
+ throw new Error(
+ `terminal focus failed (${JSON.stringify(focus.exit)}): ${new TextDecoder().decode(focus.stderr)}`
+ );
+ }
await env.typeText('hello');
await env.hotkey(['ctrl', 'a']);
await writeFile('desk.png', Buffer.from((await env.screenshot(undefined)).image));
diff --git a/docs/content/docs/cua-sdk/guides/own-harness.mdx b/docs/content/docs/cua-sdk/guides/own-harness.mdx
index 41b1d49931..bbf7789d54 100644
--- a/docs/content/docs/cua-sdk/guides/own-harness.mdx
+++ b/docs/content/docs/cua-sdk/guides/own-harness.mdx
@@ -40,7 +40,13 @@ from cua_sandbox import Image, Sandbox
# Open a terminal titled "harness" and focus it, so typed keys land there.
OPEN_TERMINAL = (
"export DISPLAY=:1; setsid -f xfce4-terminal -T harness >/dev/null 2>&1; "
- "xdotool search --sync --onlyvisible --name harness windowactivate --sync"
+ "end=$(($(date +%s) + 20)); "
+ "while [ $(date +%s) -lt $end ]; do "
+ "w=$(xdotool search --onlyvisible --limit 1 --name ^harness$) && "
+ "xprop -id $w WM_STATE | grep -q window.state && "
+ "timeout 5 xdotool windowactivate --sync $w && exit 0; "
+ "sleep 0.2; done; "
+ "echo harness terminal never got keyboard focus >&2; exit 1"
)
@@ -58,7 +64,8 @@ async def main():
# Click, then type into a terminal.
width, height = await sb.get_dimensions()
await sb.mouse.click(width // 2, height // 2)
- await sb.shell.run(OPEN_TERMINAL)
+ opened = await sb.shell.run(OPEN_TERMINAL, timeout=40)
+ assert opened.success, f"terminal focus failed ({opened.returncode}): {opened.stderr}"
await sb.keyboard.type("echo typed by my harness")
await sb.keyboard.keypress("enter")
await asyncio.sleep(1) # let the terminal draw
@@ -95,7 +102,13 @@ import { embedded, SandboxCreateOptions } from '@trycua/cua';
// Open a terminal titled "harness" and focus it, so typed keys land there.
const OPEN_TERMINAL =
'export DISPLAY=:1; setsid -f xfce4-terminal -T harness >/dev/null 2>&1; ' +
- 'xdotool search --sync --onlyvisible --name harness windowactivate --sync';
+ 'end=$(($(date +%s) + 20)); ' +
+ 'while [ $(date +%s) -lt $end ]; do ' +
+ 'w=$(xdotool search --onlyvisible --limit 1 --name ^harness$) && ' +
+ 'xprop -id $w WM_STATE | grep -q window.state && ' +
+ 'timeout 5 xdotool windowactivate --sync $w && exit 0; ' +
+ 'sleep 0.2; done; ' +
+ 'echo harness terminal never got keyboard focus >&2; exit 1';
const text = (bytes: ArrayBuffer) => new TextDecoder().decode(bytes);
// A Linux desktop on this machine.
@@ -115,7 +128,12 @@ try {
// Click, then type into a terminal.
const { width, height } = await env.screenshot(undefined);
await env.click(Math.floor(width / 2), Math.floor(height / 2));
- await env.sh(OPEN_TERMINAL, undefined);
+ const opened = await env.sh(OPEN_TERMINAL, 40_000);
+ if (!opened.exit.success) {
+ throw new Error(
+ `terminal focus failed (${JSON.stringify(opened.exit)}): ${text(opened.stderr)}`
+ );
+ }
await env.typeText('echo typed by my harness');
await env.press('enter');
await new Promise((resolve) => setTimeout(resolve, 1000)); // let the terminal draw
@@ -159,7 +177,13 @@ use serde_json::{json, Value};
// Open a terminal titled "harness" and focus it, so typed keys land there.
const OPEN_TERMINAL: &str = "export DISPLAY=:1; setsid -f xfce4-terminal -T harness >/dev/null 2>&1; \
- xdotool search --sync --onlyvisible --name harness windowactivate --sync";
+ end=$(($(date +%s) + 20)); \
+ while [ $(date +%s) -lt $end ]; do \
+ w=$(xdotool search --onlyvisible --limit 1 --name ^harness$) && \
+ xprop -id $w WM_STATE | grep -q window.state && \
+ timeout 5 xdotool windowactivate --sync $w && exit 0; \
+ sleep 0.2; done; \
+ echo harness terminal never got keyboard focus >&2; exit 1";
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -195,7 +219,15 @@ async fn drive(sb: &cua_sdk::Sandbox) -> Result<(), Box<dyn std::error::Error>>
let shot = env.screenshot(None).await?;
let (width, height) = (shot.width, shot.height);
env.click(f64::from(width / 2), f64::from(height / 2)).await?;
- env.sh(OPEN_TERMINAL.into(), None).await?;
+ let opened = env.sh(OPEN_TERMINAL.into(), Some(40_000)).await?;
+ if !opened.exit.success {
+ return Err(format!(
+ "terminal focus failed ({:?}): {}",
+ opened.exit,
+ String::from_utf8_lossy(&opened.stderr)
+ )
+ .into());
+ }
env.type_text("echo typed by my harness".into()).await?;
env.press("enter".into()).await?;
tokio::time::sleep(std::time::Duration::from_secs(1)).await; // let the terminal draw