Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): target the Cancel card in the jev-use canvas task (#4301)

Francesco Bonacci3937539c0435
diff --git a/libs/cua-driver/examples/jev-use/README.md b/libs/cua-driver/examples/jev-use/README.md
index 7ddc3a4b56..68ed9c767d 100644
--- a/libs/cua-driver/examples/jev-use/README.md
+++ b/libs/cua-driver/examples/jev-use/README.md
@@ -116,13 +116,14 @@ mode, each harness shows the same labeled controls, so candidate IDs and the
mock provider's choices are identical on every platform. Task mode is selected
with `CUA_APPKIT_TASK_STATE`, `CUA_WPF_TASK_STATE`, or `CUA_GTK3_TASK_STATE`.
WPF and GTK3 show a small dedicated task window.
-A fourth task, `canvas-save`, clicks the Save card on the cross-platform
+A fourth task, `canvas-cancel`, clicks the Cancel card on the cross-platform
visual-only canvas fixture (a custom-painted Tk window with no accessibility
tree), so it can only succeed through the visual fallback and needs the
cua-perception extension. It observes only the window's top level
-(`max_depth: 1`), so macOS does not spend the walk budget in the menu bar, and
-accepts the Save label at OCR confidence 0.7, because a 1x macOS capture reads
-it at about 0.75. Run it with `verify_native.py --harness canvas`; the
+(`max_depth: 1`), so macOS does not spend the walk budget in the menu bar. It
+targets Cancel because OmniParser reads the Save label inconsistently on a 1x
+macOS capture (`Save`, then `Saye`), and the exact text match then correctly
+offers no candidate. Send is a risk phrase. Run it with `verify_native.py --harness canvas`; the
verifier serves the loopback journal the fixture publishes its state to and
writes that state to the task's state file. Each harness rewrites that
state file on every change, and the file is the independent oracle. On
diff --git a/libs/cua-driver/examples/jev-use/python/native_tasks.py b/libs/cua-driver/examples/jev-use/python/native_tasks.py
index a5a0cbd8df..2a45dabaab 100644
--- a/libs/cua-driver/examples/jev-use/python/native_tasks.py
+++ b/libs/cua-driver/examples/jev-use/python/native_tasks.py
@@ -13,7 +13,7 @@ AppKit (``CUA_APPKIT_TASK_STATE``), WPF (``CUA_WPF_TASK_STATE``), and GTK3
Their oracle is the harness's own JSON state file, which the app rewrites on
every change; it never depends on Driver output.
-``canvas-save`` drives the cross-platform visual-only canvas fixture
+``canvas-cancel`` drives the cross-platform visual-only canvas fixture
(``tests/fixtures/apps/cross-platform/visual-only-canvas``), a custom-painted
Tk surface with no accessibility tree. Its only executable candidate is a
capture-bound visual click, so it proves the OmniParser fallback. The fixture
@@ -612,15 +612,17 @@ CANVAS = HarnessSpec(
"canvas", "any", "Cua Visual-Only Canvas Fixture", "cua.visual_canvas_task_state_v1",
"CUA_CANVAS_TASK_STATE",
)
-CANVAS_TASK_ID = "canvas-save"
-CANVAS_TARGET = "Save"
+CANVAS_TASK_ID = "canvas-cancel"
+# Cancel, not Save: OmniParser reads the 32 px "Save" label inconsistently on a
+# 1x macOS capture ("Save", then "Saye"), and the exact text match (correctly)
+# withholds the candidate. "Send" is a risk phrase. "Cancel" reads reliably.
+CANVAS_TARGET = "Cancel"
# A painted surface has no deeper tree. Walking only the window's top level
# keeps macOS from spending the walk budget in the application menu bar (which
# would truncate the tree and forbid the fallback). Any content container still
# appears at depth 1 and blocks the fallback.
CANVAS_MAX_DEPTH = 1
-# The macOS VM's 1x capture reads the canvas's "Save" label at about 0.75.
-CANVAS_MIN_CONFIDENCE = 0.7
+CANVAS_MIN_CONFIDENCE = 0.8
APPKIT_TASK_IDS = harness_task_ids("appkit")
NATIVE_TASK_IDS = (
@@ -632,7 +634,7 @@ NATIVE_TASK_IDS = (
def split_task_id(task_id: str) -> tuple[HarnessSpec, str]:
"""Return the harness and task kind of a built-in task ID."""
if task_id == CANVAS_TASK_ID:
- return CANVAS, "save"
+ return CANVAS, "cancel"
harness, _, kind = task_id.partition("-")
if harness not in HARNESSES or kind not in TASK_KINDS:
raise ValueError(f"unknown native task: {task_id}")
@@ -653,11 +655,11 @@ def native_task(
scope = WindowScope(
harness.window_title, bundle_id=harness.bundle_id, process_name=harness.process_name
)
- if kind == "save":
+ if kind == "cancel":
return NativeTask(
id=task_id,
goal=(
- f"The window is a painted canvas with {CANVAS_TARGET}, Send, and Cancel cards. "
+ "The window is a painted canvas with Save, Send, and Cancel cards. "
f"Click the {CANVAS_TARGET} card once, then stop."
),
scope=replace(scope, max_depth=CANVAS_MAX_DEPTH),
diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_native_canvas.py b/libs/cua-driver/examples/jev-use/python/tests/test_native_canvas.py
index da7d4cfcb2..aec811e9b4 100644
--- a/libs/cua-driver/examples/jev-use/python/tests/test_native_canvas.py
+++ b/libs/cua-driver/examples/jev-use/python/tests/test_native_canvas.py
@@ -148,7 +148,7 @@ class CanvasTaskTest(unittest.TestCase):
def test_scope_and_confidence(self) -> None:
task = native_task(CANVAS_TASK_ID, Path("/tmp/none.json"))
self.assertEqual(task.scope.window_state_arguments(), {"max_depth": 1})
- self.assertEqual(task.visual_min_confidence, 0.7)
+ self.assertEqual(task.visual_min_confidence, 0.8)
form = native_task("appkit-counter", Path("/tmp/none.json"))
self.assertEqual(form.scope.window_state_arguments(), {})
self.assertEqual(form.visual_min_confidence, 0.8)
@@ -160,41 +160,41 @@ class CanvasTaskTest(unittest.TestCase):
def test_registry(self) -> None:
self.assertIn(CANVAS_TASK_ID, NATIVE_TASK_IDS)
- self.assertEqual(split_task_id(CANVAS_TASK_ID), (CANVAS, "save"))
+ self.assertEqual(split_task_id(CANVAS_TASK_ID), (CANVAS, "cancel"))
task = native_task(CANVAS_TASK_ID, Path("/tmp/none.json"), pid=7)
self.assertEqual(task.scope.window_title, "Cua Visual-Only Canvas Fixture")
self.assertEqual(task.oracle.schema, "cua.visual_canvas_task_state_v1")
self.assertEqual(task.allowed_action_kinds, frozenset({"click"}))
- def test_only_the_save_region_is_executable(self) -> None:
+ def test_only_the_cancel_region_is_executable(self) -> None:
task = native_task(CANVAS_TASK_ID, Path("/tmp/none.json"))
for platform in CANVAS_FIXTURES:
with self.subTest(platform=platform):
sources = canvas_sources(platform)
step = task.plan(sources)
- self.assertEqual([c.id for c in step.candidates], ["visual:save", "reobserve", "abstain"])
+ self.assertEqual([c.id for c in step.candidates], ["visual:cancel", "reobserve", "abstain"])
save = step.candidates[0]
self.assertEqual(save.source, "visual")
self.assertEqual(save.tool, "click")
self.assertEqual(save.arguments["delivery_mode"], "background")
self.assertEqual(save.arguments["capture_id"], sources.ax.observation.capture_id)
- self.assertEqual((save.arguments["x"], save.arguments["y"]), (170.0, 265.0))
+ self.assertEqual((save.arguments["x"], save.arguments["y"]), (610.0, 265.0))
request = native_choice_request(task, sources, step, [])
validate_request(request)
self.assertEqual([c["source"] for c in request["candidates"][:1]], ["visual"])
self.assertEqual(request["elements"], [])
choice, _, _ = choose_mock_for_task(task, sources, step.candidates, [])
- self.assertEqual(choice, "visual:save")
+ self.assertEqual(choice, "visual:cancel")
def test_background_refusal_offers_an_explicit_foreground_variant(self) -> None:
- refused = frozenset({"visual:save"})
+ refused = frozenset({"visual:cancel"})
allowed = native_task(CANVAS_TASK_ID, Path("/tmp/none.json"), allow_foreground=True)
step = allowed.plan(canvas_sources("linux", foreground_ids=refused))
- self.assertEqual([c.id for c in step.candidates], ["visual:save:foreground", "reobserve", "abstain"])
+ self.assertEqual([c.id for c in step.candidates], ["visual:cancel:foreground", "reobserve", "abstain"])
self.assertEqual(step.candidates[0].arguments["delivery_mode"], "foreground")
self.assertEqual(
choose_mock_for_task(allowed, canvas_sources("linux", foreground_ids=refused), step.candidates, [])[0],
- "visual:save:foreground",
+ "visual:cancel:foreground",
)
denied = native_task(CANVAS_TASK_ID, Path("/tmp/none.json"))
self.assertEqual(
@@ -205,8 +205,9 @@ class CanvasTaskTest(unittest.TestCase):
def test_oracle(self) -> None:
task = native_task(CANVAS_TASK_ID, Path("/tmp/none.json"))
self.assertEqual(task.check({"selected": None, "action_count": 0}), "pending")
- self.assertEqual(task.check({"selected": "save", "action_count": 1}), "verified")
- self.assertEqual(task.check({"selected": "save", "action_count": 2}), "refuted")
+ self.assertEqual(task.check({"selected": "cancel", "action_count": 1}), "verified")
+ self.assertEqual(task.check({"selected": "cancel", "action_count": 2}), "refuted")
+ self.assertEqual(task.check({"selected": "save", "action_count": 1}), "refuted")
self.assertEqual(task.check({"selected": "send", "action_count": 1}), "refuted")
diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_native_platforms.py b/libs/cua-driver/examples/jev-use/python/tests/test_native_platforms.py
index 83e6a2f59b..444c6e2067 100644
--- a/libs/cua-driver/examples/jev-use/python/tests/test_native_platforms.py
+++ b/libs/cua-driver/examples/jev-use/python/tests/test_native_platforms.py
@@ -52,11 +52,11 @@ class HarnessRegistryTest(unittest.TestCase):
NATIVE_TASK_IDS,
(
*(f"{h}-{k}" for h in ("appkit", "wpf", "gtk3") for k in ("counter", "save-note", "choose-size")),
- "canvas-save",
+ "canvas-cancel",
),
)
for task_id in NATIVE_TASK_IDS:
- if task_id == "canvas-save":
+ if task_id == "canvas-cancel":
continue # the visual-only canvas is not a form harness (see test_native_canvas.py)
harness, kind = split_task_id(task_id)
task = native_task(task_id, Path("/tmp/none.json"), pid=7)
diff --git a/libs/cua-driver/examples/jev-use/typescript/native_canvas.test.ts b/libs/cua-driver/examples/jev-use/typescript/native_canvas.test.ts
index 11ea5597bf..d772ca13a6 100644
--- a/libs/cua-driver/examples/jev-use/typescript/native_canvas.test.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/native_canvas.test.ts
@@ -96,13 +96,13 @@ test('canvas trees fall back to visual regions; partial, truncated, and form cas
test('the canvas task is registered and bound to its state file', () => {
assert.ok(NATIVE_TASK_IDS.includes(CANVAS_TASK_ID));
- assert.deepEqual(splitTaskId(CANVAS_TASK_ID), [CANVAS, 'save']);
+ assert.deepEqual(splitTaskId(CANVAS_TASK_ID), [CANVAS, 'cancel']);
const task = nativeTask(CANVAS_TASK_ID, '/tmp/none.json', { pid: 7 });
assert.equal(task.scope.windowTitle, 'Cua Visual-Only Canvas Fixture');
assert.equal(task.oracle.schema, 'cua.visual_canvas_task_state_v1');
assert.equal(task.oracle.expectedPid, 7);
assert.deepEqual(task.scope.maxDepth, 1);
- assert.equal(task.visualMinConfidence, 0.7);
+ assert.equal(task.visualMinConfidence, 0.8);
assert.equal(nativeTask('appkit-counter', '/tmp/none.json').visualMinConfidence, 0.8);
const sources = canvasSources('macos');
const low = { ...sources.visual!.observation, regions: sources.visual!.observation.regions.map((r) => ({ ...r, confidence: 0.75 })) };
@@ -110,30 +110,30 @@ test('the canvas task is registered and bound to its state file', () => {
assert.ok(new VisualRegionSource(low, 'background', true, 0.7).find('button', 'Save'));
});
-test('only the Save region is executable, through a capture-bound click', () => {
+test('only the Cancel region is executable, through a capture-bound click', () => {
const task = nativeTask(CANVAS_TASK_ID, '/tmp/none.json');
for (const platform of ['linux', 'windows', 'macos'] as const) {
const sources = canvasSources(platform);
const step = task.plan(sources);
- assert.deepEqual(step.candidates.map((c) => c.id), ['visual:save', 'reobserve', 'abstain'], platform);
+ assert.deepEqual(step.candidates.map((c) => c.id), ['visual:cancel', 'reobserve', 'abstain'], platform);
const save = step.candidates[0];
assert.equal(save.source, 'visual');
assert.equal(save.tool, 'click');
assert.equal(save.arguments.delivery_mode, 'background');
assert.equal(save.arguments.capture_id, sources.ax!.observation.captureId);
- assert.deepEqual([save.arguments.x, save.arguments.y], [170, 265]);
+ assert.deepEqual([save.arguments.x, save.arguments.y], [610, 265]);
const request = nativeChoiceRequest(task, sources, step, []);
validateRequest(request);
assert.deepEqual((request.elements as unknown[]) ?? [], []);
- assert.equal(chooseMockForTask(task, sources, step.candidates, []).choice, 'visual:save');
+ assert.equal(chooseMockForTask(task, sources, step.candidates, []).choice, 'visual:cancel');
}
});
test('a background refusal offers an explicit foreground variant only when allowed', () => {
- const refused = new Set(['visual:save']);
+ const refused = new Set(['visual:cancel']);
const allowed = nativeTask(CANVAS_TASK_ID, '/tmp/none.json', { allowForeground: true });
const step = allowed.plan(canvasSources('linux', { foregroundIds: refused }));
- assert.deepEqual(step.candidates.map((c) => c.id), ['visual:save:foreground', 'reobserve', 'abstain']);
+ assert.deepEqual(step.candidates.map((c) => c.id), ['visual:cancel:foreground', 'reobserve', 'abstain']);
assert.equal(step.candidates[0].arguments.delivery_mode, 'foreground');
const denied = nativeTask(CANVAS_TASK_ID, '/tmp/none.json');
assert.deepEqual(
@@ -145,7 +145,8 @@ test('a background refusal offers an explicit foreground variant only when allow
test('the canvas oracle', () => {
const task = nativeTask(CANVAS_TASK_ID, '/tmp/none.json');
assert.equal(task.check({ selected: null, action_count: 0 }), 'pending');
- assert.equal(task.check({ selected: 'save', action_count: 1 }), 'verified');
- assert.equal(task.check({ selected: 'save', action_count: 2 }), 'refuted');
+ assert.equal(task.check({ selected: 'cancel', action_count: 1 }), 'verified');
+ assert.equal(task.check({ selected: 'cancel', action_count: 2 }), 'refuted');
+ assert.equal(task.check({ selected: 'save', action_count: 1 }), 'refuted');
assert.equal(task.check({ selected: 'send', action_count: 1 }), 'refuted');
});
diff --git a/libs/cua-driver/examples/jev-use/typescript/native_platforms.test.ts b/libs/cua-driver/examples/jev-use/typescript/native_platforms.test.ts
index a0efafb16b..40414c38d7 100644
--- a/libs/cua-driver/examples/jev-use/typescript/native_platforms.test.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/native_platforms.test.ts
@@ -44,11 +44,11 @@ test('every harness has the same three tasks', () => {
NATIVE_TASK_IDS,
[
...['appkit', 'wpf', 'gtk3'].flatMap((h) => ['counter', 'save-note', 'choose-size'].map((k) => `${h}-${k}`)),
- 'canvas-save',
+ 'canvas-cancel',
]
);
for (const taskId of NATIVE_TASK_IDS) {
- if (taskId === 'canvas-save') continue; // not a form harness (see native_canvas.test.ts)
+ if (taskId === 'canvas-cancel') continue; // not a form harness (see native_canvas.test.ts)
const [harness, kind] = splitTaskId(taskId);
const task = nativeTask(taskId, '/tmp/none.json', { pid: 7 });
assert.equal(task.scope.windowTitle, harness.windowTitle);
diff --git a/libs/cua-driver/examples/jev-use/typescript/native_tasks.ts b/libs/cua-driver/examples/jev-use/typescript/native_tasks.ts
index c37be9c633..955da3923e 100644
--- a/libs/cua-driver/examples/jev-use/typescript/native_tasks.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/native_tasks.ts
@@ -518,20 +518,21 @@ export const CANVAS: TaskHarness = {
stateSchema: 'cua.visual_canvas_task_state_v1',
stateEnv: 'CUA_CANVAS_TASK_STATE',
};
-export const CANVAS_TASK_ID = 'canvas-save';
-export const CANVAS_TARGET = 'Save';
+export const CANVAS_TASK_ID = 'canvas-cancel';
+// Cancel, not Save: see python/native_tasks.py CANVAS_TARGET.
+export const CANVAS_TARGET = 'Cancel';
// A painted surface has no deeper tree; see python/native_tasks.py CANVAS_MAX_DEPTH.
export const CANVAS_MAX_DEPTH = 1;
-export const CANVAS_MIN_CONFIDENCE = 0.7;
+export const CANVAS_MIN_CONFIDENCE = 0.8;
export const harnessTaskIds = (harness: string): string[] => TASK_KINDS.map((kind) => `${harness}-${kind}`);
export const APPKIT_TASK_IDS = harnessTaskIds('appkit');
export const NATIVE_TASK_IDS = [...Object.keys(HARNESSES).flatMap(harnessTaskIds), CANVAS_TASK_ID];
-type TaskKind = (typeof TASK_KINDS)[number] | 'save';
+type TaskKind = (typeof TASK_KINDS)[number] | 'cancel';
export function splitTaskId(taskId: string): [TaskHarness, TaskKind] {
- if (taskId === CANVAS_TASK_ID) return [CANVAS, 'save'];
+ if (taskId === CANVAS_TASK_ID) return [CANVAS, 'cancel'];
const index = taskId.indexOf('-');
const harness = index > 0 ? HARNESSES[taskId.slice(0, index)] : undefined;
const kind = taskId.slice(index + 1) as (typeof TASK_KINDS)[number];
@@ -550,11 +551,11 @@ export function nativeTask(taskId: string, statePath: string, options: HarnessTa
...(harness.processName ? { processName: harness.processName } : {}),
};
const allowForeground = options.allowForeground ?? false;
- if (kind === 'save') {
+ if (kind === 'cancel') {
return new NativeTask({
id: taskId,
goal:
- `The window is a painted canvas with ${CANVAS_TARGET}, Send, and Cancel cards. ` +
+ 'The window is a painted canvas with Save, Send, and Cancel cards. ' +
`Click the ${CANVAS_TARGET} card once, then stop.`,
scope: { ...scope, maxDepth: CANVAS_MAX_DEPTH },
allowedActions: new Set<ActionKind>(['visual_click']),
diff --git a/libs/cua-driver/examples/jev-use/verify_native.py b/libs/cua-driver/examples/jev-use/verify_native.py
index a467a68d3b..87e8f26e46 100644
--- a/libs/cua-driver/examples/jev-use/verify_native.py
+++ b/libs/cua-driver/examples/jev-use/verify_native.py
@@ -73,21 +73,21 @@ HARNESSES = {
),
}
FORM_KINDS = ("choose-size", "counter", "save-note")
-HARNESS_KINDS = {name: (("save",) if name == "canvas" else FORM_KINDS) for name in HARNESSES}
+HARNESS_KINDS = {name: (("cancel",) if name == "canvas" else FORM_KINDS) for name in HARNESSES}
# The independent end-state check for each task kind, read from the app's own file.
EXPECTED = {
"counter": lambda state: state.get("counter") == 3,
"save-note": lambda state: state.get("note_saved") == NOTE_TEXT,
"choose-size": lambda state: state.get("size") == "large" and state.get("agreed") is True,
- "save": lambda state: state.get("selected") == "save" and state.get("action_count") == 1,
+ "cancel": lambda state: state.get("selected") == "cancel" and state.get("action_count") == 1,
}
# The executable candidates the deterministic mock provider must have acted on.
MOCK_ACTIONS = {
"counter": ["ax:button:increment"] * 3,
"save-note": ["ax:text_input:note:set:note", "ax:button:save-note"],
"choose-size": ["ax:radio:large", "ax:checkbox:i-agree"],
- "save": ["visual:save"],
+ "cancel": ["visual:cancel"],
}
diff --git a/rfcs/4268-jev-use-native-candidates.md b/rfcs/4268-jev-use-native-candidates.md
index 7242a88682..fc3cd929d5 100644
--- a/rfcs/4268-jev-use-native-candidates.md
+++ b/rfcs/4268-jev-use-native-candidates.md
@@ -813,6 +813,11 @@ Implementation:
canvas task scopes its walk to `max_depth: 1`, because the menu bar
otherwise exhausts the walk budget and truncates the tree. A partial tree
with any application element still never falls back.
+ - The canvas task targets the Cancel card. On a 1x macOS capture, OmniParser
+ read the Save label as `Save` once and then as `Saye`, so the exact text
+ match offered no candidate and live Jev abstained (1/10 official runs).
+ Fuzzy matching was rejected because it would weaken the bounded-choice
+ property.
- A radio option that is already selected is not offered, and a `set_text`
candidate is offered only while the field differs from its parameter.
- `close_unsaved` matches its phrases whether or not the window reports