Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): accept affine visual coordinate spaces in jev-use (#4319)
Francesco Bonacci032e38de8697
diff --git a/libs/cua-driver/examples/jev-use/README.md b/libs/cua-driver/examples/jev-use/README.mdindex 68ed9c767d..c72a3184be 100644--- a/libs/cua-driver/examples/jev-use/README.md+++ b/libs/cua-driver/examples/jev-use/README.md@@ -445,6 +445,10 @@ removes the Submit button ref so only the visual candidate can submit. The optioDriver internals to this example. It validates capture identity, PNG geometry,coordinate mapping, region IDs, bounds, content, confidence, and ambiguitybefore constructing a `click` candidate containing the exact `capture_id`.+The coordinate mapping is `screenshot_pixels` for an identity capture or a+finite, invertible `affine` transform otherwise (for example, a Retina window).+The click still carries the original screenshot-pixel point; Driver applies the+capture's mapping itself, so the example never maps a point twice.Visual evidence never replaces the semantic editable ref required by`browser_type`.diff --git a/libs/cua-driver/examples/jev-use/fixtures/parse-visual-regions-submit-v1.json b/libs/cua-driver/examples/jev-use/fixtures/parse-visual-regions-submit-v1.jsonindex b3df1674ac..771f3f5835 100644--- a/libs/cua-driver/examples/jev-use/fixtures/parse-visual-regions-submit-v1.json+++ b/libs/cua-driver/examples/jev-use/fixtures/parse-visual-regions-submit-v1.json@@ -14,11 +14,13 @@"mime_type": "image/png"},"action_coordinate_space": {- "kind": "scaled_top_left",- "action_origin_x": 100.0,- "action_origin_y": 200.0,- "action_units_per_pixel_x": 0.5,- "action_units_per_pixel_y": 2.0+ "kind": "affine",+ "m11": 0.5,+ "m12": 0.0,+ "m21": 0.0,+ "m22": 0.5,+ "tx": 100.0,+ "ty": 200.0},"captured_at": "2026-09-17T12:00:00Z"},diff --git a/libs/cua-driver/examples/jev-use/python/action_policy.py b/libs/cua-driver/examples/jev-use/python/action_policy.pyindex cf17dcc918..75c11a3bee 100644--- a/libs/cua-driver/examples/jev-use/python/action_policy.py+++ b/libs/cua-driver/examples/jev-use/python/action_policy.py@@ -16,6 +16,55 @@ class ActionAuthorizationError(ValueError):"""The selected candidate is not authorized by the current observation."""+class CoordinateMappingError(ValueError):+ """A capture's ``action_coordinate_space`` cannot be represented exactly."""+++AffineCoefficients = tuple[float, float, float, float, float, float]+IDENTITY_MAPPING: AffineCoefficients = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0)+++def action_coordinate_mapping(coordinate_space: Any) -> AffineCoefficients:+ """Return Driver's screenshot-to-action affine as ``(m11, m12, m21, m22, tx, ty)``.++ ``parse_visual_regions`` reports ``screenshot_pixels`` for an identity+ transform and ``affine`` otherwise (for example, a Retina window capture).+ The mapping must be finite and invertible, matching Driver's own contract+ validation. Callers still send the original screenshot point with the+ ``capture_id``; Driver applies this mapping itself, so it is validated here+ but never applied to the dispatched coordinates.+ """+ if not isinstance(coordinate_space, Mapping):+ raise CoordinateMappingError("unsupported action coordinate space")+ kind = coordinate_space.get("kind")+ if kind == "screenshot_pixels":+ return IDENTITY_MAPPING+ if kind != "affine":+ raise CoordinateMappingError("unsupported action coordinate space")+ values = [coordinate_space.get(key) for key in ("m11", "m12", "m21", "m22", "tx", "ty")]+ if any(+ not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value)+ for value in values+ ):+ raise CoordinateMappingError("malformed action coordinate mapping")+ m11, m12, m21, m22, tx, ty = (float(value) for value in values)+ if abs(m11 * m22 - m12 * m21) <= sys.float_info.epsilon:+ raise CoordinateMappingError("malformed action coordinate mapping")+ return (m11, m12, m21, m22, tx, ty)+++def map_screenshot_point(+ mapping: AffineCoefficients, x: float, y: float+) -> tuple[float, float]:+ """Map one screenshot point, refusing a mapping that overflows it."""+ m11, m12, m21, m22, tx, ty = mapping+ action_x = m11 * x + m12 * y + tx+ action_y = m21 * x + m22 * y + ty+ if not math.isfinite(action_x) or not math.isfinite(action_y):+ raise CoordinateMappingError("action coordinate mapping produced a non-finite point")+ return (action_x, action_y)++@dataclass(frozen=True)class AuthorizedVisualClick:capture_id: str@@ -121,25 +170,10 @@ def authorize_exact_region_text_action()):raise ActionAuthorizationError("invalid screenshot provenance")- coordinate_space = capture.get("action_coordinate_space")- if not isinstance(coordinate_space, Mapping) or coordinate_space.get("kind") not in {- "screenshot_pixels",- "affine",- }:- raise ActionAuthorizationError("unsupported action coordinate space")- values = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]- if coordinate_space["kind"] == "affine":- values = [coordinate_space.get(key) for key in ("m11", "m12", "m21", "m22", "tx", "ty")]- if (- any(- not isinstance(value, (int, float))- or isinstance(value, bool)- or not math.isfinite(value)- for value in values- )- or abs(values[0] * values[3] - values[1] * values[2]) <= sys.float_info.epsilon- ):- raise ActionAuthorizationError("malformed action coordinate mapping")+ try:+ values = action_coordinate_mapping(capture.get("action_coordinate_space"))+ except CoordinateMappingError as error:+ raise ActionAuthorizationError(str(error)) from errorif (decision.get("schema") != "cua.decision_choice_v1"or decision.get("kind") != "selected"@@ -243,9 +277,8 @@ def authorize_exact_region_text_action(raise ActionAuthorizationError("malformed visual bounds")px = bounds["x"] + bounds["width"] / 2py = bounds["y"] + bounds["height"] / 2- m11, m12, m21, m22, tx, ty = values- x = m11 * px + m12 * py + tx- y = m21 * px + m22 * py + ty- if not math.isfinite(x) or not math.isfinite(y):- raise ActionAuthorizationError("action coordinate mapping produced a non-finite point")+ try:+ map_screenshot_point(values, px, py)+ except CoordinateMappingError as error:+ raise ActionAuthorizationError(str(error)) from errorreturn AuthorizedVisualClick(current_capture_id, px, py)diff --git a/libs/cua-driver/examples/jev-use/python/core.py b/libs/cua-driver/examples/jev-use/python/core.pyindex b3c9781191..c878fa8e99 100644--- a/libs/cua-driver/examples/jev-use/python/core.py+++ b/libs/cua-driver/examples/jev-use/python/core.py@@ -11,6 +11,12 @@ import mathfrom dataclasses import dataclassfrom typing import Any, Literal, Mapping+from action_policy import (+ AffineCoefficients,+ CoordinateMappingError,+ action_coordinate_mapping,+ map_screenshot_point,+)from sources import Candidatefrom tasks import (FIELD_NAME,@@ -76,10 +82,10 @@ class VisualObservation:screenshot_height: intpid: intwindow_id: int- action_origin_x: float- action_origin_y: float- action_units_per_pixel_x: float- action_units_per_pixel_y: float+ # Driver's screenshot-to-action affine ``(m11, m12, m21, m22, tx, ty)``.+ # It is validated but never applied here: a capture-bound click sends the+ # original screenshot point and ``capture_id`` and Driver maps it once.+ screenshot_to_action: AffineCoefficientsregions: tuple[VisualRegion, ...]def screenshot_center(self, region: VisualRegion) -> tuple[float, float]:@@ -141,27 +147,17 @@ def parse_visual_regions(screenshot_width = _positive_int(screenshot.get("width"))screenshot_height = _positive_int(screenshot.get("height"))- coordinate_space = capture.get("action_coordinate_space")- if not isinstance(coordinate_space, dict):- raise VisualObservationError("visual result has no action coordinate space")- if coordinate_space.get("kind") == "screenshot_pixels":- origin_x, origin_y, scale_x, scale_y = 0.0, 0.0, 1.0, 1.0- elif coordinate_space.get("kind") == "scaled_top_left":- values = (- coordinate_space.get("action_origin_x"),- coordinate_space.get("action_origin_y"),- coordinate_space.get("action_units_per_pixel_x"),- coordinate_space.get("action_units_per_pixel_y"),- )- if any(not isinstance(value, (int, float)) or isinstance(value, bool) for value in values):- raise VisualObservationError("visual result has malformed coordinate mapping")- origin_x, origin_y, scale_x, scale_y = (float(value) for value in values)- if not all(math.isfinite(value) for value in (origin_x, origin_y, scale_x, scale_y)):- raise VisualObservationError("visual result has non-finite coordinate mapping")- if scale_x <= 0 or scale_y <= 0:- raise VisualObservationError("visual result has non-positive coordinate scale")- else:- raise VisualObservationError("visual result has unsupported coordinate mapping")+ try:+ screenshot_to_action = action_coordinate_mapping(capture.get("action_coordinate_space"))+ for corner_x, corner_y in (+ (0, 0),+ (screenshot_width, 0),+ (0, screenshot_height),+ (screenshot_width, screenshot_height),+ ):+ map_screenshot_point(screenshot_to_action, corner_x, corner_y)+ except CoordinateMappingError as error:+ raise VisualObservationError(f"visual result has {error}") from errorraw_regions = payload.get("regions")if not isinstance(raw_regions, list):@@ -227,10 +223,7 @@ def parse_visual_regions(screenshot_height=screenshot_height,pid=expected_pid,window_id=expected_window_id,- action_origin_x=origin_x,- action_origin_y=origin_y,- action_units_per_pixel_x=scale_x,- action_units_per_pixel_y=scale_y,+ screenshot_to_action=screenshot_to_action,regions=tuple(regions),)diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_core.py b/libs/cua-driver/examples/jev-use/python/tests/test_core.pyindex 6581402189..f2b74c31db 100644--- a/libs/cua-driver/examples/jev-use/python/tests/test_core.py+++ b/libs/cua-driver/examples/jev-use/python/tests/test_core.py@@ -269,6 +269,94 @@ class CoreTest(unittest.TestCase):with self.assertRaisesRegex(ValueError, "duplicate"):validate_choice("duplicate", [duplicate, duplicate])+ def test_retina_affine_mapping_keeps_the_original_screenshot_point(self) -> None:+ # Issue #4289: Driver reports a non-identity capture as ``affine``+ # (never ``scaled_top_left``). A Retina-like 2x capture of a window at+ # (100, 200) maps pixels to points with scale 0.5 plus that offset.+ payload = json.loads((FIXTURES / "parse-visual-regions-submit-v1.json").read_text())+ self.assertEqual(payload["capture"]["action_coordinate_space"]["kind"], "affine")+ visual = parse_visual_regions(+ payload,+ expected_capture_id="capture-submit",+ expected_pid=7,+ expected_window_id=9,+ )+ self.assertEqual(visual.screenshot_to_action, (0.5, 0.0, 0.0, 0.5, 100.0, 200.0))+ page = self.snapshot("expected")+ page["refs"] = page["refs"][:1]+ selected = validate_choice(+ "submit-form",+ build_candidates(page, "expected", visual, capture_bound_click=True),+ current_capture_id="capture-submit",+ )+ # Driver applies the mapping for the capture ID, so the click carries+ # the region center in screenshot pixels, not mapped (275, 330).+ self.assertEqual(+ (selected.arguments["x"], selected.arguments["y"], selected.arguments["capture_id"]),+ (350.0, 260.0, "capture-submit"),+ )++ # The reporter's observed macOS Retina mapping from #4289.+ payload["capture"]["action_coordinate_space"] = {+ "kind": "affine",+ "m11": 1.530612244897959,+ "m12": 0.0,+ "m21": 0.0,+ "m22": 1.5304487179487178,+ "tx": 0.0,+ "ty": 0.0,+ }+ observed = parse_visual_regions(+ payload,+ expected_capture_id="capture-submit",+ expected_pid=7,+ expected_window_id=9,+ )+ self.assertEqual(observed.screenshot_to_action[0], 1.530612244897959)++ def test_identity_screenshot_pixels_mapping(self) -> None:+ payload = json.loads((FIXTURES / "parse-visual-regions-ambiguous-v1.json").read_text())+ visual = parse_visual_regions(+ payload,+ expected_capture_id="capture-ambiguous",+ expected_pid=7,+ expected_window_id=9,+ )+ self.assertEqual(visual.screenshot_to_action, (1.0, 0.0, 0.0, 1.0, 0.0, 0.0))++ def test_unrepresentable_coordinate_mappings_fail_closed(self) -> None:+ base = json.loads((FIXTURES / "parse-visual-regions-submit-v1.json").read_text())+ affine = base["capture"]["action_coordinate_space"]+ cases = {+ "non-invertible": {**affine, "m11": 0.0, "m22": 0.0},+ "singular": {**affine, "m11": 1.0, "m12": 2.0, "m21": 2.0, "m22": 4.0},+ "nan": {**affine, "m11": float("nan")},+ "infinite": {**affine, "tx": float("inf")},+ "missing coefficient": {key: value for key, value in affine.items() if key != "ty"},+ "boolean coefficient": {**affine, "m12": False},+ "overflowing": {**affine, "m11": 1e308, "m22": 1e308},+ "scaled_top_left": {+ "kind": "scaled_top_left",+ "action_origin_x": 100.0,+ "action_origin_y": 200.0,+ "action_units_per_pixel_x": 0.5,+ "action_units_per_pixel_y": 0.5,+ },+ "unknown kind": {"kind": "projective"},+ "absent": None,+ }+ for name, space in cases.items():+ with self.subTest(name):+ payload = json.loads(json.dumps(base))+ payload["capture"]["action_coordinate_space"] = space+ with self.assertRaisesRegex(ValueError, "coordinate"):+ parse_visual_regions(+ payload,+ expected_capture_id="capture-submit",+ expected_pid=7,+ expected_window_id=9,+ )+def test_capture_bound_choice_rejects_a_newer_capture(self) -> None:payload = json.loads((FIXTURES / "parse-visual-regions-submit-v1.json").read_text())visual = parse_visual_regions(diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_decision_state.py b/libs/cua-driver/examples/jev-use/python/tests/test_decision_state.pyindex 530ceae0af..faa239e41a 100644--- a/libs/cua-driver/examples/jev-use/python/tests/test_decision_state.py+++ b/libs/cua-driver/examples/jev-use/python/tests/test_decision_state.py@@ -131,7 +131,9 @@ class SubmitStepObservationTest(unittest.TestCase):def test_visual_text_is_redacted_too(self) -> None:region = VisualRegion("r1", "text", TOKEN, None, 0.9, False, 1, 1, 10, 10)- visual = VisualObservation("cap", "ref", 100, 100, 7, 9, 0.0, 0.0, 1.0, 1.0, (region,))+ visual = VisualObservation(+ "cap", "ref", 100, 100, 7, 9, (1.0, 0.0, 0.0, 1.0, 0.0, 0.0), (region,)+ )state = decision_state(AFTER, visual, [], TOKEN)self.assertEqual(state["observation"]["visual"]["regions"][0]["text"], REDACTED_TOKEN)self.assertNotIn(TOKEN, json.dumps(state))diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_native_canvas.py b/libs/cua-driver/examples/jev-use/python/tests/test_native_canvas.pyindex aec811e9b4..57da5c490a 100644--- a/libs/cua-driver/examples/jev-use/python/tests/test_native_canvas.py+++ b/libs/cua-driver/examples/jev-use/python/tests/test_native_canvas.py@@ -62,7 +62,7 @@ def canvas_sources(platform: str, *, visual: bool = True, foreground_ids=frozensregions += (VisualRegion("r9", "text", "CHOOSE A SIGNAL", None, 0.95, False, 40, 30, 300, 30),)parsed = VisualObservation(observed.capture_id or "", "ref", 762, 492, observed.pid, observed.window_id,- 0.0, 0.0, 1.0, 1.0, regions,+ (1.0, 0.0, 0.0, 1.0, 0.0, 0.0), regions,)return TaskSources(ax=ax,diff --git a/libs/cua-driver/examples/jev-use/typescript/core.test.ts b/libs/cua-driver/examples/jev-use/typescript/core.test.tsindex dbc205e4a9..efdb77b309 100644--- a/libs/cua-driver/examples/jev-use/typescript/core.test.ts+++ b/libs/cua-driver/examples/jev-use/typescript/core.test.ts@@ -108,6 +108,75 @@ test('visual fixture builds a candidate without claiming interactivity', () => {});});+test('Retina affine mapping keeps the original screenshot point (#4289)', () => {+ // Driver reports a non-identity capture as `affine` (never `scaled_top_left`).+ // A Retina-like 2x capture of a window at (100, 200) maps pixels to points+ // with scale 0.5 plus that offset.+ const payload = fixture('parse-visual-regions-submit-v1.json');+ assert.equal(payload.capture.action_coordinate_space.kind, 'affine');+ const visual = parseVisualRegions(payload, 'capture-submit', 7, 9);+ assert.deepEqual(visual.screenshotToAction, [0.5, 0, 0, 0.5, 100, 200]);+ const page = snapshot('expected');+ page.refs = page.refs.slice(0, 1);+ const selected = validateChoice(+ 'submit-form',+ buildCandidates(page, 'expected', visual, true),+ 'capture-submit'+ );+ // Driver applies the mapping for the capture ID, so the click carries the+ // region center in screenshot pixels, not the mapped (275, 330).+ assert.deepEqual(+ [selected.arguments.x, selected.arguments.y, selected.arguments.capture_id],+ [350, 260, 'capture-submit']+ );++ // The reporter's observed macOS Retina mapping from #4289.+ payload.capture.action_coordinate_space = {+ kind: 'affine',+ m11: 1.530612244897959,+ m12: 0,+ m21: 0,+ m22: 1.5304487179487178,+ tx: 0,+ ty: 0,+ };+ assert.equal(parseVisualRegions(payload, 'capture-submit', 7, 9).screenshotToAction[0], 1.530612244897959);+});++test('identity screenshot_pixels mapping', () => {+ const visual = parseVisualRegions(fixture('parse-visual-regions-ambiguous-v1.json'), 'capture-ambiguous', 7, 9);+ assert.deepEqual(visual.screenshotToAction, [1, 0, 0, 1, 0, 0]);+});++test('unrepresentable coordinate mappings fail closed', () => {+ const affine = fixture('parse-visual-regions-submit-v1.json').capture.action_coordinate_space;+ const { ty: _ty, ...missing } = affine;+ const cases: Record<string, unknown> = {+ 'non-invertible': { ...affine, m11: 0, m22: 0 },+ singular: { ...affine, m11: 1, m12: 2, m21: 2, m22: 4 },+ nan: { ...affine, m11: Number.NaN },+ infinite: { ...affine, tx: Number.POSITIVE_INFINITY },+ 'missing coefficient': missing,+ 'boolean coefficient': { ...affine, m12: false },+ 'string coefficient': { ...affine, m12: '0' },+ overflowing: { ...affine, m11: 1e308, m22: 1e308 },+ scaled_top_left: {+ kind: 'scaled_top_left',+ action_origin_x: 100,+ action_origin_y: 200,+ action_units_per_pixel_x: 0.5,+ action_units_per_pixel_y: 0.5,+ },+ 'unknown kind': { kind: 'projective' },+ absent: undefined,+ };+ for (const [name, space] of Object.entries(cases)) {+ const payload = fixture('parse-visual-regions-submit-v1.json');+ payload.capture.action_coordinate_space = space;+ assert.throws(() => parseVisualRegions(payload, 'capture-submit', 7, 9), /coordinate/, name);+ }+});+test('submit path depends on the DOM button ref', () => {const visual = parseVisualRegions(fixture('parse-visual-regions-submit-v1.json'),diff --git a/libs/cua-driver/examples/jev-use/typescript/core.ts b/libs/cua-driver/examples/jev-use/typescript/core.tsindex 108cad6f30..706965e784 100644--- a/libs/cua-driver/examples/jev-use/typescript/core.ts+++ b/libs/cua-driver/examples/jev-use/typescript/core.ts@@ -63,13 +63,60 @@ export type VisualObservation = Readonly<{screenshotHeight: number;pid: number;windowId: number;- actionOriginX: number;- actionOriginY: number;- actionUnitsPerPixelX: number;- actionUnitsPerPixelY: number;+ /**+ * Driver's screenshot-to-action affine `[m11, m12, m21, m22, tx, ty]`. It is+ * validated but never applied here: a capture-bound click sends the original+ * screenshot point and `capture_id`, and Driver maps it once.+ */+ screenshotToAction: AffineCoefficients;regions: readonly VisualRegion[];}>;+export type AffineCoefficients = readonly [number, number, number, number, number, number];++export const IDENTITY_MAPPING: AffineCoefficients = Object.freeze([1, 0, 0, 1, 0, 0] as const);++/**+ * Return Driver's screenshot-to-action affine from `parse_visual_regions`.+ *+ * Driver reports `screenshot_pixels` for an identity transform and `affine`+ * otherwise (for example, a Retina window capture). The mapping must be finite+ * and invertible, matching Driver's own contract validation. Mirrors+ * `action_coordinate_mapping` in python/action_policy.py.+ */+export function actionCoordinateMapping(space: unknown): AffineCoefficients {+ if (!space || typeof space !== 'object' || Array.isArray(space)) {+ throw new Error('unsupported action coordinate space');+ }+ const value = space as Record<string, unknown>;+ if (value.kind === 'screenshot_pixels') return IDENTITY_MAPPING;+ if (value.kind !== 'affine') throw new Error('unsupported action coordinate space');+ const coefficients = (['m11', 'm12', 'm21', 'm22', 'tx', 'ty'] as const).map((key) => value[key]);+ if (coefficients.some((item) => typeof item !== 'number' || !Number.isFinite(item))) {+ throw new Error('malformed action coordinate mapping');+ }+ const [m11, m12, m21, m22, tx, ty] = coefficients as number[];+ if (Math.abs(m11 * m22 - m12 * m21) <= Number.EPSILON) {+ throw new Error('malformed action coordinate mapping');+ }+ return Object.freeze([m11, m12, m21, m22, tx, ty] as const);+}++/** Map one screenshot point, refusing a mapping that overflows it. */+export function mapScreenshotPoint(+ mapping: AffineCoefficients,+ x: number,+ y: number+): readonly [number, number] {+ const [m11, m12, m21, m22, tx, ty] = mapping;+ const actionX = m11 * x + m12 * y + tx;+ const actionY = m21 * x + m22 * y + ty;+ if (!Number.isFinite(actionX) || !Number.isFinite(actionY)) {+ throw new Error('action coordinate mapping produced a non-finite point');+ }+ return [actionX, actionY];+}+export class VisualObservationError extends Error {constructor(message: string,@@ -140,27 +187,19 @@ export function parseVisualRegions(const screenshotWidth = positiveInt(screenshot.width);const screenshotHeight = positiveInt(screenshot.height);- const space = record(capture.action_coordinate_space, 'visual result has no coordinate mapping');- let actionOriginX = 0;- let actionOriginY = 0;- let actionUnitsPerPixelX = 1;- let actionUnitsPerPixelY = 1;- if (space.kind === 'scaled_top_left') {- const values = [- space.action_origin_x,- space.action_origin_y,- space.action_units_per_pixel_x,- space.action_units_per_pixel_y,- ];- if (values.some((value) => typeof value !== 'number' || !Number.isFinite(value))) {- throw new Error('visual result has malformed coordinate mapping');- }- [actionOriginX, actionOriginY, actionUnitsPerPixelX, actionUnitsPerPixelY] = values as number[];- if (actionUnitsPerPixelX <= 0 || actionUnitsPerPixelY <= 0) {- throw new Error('visual result has non-positive coordinate scale');+ let screenshotToAction: AffineCoefficients;+ try {+ screenshotToAction = actionCoordinateMapping(capture.action_coordinate_space);+ for (const [x, y] of [+ [0, 0],+ [screenshotWidth, 0],+ [0, screenshotHeight],+ [screenshotWidth, screenshotHeight],+ ]) {+ mapScreenshotPoint(screenshotToAction, x, y);}- } else if (space.kind !== 'screenshot_pixels') {- throw new Error('visual result has unsupported coordinate mapping');+ } catch (error) {+ throw new Error(`visual result has ${(error as Error).message}`);}if (!Array.isArray(root.regions)) throw new Error('visual result has no region list');@@ -213,10 +252,7 @@ export function parseVisualRegions(screenshotHeight,pid: expectedPid,windowId: expectedWindowId,- actionOriginX,- actionOriginY,- actionUnitsPerPixelX,- actionUnitsPerPixelY,+ screenshotToAction,regions: Object.freeze(regions),});}diff --git a/libs/cua-driver/examples/jev-use/typescript/decision_state.test.ts b/libs/cua-driver/examples/jev-use/typescript/decision_state.test.tsindex 4e805b7ca3..4e3bf4545b 100644--- a/libs/cua-driver/examples/jev-use/typescript/decision_state.test.ts+++ b/libs/cua-driver/examples/jev-use/typescript/decision_state.test.ts@@ -126,10 +126,7 @@ test('visual text is redacted too', () => {screenshotHeight: 100,pid: 7,windowId: 9,- actionOriginX: 0,- actionOriginY: 0,- actionUnitsPerPixelX: 1,- actionUnitsPerPixelY: 1,+ screenshotToAction: [1, 0, 0, 1, 0, 0],regions: [{id: 'r1',diff --git a/libs/cua-driver/examples/jev-use/typescript/native_canvas.test.ts b/libs/cua-driver/examples/jev-use/typescript/native_canvas.test.tsindex d772ca13a6..513115157f 100644--- a/libs/cua-driver/examples/jev-use/typescript/native_canvas.test.ts+++ b/libs/cua-driver/examples/jev-use/typescript/native_canvas.test.ts@@ -47,8 +47,7 @@ function canvasSources(];const parsed: VisualObservation = {captureId: observed.captureId ?? '', screenshotReference: 'ref', screenshotWidth: 762, screenshotHeight: 492,- pid: observed.pid, windowId: observed.windowId, actionOriginX: 0, actionOriginY: 0,- actionUnitsPerPixelX: 1, actionUnitsPerPixelY: 1, regions,+ pid: observed.pid, windowId: observed.windowId, screenshotToAction: [1, 0, 0, 1, 0, 0], regions,};return { ax, visual: new VisualRegionSource(parsed, 'background', true), visualPath: true, foregroundIds };}