Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
refactor(cua-driver): introduce jev-use candidate sources and task spec (#4270)
Francesco Bonacci8bed9fd1b334
diff --git a/libs/cua-driver/examples/jev-use/README.md b/libs/cua-driver/examples/jev-use/README.mdindex 4c351c8537..f85f5cfb76 100644--- a/libs/cua-driver/examples/jev-use/README.md+++ b/libs/cua-driver/examples/jev-use/README.md@@ -56,6 +56,25 @@ through an independent `/state` endpoint. Each runner:8. checks the fixture's `/state` endpoint instead of treating the actionresponse or a screenshot as proof of success.+### Candidate sources and task specs++The loop separates where candidates come from from what the task needs. A+candidate source (`python/sources.py`, `typescript/sources.ts`) finds controls+in one observation and builds fully specified candidates for them:+`BrowserSemanticSource` reads the `get_browser_state` snapshot and acts through+page refs, and `VisualRegionSource` reads validated `parse_visual_regions`+regions and acts through a capture-bound `click`. A task spec+(`python/tasks.py`, `typescript/tasks.ts`) holds the goal, the parameters with+the secret token marked for redaction, the step budget, the allowed Driver+action kinds, the `form` state summary, the candidate IDs and descriptions, and+the success oracle. The fixture above is the one built-in task,+`FixtureFormTask`, and its oracle is the fixture's `/state` endpoint. A native+accessibility source for desktop applications is planned in+[RFC #4268](https://github.com/trycua/cua/issues/4268) and is not implemented.+The model request and response are unchanged: the golden files+`fixtures/jev-provider-request-golden-*-v1.json` pin the exact provider+requests and candidate tables for the page-structure and visual fixtures.+The MCP connection stays open across the entire loop. This preserves theexplicit named Cua Driver session and avoids rebuilding tool state for everystep.diff --git a/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-python-v1.json b/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-python-v1.jsonnew file mode 100644index 0000000000..55a92a0199--- /dev/null+++ b/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-python-v1.json@@ -0,0 +1,6338 @@+{+ "schema": "cua.jev_use_provider_request_golden_v1",+ "language": "python",+ "runs": {+ "page/windows-page-1-python": [+ {+ "step": 1,+ "request": "6b7824e66e1328bf",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "871e01954513c7d7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "be0bdd31f3b11cbe",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "00f3ee95dc3813ad",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page/linux-page-3-python": [+ {+ "step": 1,+ "request": "6b7824e66e1328bf",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "871e01954513c7d7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "be0bdd31f3b11cbe",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "00f3ee95dc3813ad",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page/linux-page-7-typescript": [+ {+ "step": 1,+ "request": "6b7824e66e1328bf",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "871e01954513c7d7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "be0bdd31f3b11cbe",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "00f3ee95dc3813ad",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page/linux-page-2-python": [+ {+ "step": 1,+ "request": "6b7824e66e1328bf",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "871e01954513c7d7",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_auto/windows-page-1-python": [+ {+ "step": 1,+ "request": "6b7824e66e1328bf",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "871e01954513c7d7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "be0bdd31f3b11cbe",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "00f3ee95dc3813ad",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_auto/linux-page-3-python": [+ {+ "step": 1,+ "request": "6b7824e66e1328bf",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "871e01954513c7d7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "be0bdd31f3b11cbe",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "00f3ee95dc3813ad",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_auto/linux-page-7-typescript": [+ {+ "step": 1,+ "request": "6b7824e66e1328bf",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "871e01954513c7d7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "be0bdd31f3b11cbe",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "00f3ee95dc3813ad",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_auto/linux-page-2-python": [+ {+ "step": 1,+ "request": "6b7824e66e1328bf",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "871e01954513c7d7",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_always/windows-page-1-python": [+ {+ "step": 1,+ "request": "7d7f17000d74bdb5",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "64ad866cbb7de3a0",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "55fa0c6279c5998f",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "7ed26ecebd151c79",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_always/linux-page-3-python": [+ {+ "step": 1,+ "request": "7d7f17000d74bdb5",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "64ad866cbb7de3a0",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "55fa0c6279c5998f",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "7ed26ecebd151c79",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_always/linux-page-7-typescript": [+ {+ "step": 1,+ "request": "7d7f17000d74bdb5",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "64ad866cbb7de3a0",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "55fa0c6279c5998f",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "7ed26ecebd151c79",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_always/linux-page-2-python": [+ {+ "step": 1,+ "request": "7d7f17000d74bdb5",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "64ad866cbb7de3a0",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "visual_capture_bound_auto/linux_python_pre_4255_a": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "7b92846b88b799cd",+ "candidates": "0a86001ae911a77b"+ }+ ],+ "visual_capture_bound_auto/linux_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "40216332a214fafb",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 4,+ "request": "f76646d52770d00e",+ "candidates": "0a86001ae911a77b"+ }+ ],+ "visual_capture_bound_auto/linux_python_pre_4255_b": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "7b92846b88b799cd",+ "candidates": "0a86001ae911a77b"+ }+ ],+ "visual_capture_bound_auto/linux_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "7b92846b88b799cd",+ "candidates": "0a86001ae911a77b"+ }+ ],+ "visual_capture_bound_auto/windows_python_pre_4255_a": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "40216332a214fafb",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 4,+ "request": "8f69d7cab750cca9",+ "candidates": "d11ea8f49c30445a"+ }+ ],+ "visual_capture_bound_auto/windows_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "40216332a214fafb",+ "candidates": "d11ea8f49c30445a"+ }+ ],+ "visual_capture_bound_auto/windows_python_pre_4255_b": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "40216332a214fafb",+ "candidates": "d11ea8f49c30445a"+ }+ ],+ "visual_capture_bound_auto/windows_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "40216332a214fafb",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 4,+ "request": "8f69d7cab750cca9",+ "candidates": "d11ea8f49c30445a"+ }+ ],+ "visual_capture_bound_auto/linux_python_post_4255_regression_1": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "ae88c22f9ca831ad",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "2b38e5d032f55588",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "c11f02c7d592f865",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_auto/linux_python_post_4255_regression_2": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "ae88c22f9ca831ad",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "2b38e5d032f55588",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "c11f02c7d592f865",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_auto/linux_python_post_4255_regression_3": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "ae88c22f9ca831ad",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "2b38e5d032f55588",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "c11f02c7d592f865",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_auto/linux_python_post_4255_regression_4": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "ae88c22f9ca831ad",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "2b38e5d032f55588",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "c11f02c7d592f865",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_auto/linux_python_post_4255_regression_5": [+ {+ "step": 1,+ "request": "c76f8a530fa915f0",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "ae88c22f9ca831ad",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "2b38e5d032f55588",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "c11f02c7d592f865",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_always/linux_python_pre_4255_a": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "7b92846b88b799cd",+ "candidates": "0a86001ae911a77b"+ }+ ],+ "visual_capture_bound_always/linux_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "40216332a214fafb",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 4,+ "request": "f76646d52770d00e",+ "candidates": "0a86001ae911a77b"+ }+ ],+ "visual_capture_bound_always/linux_python_pre_4255_b": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "7b92846b88b799cd",+ "candidates": "0a86001ae911a77b"+ }+ ],+ "visual_capture_bound_always/linux_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "7b92846b88b799cd",+ "candidates": "0a86001ae911a77b"+ }+ ],+ "visual_capture_bound_always/windows_python_pre_4255_a": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "40216332a214fafb",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 4,+ "request": "8f69d7cab750cca9",+ "candidates": "d11ea8f49c30445a"+ }+ ],+ "visual_capture_bound_always/windows_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "40216332a214fafb",+ "candidates": "d11ea8f49c30445a"+ }+ ],+ "visual_capture_bound_always/windows_python_pre_4255_b": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "40216332a214fafb",+ "candidates": "d11ea8f49c30445a"+ }+ ],+ "visual_capture_bound_always/windows_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "0d32ec5c61a4a8b5",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 3,+ "request": "40216332a214fafb",+ "candidates": "d11ea8f49c30445a"+ },+ {+ "step": 4,+ "request": "8f69d7cab750cca9",+ "candidates": "d11ea8f49c30445a"+ }+ ],+ "visual_capture_bound_always/linux_python_post_4255_regression_1": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1702e777270e55d9",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "13ad158a26367929",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "b3bd04ea455a1283",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_always/linux_python_post_4255_regression_2": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1702e777270e55d9",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "13ad158a26367929",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "b3bd04ea455a1283",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_always/linux_python_post_4255_regression_3": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1702e777270e55d9",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "13ad158a26367929",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "b3bd04ea455a1283",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_always/linux_python_post_4255_regression_4": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1702e777270e55d9",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "13ad158a26367929",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "b3bd04ea455a1283",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_always/linux_python_post_4255_regression_5": [+ {+ "step": 1,+ "request": "b645dd31d80e6d4b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1702e777270e55d9",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "13ad158a26367929",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "b3bd04ea455a1283",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_off/linux_python_pre_4255_a": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1dc3f220096db95a",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "566457a742192f26",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/linux_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1dc3f220096db95a",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "d5821263e5bced53",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 4,+ "request": "a415ac151c2ee8a0",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/linux_python_pre_4255_b": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1dc3f220096db95a",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "566457a742192f26",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/linux_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1dc3f220096db95a",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "566457a742192f26",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/windows_python_pre_4255_a": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1dc3f220096db95a",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "d5821263e5bced53",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 4,+ "request": "1114de9dcd19be4c",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/windows_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1dc3f220096db95a",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "d5821263e5bced53",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/windows_python_pre_4255_b": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1dc3f220096db95a",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "d5821263e5bced53",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/windows_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "1dc3f220096db95a",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "d5821263e5bced53",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 4,+ "request": "1114de9dcd19be4c",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/linux_python_post_4255_regression_1": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "e6d4d6a95565d28c",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "20599cb6821eb742",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "8a03f8eedacfd33e",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_off/linux_python_post_4255_regression_2": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "e6d4d6a95565d28c",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "20599cb6821eb742",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "8a03f8eedacfd33e",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_off/linux_python_post_4255_regression_3": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "e6d4d6a95565d28c",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "20599cb6821eb742",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "8a03f8eedacfd33e",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_off/linux_python_post_4255_regression_4": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "e6d4d6a95565d28c",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "20599cb6821eb742",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "8a03f8eedacfd33e",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_off/linux_python_post_4255_regression_5": [+ {+ "step": 1,+ "request": "8dfe7fa95bf30864",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "e6d4d6a95565d28c",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "20599cb6821eb742",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "8a03f8eedacfd33e",+ "candidates": "70875f146d859b36"+ }+ ]+ },+ "single_requests": {+ "page_other_value": {+ "request": "900887082b50ecd6",+ "candidates": "ec571c9800d7b4e8"+ },+ "page_after_with_visual": {+ "request": "9d507ebbaad75a8e",+ "candidates": "d229b9dd4f676e88"+ },+ "page_after_no_button": {+ "request": "77f2b9531e52e341",+ "candidates": "daba6141ff108d4e"+ },+ "page_after_no_button_visual_path": {+ "request": "2a0f4a3e90da5dab",+ "candidates": "daba6141ff108d4e"+ },+ "no_form_refs": {+ "request": "3905fd10ee1e16ae",+ "candidates": "daba6141ff108d4e"+ },+ "visual_after_no_capture_bound_click": {+ "request": "4a5f5196307c7a85",+ "candidates": "daba6141ff108d4e"+ },+ "visual_after_without_submit_region": {+ "request": "966403408348763b",+ "candidates": "daba6141ff108d4e"+ },+ "visual_after_duplicate_submit_region": {+ "request": "38460b37f9ee6b63",+ "candidates": "daba6141ff108d4e"+ },+ "visual_after_foreground": {+ "request": "3cd06681ec4e772d",+ "candidates": "0a86001ae911a77b"+ },+ "visual_after_other_value": {+ "request": "c6a54b6b1940bb00",+ "candidates": "3c210f06a8903bc9"+ }+ },+ "requests": {+ "6b7824e66e1328bf": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "available"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "871e01954513c7d7": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "available"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "be0bdd31f3b11cbe": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "available"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "00f3ee95dc3813ad": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "available"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 3,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "7d7f17000d74bdb5": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "available"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "64ad866cbb7de3a0": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "available"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "55fa0c6279c5998f": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "available"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "7ed26ecebd151c79": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "available"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 3,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "c76f8a530fa915f0": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "visual_check_pending"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "0d32ec5c61a4a8b5": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "7b92846b88b799cd": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "submit-form",+ "outcome": "Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "40216332a214fafb": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "f76646d52770d00e": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 3,+ "selected_id": "submit-form",+ "outcome": "Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "8f69d7cab750cca9": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 3,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "ae88c22f9ca831ad": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "visual_check_pending"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "2b38e5d032f55588": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "visual_check_pending"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "c11f02c7d592f865": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "visual_check_pending"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 3,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "b645dd31d80e6d4b": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "1702e777270e55d9": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": [+ {+ "step": 1,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "13ad158a26367929": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": [+ {+ "step": 1,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "b3bd04ea455a1283": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": [+ {+ "step": 1,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 3,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "8dfe7fa95bf30864": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "not_in_page_structure"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "1dc3f220096db95a": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "not_in_page_structure"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "566457a742192f26": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "not_in_page_structure"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "d5821263e5bced53": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "not_in_page_structure"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "a415ac151c2ee8a0": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "not_in_page_structure"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 3,+ "selected_id": "reobserve",+ "outcome": "Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "1114de9dcd19be4c": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "not_in_page_structure"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 3,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "e6d4d6a95565d28c": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "not_in_page_structure"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "20599cb6821eb742": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "not_in_page_structure"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "8a03f8eedacfd33e": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "empty",+ "submit_button": "not_in_page_structure"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "step": 1,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ },+ {+ "step": 3,+ "selected_id": "reobserve",+ "outcome": "took no action and requested a fresh observation"+ }+ ]+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "900887082b50ecd6": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "contains_other_value",+ "submit_button": "available"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "9d507ebbaad75a8e": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "available"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "77f2b9531e52e341": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "not_in_page_structure"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "2a0f4a3e90da5dab": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "visual_check_pending"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "3905fd10ee1e16ae": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": null,+ "form": {+ "verification_field": "not_found",+ "submit_button": "not_in_page_structure"+ },+ "outline": null,+ "visual": null+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "4a5f5196307c7a85": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "966403408348763b": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "not_found_visually"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ }+ ]+ }+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "38460b37f9ee6b63": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "not_found_visually"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ },+ {+ "id": "dup",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "3cd06681ec4e772d": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_required_token",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "c6a54b6b1940bb00": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:41605/"+ },+ "form": {+ "verification_field": "contains_other_value",+ "submit_button": "visual_only"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": {+ "schema": "cua.visual_regions_v1",+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",+ "source": {+ "kind": "window",+ "pid": 2539,+ "window_id": 52428804+ },+ "regions": [+ {+ "id": "text-9",+ "kind": "text",+ "text": "×",+ "label": null,+ "confidence": 0.8168784976005554,+ "interactive": false,+ "bounds": {+ "x": 252,+ "y": 10,+ "width": 18,+ "height": 19+ }+ },+ {+ "id": "text-11",+ "kind": "text",+ "text": "+",+ "label": null,+ "confidence": 0.8311772346496582,+ "interactive": false,+ "bounds": {+ "x": 288,+ "y": 13,+ "width": 14,+ "height": 14+ }+ },+ {+ "id": "icon-15",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8110984563827515,+ "interactive": false,+ "bounds": {+ "x": 10,+ "y": 45,+ "width": 34,+ "height": 36+ }+ },+ {+ "id": "icon-17",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8079566955566406,+ "interactive": false,+ "bounds": {+ "x": 83,+ "y": 46,+ "width": 32,+ "height": 36+ }+ },+ {+ "id": "icon-18",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.8043645024299622,+ "interactive": false,+ "bounds": {+ "x": 47,+ "y": 46,+ "width": 31,+ "height": 34+ }+ },+ {+ "id": "text-26",+ "kind": "text",+ "text": "←",+ "label": null,+ "confidence": 0.8558894395828247,+ "interactive": false,+ "bounds": {+ "x": 17,+ "y": 54,+ "width": 20,+ "height": 18+ }+ },+ {+ "id": "text-28",+ "kind": "text",+ "text": "Cua",+ "label": null,+ "confidence": 0.8023205995559692,+ "interactive": false,+ "bounds": {+ "x": 175,+ "y": 389,+ "width": 118,+ "height": 47+ }+ },+ {+ "id": "text-29",+ "kind": "text",+ "text": "TypeSafe Jev",+ "label": null,+ "confidence": 0.8917925357818604,+ "interactive": false,+ "bounds": {+ "x": 471,+ "y": 389,+ "width": 302,+ "height": 52+ }+ },+ {+ "id": "text-30",+ "kind": "text",+ "text": "Driver +",+ "label": null,+ "confidence": 0.865290641784668,+ "interactive": false,+ "bounds": {+ "x": 270,+ "y": 390,+ "width": 207,+ "height": 46+ }+ },+ {+ "id": "icon-33",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9488332271575928,+ "interactive": false,+ "bounds": {+ "x": 637,+ "y": 561,+ "width": 132,+ "height": 63+ }+ },+ {+ "id": "icon-34",+ "kind": "icon",+ "text": null,+ "label": "icon-class-0",+ "confidence": 0.9535441398620604,+ "interactive": false,+ "bounds": {+ "x": 172,+ "y": 562,+ "width": 449,+ "height": 65+ }+ },+ {+ "id": "text-35",+ "kind": "text",+ "text": "[verification token]",+ "label": null,+ "confidence": 0.8249707818031311,+ "interactive": false,+ "bounds": {+ "x": 188,+ "y": 575,+ "width": 149,+ "height": 35+ }+ },+ {+ "id": "text-36",+ "kind": "text",+ "text": "Submit",+ "label": null,+ "confidence": 0.8971455097198486,+ "interactive": false,+ "bounds": {+ "x": 662,+ "y": 575,+ "width": 85,+ "height": 32+ }+ }+ ]+ }+ },+ "history": []+ },+ "questions": {+ "driver_action": {+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ }+ },+ "candidate_tables": {+ "ec571c9800d7b4e8": [+ {+ "id": "type-verification-value",+ "description": "Type the required verification token into the verification field, replacing its current contents.",+ "tool": "browser_type",+ "arguments": {+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",+ "ref": "p3:0",+ "text": "jev-guide-live",+ "replace": true+ },+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "d229b9dd4f676e88": [+ {+ "id": "submit-form",+ "description": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "tool": "browser_click",+ "arguments": {+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",+ "ref": "p4:2",+ "input_route": "dom_event"+ },+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "70875f146d859b36": [+ {+ "id": "type-verification-value",+ "description": "Type the required verification token into the verification field, replacing its current contents.",+ "tool": "browser_type",+ "arguments": {+ "target_id": "bt-aeb40534-c16c-4da9-9567-aee3d2d4958a",+ "tab_id": "tab-11edc8ea-1185-4d45-be26-6df2f7082d7e",+ "ref": "p1:0",+ "text": "jev-guide-live",+ "replace": true+ },+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "d11ea8f49c30445a": [+ {+ "id": "submit-form",+ "description": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",+ "tool": "click",+ "arguments": {+ "pid": 2539,+ "window_id": 52428804,+ "x": 704.5,+ "y": 591.0,+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "delivery_mode": "background"+ },+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a"+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "0a86001ae911a77b": [+ {+ "id": "submit-form-foreground",+ "description": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",+ "tool": "click",+ "arguments": {+ "pid": 2539,+ "window_id": 52428804,+ "x": 704.5,+ "y": 591.0,+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "delivery_mode": "foreground"+ },+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a"+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "daba6141ff108d4e": [+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "3c210f06a8903bc9": [+ {+ "id": "type-verification-value",+ "description": "Type the required verification token into the verification field, replacing its current contents.",+ "tool": "browser_type",+ "arguments": {+ "target_id": "bt-aeb40534-c16c-4da9-9567-aee3d2d4958a",+ "tab_id": "tab-11edc8ea-1185-4d45-be26-6df2f7082d7e",+ "ref": "p2:1",+ "text": "jev-guide-live",+ "replace": true+ },+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ]+ }+}diff --git a/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-typescript-v1.json b/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-typescript-v1.jsonnew file mode 100644index 0000000000..a88c0db8b5--- /dev/null+++ b/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-typescript-v1.json@@ -0,0 +1,2189 @@+{+ "schema": "cua.jev_use_provider_request_golden_v1",+ "language": "typescript",+ "runs": {+ "page/windows-page-1-python": [+ {+ "step": 1,+ "request": "d144a9a07f2f3c01",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "7dca951dd791e8c7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "3826ca5aa3a4582f",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "60d9f833a0fb1977",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page/linux-page-3-python": [+ {+ "step": 1,+ "request": "d144a9a07f2f3c01",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "7dca951dd791e8c7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "3826ca5aa3a4582f",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "60d9f833a0fb1977",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page/linux-page-7-typescript": [+ {+ "step": 1,+ "request": "d144a9a07f2f3c01",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "7dca951dd791e8c7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "3826ca5aa3a4582f",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "60d9f833a0fb1977",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page/linux-page-2-python": [+ {+ "step": 1,+ "request": "d144a9a07f2f3c01",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "7dca951dd791e8c7",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_auto/windows-page-1-python": [+ {+ "step": 1,+ "request": "d144a9a07f2f3c01",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "7dca951dd791e8c7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "3826ca5aa3a4582f",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "60d9f833a0fb1977",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_auto/linux-page-3-python": [+ {+ "step": 1,+ "request": "d144a9a07f2f3c01",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "7dca951dd791e8c7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "3826ca5aa3a4582f",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "60d9f833a0fb1977",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_auto/linux-page-7-typescript": [+ {+ "step": 1,+ "request": "d144a9a07f2f3c01",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "7dca951dd791e8c7",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "3826ca5aa3a4582f",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "60d9f833a0fb1977",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_auto/linux-page-2-python": [+ {+ "step": 1,+ "request": "d144a9a07f2f3c01",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "7dca951dd791e8c7",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_always/windows-page-1-python": [+ {+ "step": 1,+ "request": "f05e1d864efb3c23",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "6cbb6199137b0925",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "13d9c8d27fdc73d1",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "5c30c0fb3c4b7442",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_always/linux-page-3-python": [+ {+ "step": 1,+ "request": "f05e1d864efb3c23",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "6cbb6199137b0925",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "13d9c8d27fdc73d1",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "5c30c0fb3c4b7442",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_always/linux-page-7-typescript": [+ {+ "step": 1,+ "request": "f05e1d864efb3c23",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "6cbb6199137b0925",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 3,+ "request": "13d9c8d27fdc73d1",+ "candidates": "d229b9dd4f676e88"+ },+ {+ "step": 4,+ "request": "5c30c0fb3c4b7442",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "page_capture_bound_always/linux-page-2-python": [+ {+ "step": 1,+ "request": "f05e1d864efb3c23",+ "candidates": "ec571c9800d7b4e8"+ },+ {+ "step": 2,+ "request": "6cbb6199137b0925",+ "candidates": "d229b9dd4f676e88"+ }+ ],+ "visual_capture_bound_auto/linux_python_pre_4255_a": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "1b28206125d32608",+ "candidates": "706d6b86ab8b3002"+ }+ ],+ "visual_capture_bound_auto/linux_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "e9a218c29722d446",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 4,+ "request": "8ab9a17f015b0fcc",+ "candidates": "706d6b86ab8b3002"+ }+ ],+ "visual_capture_bound_auto/linux_python_pre_4255_b": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "1b28206125d32608",+ "candidates": "706d6b86ab8b3002"+ }+ ],+ "visual_capture_bound_auto/linux_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "1b28206125d32608",+ "candidates": "706d6b86ab8b3002"+ }+ ],+ "visual_capture_bound_auto/windows_python_pre_4255_a": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "e9a218c29722d446",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 4,+ "request": "695d64df7aea71d1",+ "candidates": "366d1fa45767f12c"+ }+ ],+ "visual_capture_bound_auto/windows_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "e9a218c29722d446",+ "candidates": "366d1fa45767f12c"+ }+ ],+ "visual_capture_bound_auto/windows_python_pre_4255_b": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "e9a218c29722d446",+ "candidates": "366d1fa45767f12c"+ }+ ],+ "visual_capture_bound_auto/windows_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "e9a218c29722d446",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 4,+ "request": "695d64df7aea71d1",+ "candidates": "366d1fa45767f12c"+ }+ ],+ "visual_capture_bound_auto/linux_python_post_4255_regression_1": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "5423b86e68446d49",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "d5a8cd04d8624014",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "a68eda5180158762",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_auto/linux_python_post_4255_regression_2": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "5423b86e68446d49",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "d5a8cd04d8624014",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "a68eda5180158762",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_auto/linux_python_post_4255_regression_3": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "5423b86e68446d49",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "d5a8cd04d8624014",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "a68eda5180158762",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_auto/linux_python_post_4255_regression_4": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "5423b86e68446d49",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "d5a8cd04d8624014",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "a68eda5180158762",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_auto/linux_python_post_4255_regression_5": [+ {+ "step": 1,+ "request": "e8d87e3e15e30453",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "5423b86e68446d49",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "d5a8cd04d8624014",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "a68eda5180158762",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_always/linux_python_pre_4255_a": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "1b28206125d32608",+ "candidates": "706d6b86ab8b3002"+ }+ ],+ "visual_capture_bound_always/linux_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "e9a218c29722d446",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 4,+ "request": "8ab9a17f015b0fcc",+ "candidates": "706d6b86ab8b3002"+ }+ ],+ "visual_capture_bound_always/linux_python_pre_4255_b": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "1b28206125d32608",+ "candidates": "706d6b86ab8b3002"+ }+ ],+ "visual_capture_bound_always/linux_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "1b28206125d32608",+ "candidates": "706d6b86ab8b3002"+ }+ ],+ "visual_capture_bound_always/windows_python_pre_4255_a": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "e9a218c29722d446",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 4,+ "request": "695d64df7aea71d1",+ "candidates": "366d1fa45767f12c"+ }+ ],+ "visual_capture_bound_always/windows_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "e9a218c29722d446",+ "candidates": "366d1fa45767f12c"+ }+ ],+ "visual_capture_bound_always/windows_python_pre_4255_b": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "e9a218c29722d446",+ "candidates": "366d1fa45767f12c"+ }+ ],+ "visual_capture_bound_always/windows_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "00bfc515ce7e492a",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 3,+ "request": "e9a218c29722d446",+ "candidates": "366d1fa45767f12c"+ },+ {+ "step": 4,+ "request": "695d64df7aea71d1",+ "candidates": "366d1fa45767f12c"+ }+ ],+ "visual_capture_bound_always/linux_python_post_4255_regression_1": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "b4e5d58c3c0e8f93",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "e6ee50e7e55aee8d",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "cca37a534df52ebb",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_always/linux_python_post_4255_regression_2": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "b4e5d58c3c0e8f93",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "e6ee50e7e55aee8d",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "cca37a534df52ebb",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_always/linux_python_post_4255_regression_3": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "b4e5d58c3c0e8f93",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "e6ee50e7e55aee8d",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "cca37a534df52ebb",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_always/linux_python_post_4255_regression_4": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "b4e5d58c3c0e8f93",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "e6ee50e7e55aee8d",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "cca37a534df52ebb",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_always/linux_python_post_4255_regression_5": [+ {+ "step": 1,+ "request": "cc2331d3ee880e5b",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "b4e5d58c3c0e8f93",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "e6ee50e7e55aee8d",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "cca37a534df52ebb",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_off/linux_python_pre_4255_a": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "947e79fa8942aeac",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "9f13ac1d2ad5d0bd",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/linux_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "947e79fa8942aeac",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "d00f64505137d76d",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 4,+ "request": "91aed4f2a7b52459",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/linux_python_pre_4255_b": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "947e79fa8942aeac",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "9f13ac1d2ad5d0bd",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/linux_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "947e79fa8942aeac",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "9f13ac1d2ad5d0bd",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/windows_python_pre_4255_a": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "947e79fa8942aeac",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "d00f64505137d76d",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 4,+ "request": "a10e3de51047800d",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/windows_typescript_pre_4255_a": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "947e79fa8942aeac",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "d00f64505137d76d",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/windows_python_pre_4255_b": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "947e79fa8942aeac",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "d00f64505137d76d",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/windows_typescript_pre_4255_b": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "947e79fa8942aeac",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 3,+ "request": "d00f64505137d76d",+ "candidates": "daba6141ff108d4e"+ },+ {+ "step": 4,+ "request": "a10e3de51047800d",+ "candidates": "daba6141ff108d4e"+ }+ ],+ "visual_capture_bound_off/linux_python_post_4255_regression_1": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "ef44e05f270244a2",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "4b51b7ad29961b42",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "08be10eed5bf3061",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_off/linux_python_post_4255_regression_2": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "ef44e05f270244a2",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "4b51b7ad29961b42",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "08be10eed5bf3061",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_off/linux_python_post_4255_regression_3": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "ef44e05f270244a2",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "4b51b7ad29961b42",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "08be10eed5bf3061",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_off/linux_python_post_4255_regression_4": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "ef44e05f270244a2",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "4b51b7ad29961b42",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "08be10eed5bf3061",+ "candidates": "70875f146d859b36"+ }+ ],+ "visual_capture_bound_off/linux_python_post_4255_regression_5": [+ {+ "step": 1,+ "request": "81a25fcd82f06cf4",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 2,+ "request": "ef44e05f270244a2",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 3,+ "request": "4b51b7ad29961b42",+ "candidates": "70875f146d859b36"+ },+ {+ "step": 4,+ "request": "08be10eed5bf3061",+ "candidates": "70875f146d859b36"+ }+ ]+ },+ "single_requests": {+ "page_other_value": {+ "request": "e1ad8e9440c7d080",+ "candidates": "ec571c9800d7b4e8"+ },+ "page_after_with_visual": {+ "request": "ebbcae05b138d8d2",+ "candidates": "d229b9dd4f676e88"+ },+ "page_after_no_button": {+ "request": "bc0dcf345fc4ff00",+ "candidates": "daba6141ff108d4e"+ },+ "page_after_no_button_visual_path": {+ "request": "77a2b7b29d13d90b",+ "candidates": "daba6141ff108d4e"+ },+ "no_form_refs": {+ "request": "74545c6ebe3990ae",+ "candidates": "daba6141ff108d4e"+ },+ "visual_after_no_capture_bound_click": {+ "request": "de5ff5949f9e4473",+ "candidates": "daba6141ff108d4e"+ },+ "visual_after_without_submit_region": {+ "request": "476a091a64f9f2ab",+ "candidates": "daba6141ff108d4e"+ },+ "visual_after_duplicate_submit_region": {+ "request": "540fdaf0e9d0b71e",+ "candidates": "daba6141ff108d4e"+ },+ "visual_after_foreground": {+ "request": "dc62deb5acc6dacc",+ "candidates": "706d6b86ab8b3002"+ },+ "visual_after_other_value": {+ "request": "7906009abc493b1e",+ "candidates": "3c210f06a8903bc9"+ }+ },+ "requests": {+ "d144a9a07f2f3c01": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"available\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "7dca951dd791e8c7": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "3826ca5aa3a4582f": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "60d9f833a0fb1977": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "f05e1d864efb3c23": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"available\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "6cbb6199137b0925": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "13d9c8d27fdc73d1": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "5c30c0fb3c4b7442": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "e8d87e3e15e30453": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_check_pending\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "00bfc515ce7e492a": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "1b28206125d32608": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"submit-form\",\"outcome\":\"Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "e9a218c29722d446": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "8ab9a17f015b0fcc": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"submit-form\",\"outcome\":\"Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "695d64df7aea71d1": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "5423b86e68446d49": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_check_pending\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "d5a8cd04d8624014": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_check_pending\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "a68eda5180158762": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_check_pending\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "cc2331d3ee880e5b": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "b4e5d58c3c0e8f93": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "e6ee50e7e55aee8d": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "cca37a534df52ebb": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "81a25fcd82f06cf4": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"not_in_page_structure\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "947e79fa8942aeac": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "9f13ac1d2ad5d0bd": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "d00f64505137d76d": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "91aed4f2a7b52459": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "a10e3de51047800d": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "ef44e05f270244a2": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"not_in_page_structure\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "4b51b7ad29961b42": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"not_in_page_structure\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "08be10eed5bf3061": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"not_in_page_structure\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "e1ad8e9440c7d080": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"contains_other_value\",\"submit_button\":\"available\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "ebbcae05b138d8d2": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "bc0dcf345fc4ff00": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "77a2b7b29d13d90b": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_check_pending\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": "null"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "74545c6ebe3990ae": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "null",+ "form": "{\"verification_field\":\"not_found\",\"submit_button\":\"not_in_page_structure\"}",+ "outline": "",+ "visual": "null"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "de5ff5949f9e4473": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "476a091a64f9f2ab": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_found_visually\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}}]}"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "540fdaf0e9d0b71e": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_found_visually\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}},{\"id\":\"dup\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "dc62deb5acc6dacc": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ },+ "7906009abc493b1e": {+ "state": {+ "goal": "Enter the required verification token into the verification field, then submit the form.",+ "observation": {+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",+ "form": "{\"verification_field\":\"contains_other_value\",\"submit_button\":\"visual_only\"}",+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"+ },+ "history": "[]"+ },+ "questions": {+ "driver_action": {+ "type": "choice",+ "instructions": "Which complete executable action should Cua Driver run next?",+ "criteria": {+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."+ }+ }+ }+ }+ },+ "candidate_tables": {+ "ec571c9800d7b4e8": [+ {+ "id": "type-verification-value",+ "description": "Type the required verification token into the verification field, replacing its current contents.",+ "tool": "browser_type",+ "arguments": {+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",+ "ref": "p3:0",+ "text": "jev-guide-live",+ "replace": true+ },+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "d229b9dd4f676e88": [+ {+ "id": "submit-form",+ "description": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",+ "tool": "browser_click",+ "arguments": {+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",+ "ref": "p4:2",+ "input_route": "dom_event"+ },+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "70875f146d859b36": [+ {+ "id": "type-verification-value",+ "description": "Type the required verification token into the verification field, replacing its current contents.",+ "tool": "browser_type",+ "arguments": {+ "target_id": "bt-aeb40534-c16c-4da9-9567-aee3d2d4958a",+ "tab_id": "tab-11edc8ea-1185-4d45-be26-6df2f7082d7e",+ "ref": "p1:0",+ "text": "jev-guide-live",+ "replace": true+ },+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "366d1fa45767f12c": [+ {+ "id": "submit-form",+ "description": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",+ "tool": "click",+ "arguments": {+ "pid": 2539,+ "window_id": 52428804,+ "x": 704.5,+ "y": 591,+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "delivery_mode": "background"+ },+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a"+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "706d6b86ab8b3002": [+ {+ "id": "submit-form-foreground",+ "description": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",+ "tool": "click",+ "arguments": {+ "pid": 2539,+ "window_id": 52428804,+ "x": 704.5,+ "y": 591,+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "delivery_mode": "foreground"+ },+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a"+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "daba6141ff108d4e": [+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ],+ "3c210f06a8903bc9": [+ {+ "id": "type-verification-value",+ "description": "Type the required verification token into the verification field, replacing its current contents.",+ "tool": "browser_type",+ "arguments": {+ "target_id": "bt-aeb40534-c16c-4da9-9567-aee3d2d4958a",+ "tab_id": "tab-11edc8ea-1185-4d45-be26-6df2f7082d7e",+ "ref": "p2:1",+ "text": "jev-guide-live",+ "replace": true+ },+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "reobserve",+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ },+ {+ "id": "abstain",+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",+ "tool": null,+ "arguments": {},+ "capture_id": null,+ "screenshot_reference": null+ }+ ]+ }+}diff --git a/libs/cua-driver/examples/jev-use/python/core.py b/libs/cua-driver/examples/jev-use/python/core.pyindex a4c1b4cd9e..b3c9781191 100644--- a/libs/cua-driver/examples/jev-use/python/core.py+++ b/libs/cua-driver/examples/jev-use/python/core.py@@ -1,35 +1,57 @@+"""Runner primitives shared by every jev-use task.++Visual-region parsing, candidate validation, and the mock chooser live here.+Candidate sources are in ``sources.py`` and task specs in ``tasks.py``; the+names re-exported below keep existing imports from ``core`` working.+"""+from __future__ import annotationsimport mathfrom dataclasses import dataclass-from types import MappingProxyTypefrom typing import Any, Literal, Mapping-Outcome = Literal["verified", "refuted", "unknown", "abstained", "budget_exhausted"]-VisualDelivery = Literal["background", "foreground"]--SUBMIT_IDS = frozenset({"submit-form", "submit-form-foreground"})---def _freeze(value: Any) -> Any:- if isinstance(value, dict):- return MappingProxyType({key: _freeze(item) for key, item in value.items()})- if isinstance(value, list):- return tuple(_freeze(item) for item in value)- return value---@dataclass(frozen=True)-class Candidate:- id: str- description: str- tool: str | None- arguments: Mapping[str, Any]- capture_id: str | None = None- screenshot_reference: str | None = None+from sources import Candidate+from tasks import (+ FIELD_NAME,+ HISTORY_OUTCOMES,+ REDACTED_TOKEN,+ SUBMIT_IDS,+ SUBMIT_NAME,+ Outcome,+ build_candidates,+ classify,+ form_state,+ history_entry,+ redact_token,+ visual_submit_region,+)++__all__ = [+ "FIELD_NAME",+ "HISTORY_OUTCOMES",+ "REDACTED_TOKEN",+ "SUBMIT_IDS",+ "SUBMIT_NAME",+ "Candidate",+ "Outcome",+ "VisualDelivery",+ "VisualObservation",+ "VisualObservationError",+ "VisualRegion",+ "build_candidates",+ "choose_mock",+ "classify",+ "form_state",+ "has_executable_candidate",+ "history_entry",+ "parse_visual_regions",+ "redact_token",+ "validate_choice",+ "visual_submit_region",+]- def __post_init__(self) -> None:- object.__setattr__(self, "arguments", _freeze(dict(self.arguments)))+VisualDelivery = Literal["background", "foreground"]@dataclass(frozen=True)@@ -213,228 +235,10 @@ def parse_visual_regions()-FIELD_NAME = "verification value"-SUBMIT_NAME = "Submit"-REDACTED_TOKEN = "[verification token]"---def _form_refs(snapshot: Mapping[str, Any]) -> tuple[Any, Any]:- refs = snapshot.get("refs") or []- field = next(- (ref for ref in refs if ref.get("role") == "textbox" and ref.get("name") == FIELD_NAME),- None,- )- button = next(- (ref for ref in refs if ref.get("role") == "button" and ref.get("name") == SUBMIT_NAME),- None,- )- return field, button---def visual_submit_region(visual: VisualObservation | None) -> VisualRegion | None:- """Return the unique validated visual Submit region, or ``None``."""- if visual is None:- return None- matches = [- region- for region in visual.regions- if region.confidence >= 0.8- and _ascii_lower(region.text or region.label or "") == "submit"- ]- return matches[0] if len(matches) == 1 else None---def form_state(- snapshot: Mapping[str, Any],- token: str,- visual: VisualObservation | None = None,- *,- visual_path: bool = False,-) -> dict[str, str]:- """Summarize the form for the decision model without revealing the token.-- The raw field value never leaves the runner; the model receives only whether- the field is empty, holds the required token, or holds something else.-- ``submit_button`` is ``available`` for a clickable page-structure ref. When- the page structure has none and the capture-bound visual path is enabled- (``visual_path``), it is ``visual_only`` if this observation holds a unique- validated visual Submit region, ``visual_check_pending`` if no visual regions- were parsed for this step yet (the runner parses them once no page-structure- action remains), and ``not_found_visually`` otherwise. Without a visual path- it is ``not_in_page_structure``.- """- field, button = _form_refs(snapshot)- if field is None:- field_state = "not_found"- elif not field.get("value"):- field_state = "empty"- elif field.get("value") == token:- field_state = "contains_required_token"- else:- field_state = "contains_other_value"- if button is not None:- submit_state = "available"- elif not visual_path:- submit_state = "not_in_page_structure"- elif visual is None:- submit_state = "visual_check_pending"- elif visual_submit_region(visual) is not None:- submit_state = "visual_only"- else:- submit_state = "not_found_visually"- return {"verification_field": field_state, "submit_button": submit_state}---def redact_token(value: Any, token: str) -> Any:- """Replace every occurrence of the token in strings nested in ``value``."""- if not token:- return value- if isinstance(value, str):- return value.replace(token, REDACTED_TOKEN)- if isinstance(value, Mapping):- return {key: redact_token(item, token) for key, item in value.items()}- if isinstance(value, (list, tuple)):- return [redact_token(item, token) for item in value]- return value---HISTORY_OUTCOMES = {- "type-verification-value": "typed the required token into the verification field",- "submit-form": "clicked Submit; the submission was not yet confirmed",- "submit-form-foreground": "clicked Submit in the foreground; the submission was not yet confirmed",- "reobserve": "took no action and requested a fresh observation",-}---def history_entry(- step: int, candidate_id: str, *, refusal: str | None = None-) -> dict[str, Any]:- """Build the compact decision-history item shown to the model.-- It records what each step did, not timings or model probabilities, so earlier- choices do not become a signal to repeat themselves.- """- if refusal is not None:- outcome = (- f"Driver refused background delivery ({refusal}); no click happened and a "- "foreground Submit candidate is offered next"- )- else:- outcome = HISTORY_OUTCOMES.get(candidate_id, "completed")- return {"step": step, "selected_id": candidate_id, "outcome": outcome}---def _reserved_candidates() -> list[Candidate]:- return [- Candidate(- "reobserve",- "Take no action and obtain a fresh Driver observation, because the current "- "observation is stale or contradicts the reported form state. A Submit control "- "that is visual-only, or whose visual check is still pending, is not a reason to "- "reobserve: the runner parses visual regions for Submit once no page-structure "- "action remains.",- None,- {},- ),- Candidate(- "abstain",- "Stop without acting if none of the proposed actions is safe for the observed state.",- None,- {},- ),- ]---def build_candidates(- snapshot: dict[str, Any],- token: str,- visual: VisualObservation | None = None,- *,- capture_bound_click: bool = False,- visual_delivery: VisualDelivery = "background",-) -> list[Candidate]:- """Build the closed candidate set for one decision.-- Page-structure refs always win. The capture-bound visual Submit is offered- only when no Submit ref exists. ``visual_delivery="foreground"`` replaces the- background visual click with a distinct ``submit-form-foreground`` candidate- after Driver refused background delivery; the chooser must pick it explicitly.- """- common = {- "target_id": snapshot["target_id"],- "tab_id": snapshot["tab_id"],- }- field, button = _form_refs(snapshot)- candidates: list[Candidate] = []- if field and field.get("value") != token:- candidates.append(- Candidate(- "type-verification-value",- "Type the required verification token into the verification field, "- "replacing its current contents.",- "browser_type",- {**common, "ref": field["ref"], "text": token, "replace": True},- )- )- elif field and field.get("value") == token and button:- candidates.append(- Candidate(- "submit-form",- "Click the form's Submit button. The observed form state reports that the "- "verification field already contains the required token, so the form is "- "ready to submit.",- "browser_click",- {**common, "ref": button["ref"], "input_route": "dom_event"},- )- )- elif (- field- and field.get("value") == token- and visual- and capture_bound_click- ):- region = visual_submit_region(visual)- if region is not None:- x, y = visual.screenshot_center(region)- foreground = visual_delivery == "foreground"- candidates.append(- Candidate(- "submit-form-foreground" if foreground else "submit-form",- (- "Submit the form by clicking the unique validated visual Submit "- "region with foreground delivery, which activates the browser "- "window, because Driver refused background delivery for the "- "previous visual click."- if foreground- else "Submit the form by clicking the unique validated visual Submit "- "region. The observed form state reports that the verification field "- "already contains the required token."- ),- "click",- {- "pid": visual.pid,- "window_id": visual.window_id,- "x": x,- "y": y,- "capture_id": visual.capture_id,- "delivery_mode": visual_delivery,- },- capture_id=visual.capture_id,- screenshot_reference=visual.screenshot_reference,- )- )- return candidates + _reserved_candidates()--def has_executable_candidate(candidates: list[Candidate]) -> bool:return any(candidate.tool is not None for candidate in candidates)-def _ascii_lower(value: str) -> str:- return "".join(chr(ord(char) + 32) if "A" <= char <= "Z" else char for char in value)--def choose_mock(candidates: list[Candidate]) -> tuple[str | None, float, dict[str, float]]:ids = {candidate.id for candidate in candidates}selected = (@@ -469,13 +273,3 @@ def validate_choice(if candidate.capture_id is not None and candidate.capture_id != current_capture_id:raise ValueError("provider selected a stale or capture-mismatched candidate")return candidate---def classify(submitted: str | None, token: str, *, steps: int, max_steps: int) -> Outcome:- if submitted == token:- return "verified"- if submitted is not None:- return "refuted"- if steps >= max_steps:- return "budget_exhausted"- return "unknown"diff --git a/libs/cua-driver/examples/jev-use/python/jev_adapter.py b/libs/cua-driver/examples/jev-use/python/jev_adapter.pyindex 90db703b65..c263edf499 100644--- a/libs/cua-driver/examples/jev-use/python/jev_adapter.py+++ b/libs/cua-driver/examples/jev-use/python/jev_adapter.py@@ -4,7 +4,8 @@ import mathfrom dataclasses import dataclassfrom typing import Any, Mapping, Protocol-from core import Candidate, VisualObservation, choose_mock, form_state, redact_token+from core import Candidate, VisualObservation, choose_mock+from tasks import FIXTURE_GOAL, FixtureFormTask, Task, TaskSources, fixture_sourcesclass TypeSafeClientLike(Protocol):@@ -95,52 +96,66 @@ def visual_decision_state(visual: VisualObservation | None) -> dict[str, Any] |}-GOAL = "Enter the required verification token into the verification field, then submit the form."+GOAL = FIXTURE_GOAL-def decision_state(- snapshot: Mapping[str, Any],- visual: VisualObservation | None,- history: list[dict[str, Any]],- token: str,- visual_path: bool = False,+def task_decision_state(+ task: Task, sources: TaskSources, history: list[dict[str, Any]]) -> dict[str, Any]:- """Build the compact, deterministic, token-redacted state sent to Jev.+ """Build the compact, deterministic, secret-redacted state sent to Jev.- ``form`` states the field and Submit status the runner verified from the page- structure and, when ``visual_path`` is enabled, from the validated visual- regions, so the model does not have to infer it from the outline. The token- itself is replaced everywhere, including the outline and visual text.+ ``form`` is the task's state summary, which states what the runner verified+ from its candidate sources, so the model does not have to infer it from the+ outline. Every secret task parameter is replaced everywhere, including the+ outline and visual text."""+ snapshot = sources.page.snapshot+ visual = sources.visual.observation if sources.visual is not None else Nonereturn {- "goal": GOAL,+ "goal": task.goal,"observation": {- "page": redact_token(snapshot.get("page"), token),- "form": form_state(snapshot, token, visual, visual_path=visual_path),- "outline": redact_token(snapshot.get("outline"), token),- "visual": redact_token(visual_decision_state(visual), token),+ "page": task.redact(snapshot.get("page")),+ "form": task.state_summary(sources),+ "outline": task.redact(snapshot.get("outline")),+ "visual": task.redact(visual_decision_state(visual)),},"history": [dict(item) for item in history],}-def choose_with_typesafe(- client: TypeSafeClientLike,- candidates: list[Candidate],- snapshot: dict[str, Any],+def decision_state(+ snapshot: Mapping[str, Any],visual: VisualObservation | None,history: list[dict[str, Any]],token: str,visual_path: bool = False,+) -> dict[str, Any]:+ """Build the fixture task's decision state; see ``task_decision_state``."""+ return task_decision_state(+ FixtureFormTask(token),+ fixture_sources(snapshot, visual, visual_path=visual_path),+ history,+ )+++DRIVER_ACTION_INSTRUCTIONS = "Which complete executable action should Cua Driver run next?"+++def choose_for_task(+ client: TypeSafeClientLike,+ task: Task,+ sources: TaskSources,+ candidates: list[Candidate],+ history: list[dict[str, Any]],) -> tuple[str, float, dict[str, float]]:from typesafe_sdk import Choicecriteria = _candidate_criteria(candidates)response = client.system_one(- state=decision_state(snapshot, visual, history, token, visual_path),+ state=task_decision_state(task, sources, history),questions={"driver_action": Choice(- instructions="Which complete executable action should Cua Driver run next?",+ instructions=DRIVER_ACTION_INSTRUCTIONS,criteria=criteria,)},@@ -151,20 +166,59 @@ def choose_with_typesafe(return answer.choice, answer.confidence, answer.probabilities-def choose_live(+def choose_with_typesafe(+ client: TypeSafeClientLike,candidates: list[Candidate],snapshot: dict[str, Any],visual: VisualObservation | None,history: list[dict[str, Any]],token: str,visual_path: bool = False,+) -> tuple[str, float, dict[str, float]]:+ return choose_for_task(+ client,+ FixtureFormTask(token),+ fixture_sources(snapshot, visual, visual_path=visual_path),+ candidates,+ history,+ )+++def choose_live_for_task(+ task: Task,+ sources: TaskSources,+ candidates: list[Candidate],+ history: list[dict[str, Any]],) -> tuple[str, float, dict[str, float]]:from typesafe_sdk import TypeSafeClientwith TypeSafeClient() as client:- return choose_with_typesafe(- client, candidates, snapshot, visual, history, token, visual_path- )+ return choose_for_task(client, task, sources, candidates, history)+++def choose_mock_for_task(+ _task: Task,+ _sources: TaskSources,+ candidates: list[Candidate],+ _history: list[dict[str, Any]],+) -> tuple[str | None, float, dict[str, float]]:+ return choose_mock(candidates)+++def choose_live(+ candidates: list[Candidate],+ snapshot: dict[str, Any],+ visual: VisualObservation | None,+ history: list[dict[str, Any]],+ token: str,+ visual_path: bool = False,+) -> tuple[str, float, dict[str, float]]:+ return choose_live_for_task(+ FixtureFormTask(token),+ fixture_sources(snapshot, visual, visual_path=visual_path),+ candidates,+ history,+ )def choose_mock_adapter(diff --git a/libs/cua-driver/examples/jev-use/python/run.py b/libs/cua-driver/examples/jev-use/python/run.pyindex 1ec01b93b1..66f4edd454 100644--- a/libs/cua-driver/examples/jev-use/python/run.py+++ b/libs/cua-driver/examples/jev-use/python/run.py@@ -9,31 +9,31 @@ import uuidfrom pathlib import Pathfrom typing import Anyfrom urllib.parse import urlsplit-from urllib.request import Request, urlopenfrom mcp import ClientSession, StdioServerParametersfrom mcp.client.stdio import stdio_clientfrom driver_env import driver_environmentfrom core import (- SUBMIT_IDS,Candidate,VisualDelivery,VisualObservation,VisualObservationError,- build_candidates,- classify,- history_entry,has_executable_candidate,parse_visual_regions,validate_choice,)-from jev_adapter import choose_live, choose_mock_adapter-+from jev_adapter import choose_live_for_task, choose_mock_for_task+from tasks import (+ FixtureFormTask,+ Task,+ TaskSources,+ fixture_sources,+ fixture_state,+ reset_fixture,+)-def fixture_state(fixture_url: str) -> dict[str, str | None]:- with urlopen(f"{fixture_url.rstrip('/')}/state", timeout=2) as response:- return json.loads(response.read())+__all__ = ["fixture_state", "reset_fixture"]def validate_fixture_url(value: str) -> str:@@ -60,13 +60,6 @@ def select_tab_id(tabs: list[dict[str, Any]]) -> str:return str(selected["tab_id"])-def reset_fixture(fixture_url: str) -> None:- request = Request(f"{fixture_url.rstrip('/')}/reset", method="POST", data=b"")- with urlopen(request, timeout=2) as response:- if response.status != 204:- raise RuntimeError(f"fixture reset failed: HTTP {response.status}")--class DriverToolError(RuntimeError):"""A Driver tool returned an error result, optionally with a stable error code."""@@ -223,10 +216,10 @@ async def optional_visual_observation(return visual-async def candidates_for_step(+async def task_candidates_for_step(driver: Driver,+ task: Task,snapshot: dict[str, Any],- token: str,pid: int,window_id: int,available_tools: set[str],@@ -234,35 +227,68 @@ async def candidates_for_step(*,visual_mode: str = "auto",visual_delivery: VisualDelivery = "background",-) -> tuple[list[Candidate], VisualObservation | None, dict[str, Any]]:- """Build one step's candidates, parsing visual regions only when useful.+) -> tuple[list[Candidate], TaskSources, dict[str, Any]]:+ """Build one step's sources and candidates, parsing visual regions only when useful.In ``auto`` mode the capture and parse run only when the page structureoffers no executable candidate, because only then can a visual region addone. ``always`` restores the per-step parse; ``off`` never parses."""- candidates = build_candidates(+ # Whether a control missing from the page structure can still be found+ # through a capture-bound visual region; reported in the task state.+ visual_path = capture_bound_click and visual_mode != "off"+ sources = fixture_sources(snapshot,- token,None,capture_bound_click=capture_bound_click,visual_delivery=visual_delivery,+ visual_path=visual_path,)+ candidates = task.candidates(sources)if visual_mode == "off":- return candidates, None, visual_status("skipped", reason="disabled")+ return candidates, sources, visual_status("skipped", reason="disabled")if visual_mode == "auto" and has_executable_candidate(candidates):- return candidates, None, visual_status("skipped", reason="page_structure_candidate")+ return candidates, sources, visual_status("skipped", reason="page_structure_candidate")visual, record = await observe_visual(driver, pid, window_id, available_tools, capture_bound_click)if visual is not None:- candidates = build_candidates(+ sources = fixture_sources(snapshot,- token,visual,capture_bound_click=capture_bound_click,visual_delivery=visual_delivery,+ visual_path=visual_path,)+ candidates = task.candidates(sources)+ return candidates, sources, record+++async def candidates_for_step(+ driver: Driver,+ snapshot: dict[str, Any],+ token: str,+ pid: int,+ window_id: int,+ available_tools: set[str],+ capture_bound_click: bool,+ *,+ visual_mode: str = "auto",+ visual_delivery: VisualDelivery = "background",+) -> tuple[list[Candidate], VisualObservation | None, dict[str, Any]]:+ """Build one step's fixture-task candidates; see ``task_candidates_for_step``."""+ candidates, sources, record = await task_candidates_for_step(+ driver,+ FixtureFormTask(token),+ snapshot,+ pid,+ window_id,+ available_tools,+ capture_bound_click,+ visual_mode=visual_mode,+ visual_delivery=visual_delivery,+ )+ visual = sources.visual.observation if sources.visual is not None else Nonereturn candidates, visual, record@@ -289,6 +315,7 @@ def write_event(log_path: Path | None, event: dict[str, Any]) -> None:async def run(args: argparse.Namespace) -> str:token = args.token or f"jev-{uuid.uuid4().hex[:10]}"+ task: Task = FixtureFormTask(token, args.fixture_url, args.max_steps)label = f"jev-python-{uuid.uuid4().hex[:8]}"# Compact what-happened record for the decision model. The full telemetry# events (timings, probabilities) go only to the JSONL log.@@ -297,7 +324,7 @@ async def run(args: argparse.Namespace) -> str:log_path = Path(args.log) if args.log else Noneif log_path:log_path.write_text("", encoding="utf-8")- reset_fixture(args.fixture_url)+ task.reset()params = StdioServerParameters(command=os.getenv("CUA_DRIVER_BIN", "cua-driver"), args=["mcp"], env=driver_environment()@@ -308,9 +335,6 @@ async def run(args: argparse.Namespace) -> str:advertised_tools = (await session.list_tools()).toolsavailable_tools = {tool.name for tool in advertised_tools}capture_bound_click = supports_capture_bound_click(advertised_tools)- # Whether a Submit missing from the page structure can still be found- # through a capture-bound visual region; reported in the form state.- visual_path = capture_bound_click and args.visual_observation != "off"driver = Driver(session, label)prepared = await driver.call("browser_prepare",@@ -328,9 +352,8 @@ async def run(args: argparse.Namespace) -> str:{"target_id": target_id, "tab_id": tab_id, "url": args.fixture_url},)- for step in range(1, args.max_steps + 1):- oracle = fixture_state(args.fixture_url)- current = classify(oracle.get("submitted"), token, steps=step - 1, max_steps=args.max_steps)+ for step in range(1, task.max_steps + 1):+ current = task.classify(task.read_oracle(), steps=step - 1)if current in {"verified", "refuted"}:write_event(log_path, {"event": "outcome", "outcome": current, "token": token})return current@@ -344,10 +367,10 @@ async def run(args: argparse.Namespace) -> str:"snapshot_format": "semantic_v2",},)- candidates, visual, visual_record = await candidates_for_step(+ candidates, sources, visual_record = await task_candidates_for_step(driver,+ task,snapshot,- token,pid,int(window["window_id"]),available_tools,@@ -362,13 +385,14 @@ async def run(args: argparse.Namespace) -> str:)return "abstained"+ visual = sources.visual.observation if sources.visual is not None else Noneif args.provider == "mock":- choice, confidence, probabilities = choose_mock_adapter(- candidates, snapshot, visual, history, token, visual_path+ choice, confidence, probabilities = choose_mock_for_task(+ task, sources, candidates, history)else:choice, confidence, probabilities = await asyncio.to_thread(- choose_live, candidates, snapshot, visual, history, token, visual_path+ choose_live_for_task, task, sources, candidates, history)if choice is None:return "abstained"@@ -392,7 +416,7 @@ async def run(args: argparse.Namespace) -> str:"tool": None,"visual": visual_record,}- history.append(history_entry(step, candidate.id))+ history.append(task.history_entry(step, candidate.id))write_event(log_path, event)continue@@ -440,7 +464,7 @@ async def run(args: argparse.Namespace) -> str:},"visual": visual_record,}- history.append(history_entry(step, candidate.id, refusal=refusal))+ history.append(task.history_entry(step, candidate.id, refusal=refusal))write_event(log_path, event)continuewrite_event(@@ -472,16 +496,13 @@ async def run(args: argparse.Namespace) -> str:"delivery_mode": candidate.arguments.get("delivery_mode"),"visual": visual_record,}- history.append(history_entry(step, candidate.id))+ history.append(task.history_entry(step, candidate.id))write_event(log_path, event)if args.dry_run:return "unknown"- if candidate.id in SUBMIT_IDS:+ if candidate.id in task.completion_candidate_ids:for _ in range(20):- oracle = fixture_state(args.fixture_url)- outcome = classify(- oracle.get("submitted"), token, steps=step, max_steps=args.max_steps- )+ outcome = task.classify(task.read_oracle(), steps=step)if outcome in {"verified", "refuted"}:write_event(log_path, {"event": "outcome", "outcome": outcome, "token": token}@@ -489,10 +510,7 @@ async def run(args: argparse.Namespace) -> str:return outcomeawait asyncio.sleep(0.1)- oracle = fixture_state(args.fixture_url)- outcome = classify(- oracle.get("submitted"), token, steps=args.max_steps, max_steps=args.max_steps- )+ outcome = task.classify(task.read_oracle(), steps=task.max_steps)write_event(log_path, {"event": "outcome", "outcome": outcome, "token": token})return outcomediff --git a/libs/cua-driver/examples/jev-use/python/sources.py b/libs/cua-driver/examples/jev-use/python/sources.pynew file mode 100644index 0000000000..3fa2531bbb--- /dev/null+++ b/libs/cua-driver/examples/jev-use/python/sources.py@@ -0,0 +1,197 @@+"""Candidate sources: where a decision's executable candidates come from.++A candidate source turns one Driver observation into addressable controls and+builds fully specified candidates for them. The model only ever sees a+candidate's ID and description; the tool name and exact arguments stay in the+runner. A task (see ``tasks.py``) decides which controls matter and which+candidate IDs and descriptions to offer; the source decides how to address a+control and which Driver tool acts on it.++Implemented sources:++- ``BrowserSemanticSource`` reads a ``get_browser_state`` ``semantic_v2``+ snapshot and acts through ``browser_type`` / ``browser_click`` on its refs.+- ``VisualRegionSource`` reads a validated ``parse_visual_regions`` result and+ acts through a capture-bound ``click`` on a region's center.++Extension point (RFC #4268, Phase 1): a ``NativeAccessibilitySource`` will read+``get_window_state`` elements, find controls by normalized role and label, and+act through the snapshot's ``element_token``. It is deliberately not+implemented here; it must satisfy ``CandidateSource`` and report+``kind = "ax"``.+"""++from __future__ import annotations++from dataclasses import dataclass+from types import MappingProxyType+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Mapping, Protocol++if TYPE_CHECKING:+ from core import VisualDelivery, VisualObservation, VisualRegion++# "ax" is reserved for the future NativeAccessibilitySource.+SourceKind = Literal["page", "visual", "ax"]+++def _freeze(value: Any) -> Any:+ if isinstance(value, dict):+ return MappingProxyType({key: _freeze(item) for key, item in value.items()})+ if isinstance(value, list):+ return tuple(_freeze(item) for item in value)+ return value+++def _ascii_lower(value: str) -> str:+ return "".join(chr(ord(char) + 32) if "A" <= char <= "Z" else char for char in value)+++@dataclass(frozen=True)+class Candidate:+ id: str+ description: str+ tool: str | None+ arguments: Mapping[str, Any]+ capture_id: str | None = None+ screenshot_reference: str | None = None++ def __post_init__(self) -> None:+ object.__setattr__(self, "arguments", _freeze(dict(self.arguments)))+++@dataclass(frozen=True)+class Control:+ """One addressable element a source found in its observation.++ ``value`` is the element's current value when the source can read it (never+ sent to a model; tasks summarize it). ``handle`` is source-private: a page+ ref for the browser source, a ``VisualRegion`` for the visual source.+ """++ source: SourceKind+ role: str+ name: str+ value: Any+ handle: Any+++class CandidateSource(Protocol):+ """The interface every candidate source implements.++ ``find`` returns the unique control matching a role and accessible name, or+ ``None``. ``click`` and ``type_text`` return a fully specified candidate for+ that control, or ``None`` when this source cannot perform the action in the+ current observation (for example a visual region cannot receive text, and a+ visual click needs a capture-bound Driver ``click``).+ """++ kind: ClassVar[SourceKind]++ def find(self, role: str, name: str) -> Control | None: ...++ def click(self, control: Control, *, candidate_id: str, description: str) -> Candidate | None: ...++ def type_text(+ self, control: Control, text: str, *, candidate_id: str, description: str+ ) -> Candidate | None: ...+++@dataclass(frozen=True)+class BrowserSemanticSource:+ """Controls from a ``get_browser_state`` snapshot, acted on through its refs."""++ snapshot: Mapping[str, Any]+ kind: ClassVar[SourceKind] = "page"++ def require_target(self) -> dict[str, Any]:+ """Return the browser target every page candidate addresses.++ Raises ``KeyError`` when the snapshot has no target or tab.+ """+ return {"target_id": self.snapshot["target_id"], "tab_id": self.snapshot["tab_id"]}++ def find(self, role: str, name: str) -> Control | None:+ refs = self.snapshot.get("refs") or []+ ref = next(+ (item for item in refs if item.get("role") == role and item.get("name") == name),+ None,+ )+ if ref is None:+ return None+ return Control("page", role, name, ref.get("value"), ref)++ def click(self, control: Control, *, candidate_id: str, description: str) -> Candidate:+ return Candidate(+ candidate_id,+ description,+ "browser_click",+ {**self.require_target(), "ref": control.handle["ref"], "input_route": "dom_event"},+ )++ def type_text(+ self, control: Control, text: str, *, candidate_id: str, description: str+ ) -> Candidate:+ return Candidate(+ candidate_id,+ description,+ "browser_type",+ {**self.require_target(), "ref": control.handle["ref"], "text": text, "replace": True},+ )+++@dataclass(frozen=True)+class VisualRegionSource:+ """Controls from validated OmniParser regions, clicked through their capture.++ Regions carry no role, so ``find`` matches the visible text or icon label+ (ASCII case-insensitive) of regions at or above ``MIN_CONFIDENCE`` and+ returns a control only when exactly one region matches. ``click`` is offered+ only when Driver advertises a capture-bound ``click`` (``capture_bound``); it+ targets the region's screenshot-pixel center with the exact ``capture_id``+ and the source's ``delivery`` mode.+ """++ observation: VisualObservation+ delivery: VisualDelivery = "background"+ capture_bound: bool = False+ kind: ClassVar[SourceKind] = "visual"+ MIN_CONFIDENCE: ClassVar[float] = 0.8++ def find(self, role: str, name: str) -> Control | None:+ wanted = _ascii_lower(name)+ matches = [+ region+ for region in self.observation.regions+ if region.confidence >= self.MIN_CONFIDENCE+ and _ascii_lower(region.text or region.label or "") == wanted+ ]+ if len(matches) != 1:+ return None+ region: VisualRegion = matches[0]+ return Control("visual", role, region.text or region.label or "", None, region)++ def click(self, control: Control, *, candidate_id: str, description: str) -> Candidate | None:+ if not self.capture_bound:+ return None+ visual = self.observation+ x, y = visual.screenshot_center(control.handle)+ return Candidate(+ candidate_id,+ description,+ "click",+ {+ "pid": visual.pid,+ "window_id": visual.window_id,+ "x": x,+ "y": y,+ "capture_id": visual.capture_id,+ "delivery_mode": self.delivery,+ },+ capture_id=visual.capture_id,+ screenshot_reference=visual.screenshot_reference,+ )++ def type_text(+ self, control: Control, text: str, *, candidate_id: str, description: str+ ) -> None:+ return Nonediff --git a/libs/cua-driver/examples/jev-use/python/tasks.py b/libs/cua-driver/examples/jev-use/python/tasks.pynew file mode 100644index 0000000000..5cb1bbe923--- /dev/null+++ b/libs/cua-driver/examples/jev-use/python/tasks.py@@ -0,0 +1,382 @@+"""Task specs: what one jev-use run is trying to achieve and how it is checked.++A task spec owns everything that is specific to one application task: the goal+shown to the model, its parameters (secrets are redacted from all model input),+the step budget, the Driver action kinds it may use, the compact state summary+the model sees, the candidate IDs and descriptions it offers, and the success+oracle that verifies the outcome independently of the runner's own events.++Candidate sources (``sources.py``) supply the controls and build the executable+candidates. The runner stays task-agnostic: it observes, asks the task for+candidates and a state summary, lets the model choose one supplied ID, acts,+and asks the task's oracle whether the task is done.++``FixtureFormTask`` is the one built-in task: type a secret verification token+into the loopback fixture's form and submit it, verified through the fixture's+``/state`` endpoint.+"""++from __future__ import annotations++import json+from dataclasses import dataclass, field+from typing import TYPE_CHECKING, Any, Literal, Mapping, Protocol+from urllib.request import Request, urlopen++from sources import BrowserSemanticSource, Candidate, VisualRegionSource++if TYPE_CHECKING:+ from core import VisualDelivery, VisualObservation, VisualRegion++Outcome = Literal["verified", "refuted", "unknown", "abstained", "budget_exhausted"]++REDACTED_TOKEN = "[verification token]"+++def redact_token(value: Any, token: str, placeholder: str = REDACTED_TOKEN) -> Any:+ """Replace every occurrence of the token in strings nested in ``value``."""+ if not token:+ return value+ if isinstance(value, str):+ return value.replace(token, placeholder)+ if isinstance(value, Mapping):+ return {key: redact_token(item, token, placeholder) for key, item in value.items()}+ if isinstance(value, (list, tuple)):+ return [redact_token(item, token, placeholder) for item in value]+ return value+++@dataclass(frozen=True)+class TaskParameter:+ """A named task input. A ``secret`` value is replaced by ``redaction``+ everywhere in model input; it may still appear in executable arguments."""++ name: str+ value: str+ secret: bool = False+ redaction: str = REDACTED_TOKEN+++@dataclass(frozen=True)+class TaskSources:+ """The candidate sources available for one decision step.++ ``visual`` is present only when this step parsed visual regions.+ ``visual_path`` reports whether a capture-bound visual path exists at all,+ so a task can say a control is still pending a visual check.+ """++ page: BrowserSemanticSource+ visual: VisualRegionSource | None = None+ visual_path: bool = False+++class Task(Protocol):+ """The interface the runner uses for every task."""++ id: str+ goal: str+ parameters: tuple[TaskParameter, ...]+ max_steps: int+ allowed_action_kinds: frozenset[str]+ # Candidate IDs after which the runner polls the oracle for completion.+ completion_candidate_ids: frozenset[str]++ def redact(self, value: Any) -> Any: ...++ def candidates(self, sources: TaskSources) -> list[Candidate]: ...++ def state_summary(self, sources: TaskSources) -> dict[str, str]: ...++ def history_entry(+ self, step: int, candidate_id: str, *, refusal: str | None = None+ ) -> dict[str, Any]: ...++ def reset(self) -> None: ...++ def read_oracle(self) -> Mapping[str, Any]: ...++ def classify(self, oracle: Mapping[str, Any], *, steps: int) -> Outcome: ...+++# The built-in browser fixture task.++FIXTURE_TASK_ID = "browser-fixture-form"+FIXTURE_GOAL = (+ "Enter the required verification token into the verification field, then submit the form."+)+DEFAULT_FIXTURE_URL = "http://127.0.0.1:8765/"+DEFAULT_MAX_STEPS = 4+FIELD_NAME = "verification value"+SUBMIT_NAME = "Submit"+SUBMIT_IDS = frozenset({"submit-form", "submit-form-foreground"})+FIXTURE_ACTION_KINDS = frozenset({"browser_type", "browser_click", "click"})++HISTORY_OUTCOMES = {+ "type-verification-value": "typed the required token into the verification field",+ "submit-form": "clicked Submit; the submission was not yet confirmed",+ "submit-form-foreground": "clicked Submit in the foreground; the submission was not yet confirmed",+ "reobserve": "took no action and requested a fresh observation",+}+++def fixture_state(fixture_url: str) -> dict[str, str | None]:+ with urlopen(f"{fixture_url.rstrip('/')}/state", timeout=2) as response:+ return json.loads(response.read())+++def reset_fixture(fixture_url: str) -> None:+ request = Request(f"{fixture_url.rstrip('/')}/reset", method="POST", data=b"")+ with urlopen(request, timeout=2) as response:+ if response.status != 204:+ raise RuntimeError(f"fixture reset failed: HTTP {response.status}")+++def classify(submitted: str | None, token: str, *, steps: int, max_steps: int) -> Outcome:+ if submitted == token:+ return "verified"+ if submitted is not None:+ return "refuted"+ if steps >= max_steps:+ return "budget_exhausted"+ return "unknown"+++def history_entry(+ step: int, candidate_id: str, *, refusal: str | None = None+) -> dict[str, Any]:+ """Build the compact decision-history item shown to the model.++ It records what each step did, not timings or model probabilities, so earlier+ choices do not become a signal to repeat themselves.+ """+ if refusal is not None:+ outcome = (+ f"Driver refused background delivery ({refusal}); no click happened and a "+ "foreground Submit candidate is offered next"+ )+ else:+ outcome = HISTORY_OUTCOMES.get(candidate_id, "completed")+ return {"step": step, "selected_id": candidate_id, "outcome": outcome}+++def _reserved_candidates() -> list[Candidate]:+ return [+ Candidate(+ "reobserve",+ "Take no action and obtain a fresh Driver observation, because the current "+ "observation is stale or contradicts the reported form state. A Submit control "+ "that is visual-only, or whose visual check is still pending, is not a reason to "+ "reobserve: the runner parses visual regions for Submit once no page-structure "+ "action remains.",+ None,+ {},+ ),+ Candidate(+ "abstain",+ "Stop without acting if none of the proposed actions is safe for the observed state.",+ None,+ {},+ ),+ ]+++@dataclass(frozen=True)+class FixtureFormTask:+ """Type the secret token into the fixture form and submit it.++ The oracle is the fixture's ``/state`` endpoint: the task is verified only+ when the fixture recorded exactly this token as submitted.+ """++ token: str+ fixture_url: str = DEFAULT_FIXTURE_URL+ max_steps: int = DEFAULT_MAX_STEPS+ id: str = field(default=FIXTURE_TASK_ID, init=False)+ goal: str = field(default=FIXTURE_GOAL, init=False)+ allowed_action_kinds: frozenset[str] = field(default=FIXTURE_ACTION_KINDS, init=False)+ completion_candidate_ids: frozenset[str] = field(default=SUBMIT_IDS, init=False)++ @property+ def parameters(self) -> tuple[TaskParameter, ...]:+ return (TaskParameter("verification_token", self.token, secret=True),)++ def redact(self, value: Any) -> Any:+ for parameter in self.parameters:+ if parameter.secret:+ value = redact_token(value, parameter.value, parameter.redaction)+ return value++ def candidates(self, sources: TaskSources) -> list[Candidate]:+ """Build the closed candidate set for one decision.++ Page-structure refs always win. The capture-bound visual Submit is offered+ only when no Submit ref exists. A visual source with ``foreground``+ delivery yields a distinct ``submit-form-foreground`` candidate after+ Driver refused background delivery; the chooser must pick it explicitly.+ """+ page = sources.page+ # Every page candidate addresses the snapshot's target; reject a snapshot+ # without one before offering anything.+ page.require_target()+ token = self.token+ field_control = page.find("textbox", FIELD_NAME)+ button = page.find("button", SUBMIT_NAME)+ candidates: list[Candidate] = []+ if field_control and field_control.value != token:+ candidates.append(+ page.type_text(+ field_control,+ token,+ candidate_id="type-verification-value",+ description="Type the required verification token into the verification "+ "field, replacing its current contents.",+ )+ )+ elif field_control and field_control.value == token and button:+ candidates.append(+ page.click(+ button,+ candidate_id="submit-form",+ description="Click the form's Submit button. The observed form state "+ "reports that the verification field already contains the required "+ "token, so the form is ready to submit.",+ )+ )+ elif field_control and field_control.value == token and sources.visual is not None:+ visual = sources.visual+ region = visual.find("button", SUBMIT_NAME)+ if region is not None:+ foreground = visual.delivery == "foreground"+ candidate = visual.click(+ region,+ candidate_id="submit-form-foreground" if foreground else "submit-form",+ description=(+ "Submit the form by clicking the unique validated visual Submit "+ "region with foreground delivery, which activates the browser "+ "window, because Driver refused background delivery for the "+ "previous visual click."+ if foreground+ else "Submit the form by clicking the unique validated visual Submit "+ "region. The observed form state reports that the verification field "+ "already contains the required token."+ ),+ )+ if candidate is not None:+ candidates.append(candidate)+ for candidate in candidates:+ if candidate.tool not in self.allowed_action_kinds:+ raise ValueError(f"task {self.id} does not allow action kind {candidate.tool}")+ return candidates + _reserved_candidates()++ def state_summary(self, sources: TaskSources) -> dict[str, str]:+ """Summarize the form for the decision model without revealing the token.++ The raw field value never leaves the runner; the model receives only whether+ the field is empty, holds the required token, or holds something else.++ ``submit_button`` is ``available`` for a clickable page-structure ref. When+ the page structure has none and the capture-bound visual path is enabled+ (``visual_path``), it is ``visual_only`` if this observation holds a unique+ validated visual Submit region, ``visual_check_pending`` if no visual regions+ were parsed for this step yet (the runner parses them once no page-structure+ action remains), and ``not_found_visually`` otherwise. Without a visual path+ it is ``not_in_page_structure``.+ """+ field_control = sources.page.find("textbox", FIELD_NAME)+ button = sources.page.find("button", SUBMIT_NAME)+ if field_control is None:+ field_state = "not_found"+ elif not field_control.value:+ field_state = "empty"+ elif field_control.value == self.token:+ field_state = "contains_required_token"+ else:+ field_state = "contains_other_value"+ if button is not None:+ submit_state = "available"+ elif not sources.visual_path:+ submit_state = "not_in_page_structure"+ elif sources.visual is None:+ submit_state = "visual_check_pending"+ elif sources.visual.find("button", SUBMIT_NAME) is not None:+ submit_state = "visual_only"+ else:+ submit_state = "not_found_visually"+ return {"verification_field": field_state, "submit_button": submit_state}++ def history_entry(+ self, step: int, candidate_id: str, *, refusal: str | None = None+ ) -> dict[str, Any]:+ return history_entry(step, candidate_id, refusal=refusal)++ def reset(self) -> None:+ reset_fixture(self.fixture_url)++ def read_oracle(self) -> Mapping[str, Any]:+ return fixture_state(self.fixture_url)++ def classify(self, oracle: Mapping[str, Any], *, steps: int) -> Outcome:+ return classify(oracle.get("submitted"), self.token, steps=steps, max_steps=self.max_steps)+++def fixture_sources(+ snapshot: Mapping[str, Any],+ visual: VisualObservation | None = None,+ *,+ capture_bound_click: bool = False,+ visual_delivery: VisualDelivery = "background",+ visual_path: bool = False,+) -> TaskSources:+ """Build one step's sources from a browser snapshot and optional visual parse."""+ return TaskSources(+ BrowserSemanticSource(snapshot),+ None+ if visual is None+ else VisualRegionSource(visual, visual_delivery, capture_bound_click),+ visual_path,+ )+++# Compatibility entry points. They keep the pre-task-spec signatures and+# delegate to the built-in fixture task, so existing callers are unchanged.+++def visual_submit_region(visual: VisualObservation | None) -> VisualRegion | None:+ """Return the unique validated visual Submit region, or ``None``."""+ if visual is None:+ return None+ control = VisualRegionSource(visual).find("button", SUBMIT_NAME)+ return control.handle if control is not None else None+++def form_state(+ snapshot: Mapping[str, Any],+ token: str,+ visual: VisualObservation | None = None,+ *,+ visual_path: bool = False,+) -> dict[str, str]:+ """Summarize the fixture form; see ``FixtureFormTask.state_summary``."""+ return FixtureFormTask(token).state_summary(+ fixture_sources(snapshot, visual, visual_path=visual_path)+ )+++def build_candidates(+ snapshot: Mapping[str, Any],+ token: str,+ visual: VisualObservation | None = None,+ *,+ capture_bound_click: bool = False,+ visual_delivery: VisualDelivery = "background",+) -> list[Candidate]:+ """Build the fixture candidate set; see ``FixtureFormTask.candidates``."""+ return FixtureFormTask(token).candidates(+ fixture_sources(+ snapshot,+ visual,+ capture_bound_click=capture_bound_click,+ visual_delivery=visual_delivery,+ )+ )diff --git a/libs/cua-driver/examples/jev-use/python/tests/provider_request_golden.py b/libs/cua-driver/examples/jev-use/python/tests/provider_request_golden.pynew file mode 100644index 0000000000..5f377fce7b--- /dev/null+++ b/libs/cua-driver/examples/jev-use/python/tests/provider_request_golden.py@@ -0,0 +1,306 @@+"""Record the exact TypeSafe Jev request payloads for the page and visual fixtures.++The golden file ``fixtures/jev-provider-request-golden-python-v1.json`` was+captured from this module on ``main`` before the candidate-source and task-spec+refactor (RFC #4268 Phase 0). ``test_provider_request_golden.py`` replays the+same scenarios and requires byte-identical payloads, so any change to the model+input or to the candidate table is a test failure, not a silent drift.++Regenerate only for an intentional contract change::++ uv run python python/tests/provider_request_golden.py --write+"""++from __future__ import annotations++import argparse+import copy+import hashlib+import json+import sys+from dataclasses import replace+from pathlib import Path+from types import SimpleNamespace+from typing import Any, Callable++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from core import ( # noqa: E402+ SUBMIT_IDS,+ build_candidates,+ has_executable_candidate,+ history_entry,+ parse_visual_regions,+ validate_choice,+)+from jev_adapter import choose_with_typesafe # noqa: E402++ROOT = Path(__file__).resolve().parents[2]+FIXTURES = ROOT / "fixtures"+GOLDEN = FIXTURES / "jev-provider-request-golden-python-v1.json"+PAGE = json.loads((FIXTURES / "jev-page-structure-replay-v1.json").read_text(encoding="utf-8"))+VISUAL = json.loads((FIXTURES / "jev-visual-replay-v1.json").read_text(encoding="utf-8"))++# build(snapshot, token, visual, capture_bound_click, visual_delivery) -> candidates+Build = Callable[..., list]+# choose(client, candidates, snapshot, visual, history, token, visual_path) -> choice tuple+Choose = Callable[..., tuple]+++def observation(payload: dict[str, Any] | None = None):+ payload = payload or VISUAL["visual_regions"]+ source = payload["capture"]["source"]+ return parse_visual_regions(+ payload,+ expected_capture_id=payload["capture"]["capture_id"],+ expected_pid=source["pid"],+ expected_window_id=source["window_id"],+ )+++class RecordingClient:+ """Answer with scripted IDs and keep every request exactly as sent.++ A scripted ID absent from the offered criteria (for example a visual Submit+ when visual parsing is off) is answered with ``reobserve``.+ """++ def __init__(self, choices: list[str]) -> None:+ self.choices = list(choices)+ self.requests: list[dict[str, Any]] = []++ def system_one(self, **request: Any) -> Any:+ self.requests.append(request)+ choice = self.choices.pop(0)+ if choice not in request["questions"]["driver_action"].criteria:+ choice = "reobserve"+ answer = SimpleNamespace(choice=choice, confidence=1.0, probabilities={choice: 1.0})+ return SimpleNamespace(choices={"driver_action": answer})+++def wire_request(request: dict[str, Any]) -> dict[str, Any]:+ question = request["questions"]["driver_action"]+ return {+ "state": request["state"],+ "questions": {+ "driver_action": {+ "instructions": question.instructions,+ "criteria": question.criteria,+ }+ },+ }+++def wire_candidates(candidates: list) -> list[dict[str, Any]]:+ return [+ {+ "id": candidate.id,+ "description": candidate.description,+ "tool": candidate.tool,+ "arguments": json.loads(json.dumps(dict(candidate.arguments))),+ "capture_id": candidate.capture_id,+ "screenshot_reference": candidate.screenshot_reference,+ }+ for candidate in candidates+ ]+++def replay(+ build: Build,+ choose: Choose,+ fixture: dict[str, Any],+ run: dict[str, Any],+ *,+ capture_bound_click: bool,+ visual_mode: str,+) -> list[dict[str, Any]]:+ """Replay one recorded run the way the runner steps through it."""+ token = fixture["token"]+ before = fixture["snapshots"]["before_typing"]+ after = fixture["snapshots"]["after_typing"]+ visual_path = capture_bound_click and visual_mode != "off"+ history: list[dict[str, Any]] = []+ typed = False+ delivery = "background"+ records: list[dict[str, Any]] = []+ client = RecordingClient([step["selected_id"] for step in run["steps"]])+ for index, recorded in enumerate(run["steps"], start=1):+ step = recorded.get("step", index)+ snapshot = after if typed else before+ candidates = build(snapshot, token, None, capture_bound_click, delivery)+ visual = None+ if visual_mode == "always" or (+ visual_mode == "auto" and not has_executable_candidate(candidates)+ ):+ if capture_bound_click:+ visual = observation()+ candidates = build(snapshot, token, visual, capture_bound_click, delivery)+ choice, _, _ = choose(client, candidates, snapshot, visual, history, token, visual_path)+ candidate = validate_choice(+ choice, candidates, current_capture_id=visual.capture_id if visual else None+ )+ records.append(+ {+ "step": step,+ "request": wire_request(client.requests[-1]),+ "candidates": wire_candidates(candidates),+ }+ )+ refusal = recorded.get("action_error")+ history.append(history_entry(step, candidate.id, refusal=refusal))+ if candidate.id == "abstain":+ break+ if candidate.id == "type-verification-value":+ typed = True+ elif candidate.id in SUBMIT_IDS:+ if refusal:+ delivery = "foreground"+ else:+ break+ return records+++def with_field_value(snapshot: dict[str, Any], value: str | None) -> dict[str, Any]:+ changed = copy.deepcopy(snapshot)+ for ref in changed["refs"]:+ if ref.get("role") == "textbox" and ref.get("name") == "verification value":+ ref["value"] = value+ return changed+++def without_button(snapshot: dict[str, Any]) -> dict[str, Any]:+ changed = copy.deepcopy(snapshot)+ changed["refs"] = [ref for ref in changed["refs"] if ref.get("role") != "button"]+ return changed+++def single_requests(build: Build, choose: Choose) -> dict[str, Any]:+ """One request per distinct form and candidate state, with no history."""+ page_token = PAGE["token"]+ page_before = PAGE["snapshots"]["before_typing"]+ page_after = PAGE["snapshots"]["after_typing"]+ visual_token = VISUAL["token"]+ visual_after = VISUAL["snapshots"]["after_typing"]+ visual = observation()+ no_submit = replace(visual, regions=tuple(r for r in visual.regions if r.text != "Submit"))+ submit = next(r for r in visual.regions if r.text == "Submit")+ duplicated = replace(visual, regions=visual.regions + (replace(submit, id="dup"),))+ empty = {"target_id": "target", "tab_id": "tab", "refs": []}+ cases = {+ "page_other_value": (with_field_value(page_before, "other"), page_token, None, False, False, "background"),+ "page_after_with_visual": (page_after, page_token, visual, True, True, "background"),+ "page_after_no_button": (without_button(page_after), page_token, None, False, False, "background"),+ "page_after_no_button_visual_path": (without_button(page_after), page_token, None, True, True, "background"),+ "no_form_refs": (empty, page_token, None, False, False, "background"),+ "visual_after_no_capture_bound_click": (visual_after, visual_token, visual, False, True, "background"),+ "visual_after_without_submit_region": (visual_after, visual_token, no_submit, True, True, "background"),+ "visual_after_duplicate_submit_region": (visual_after, visual_token, duplicated, True, True, "background"),+ "visual_after_foreground": (visual_after, visual_token, visual, True, True, "foreground"),+ "visual_after_other_value": (with_field_value(visual_after, "other"), visual_token, visual, True, True, "background"),+ }+ result: dict[str, Any] = {}+ for name, (snapshot, token, current, cbc, visual_path, delivery) in cases.items():+ candidates = build(snapshot, token, current, cbc, delivery)+ client = RecordingClient(["abstain"])+ choose(client, candidates, snapshot, current, [], token, visual_path)+ result[name] = {+ "request": wire_request(client.requests[-1]),+ "candidates": wire_candidates(candidates),+ }+ return result+++def payloads(build: Build, choose: Choose) -> dict[str, Any]:+ runs: dict[str, Any] = {}+ modes = (+ ("page", PAGE, False, "auto"),+ ("page_capture_bound_auto", PAGE, True, "auto"),+ ("page_capture_bound_always", PAGE, True, "always"),+ ("visual_capture_bound_auto", VISUAL, True, "auto"),+ ("visual_capture_bound_always", VISUAL, True, "always"),+ ("visual_capture_bound_off", VISUAL, True, "off"),+ )+ for prefix, fixture, cbc, mode in modes:+ for name, run in fixture["recorded_live_runs"].items():+ runs[f"{prefix}/{name}"] = replay(+ build, choose, fixture, run, capture_bound_click=cbc, visual_mode=mode+ )+ return deduplicate(+ {+ "schema": "cua.jev_use_provider_request_golden_v1",+ "language": "python",+ "runs": runs,+ "single_requests": single_requests(build, choose),+ }+ )+++def _digest(value: Any) -> str:+ return hashlib.sha256(+ json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8")+ ).hexdigest()[:16]+++def deduplicate(result: dict[str, Any]) -> dict[str, Any]:+ """Store each distinct request and candidate table once, keyed by digest.++ Scenario records keep their order and reference the exact payloads, so the+ golden stays reviewable without repeating the visual regions per step.+ """+ requests: dict[str, Any] = {}+ tables: dict[str, Any] = {}++ def reference(record: dict[str, Any]) -> dict[str, Any]:+ request_id = _digest(record["request"])+ table_id = _digest(record["candidates"])+ requests.setdefault(request_id, record["request"])+ tables.setdefault(table_id, record["candidates"])+ refs = {"request": request_id, "candidates": table_id}+ return {"step": record["step"], **refs} if "step" in record else refs++ runs = {+ name: [reference(record) for record in records]+ for name, records in result["runs"].items()+ }+ singles = {name: reference(record) for name, record in result["single_requests"].items()}+ return {+ "schema": result["schema"],+ "language": result["language"],+ "runs": runs,+ "single_requests": singles,+ "requests": requests,+ "candidate_tables": tables,+ }+++def legacy_build(snapshot, token, visual, capture_bound_click, visual_delivery):+ return build_candidates(+ snapshot,+ token,+ visual,+ capture_bound_click=capture_bound_click,+ visual_delivery=visual_delivery,+ )+++def legacy_choose(client, candidates, snapshot, visual, history, token, visual_path):+ return choose_with_typesafe(client, candidates, snapshot, visual, history, token, visual_path)+++def encode(value: Any) -> str:+ return json.dumps(value, indent=1, ensure_ascii=False) + "\n"+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--write", action="store_true")+ args = parser.parse_args()+ text = encode(payloads(legacy_build, legacy_choose))+ if args.write:+ GOLDEN.write_text(text, encoding="utf-8")+ else:+ sys.stdout.write(text)+++if __name__ == "__main__":+ main()diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_provider_request_golden.py b/libs/cua-driver/examples/jev-use/python/tests/test_provider_request_golden.pynew file mode 100644index 0000000000..b9512f44f6--- /dev/null+++ b/libs/cua-driver/examples/jev-use/python/tests/test_provider_request_golden.py@@ -0,0 +1,63 @@+from __future__ import annotations++import sys+import unittest+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parent))+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from jev_adapter import choose_for_task # noqa: E402+from provider_request_golden import ( # noqa: E402+ GOLDEN,+ encode,+ legacy_build,+ legacy_choose,+ payloads,+)+from tasks import FixtureFormTask, fixture_sources # noqa: E402+++def task_build(snapshot, token, visual, capture_bound_click, visual_delivery):+ return FixtureFormTask(token).candidates(+ fixture_sources(+ snapshot,+ visual,+ capture_bound_click=capture_bound_click,+ visual_delivery=visual_delivery,+ )+ )+++def task_choose(client, candidates, snapshot, visual, history, token, visual_path):+ return choose_for_task(+ client,+ FixtureFormTask(token),+ fixture_sources(snapshot, visual, visual_path=visual_path),+ candidates,+ history,+ )+++class ProviderRequestGoldenTest(unittest.TestCase):+ """Provider requests and candidate tables match the capture from main.++ The golden was recorded before the candidate-source and task-spec refactor+ (RFC #4268 Phase 0), so equality proves the refactor sends byte-identical+ model input and offers identical executable candidates.+ """++ maxDiff = None++ def assert_golden(self, build, choose) -> None:+ self.assertEqual(encode(payloads(build, choose)), GOLDEN.read_text(encoding="utf-8"))++ def test_core_entry_points_send_byte_identical_requests(self) -> None:+ self.assert_golden(legacy_build, legacy_choose)++ def test_task_spec_and_sources_send_byte_identical_requests(self) -> None:+ self.assert_golden(task_build, task_choose)+++if __name__ == "__main__":+ unittest.main()diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_sources_and_tasks.py b/libs/cua-driver/examples/jev-use/python/tests/test_sources_and_tasks.pynew file mode 100644index 0000000000..89deecace1--- /dev/null+++ b/libs/cua-driver/examples/jev-use/python/tests/test_sources_and_tasks.py@@ -0,0 +1,125 @@+from __future__ import annotations++import json+import sys+import unittest+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from core import REDACTED_TOKEN, parse_visual_regions+from sources import BrowserSemanticSource, CandidateSource, VisualRegionSource+from tasks import (+ FIXTURE_GOAL,+ SUBMIT_IDS,+ FixtureFormTask,+ Task,+ TaskSources,+ fixture_sources,+)++FIXTURES = Path(__file__).resolve().parents[2] / "fixtures"+PAGE = json.loads((FIXTURES / "jev-page-structure-replay-v1.json").read_text(encoding="utf-8"))+VISUAL = json.loads((FIXTURES / "jev-visual-replay-v1.json").read_text(encoding="utf-8"))+PAYLOAD = VISUAL["visual_regions"]+++def observation():+ source = PAYLOAD["capture"]["source"]+ return parse_visual_regions(+ PAYLOAD,+ expected_capture_id=PAYLOAD["capture"]["capture_id"],+ expected_pid=source["pid"],+ expected_window_id=source["window_id"],+ )+++class CandidateSourceTest(unittest.TestCase):+ def test_sources_implement_one_interface_with_distinct_kinds(self) -> None:+ page: CandidateSource = BrowserSemanticSource(PAGE["snapshots"]["before_typing"])+ visual: CandidateSource = VisualRegionSource(observation())+ self.assertEqual((page.kind, visual.kind), ("page", "visual"))++ def test_page_source_addresses_refs_with_exact_browser_arguments(self) -> None:+ snapshot = PAGE["snapshots"]["after_typing"]+ page = BrowserSemanticSource(snapshot)+ button = page.find("button", "Submit")+ self.assertIsNotNone(button)+ self.assertEqual((button.source, button.handle["ref"]), ("page", "p4:2"))+ candidate = page.click(button, candidate_id="c", description="d")+ self.assertEqual(candidate.tool, "browser_click")+ self.assertEqual(+ dict(candidate.arguments),+ {+ "target_id": snapshot["target_id"],+ "tab_id": snapshot["tab_id"],+ "ref": "p4:2",+ "input_route": "dom_event",+ },+ )+ self.assertIsNone(page.find("button", "Cancel"))++ def test_visual_source_clicks_only_capture_bound_and_never_types(self) -> None:+ visual = observation()+ source = VisualRegionSource(visual)+ submit = source.find("button", "submit")+ self.assertIsNotNone(submit)+ self.assertIsNone(source.click(submit, candidate_id="c", description="d"))+ self.assertIsNone(source.type_text(submit, "x", candidate_id="c", description="d"))+ bound = VisualRegionSource(visual, "foreground", capture_bound=True)+ candidate = bound.click(submit, candidate_id="c", description="d")+ self.assertEqual(candidate.tool, "click")+ self.assertEqual(candidate.capture_id, visual.capture_id)+ self.assertEqual(candidate.arguments["delivery_mode"], "foreground")+ self.assertEqual(+ (candidate.arguments["x"], candidate.arguments["y"]),+ visual.screenshot_center(submit.handle),+ )+++class FixtureTaskSpecTest(unittest.TestCase):+ def test_built_in_task_declares_its_spec(self) -> None:+ task: Task = FixtureFormTask("secret-token", "http://127.0.0.1:9/", 3)+ self.assertEqual(task.goal, FIXTURE_GOAL)+ self.assertEqual(task.max_steps, 3)+ self.assertEqual(task.completion_candidate_ids, SUBMIT_IDS)+ self.assertEqual(task.allowed_action_kinds, {"browser_type", "browser_click", "click"})+ [parameter] = task.parameters+ self.assertTrue(parameter.secret)+ self.assertEqual((parameter.value, parameter.redaction), ("secret-token", REDACTED_TOKEN))+ self.assertEqual(+ task.redact({"a": ["x secret-token y"]}), {"a": [f"x {REDACTED_TOKEN} y"]}+ )++ def test_oracle_classification_uses_the_task_budget(self) -> None:+ task = FixtureFormTask("t", max_steps=2)+ self.assertEqual(task.classify({"submitted": "t"}, steps=0), "verified")+ self.assertEqual(task.classify({"submitted": "other"}, steps=0), "refuted")+ self.assertEqual(task.classify({"submitted": None}, steps=1), "unknown")+ self.assertEqual(task.classify({"submitted": None}, steps=2), "budget_exhausted")++ def test_candidates_outside_the_allowed_action_kinds_are_refused(self) -> None:+ task = FixtureFormTask(PAGE["token"])+ object.__setattr__(task, "allowed_action_kinds", frozenset({"browser_type"}))+ sources = fixture_sources(PAGE["snapshots"]["after_typing"])+ with self.assertRaisesRegex(ValueError, "browser_click"):+ task.candidates(sources)+ before = task.candidates(fixture_sources(PAGE["snapshots"]["before_typing"]))+ self.assertEqual(before[0].tool, "browser_type")++ def test_state_summary_reads_the_step_sources(self) -> None:+ task = FixtureFormTask(VISUAL["token"])+ after = VISUAL["snapshots"]["after_typing"]+ pending = TaskSources(BrowserSemanticSource(after), None, visual_path=True)+ self.assertEqual(task.state_summary(pending)["submit_button"], "visual_check_pending")+ parsed = TaskSources(+ BrowserSemanticSource(after), VisualRegionSource(observation()), visual_path=True+ )+ self.assertEqual(+ task.state_summary(parsed),+ {"verification_field": "contains_required_token", "submit_button": "visual_only"},+ )+++if __name__ == "__main__":+ unittest.main()diff --git a/libs/cua-driver/examples/jev-use/typescript/core.ts b/libs/cua-driver/examples/jev-use/typescript/core.tsindex 55d860917e..108cad6f30 100644--- a/libs/cua-driver/examples/jev-use/typescript/core.ts+++ b/libs/cua-driver/examples/jev-use/typescript/core.ts@@ -1,18 +1,33 @@-export type Outcome = 'verified' | 'refuted' | 'unknown' | 'abstained' | 'budget_exhausted';-export type VisualDelivery = 'background' | 'foreground';--export const SUBMIT_IDS: ReadonlySet<string> = new Set(['submit-form', 'submit-form-foreground']);+/**+ * Runner primitives shared by every jev-use task: visual-region parsing,+ * candidate validation, and the mock chooser. Candidate sources are in+ * sources.ts and task specs in tasks.ts; the names re-exported below keep+ * existing imports from core.ts working.+ */+import type { Candidate } from './sources.js';++export type { Candidate } from './sources.js';+export {+ FIELD_NAME,+ HISTORY_OUTCOMES,+ REDACTED_TOKEN,+ SUBMIT_IDS,+ SUBMIT_NAME,+ buildCandidates,+ classify,+ formState,+ historyEntry,+ redactToken,+ visualSubmitRegion,+ type FormState,+ type HistoryEntry,+ type Outcome,+ type SubmitButtonState,+} from './tasks.js';-export type Candidate = Readonly<{- id: string;- description: string;- tool: string | null;- arguments: Readonly<Record<string, unknown>>;- captureId?: string;- screenshotReference?: string;-}>;+export type VisualDelivery = 'background' | 'foreground';-type PageRef = {+export type PageRef = {role?: string;name?: string | null;ref?: string;@@ -91,10 +106,6 @@ function pixelInt(value: unknown): number {return Number(value);}-function immutableCandidate(candidate: Candidate): Candidate {- return Object.freeze({ ...candidate, arguments: Object.freeze({ ...candidate.arguments }) });-}-export function parseVisualRegions(payload: unknown,expectedCaptureId: string,@@ -210,223 +221,10 @@ export function parseVisualRegions(});}-export const REDACTED_TOKEN = '[verification token]';--function formRefs(snapshot: BrowserSnapshot) {- const refs = snapshot.refs ?? [];- const field = refs.find(- (item) => item.role === 'textbox' && item.name === 'verification value' && item.ref- );- const button = refs.find((item) => item.role === 'button' && item.name === 'Submit' && item.ref);- return { field, button };-}--export type SubmitButtonState =- | 'available'- | 'visual_only'- | 'visual_check_pending'- | 'not_found_visually'- | 'not_in_page_structure';--export type FormState = Readonly<{- verification_field: 'not_found' | 'empty' | 'contains_required_token' | 'contains_other_value';- submit_button: SubmitButtonState;-}>;--/** Return the unique validated visual Submit region, or undefined. */-export function visualSubmitRegion(visual?: VisualObservation) {- if (!visual) return undefined;- const matches = visual.regions.filter(- (region) =>- region.confidence >= 0.8 && asciiLower(region.text ?? region.label ?? '') === 'submit'- );- return matches.length === 1 ? matches[0] : undefined;-}--/**- * Summarize the form for the decision model without revealing the token. The- * raw field value never leaves the runner.- *- * submit_button is 'available' for a clickable page-structure ref. When the- * page structure has none and the capture-bound visual path is enabled- * (visualPath), it is 'visual_only' if this observation holds a unique- * validated visual Submit region, 'visual_check_pending' if no visual regions- * were parsed for this step yet (the runner parses them once no page-structure- * action remains), and 'not_found_visually' otherwise. Without a visual path it- * is 'not_in_page_structure'.- */-export function formState(- snapshot: BrowserSnapshot,- token: string,- visual?: VisualObservation,- visualPath = false-): FormState {- const { field, button } = formRefs(snapshot);- const verificationField = !field- ? 'not_found'- : !field.value- ? 'empty'- : field.value === token- ? 'contains_required_token'- : 'contains_other_value';- const submitButton: SubmitButtonState = button- ? 'available'- : !visualPath- ? 'not_in_page_structure'- : !visual- ? 'visual_check_pending'- : visualSubmitRegion(visual)- ? 'visual_only'- : 'not_found_visually';- return { verification_field: verificationField, submit_button: submitButton };-}--/** Replace every occurrence of the token in strings nested in value. */-export function redactToken(value: unknown, token: string): unknown {- if (!token) return value;- if (typeof value === 'string') return value.split(token).join(REDACTED_TOKEN);- if (Array.isArray(value)) return value.map((item) => redactToken(item, token));- if (value && typeof value === 'object') {- return Object.fromEntries(- Object.entries(value).map(([key, item]) => [key, redactToken(item, token)])- );- }- return value;-}--export type HistoryEntry = Readonly<{ step: number; selected_id: string; outcome: string }>;--const HISTORY_OUTCOMES: Readonly<Record<string, string>> = {- 'type-verification-value': 'typed the required token into the verification field',- 'submit-form': 'clicked Submit; the submission was not yet confirmed',- 'submit-form-foreground':- 'clicked Submit in the foreground; the submission was not yet confirmed',- reobserve: 'took no action and requested a fresh observation',-};--/**- * Build the compact decision-history item shown to the model. It records what- * each step did, not timings or model probabilities, so earlier choices do not- * become a signal to repeat themselves.- */-export function historyEntry(step: number, candidateId: string, refusal?: string): HistoryEntry {- const outcome = refusal- ? `Driver refused background delivery (${refusal}); no click happened and a ` +- 'foreground Submit candidate is offered next'- : (HISTORY_OUTCOMES[candidateId] ?? 'completed');- return { step, selected_id: candidateId, outcome };-}--function reservedCandidates(): Candidate[] {- return [- immutableCandidate({- id: 'reobserve',- description:- 'Take no action and obtain a fresh Driver observation, because the current ' +- 'observation is stale or contradicts the reported form state. A Submit control ' +- 'that is visual-only, or whose visual check is still pending, is not a reason to ' +- 'reobserve: the runner parses visual regions for Submit once no page-structure ' +- 'action remains.',- tool: null,- arguments: {},- }),- immutableCandidate({- id: 'abstain',- description: 'Stop without acting if none of the proposed actions is safe for the observed state.',- tool: null,- arguments: {},- }),- ];-}--/**- * Build the closed candidate set for one decision. Page-structure refs always- * win. The capture-bound visual Submit is offered only when no Submit ref- * exists. visualDelivery 'foreground' replaces the background visual click with- * a distinct submit-form-foreground candidate after Driver refused background- * delivery; the chooser must pick it explicitly.- */-export function buildCandidates(- snapshot: BrowserSnapshot,- token: string,- visual?: VisualObservation,- captureBoundClick = false,- visualDelivery: VisualDelivery = 'background'-): Candidate[] {- const common = { target_id: snapshot.target_id, tab_id: snapshot.tab_id };- const { field, button } = formRefs(snapshot);- const candidates: Candidate[] = [];- if (field?.value !== token && field?.ref) {- candidates.push(- immutableCandidate({- id: 'type-verification-value',- description:- 'Type the required verification token into the verification field, ' +- 'replacing its current contents.',- tool: 'browser_type',- arguments: { ...common, ref: field.ref, text: token, replace: true },- })- );- } else if (field?.value === token && button?.ref) {- candidates.push(- immutableCandidate({- id: 'submit-form',- description:- "Click the form's Submit button. The observed form state reports that the " +- 'verification field already contains the required token, so the form is ' +- 'ready to submit.',- tool: 'browser_click',- arguments: { ...common, ref: button.ref, input_route: 'dom_event' },- })- );- } else if (- field?.value === token &&- visual &&- captureBoundClick- ) {- const region = visualSubmitRegion(visual);- if (region) {- const x = region.x + region.width / 2;- const y = region.y + region.height / 2;- const foreground = visualDelivery === 'foreground';- candidates.push(- immutableCandidate({- id: foreground ? 'submit-form-foreground' : 'submit-form',- description: foreground- ? 'Submit the form by clicking the unique validated visual Submit region with ' +- 'foreground delivery, which activates the browser window, because Driver ' +- 'refused background delivery for the previous visual click.'- : 'Submit the form by clicking the unique validated visual Submit region. ' +- 'The observed form state reports that the verification field already ' +- 'contains the required token.',- tool: 'click',- arguments: {- pid: visual.pid,- window_id: visual.windowId,- x,- y,- capture_id: visual.captureId,- delivery_mode: visualDelivery,- },- captureId: visual.captureId,- screenshotReference: visual.screenshotReference,- })- );- }- }- return [...candidates, ...reservedCandidates()];-}-export function hasExecutableCandidate(candidates: readonly Candidate[]): boolean {return candidates.some((candidate) => candidate.tool !== null);}-function asciiLower(value: string): string {- return value.replace(/[A-Z]/g, (character) =>- String.fromCharCode(character.charCodeAt(0) + 32)- );-}-export function chooseMock(candidates: Candidate[]) {const ids = new Set(candidates.map((candidate) => candidate.id));const selected = ids.has('type-verification-value')@@ -462,15 +260,3 @@ export function validateChoice(}return candidate;}--export function classify(- submitted: string | null,- token: string,- steps: number,- maxSteps: number-): Outcome {- if (submitted === token) return 'verified';- if (submitted !== null) return 'refuted';- if (steps >= maxSteps) return 'budget_exhausted';- return 'unknown';-}diff --git a/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts b/libs/cua-driver/examples/jev-use/typescript/jev_adapter.tsindex 5cec3b910c..482e20bd28 100644--- a/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts+++ b/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts@@ -2,13 +2,12 @@ import { choice, TypeSafeClient } from '@typesafe-ai/sdk';import {chooseMock,- formState,- redactToken,type BrowserSnapshot,type HistoryEntry,type Candidate,type VisualObservation,} from './core.js';+import { FIXTURE_GOAL, FixtureFormTask, fixtureSources, type Task, type TaskSources } from './tasks.js';type TypeSafeClientLike = Pick<TypeSafeClient, 'systemOne'>;@@ -99,51 +98,63 @@ export function visualDecisionState(visual?: VisualObservation) {};}-export const GOAL =- 'Enter the required verification token into the verification field, then submit the form.';+export const GOAL = FIXTURE_GOAL;/**- * Build the compact, deterministic, token-redacted state sent to Jev. `form`- * states the field and Submit status the runner verified from the page- * structure and, when visualPath is enabled, from the validated visual regions;- * the token is replaced everywhere, including outline and visual text.+ * Build the compact, deterministic, secret-redacted state sent to Jev. `form`+ * is the task's state summary, which states what the runner verified from its+ * candidate sources, so the model does not have to infer it from the outline.+ * Every secret task parameter is replaced everywhere, including outline and+ * visual text.*/-export function decisionState(- snapshot: BrowserSnapshot,- visual: VisualObservation | undefined,- history: readonly HistoryEntry[],- token: string,- visualPath = false+export function taskDecisionState(+ task: Task,+ sources: TaskSources,+ history: readonly HistoryEntry[]) {+ const snapshot = sources.page.snapshot;return {- goal: GOAL,+ goal: task.goal,observation: {- page: JSON.stringify(redactToken(snapshot.page ?? null, token)),- form: JSON.stringify(formState(snapshot, token, visual, visualPath)),- outline: redactToken(snapshot.outline ?? '', token) as string,- visual: JSON.stringify(redactToken(visualDecisionState(visual), token)),+ page: JSON.stringify(task.redact(snapshot.page ?? null)),+ form: JSON.stringify(task.stateSummary(sources)),+ outline: task.redact(snapshot.outline ?? '') as string,+ visual: JSON.stringify(task.redact(visualDecisionState(sources.visual?.observation))),},history: JSON.stringify(history),};}-export async function chooseWithTypeSafe(- client: TypeSafeClientLike,- candidates: Candidate[],+/** Build the fixture task's decision state; see taskDecisionState. */+export function decisionState(snapshot: BrowserSnapshot,visual: VisualObservation | undefined,history: readonly HistoryEntry[],token: string,visualPath = false+) {+ return taskDecisionState(+ new FixtureFormTask(token),+ fixtureSources(snapshot, visual, false, 'background', visualPath),+ history+ );+}++export const DRIVER_ACTION_INSTRUCTIONS =+ 'Which complete executable action should Cua Driver run next?';++export async function chooseForTask(+ client: TypeSafeClientLike,+ task: Task,+ sources: TaskSources,+ candidates: Candidate[],+ history: readonly HistoryEntry[]) {const criteria = candidateCriteria(candidates);const response = await client.systemOne({- state: decisionState(snapshot, visual, history, token, visualPath),+ state: taskDecisionState(task, sources, history),questions: {- driver_action: choice(- 'Which complete executable action should Cua Driver run next?',- criteria- ),+ driver_action: choice(DRIVER_ACTION_INSTRUCTIONS, criteria),},});const answer = response.answers.driver_action;@@ -154,6 +165,42 @@ export async function chooseWithTypeSafe(return answer;}+export function chooseWithTypeSafe(+ client: TypeSafeClientLike,+ candidates: Candidate[],+ snapshot: BrowserSnapshot,+ visual: VisualObservation | undefined,+ history: readonly HistoryEntry[],+ token: string,+ visualPath = false+) {+ return chooseForTask(+ client,+ new FixtureFormTask(token),+ fixtureSources(snapshot, visual, false, 'background', visualPath),+ candidates,+ history+ );+}++export function chooseLiveForTask(+ task: Task,+ sources: TaskSources,+ candidates: Candidate[],+ history: readonly HistoryEntry[]+) {+ return chooseForTask(new TypeSafeClient(), task, sources, candidates, history);+}++export function chooseMockForTask(+ _task: Task,+ _sources: TaskSources,+ candidates: Candidate[],+ _history: readonly HistoryEntry[]+) {+ return chooseMock(candidates);+}+export function chooseLive(candidates: Candidate[],snapshot: BrowserSnapshot,@@ -162,14 +209,11 @@ export function chooseLive(token: string,visualPath = false) {- return chooseWithTypeSafe(- new TypeSafeClient(),+ return chooseLiveForTask(+ new FixtureFormTask(token),+ fixtureSources(snapshot, visual, false, 'background', visualPath),candidates,- snapshot,- visual,- history,- token,- visualPath+ history);}diff --git a/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.test.ts b/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.test.tsnew file mode 100644index 0000000000..d848a94207--- /dev/null+++ b/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.test.ts@@ -0,0 +1,44 @@+import assert from 'node:assert/strict';+import { readFileSync } from 'node:fs';+import test from 'node:test';++import { chooseForTask } from './jev_adapter.js';+import {+ GOLDEN,+ encode,+ legacyBuild,+ legacyChoose,+ payloads,+ type Build,+ type Choose,+} from './provider_request_golden.js';+import { FixtureFormTask, fixtureSources } from './tasks.js';++// The golden was recorded before the candidate-source and task-spec refactor+// (RFC #4268 Phase 0), so equality proves the refactor sends byte-identical+// model input and offers identical executable candidates.+async function assertGolden(build: Build, choose: Choose) {+ assert.equal(encode(await payloads(build, choose)), readFileSync(GOLDEN, 'utf8'));+}++const taskBuild: Build = (snapshot, token, visual, captureBoundClick, visualDelivery) =>+ new FixtureFormTask(token).candidates(+ fixtureSources(snapshot, visual, captureBoundClick, visualDelivery)+ );++const taskChoose: Choose = (client, candidates, snapshot, visual, history, token, visualPath) =>+ chooseForTask(+ client as never,+ new FixtureFormTask(token),+ fixtureSources(snapshot, visual, false, 'background', visualPath),+ candidates,+ history+ );++test('core entry points send byte-identical provider requests', async () => {+ await assertGolden(legacyBuild, legacyChoose);+});++test('task spec and sources send byte-identical provider requests', async () => {+ await assertGolden(taskBuild, taskChoose);+});diff --git a/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.ts b/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.tsnew file mode 100644index 0000000000..a296578df4--- /dev/null+++ b/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.ts@@ -0,0 +1,303 @@+/**+ * Record the exact TypeSafe Jev request payloads for the page and visual+ * fixtures.+ *+ * fixtures/jev-provider-request-golden-typescript-v1.json was captured from this+ * module on main before the candidate-source and task-spec refactor (RFC #4268+ * Phase 0). provider_request_golden.test.ts replays the same scenarios and+ * requires byte-identical payloads, so any change to the model input or the+ * candidate table is a test failure, not a silent drift.+ *+ * Regenerate only for an intentional contract change:+ *+ * npx tsx typescript/provider_request_golden.ts --write+ */+import { createHash } from 'node:crypto';+import { readFileSync, writeFileSync } from 'node:fs';+import process from 'node:process';+import { pathToFileURL } from 'node:url';++import {+ SUBMIT_IDS,+ buildCandidates,+ hasExecutableCandidate,+ historyEntry,+ parseVisualRegions,+ validateChoice,+ type BrowserSnapshot,+ type Candidate,+ type HistoryEntry,+ type VisualDelivery,+ type VisualObservation,+} from './core.js';+import { chooseWithTypeSafe } from './jev_adapter.js';++export const GOLDEN = new URL(+ '../fixtures/jev-provider-request-golden-typescript-v1.json',+ import.meta.url+);++const load = (name: string) =>+ JSON.parse(readFileSync(new URL(`../fixtures/${name}`, import.meta.url), 'utf8'));+const PAGE = load('jev-page-structure-replay-v1.json');+const VISUAL = load('jev-visual-replay-v1.json');++export type Build = (+ snapshot: BrowserSnapshot,+ token: string,+ visual: VisualObservation | undefined,+ captureBoundClick: boolean,+ visualDelivery: VisualDelivery+) => Candidate[];++export type Choose = (+ client: RecordingClient,+ candidates: Candidate[],+ snapshot: BrowserSnapshot,+ visual: VisualObservation | undefined,+ history: readonly HistoryEntry[],+ token: string,+ visualPath: boolean+) => Promise<{ choice: string }>;++type Fixture = {+ token: string;+ snapshots: { before_typing: BrowserSnapshot; after_typing: BrowserSnapshot };+ recorded_live_runs: Record<string, { steps: RecordedStep[] }>;+};++type RecordedStep = { step?: number; selected_id: string; action_error?: string };++type WireRecord = { step?: number; request: unknown; candidates: unknown };++export function observation(payload = VISUAL.visual_regions): VisualObservation {+ const source = payload.capture.source;+ return parseVisualRegions(payload, payload.capture.capture_id, source.pid, source.window_id);+}++/**+ * Answer with scripted IDs and keep every request exactly as sent. A scripted+ * ID absent from the offered criteria (for example a visual Submit when visual+ * parsing is off) is answered with reobserve.+ */+export class RecordingClient {+ readonly requests: any[] = [];+ private readonly choices: string[];++ constructor(choices: string[]) {+ this.choices = [...choices];+ }++ async systemOne(request: any) {+ this.requests.push(request);+ let selected = this.choices.shift()!;+ if (!Object.hasOwn(request.questions.driver_action.criteria, selected)) selected = 'reobserve';+ return {+ answers: {+ driver_action: {+ type: 'choice' as const,+ choice: selected,+ confidence: 1,+ probabilities: { [selected]: 1 },+ },+ },+ };+ }+}++function wireCandidates(candidates: readonly Candidate[]) {+ return candidates.map((candidate) => ({+ id: candidate.id,+ description: candidate.description,+ tool: candidate.tool,+ arguments: JSON.parse(JSON.stringify(candidate.arguments)),+ capture_id: candidate.captureId ?? null,+ screenshot_reference: candidate.screenshotReference ?? null,+ }));+}++function wireRequest(request: any) {+ return JSON.parse(JSON.stringify(request));+}++async function replay(+ build: Build,+ choose: Choose,+ fixture: Fixture,+ run: { steps: RecordedStep[] },+ captureBoundClick: boolean,+ visualMode: 'auto' | 'always' | 'off'+): Promise<WireRecord[]> {+ const token = fixture.token;+ const before = fixture.snapshots.before_typing;+ const after = fixture.snapshots.after_typing;+ const visualPath = captureBoundClick && visualMode !== 'off';+ const history: HistoryEntry[] = [];+ let typed = false;+ let delivery: VisualDelivery = 'background';+ const records: WireRecord[] = [];+ const client = new RecordingClient(run.steps.map((step) => step.selected_id));+ for (const [offset, recorded] of run.steps.entries()) {+ const step = recorded.step ?? offset + 1;+ const snapshot = typed ? after : before;+ let candidates = build(snapshot, token, undefined, captureBoundClick, delivery);+ let visual: VisualObservation | undefined;+ if (+ visualMode === 'always' ||+ (visualMode === 'auto' && !hasExecutableCandidate(candidates))+ ) {+ if (captureBoundClick) {+ visual = observation();+ candidates = build(snapshot, token, visual, captureBoundClick, delivery);+ }+ }+ const answer = await choose(client, candidates, snapshot, visual, history, token, visualPath);+ const candidate = validateChoice(answer.choice, candidates, visual?.captureId);+ records.push({+ step,+ request: wireRequest(client.requests.at(-1)),+ candidates: wireCandidates(candidates),+ });+ const refusal = recorded.action_error;+ history.push(historyEntry(step, candidate.id, refusal));+ if (candidate.id === 'abstain') break;+ if (candidate.id === 'type-verification-value') {+ typed = true;+ } else if (SUBMIT_IDS.has(candidate.id)) {+ if (refusal) delivery = 'foreground';+ else break;+ }+ }+ return records;+}++function withFieldValue(snapshot: BrowserSnapshot, value: string | null): BrowserSnapshot {+ const changed = structuredClone(snapshot);+ for (const ref of changed.refs ?? []) {+ if (ref.role === 'textbox' && ref.name === 'verification value') ref.value = value;+ }+ return changed;+}++function withoutButton(snapshot: BrowserSnapshot): BrowserSnapshot {+ const changed = structuredClone(snapshot);+ changed.refs = (changed.refs ?? []).filter((ref) => ref.role !== 'button');+ return changed;+}++async function singleRequests(build: Build, choose: Choose) {+ const pageToken: string = PAGE.token;+ const pageBefore: BrowserSnapshot = PAGE.snapshots.before_typing;+ const pageAfter: BrowserSnapshot = PAGE.snapshots.after_typing;+ const visualToken: string = VISUAL.token;+ const visualAfter: BrowserSnapshot = VISUAL.snapshots.after_typing;+ const visual = observation();+ const noSubmit = { ...visual, regions: visual.regions.filter((r) => r.text !== 'Submit') };+ const submit = visual.regions.find((r) => r.text === 'Submit')!;+ const duplicated = { ...visual, regions: [...visual.regions, { ...submit, id: 'dup' }] };+ const empty: BrowserSnapshot = { target_id: 'target', tab_id: 'tab', refs: [] };+ const cases: Record<+ string,+ [BrowserSnapshot, string, VisualObservation | undefined, boolean, boolean, VisualDelivery]+ > = {+ page_other_value: [withFieldValue(pageBefore, 'other'), pageToken, undefined, false, false, 'background'],+ page_after_with_visual: [pageAfter, pageToken, visual, true, true, 'background'],+ page_after_no_button: [withoutButton(pageAfter), pageToken, undefined, false, false, 'background'],+ page_after_no_button_visual_path: [withoutButton(pageAfter), pageToken, undefined, true, true, 'background'],+ no_form_refs: [empty, pageToken, undefined, false, false, 'background'],+ visual_after_no_capture_bound_click: [visualAfter, visualToken, visual, false, true, 'background'],+ visual_after_without_submit_region: [visualAfter, visualToken, noSubmit, true, true, 'background'],+ visual_after_duplicate_submit_region: [visualAfter, visualToken, duplicated, true, true, 'background'],+ visual_after_foreground: [visualAfter, visualToken, visual, true, true, 'foreground'],+ visual_after_other_value: [withFieldValue(visualAfter, 'other'), visualToken, visual, true, true, 'background'],+ };+ const result: Record<string, WireRecord> = {};+ for (const [name, [snapshot, token, current, cbc, visualPath, delivery]] of Object.entries(cases)) {+ const candidates = build(snapshot, token, current, cbc, delivery);+ const client = new RecordingClient(['abstain']);+ await choose(client, candidates, snapshot, current, [], token, visualPath);+ result[name] = {+ request: wireRequest(client.requests.at(-1)),+ candidates: wireCandidates(candidates),+ };+ }+ return result;+}++function digest(value: unknown): string {+ return createHash('sha256').update(JSON.stringify(value), 'utf8').digest('hex').slice(0, 16);+}++/**+ * Store each distinct request and candidate table once, keyed by digest.+ * Scenario records keep their order and reference the exact payloads.+ */+function deduplicate(+ runs: Record<string, WireRecord[]>,+ singles: Record<string, WireRecord>+) {+ const requests: Record<string, unknown> = {};+ const tables: Record<string, unknown> = {};+ const reference = (record: WireRecord) => {+ const requestId = digest(record.request);+ const tableId = digest(record.candidates);+ requests[requestId] ??= record.request;+ tables[tableId] ??= record.candidates;+ const refs = { request: requestId, candidates: tableId };+ return record.step === undefined ? refs : { step: record.step, ...refs };+ };+ return {+ schema: 'cua.jev_use_provider_request_golden_v1',+ language: 'typescript',+ runs: Object.fromEntries(+ Object.entries(runs).map(([name, records]) => [name, records.map(reference)])+ ),+ single_requests: Object.fromEntries(+ Object.entries(singles).map(([name, record]) => [name, reference(record)])+ ),+ requests,+ candidate_tables: tables,+ };+}++export async function payloads(build: Build, choose: Choose) {+ const modes: [string, Fixture, boolean, 'auto' | 'always' | 'off'][] = [+ ['page', PAGE, false, 'auto'],+ ['page_capture_bound_auto', PAGE, true, 'auto'],+ ['page_capture_bound_always', PAGE, true, 'always'],+ ['visual_capture_bound_auto', VISUAL, true, 'auto'],+ ['visual_capture_bound_always', VISUAL, true, 'always'],+ ['visual_capture_bound_off', VISUAL, true, 'off'],+ ];+ const runs: Record<string, WireRecord[]> = {};+ for (const [prefix, fixture, cbc, mode] of modes) {+ for (const [name, run] of Object.entries(fixture.recorded_live_runs)) {+ runs[`${prefix}/${name}`] = await replay(build, choose, fixture, run, cbc, mode);+ }+ }+ return deduplicate(runs, await singleRequests(build, choose));+}++export const legacyBuild: Build = (snapshot, token, visual, captureBoundClick, visualDelivery) =>+ buildCandidates(snapshot, token, visual, captureBoundClick, visualDelivery);++export const legacyChoose: Choose = (+ client,+ candidates,+ snapshot,+ visual,+ history,+ token,+ visualPath+) =>+ chooseWithTypeSafe(client as never, candidates, snapshot, visual, history, token, visualPath);++export function encode(value: unknown): string {+ return `${JSON.stringify(value, null, 1)}\n`;+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ const text = encode(await payloads(legacyBuild, legacyChoose));+ if (process.argv.includes('--write')) writeFileSync(GOLDEN, text, 'utf8');+ else process.stdout.write(text);+}diff --git a/libs/cua-driver/examples/jev-use/typescript/run.ts b/libs/cua-driver/examples/jev-use/typescript/run.tsindex d34b97da92..853153b5d6 100644--- a/libs/cua-driver/examples/jev-use/typescript/run.ts+++ b/libs/cua-driver/examples/jev-use/typescript/run.ts@@ -6,12 +6,8 @@ import { pathToFileURL } from 'node:url';import { Client } from '@modelcontextprotocol/sdk/client/index.js';import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';import {- buildCandidates,- classify,hasExecutableCandidate,- historyEntry,parseVisualRegions,- SUBMIT_IDS,validateChoice,VisualObservationError,type BrowserSnapshot,@@ -22,7 +18,8 @@ import {type VisualObservation,} from './core.js';import { driverEnvironment } from './driver_env.js';-import { chooseLive, chooseMockAdapter } from './jev_adapter.js';+import { chooseLiveForTask, chooseMockForTask } from './jev_adapter.js';+import { FixtureFormTask, fixtureSources, type Task, type TaskSources } from './tasks.js';type VisualMode = 'auto' | 'always' | 'off';@@ -248,29 +245,34 @@ export async function optionalVisualObservation(}/**- * Build one step's candidates, parsing visual regions only when useful. In- * auto mode the capture and parse run only when the page structure offers no- * executable candidate, because only then can a visual region add one. always- * restores the per-step parse; off never parses.+ * Build one step's sources and candidates, parsing visual regions only when+ * useful. In auto mode the capture and parse run only when the page structure+ * offers no executable candidate, because only then can a visual region add+ * one. always restores the per-step parse; off never parses.*/-export async function candidatesForStep(+export async function taskCandidatesForStep(driver: Driver,+ task: Task,snapshot: BrowserSnapshot,- token: string,pid: number,windowId: number,availableTools: ReadonlySet<string>,captureBoundClick: boolean,visualMode: VisualMode = 'auto',visualDelivery: VisualDelivery = 'background'-): Promise<{ candidates: Candidate[]; visual?: VisualObservation; status: VisualStatus }> {- let candidates = buildCandidates(snapshot, token, undefined, captureBoundClick, visualDelivery);+): Promise<{ candidates: Candidate[]; sources: TaskSources; status: VisualStatus }> {+ // Whether a control missing from the page structure can still be found+ // through a capture-bound visual region; reported in the task state.+ const visualPath = captureBoundClick && visualMode !== 'off';+ let sources = fixtureSources(snapshot, undefined, captureBoundClick, visualDelivery, visualPath);+ let candidates = task.candidates(sources);if (visualMode === 'off') {- return { candidates, status: visualStatus('skipped', undefined, undefined, 'disabled') };+ return { candidates, sources, status: visualStatus('skipped', undefined, undefined, 'disabled') };}if (visualMode === 'auto' && hasExecutableCandidate(candidates)) {return {candidates,+ sources,status: visualStatus('skipped', undefined, undefined, 'page_structure_candidate'),};}@@ -282,20 +284,36 @@ export async function candidatesForStep(captureBoundClick);if (visual) {- candidates = buildCandidates(snapshot, token, visual, captureBoundClick, visualDelivery);+ sources = fixtureSources(snapshot, visual, captureBoundClick, visualDelivery, visualPath);+ candidates = task.candidates(sources);}- return { candidates, visual, status };+ return { candidates, sources, status };}-async function fixtureState(fixtureUrl: string): Promise<{ submitted: string | null }> {- const response = await fetch(new URL('state', fixtureUrl));- if (!response.ok) throw new Error(`fixture state failed: HTTP ${response.status}`);- return (await response.json()) as { submitted: string | null };-}--async function resetFixture(fixtureUrl: string): Promise<void> {- const response = await fetch(new URL('reset', fixtureUrl), { method: 'POST' });- if (response.status !== 204) throw new Error(`fixture reset failed: HTTP ${response.status}`);+/** Build one step's fixture-task candidates; see taskCandidatesForStep. */+export async function candidatesForStep(+ driver: Driver,+ snapshot: BrowserSnapshot,+ token: string,+ pid: number,+ windowId: number,+ availableTools: ReadonlySet<string>,+ captureBoundClick: boolean,+ visualMode: VisualMode = 'auto',+ visualDelivery: VisualDelivery = 'background'+): Promise<{ candidates: Candidate[]; visual?: VisualObservation; status: VisualStatus }> {+ const { candidates, sources, status } = await taskCandidatesForStep(+ driver,+ new FixtureFormTask(token),+ snapshot,+ pid,+ windowId,+ availableTools,+ captureBoundClick,+ visualMode,+ visualDelivery+ );+ return { candidates, visual: sources.visual?.observation, status };}async function waitForWindow(driver: Driver, pid: number) {@@ -323,6 +341,7 @@ async function writeEvent(path: string | undefined, event: Record<string, unknowasync function run(args: Arguments): Promise<Outcome> {const token = args.token ?? `jev-${randomUUID().replaceAll('-', '').slice(0, 10)}`;+ const task: Task = new FixtureFormTask(token, args.fixtureUrl, args.maxSteps);const transport = new StdioClientTransport({command: process.env.CUA_DRIVER_BIN ?? 'cua-driver',args: ['mcp'],@@ -334,16 +353,13 @@ async function run(args: Arguments): Promise<Outcome> {const history: HistoryEntry[] = [];let visualDelivery: VisualDelivery = 'background';if (args.log) await writeFile(args.log, '', 'utf8');- await resetFixture(args.fixtureUrl);+ await task.reset();try {await client.connect(transport);const advertisedTools = (await client.listTools()).tools;const availableTools = new Set(advertisedTools.map((tool) => tool.name));const captureBoundClick = supportsCaptureBoundClick(advertisedTools);- // Whether a Submit missing from the page structure can still be found through- // a capture-bound visual region; reported in the form state.- const visualPath = captureBoundClick && args.visualObservation !== 'off';const driver = new Driver(client, `jev-typescript-${randomUUID().slice(0, 8)}`);const prepared = await driver.call('browser_prepare', {allow_launch: true,@@ -363,9 +379,8 @@ async function run(args: Arguments): Promise<Outcome> {url: args.fixtureUrl,});- for (let step = 1; step <= args.maxSteps; step += 1) {- const oracle = await fixtureState(args.fixtureUrl);- const current = classify(oracle.submitted, token, step - 1, args.maxSteps);+ for (let step = 1; step <= task.maxSteps; step += 1) {+ const current = task.classify(await task.readOracle(), step - 1);if (current === 'verified' || current === 'refuted') {await writeEvent(args.log, { event: 'outcome', outcome: current, token });return current;@@ -379,12 +394,12 @@ async function run(args: Arguments): Promise<Outcome> {})) as BrowserSnapshot;const {candidates,- visual,+ sources,status: visualRecord,- } = await candidatesForStep(+ } = await taskCandidatesForStep(driver,+ task,snapshot,- token,pid,Number(window.window_id),availableTools,@@ -401,10 +416,11 @@ async function run(args: Arguments): Promise<Outcome> {});return 'abstained';}+ const visual = sources.visual?.observation;const answer =args.provider === 'mock'- ? chooseMockAdapter(candidates, snapshot, visual, history, token, visualPath)- : await chooseLive(candidates, snapshot, visual, history, token, visualPath);+ ? chooseMockForTask(task, sources, candidates, history)+ : await chooseLiveForTask(task, sources, candidates, history);if (!answer.choice) return 'abstained';const candidate = validateChoice(answer.choice, candidates, visual?.captureId);const decisionMs = Math.round((performance.now() - decisionStarted) * 100) / 100;@@ -422,7 +438,7 @@ async function run(args: Arguments): Promise<Outcome> {tool: null,visual: visualRecord,};- history.push(historyEntry(step, candidate.id));+ history.push(task.historyEntry(step, candidate.id));await writeEvent(args.log, event);continue;}@@ -466,7 +482,7 @@ async function run(args: Arguments): Promise<Outcome> {escalation: { from: 'background', to: 'foreground', reason: refusal },visual: visualRecord,};- history.push(historyEntry(step, candidate.id, refusal));+ history.push(task.historyEntry(step, candidate.id, refusal));await writeEvent(args.log, event);continue;}@@ -496,13 +512,12 @@ async function run(args: Arguments): Promise<Outcome> {delivery_mode: candidate.arguments.delivery_mode ?? null,visual: visualRecord,};- history.push(historyEntry(step, candidate.id));+ history.push(task.historyEntry(step, candidate.id));await writeEvent(args.log, event);if (args.dryRun) return 'unknown';- if (SUBMIT_IDS.has(candidate.id)) {+ if (task.completionCandidateIds.has(candidate.id)) {for (let attempt = 0; attempt < 20; attempt += 1) {- const oracle = await fixtureState(args.fixtureUrl);- const outcome = classify(oracle.submitted, token, step, args.maxSteps);+ const outcome = task.classify(await task.readOracle(), step);if (outcome === 'verified' || outcome === 'refuted') {await writeEvent(args.log, { event: 'outcome', outcome, token });return outcome;@@ -512,8 +527,7 @@ async function run(args: Arguments): Promise<Outcome> {}}- const oracle = await fixtureState(args.fixtureUrl);- const outcome = classify(oracle.submitted, token, args.maxSteps, args.maxSteps);+ const outcome = task.classify(await task.readOracle(), task.maxSteps);await writeEvent(args.log, { event: 'outcome', outcome, token });return outcome;} finally {diff --git a/libs/cua-driver/examples/jev-use/typescript/sources.ts b/libs/cua-driver/examples/jev-use/typescript/sources.tsnew file mode 100644index 0000000000..9e3c5b3e43--- /dev/null+++ b/libs/cua-driver/examples/jev-use/typescript/sources.ts@@ -0,0 +1,195 @@+/**+ * Candidate sources: where a decision's executable candidates come from.+ *+ * A candidate source turns one Driver observation into addressable controls+ * and builds fully specified candidates for them. The model only ever sees a+ * candidate's ID and description; the tool name and exact arguments stay in+ * the runner. A task (see tasks.ts) decides which controls matter and which+ * candidate IDs and descriptions to offer; the source decides how to address a+ * control and which Driver tool acts on it.+ *+ * Implemented sources:+ *+ * - BrowserSemanticSource reads a get_browser_state semantic_v2 snapshot and+ * acts through browser_type / browser_click on its refs.+ * - VisualRegionSource reads a validated parse_visual_regions result and acts+ * through a capture-bound click on a region's center.+ *+ * Extension point (RFC #4268, Phase 1): a NativeAccessibilitySource will read+ * get_window_state elements, find controls by normalized role and label, and+ * act through the snapshot's element_token. It is deliberately not implemented+ * here; it must implement CandidateSource with kind 'ax'.+ */+import type {+ BrowserSnapshot,+ PageRef,+ VisualDelivery,+ VisualObservation,+ VisualRegion,+} from './core.js';++/** 'ax' is reserved for the future NativeAccessibilitySource. */+export type SourceKind = 'page' | 'visual' | 'ax';++export type Candidate = Readonly<{+ id: string;+ description: string;+ tool: string | null;+ arguments: Readonly<Record<string, unknown>>;+ captureId?: string;+ screenshotReference?: string;+}>;++export function immutableCandidate(candidate: Candidate): Candidate {+ return Object.freeze({ ...candidate, arguments: Object.freeze({ ...candidate.arguments }) });+}++export function asciiLower(value: string): string {+ return value.replace(/[A-Z]/g, (character) =>+ String.fromCharCode(character.charCodeAt(0) + 32)+ );+}++/**+ * One addressable element a source found in its observation. value is the+ * element's current value when the source can read it (never sent to a model;+ * tasks summarize it). handle is source-private: a page ref for the browser+ * source, a VisualRegion for the visual source.+ */+export type Control<Handle = unknown> = Readonly<{+ source: SourceKind;+ role: string;+ name: string;+ value: unknown;+ handle: Handle;+}>;++/**+ * The interface every candidate source implements. find returns the unique+ * control matching a role and accessible name, or undefined. click and+ * typeText return a fully specified candidate for that control, or undefined+ * when this source cannot perform the action in the current observation (for+ * example a visual region cannot receive text, and a visual click needs a+ * capture-bound Driver click).+ */+export interface CandidateSource {+ readonly kind: SourceKind;+ find(role: string, name: string): Control | undefined;+ click(control: Control, candidateId: string, description: string): Candidate | undefined;+ typeText(+ control: Control,+ text: string,+ candidateId: string,+ description: string+ ): Candidate | undefined;+}++/** Controls from a get_browser_state snapshot, acted on through its refs. */+export class BrowserSemanticSource implements CandidateSource {+ readonly kind = 'page' as const;++ constructor(readonly snapshot: BrowserSnapshot) {}++ /** The browser target every page candidate addresses. */+ target() {+ return { target_id: this.snapshot.target_id, tab_id: this.snapshot.tab_id };+ }++ find(role: string, name: string): Control<PageRef> | undefined {+ const ref = (this.snapshot.refs ?? []).find(+ (item) => item.role === role && item.name === name && item.ref+ );+ if (!ref) return undefined;+ return { source: 'page', role, name, value: ref.value, handle: ref };+ }++ click(control: Control, candidateId: string, description: string): Candidate {+ return immutableCandidate({+ id: candidateId,+ description,+ tool: 'browser_click',+ arguments: {+ ...this.target(),+ ref: (control.handle as PageRef).ref,+ input_route: 'dom_event',+ },+ });+ }++ typeText(control: Control, text: string, candidateId: string, description: string): Candidate {+ return immutableCandidate({+ id: candidateId,+ description,+ tool: 'browser_type',+ arguments: {+ ...this.target(),+ ref: (control.handle as PageRef).ref,+ text,+ replace: true,+ },+ });+ }+}++/**+ * Controls from validated OmniParser regions, clicked through their capture.+ * Regions carry no role, so find matches the visible text or icon label (ASCII+ * case-insensitive) of regions at or above MIN_CONFIDENCE and returns a+ * control only when exactly one region matches. click is offered only when+ * Driver advertises a capture-bound click (captureBound); it targets the+ * region's screenshot-pixel center with the exact capture_id and the source's+ * delivery mode.+ */+export class VisualRegionSource implements CandidateSource {+ static readonly MIN_CONFIDENCE = 0.8;+ readonly kind = 'visual' as const;++ constructor(+ readonly observation: VisualObservation,+ readonly delivery: VisualDelivery = 'background',+ readonly captureBound = false+ ) {}++ find(role: string, name: string): Control<VisualRegion> | undefined {+ const wanted = asciiLower(name);+ const matches = this.observation.regions.filter(+ (region) =>+ region.confidence >= VisualRegionSource.MIN_CONFIDENCE &&+ asciiLower(region.text ?? region.label ?? '') === wanted+ );+ if (matches.length !== 1) return undefined;+ const region = matches[0];+ return {+ source: 'visual',+ role,+ name: region.text ?? region.label ?? '',+ value: undefined,+ handle: region,+ };+ }++ click(control: Control, candidateId: string, description: string): Candidate | undefined {+ if (!this.captureBound) return undefined;+ const visual = this.observation;+ const region = control.handle as VisualRegion;+ return immutableCandidate({+ id: candidateId,+ description,+ tool: 'click',+ arguments: {+ pid: visual.pid,+ window_id: visual.windowId,+ x: region.x + region.width / 2,+ y: region.y + region.height / 2,+ capture_id: visual.captureId,+ delivery_mode: this.delivery,+ },+ captureId: visual.captureId,+ screenshotReference: visual.screenshotReference,+ });+ }++ typeText(): undefined {+ return undefined;+ }+}diff --git a/libs/cua-driver/examples/jev-use/typescript/sources_and_tasks.test.ts b/libs/cua-driver/examples/jev-use/typescript/sources_and_tasks.test.tsnew file mode 100644index 0000000000..4597a1b44b--- /dev/null+++ b/libs/cua-driver/examples/jev-use/typescript/sources_and_tasks.test.ts@@ -0,0 +1,112 @@+import assert from 'node:assert/strict';+import { readFileSync } from 'node:fs';+import test from 'node:test';++import { REDACTED_TOKEN, parseVisualRegions, type VisualObservation } from './core.js';+import { BrowserSemanticSource, VisualRegionSource, type CandidateSource } from './sources.js';+import {+ FIXTURE_GOAL,+ FixtureFormTask,+ SUBMIT_IDS,+ fixtureSources,+ type Task,+ type TaskSources,+} from './tasks.js';++const load = (name: string) =>+ JSON.parse(readFileSync(new URL(`../fixtures/${name}`, import.meta.url), 'utf8'));+const PAGE = load('jev-page-structure-replay-v1.json');+const VISUAL = load('jev-visual-replay-v1.json');+const PAYLOAD = VISUAL.visual_regions;++function observation(): VisualObservation {+ const source = PAYLOAD.capture.source;+ return parseVisualRegions(PAYLOAD, PAYLOAD.capture.capture_id, source.pid, source.window_id);+}++test('sources implement one interface with distinct kinds', () => {+ const page: CandidateSource = new BrowserSemanticSource(PAGE.snapshots.before_typing);+ const visual: CandidateSource = new VisualRegionSource(observation());+ assert.deepEqual([page.kind, visual.kind], ['page', 'visual']);+});++test('page source addresses refs with exact browser arguments', () => {+ const snapshot = PAGE.snapshots.after_typing;+ const page = new BrowserSemanticSource(snapshot);+ const button = page.find('button', 'Submit');+ assert.ok(button);+ assert.deepEqual([button.source, button.handle.ref], ['page', 'p4:2']);+ const candidate = page.click(button, 'c', 'd');+ assert.equal(candidate.tool, 'browser_click');+ assert.deepEqual(candidate.arguments, {+ target_id: snapshot.target_id,+ tab_id: snapshot.tab_id,+ ref: 'p4:2',+ input_route: 'dom_event',+ });+ assert.equal(page.find('button', 'Cancel'), undefined);+});++test('visual source clicks only capture-bound and never types', () => {+ const visual = observation();+ const source = new VisualRegionSource(visual);+ const submit = source.find('button', 'submit');+ assert.ok(submit);+ assert.equal(source.click(submit, 'c', 'd'), undefined);+ assert.equal(source.typeText(), undefined);+ const bound = new VisualRegionSource(visual, 'foreground', true);+ const candidate = bound.click(submit, 'c', 'd');+ assert.equal(candidate?.tool, 'click');+ assert.equal(candidate?.captureId, visual.captureId);+ assert.equal(candidate?.arguments.delivery_mode, 'foreground');+ assert.deepEqual(+ [candidate?.arguments.x, candidate?.arguments.y],+ [submit.handle.x + submit.handle.width / 2, submit.handle.y + submit.handle.height / 2]+ );+});++test('built-in task declares its spec', () => {+ const task: Task = new FixtureFormTask('secret-token', 'http://127.0.0.1:9/', 3);+ assert.equal(task.goal, FIXTURE_GOAL);+ assert.equal(task.maxSteps, 3);+ assert.equal(task.completionCandidateIds, SUBMIT_IDS);+ assert.deepEqual([...task.allowedActionKinds].sort(), ['browser_click', 'browser_type', 'click']);+ assert.equal(task.parameters.length, 1);+ const [parameter] = task.parameters;+ assert.equal(parameter.secret, true);+ assert.deepEqual([parameter.value, parameter.redaction], ['secret-token', REDACTED_TOKEN]);+ assert.deepEqual(task.redact({ a: ['x secret-token y'] }), { a: [`x ${REDACTED_TOKEN} y`] });+});++test('oracle classification uses the task budget', () => {+ const task = new FixtureFormTask('t', undefined, 2);+ assert.equal(task.classify({ submitted: 't' }, 0), 'verified');+ assert.equal(task.classify({ submitted: 'other' }, 0), 'refuted');+ assert.equal(task.classify({ submitted: null }, 1), 'unknown');+ assert.equal(task.classify({ submitted: null }, 2), 'budget_exhausted');+});++test('candidates outside the allowed action kinds are refused', () => {+ const task = new FixtureFormTask(PAGE.token);+ (task as { allowedActionKinds: ReadonlySet<string> }).allowedActionKinds = new Set([+ 'browser_type',+ ]);+ assert.throws(() => task.candidates(fixtureSources(PAGE.snapshots.after_typing)), /browser_click/);+ assert.equal(task.candidates(fixtureSources(PAGE.snapshots.before_typing))[0].tool, 'browser_type');+});++test('state summary reads the step sources', () => {+ const task = new FixtureFormTask(VISUAL.token);+ const after = VISUAL.snapshots.after_typing;+ const pending: TaskSources = { page: new BrowserSemanticSource(after), visualPath: true };+ assert.equal(task.stateSummary(pending).submit_button, 'visual_check_pending');+ const parsed: TaskSources = {+ page: new BrowserSemanticSource(after),+ visual: new VisualRegionSource(observation()),+ visualPath: true,+ };+ assert.deepEqual(task.stateSummary(parsed), {+ verification_field: 'contains_required_token',+ submit_button: 'visual_only',+ });+});diff --git a/libs/cua-driver/examples/jev-use/typescript/tasks.ts b/libs/cua-driver/examples/jev-use/typescript/tasks.tsnew file mode 100644index 0000000000..29d7718e84--- /dev/null+++ b/libs/cua-driver/examples/jev-use/typescript/tasks.ts@@ -0,0 +1,370 @@+/**+ * Task specs: what one jev-use run is trying to achieve and how it is checked.+ *+ * A task spec owns everything that is specific to one application task: the+ * goal shown to the model, its parameters (secrets are redacted from all model+ * input), the step budget, the Driver action kinds it may use, the compact+ * state summary the model sees, the candidate IDs and descriptions it offers,+ * and the success oracle that verifies the outcome independently of the+ * runner's own events.+ *+ * Candidate sources (sources.ts) supply the controls and build the executable+ * candidates. The runner stays task-agnostic: it observes, asks the task for+ * candidates and a state summary, lets the model choose one supplied ID, acts,+ * and asks the task's oracle whether the task is done.+ *+ * FixtureFormTask is the one built-in task: type a secret verification token+ * into the loopback fixture's form and submit it, verified through the+ * fixture's /state endpoint.+ */+import type { BrowserSnapshot, VisualDelivery, VisualObservation, VisualRegion } from './core.js';+import {+ BrowserSemanticSource,+ immutableCandidate,+ VisualRegionSource,+ type Candidate,+} from './sources.js';++export type Outcome = 'verified' | 'refuted' | 'unknown' | 'abstained' | 'budget_exhausted';++export const REDACTED_TOKEN = '[verification token]';++/** Replace every occurrence of the token in strings nested in value. */+export function redactToken(value: unknown, token: string, placeholder = REDACTED_TOKEN): unknown {+ if (!token) return value;+ if (typeof value === 'string') return value.split(token).join(placeholder);+ if (Array.isArray(value)) return value.map((item) => redactToken(item, token, placeholder));+ if (value && typeof value === 'object') {+ return Object.fromEntries(+ Object.entries(value).map(([key, item]) => [key, redactToken(item, token, placeholder)])+ );+ }+ return value;+}++/**+ * A named task input. A secret value is replaced by redaction everywhere in+ * model input; it may still appear in executable arguments.+ */+export type TaskParameter = Readonly<{+ name: string;+ value: string;+ secret: boolean;+ redaction: string;+}>;++/**+ * The candidate sources available for one decision step. visual is present+ * only when this step parsed visual regions. visualPath reports whether a+ * capture-bound visual path exists at all, so a task can say a control is still+ * pending a visual check.+ */+export type TaskSources = Readonly<{+ page: BrowserSemanticSource;+ visual?: VisualRegionSource;+ visualPath: boolean;+}>;++export type HistoryEntry = Readonly<{ step: number; selected_id: string; outcome: string }>;++/** The interface the runner uses for every task. */+export interface Task {+ readonly id: string;+ readonly goal: string;+ readonly parameters: readonly TaskParameter[];+ readonly maxSteps: number;+ readonly allowedActionKinds: ReadonlySet<string>;+ /** Candidate IDs after which the runner polls the oracle for completion. */+ readonly completionCandidateIds: ReadonlySet<string>;+ redact(value: unknown): unknown;+ candidates(sources: TaskSources): Candidate[];+ stateSummary(sources: TaskSources): Readonly<Record<string, string>>;+ historyEntry(step: number, candidateId: string, refusal?: string): HistoryEntry;+ reset(): Promise<void>;+ readOracle(): Promise<Readonly<Record<string, unknown>>>;+ classify(oracle: Readonly<Record<string, unknown>>, steps: number): Outcome;+}++// The built-in browser fixture task.++export const FIXTURE_TASK_ID = 'browser-fixture-form';+export const FIXTURE_GOAL =+ 'Enter the required verification token into the verification field, then submit the form.';+export const DEFAULT_FIXTURE_URL = 'http://127.0.0.1:8765/';+export const DEFAULT_MAX_STEPS = 4;+export const FIELD_NAME = 'verification value';+export const SUBMIT_NAME = 'Submit';+export const SUBMIT_IDS: ReadonlySet<string> = new Set(['submit-form', 'submit-form-foreground']);+export const FIXTURE_ACTION_KINDS: ReadonlySet<string> = new Set([+ 'browser_type',+ 'browser_click',+ 'click',+]);++export type SubmitButtonState =+ | 'available'+ | 'visual_only'+ | 'visual_check_pending'+ | 'not_found_visually'+ | 'not_in_page_structure';++export type FormState = Readonly<{+ verification_field: 'not_found' | 'empty' | 'contains_required_token' | 'contains_other_value';+ submit_button: SubmitButtonState;+}>;++export async function fixtureState(fixtureUrl: string): Promise<{ submitted: string | null }> {+ const response = await fetch(new URL('state', fixtureUrl));+ if (!response.ok) throw new Error(`fixture state failed: HTTP ${response.status}`);+ return (await response.json()) as { submitted: string | null };+}++export async function resetFixture(fixtureUrl: string): Promise<void> {+ const response = await fetch(new URL('reset', fixtureUrl), { method: 'POST' });+ if (response.status !== 204) throw new Error(`fixture reset failed: HTTP ${response.status}`);+}++export function classify(+ submitted: string | null,+ token: string,+ steps: number,+ maxSteps: number+): Outcome {+ if (submitted === token) return 'verified';+ if (submitted !== null) return 'refuted';+ if (steps >= maxSteps) return 'budget_exhausted';+ return 'unknown';+}++export const HISTORY_OUTCOMES: Readonly<Record<string, string>> = {+ 'type-verification-value': 'typed the required token into the verification field',+ 'submit-form': 'clicked Submit; the submission was not yet confirmed',+ 'submit-form-foreground':+ 'clicked Submit in the foreground; the submission was not yet confirmed',+ reobserve: 'took no action and requested a fresh observation',+};++/**+ * Build the compact decision-history item shown to the model. It records what+ * each step did, not timings or model probabilities, so earlier choices do not+ * become a signal to repeat themselves.+ */+export function historyEntry(step: number, candidateId: string, refusal?: string): HistoryEntry {+ const outcome = refusal+ ? `Driver refused background delivery (${refusal}); no click happened and a ` ++ 'foreground Submit candidate is offered next'+ : (HISTORY_OUTCOMES[candidateId] ?? 'completed');+ return { step, selected_id: candidateId, outcome };+}++function reservedCandidates(): Candidate[] {+ return [+ immutableCandidate({+ id: 'reobserve',+ description:+ 'Take no action and obtain a fresh Driver observation, because the current ' ++ 'observation is stale or contradicts the reported form state. A Submit control ' ++ 'that is visual-only, or whose visual check is still pending, is not a reason to ' ++ 'reobserve: the runner parses visual regions for Submit once no page-structure ' ++ 'action remains.',+ tool: null,+ arguments: {},+ }),+ immutableCandidate({+ id: 'abstain',+ description: 'Stop without acting if none of the proposed actions is safe for the observed state.',+ tool: null,+ arguments: {},+ }),+ ];+}++/**+ * Type the secret token into the fixture form and submit it. The oracle is the+ * fixture's /state endpoint: the task is verified only when the fixture+ * recorded exactly this token as submitted.+ */+export class FixtureFormTask implements Task {+ readonly id = FIXTURE_TASK_ID;+ readonly goal = FIXTURE_GOAL;+ readonly allowedActionKinds = FIXTURE_ACTION_KINDS;+ readonly completionCandidateIds = SUBMIT_IDS;++ constructor(+ readonly token: string,+ readonly fixtureUrl: string = DEFAULT_FIXTURE_URL,+ readonly maxSteps: number = DEFAULT_MAX_STEPS+ ) {}++ get parameters(): readonly TaskParameter[] {+ return [+ { name: 'verification_token', value: this.token, secret: true, redaction: REDACTED_TOKEN },+ ];+ }++ redact(value: unknown): unknown {+ let result = value;+ for (const parameter of this.parameters) {+ if (parameter.secret) result = redactToken(result, parameter.value, parameter.redaction);+ }+ return result;+ }++ /**+ * Build the closed candidate set for one decision. Page-structure refs always+ * win. The capture-bound visual Submit is offered only when no Submit ref+ * exists. A visual source with foreground delivery yields a distinct+ * submit-form-foreground candidate after Driver refused background delivery;+ * the chooser must pick it explicitly.+ */+ candidates(sources: TaskSources): Candidate[] {+ const { page, visual } = sources;+ const token = this.token;+ const field = page.find('textbox', FIELD_NAME);+ const button = page.find('button', SUBMIT_NAME);+ const candidates: Candidate[] = [];+ if (field && field.value !== token) {+ candidates.push(+ page.typeText(+ field,+ token,+ 'type-verification-value',+ 'Type the required verification token into the verification field, ' ++ 'replacing its current contents.'+ )+ );+ } else if (field && field.value === token && button) {+ candidates.push(+ page.click(+ button,+ 'submit-form',+ "Click the form's Submit button. The observed form state reports that the " ++ 'verification field already contains the required token, so the form is ' ++ 'ready to submit.'+ )+ );+ } else if (field && field.value === token && visual) {+ const region = visual.find('button', SUBMIT_NAME);+ if (region) {+ const foreground = visual.delivery === 'foreground';+ const candidate = visual.click(+ region,+ foreground ? 'submit-form-foreground' : 'submit-form',+ foreground+ ? 'Submit the form by clicking the unique validated visual Submit region with ' ++ 'foreground delivery, which activates the browser window, because Driver ' ++ 'refused background delivery for the previous visual click.'+ : 'Submit the form by clicking the unique validated visual Submit region. ' ++ 'The observed form state reports that the verification field already ' ++ 'contains the required token.'+ );+ if (candidate) candidates.push(candidate);+ }+ }+ for (const candidate of candidates) {+ if (candidate.tool === null || !this.allowedActionKinds.has(candidate.tool)) {+ throw new Error(`task ${this.id} does not allow action kind ${candidate.tool}`);+ }+ }+ return [...candidates, ...reservedCandidates()];+ }++ /**+ * Summarize the form for the decision model without revealing the token. The+ * raw field value never leaves the runner.+ *+ * submit_button is 'available' for a clickable page-structure ref. When the+ * page structure has none and the capture-bound visual path is enabled+ * (visualPath), it is 'visual_only' if this observation holds a unique+ * validated visual Submit region, 'visual_check_pending' if no visual regions+ * were parsed for this step yet (the runner parses them once no page-structure+ * action remains), and 'not_found_visually' otherwise. Without a visual path+ * it is 'not_in_page_structure'.+ */+ stateSummary(sources: TaskSources): FormState {+ const field = sources.page.find('textbox', FIELD_NAME);+ const button = sources.page.find('button', SUBMIT_NAME);+ const verificationField = !field+ ? 'not_found'+ : !field.value+ ? 'empty'+ : field.value === this.token+ ? 'contains_required_token'+ : 'contains_other_value';+ const submitButton: SubmitButtonState = button+ ? 'available'+ : !sources.visualPath+ ? 'not_in_page_structure'+ : !sources.visual+ ? 'visual_check_pending'+ : sources.visual.find('button', SUBMIT_NAME)+ ? 'visual_only'+ : 'not_found_visually';+ return { verification_field: verificationField, submit_button: submitButton };+ }++ historyEntry(step: number, candidateId: string, refusal?: string): HistoryEntry {+ return historyEntry(step, candidateId, refusal);+ }++ reset(): Promise<void> {+ return resetFixture(this.fixtureUrl);+ }++ readOracle(): Promise<{ submitted: string | null }> {+ return fixtureState(this.fixtureUrl);+ }++ classify(oracle: Readonly<Record<string, unknown>>, steps: number): Outcome {+ return classify(oracle.submitted as string | null, this.token, steps, this.maxSteps);+ }+}++/** Build one step's sources from a browser snapshot and optional visual parse. */+export function fixtureSources(+ snapshot: BrowserSnapshot,+ visual?: VisualObservation,+ captureBoundClick = false,+ visualDelivery: VisualDelivery = 'background',+ visualPath = false+): TaskSources {+ return {+ page: new BrowserSemanticSource(snapshot),+ visual: visual ? new VisualRegionSource(visual, visualDelivery, captureBoundClick) : undefined,+ visualPath,+ };+}++// Compatibility entry points. They keep the pre-task-spec signatures and+// delegate to the built-in fixture task, so existing callers are unchanged.++/** Return the unique validated visual Submit region, or undefined. */+export function visualSubmitRegion(visual?: VisualObservation): VisualRegion | undefined {+ if (!visual) return undefined;+ return new VisualRegionSource(visual).find('button', SUBMIT_NAME)?.handle;+}++/** Summarize the fixture form; see FixtureFormTask.stateSummary. */+export function formState(+ snapshot: BrowserSnapshot,+ token: string,+ visual?: VisualObservation,+ visualPath = false+): FormState {+ return new FixtureFormTask(token).stateSummary(+ fixtureSources(snapshot, visual, false, 'background', visualPath)+ );+}++/** Build the fixture candidate set; see FixtureFormTask.candidates. */+export function buildCandidates(+ snapshot: BrowserSnapshot,+ token: string,+ visual?: VisualObservation,+ captureBoundClick = false,+ visualDelivery: VisualDelivery = 'background'+): Candidate[] {+ return new FixtureFormTask(token).candidates(+ fixtureSources(snapshot, visual, captureBoundClick, visualDelivery)+ );+}