Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

refactor(cua-driver): introduce jev-use candidate sources and task spec (#4270)

Francesco Bonacci8bed9fd1b334
diff --git a/libs/cua-driver/examples/jev-use/README.md b/libs/cua-driver/examples/jev-use/README.md
index 4c351c8537..f85f5cfb76 100644
--- a/libs/cua-driver/examples/jev-use/README.md
+++ b/libs/cua-driver/examples/jev-use/README.md
@@ -56,6 +56,25 @@ through an independent `/state` endpoint. Each runner:
8. checks the fixture's `/state` endpoint instead of treating the action
response or a screenshot as proof of success.
+### Candidate sources and task specs
+
+The loop separates where candidates come from from what the task needs. A
+candidate source (`python/sources.py`, `typescript/sources.ts`) finds controls
+in one observation and builds fully specified candidates for them:
+`BrowserSemanticSource` reads the `get_browser_state` snapshot and acts through
+page refs, and `VisualRegionSource` reads validated `parse_visual_regions`
+regions and acts through a capture-bound `click`. A task spec
+(`python/tasks.py`, `typescript/tasks.ts`) holds the goal, the parameters with
+the secret token marked for redaction, the step budget, the allowed Driver
+action kinds, the `form` state summary, the candidate IDs and descriptions, and
+the success oracle. The fixture above is the one built-in task,
+`FixtureFormTask`, and its oracle is the fixture's `/state` endpoint. A native
+accessibility source for desktop applications is planned in
+[RFC #4268](https://github.com/trycua/cua/issues/4268) and is not implemented.
+The model request and response are unchanged: the golden files
+`fixtures/jev-provider-request-golden-*-v1.json` pin the exact provider
+requests and candidate tables for the page-structure and visual fixtures.
+
The MCP connection stays open across the entire loop. This preserves the
explicit named Cua Driver session and avoids rebuilding tool state for every
step.
diff --git a/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-python-v1.json b/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-python-v1.json
new file mode 100644
index 0000000000..55a92a0199
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-python-v1.json
@@ -0,0 +1,6338 @@
+{
+ "schema": "cua.jev_use_provider_request_golden_v1",
+ "language": "python",
+ "runs": {
+ "page/windows-page-1-python": [
+ {
+ "step": 1,
+ "request": "6b7824e66e1328bf",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "871e01954513c7d7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "be0bdd31f3b11cbe",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "00f3ee95dc3813ad",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page/linux-page-3-python": [
+ {
+ "step": 1,
+ "request": "6b7824e66e1328bf",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "871e01954513c7d7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "be0bdd31f3b11cbe",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "00f3ee95dc3813ad",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page/linux-page-7-typescript": [
+ {
+ "step": 1,
+ "request": "6b7824e66e1328bf",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "871e01954513c7d7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "be0bdd31f3b11cbe",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "00f3ee95dc3813ad",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page/linux-page-2-python": [
+ {
+ "step": 1,
+ "request": "6b7824e66e1328bf",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "871e01954513c7d7",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_auto/windows-page-1-python": [
+ {
+ "step": 1,
+ "request": "6b7824e66e1328bf",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "871e01954513c7d7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "be0bdd31f3b11cbe",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "00f3ee95dc3813ad",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_auto/linux-page-3-python": [
+ {
+ "step": 1,
+ "request": "6b7824e66e1328bf",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "871e01954513c7d7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "be0bdd31f3b11cbe",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "00f3ee95dc3813ad",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_auto/linux-page-7-typescript": [
+ {
+ "step": 1,
+ "request": "6b7824e66e1328bf",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "871e01954513c7d7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "be0bdd31f3b11cbe",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "00f3ee95dc3813ad",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_auto/linux-page-2-python": [
+ {
+ "step": 1,
+ "request": "6b7824e66e1328bf",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "871e01954513c7d7",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_always/windows-page-1-python": [
+ {
+ "step": 1,
+ "request": "7d7f17000d74bdb5",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "64ad866cbb7de3a0",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "55fa0c6279c5998f",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "7ed26ecebd151c79",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_always/linux-page-3-python": [
+ {
+ "step": 1,
+ "request": "7d7f17000d74bdb5",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "64ad866cbb7de3a0",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "55fa0c6279c5998f",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "7ed26ecebd151c79",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_always/linux-page-7-typescript": [
+ {
+ "step": 1,
+ "request": "7d7f17000d74bdb5",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "64ad866cbb7de3a0",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "55fa0c6279c5998f",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "7ed26ecebd151c79",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_always/linux-page-2-python": [
+ {
+ "step": 1,
+ "request": "7d7f17000d74bdb5",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "64ad866cbb7de3a0",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "7b92846b88b799cd",
+ "candidates": "0a86001ae911a77b"
+ }
+ ],
+ "visual_capture_bound_auto/linux_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "40216332a214fafb",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 4,
+ "request": "f76646d52770d00e",
+ "candidates": "0a86001ae911a77b"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "7b92846b88b799cd",
+ "candidates": "0a86001ae911a77b"
+ }
+ ],
+ "visual_capture_bound_auto/linux_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "7b92846b88b799cd",
+ "candidates": "0a86001ae911a77b"
+ }
+ ],
+ "visual_capture_bound_auto/windows_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "40216332a214fafb",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 4,
+ "request": "8f69d7cab750cca9",
+ "candidates": "d11ea8f49c30445a"
+ }
+ ],
+ "visual_capture_bound_auto/windows_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "40216332a214fafb",
+ "candidates": "d11ea8f49c30445a"
+ }
+ ],
+ "visual_capture_bound_auto/windows_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "40216332a214fafb",
+ "candidates": "d11ea8f49c30445a"
+ }
+ ],
+ "visual_capture_bound_auto/windows_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "40216332a214fafb",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 4,
+ "request": "8f69d7cab750cca9",
+ "candidates": "d11ea8f49c30445a"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_post_4255_regression_1": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "ae88c22f9ca831ad",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "2b38e5d032f55588",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "c11f02c7d592f865",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_post_4255_regression_2": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "ae88c22f9ca831ad",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "2b38e5d032f55588",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "c11f02c7d592f865",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_post_4255_regression_3": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "ae88c22f9ca831ad",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "2b38e5d032f55588",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "c11f02c7d592f865",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_post_4255_regression_4": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "ae88c22f9ca831ad",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "2b38e5d032f55588",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "c11f02c7d592f865",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_post_4255_regression_5": [
+ {
+ "step": 1,
+ "request": "c76f8a530fa915f0",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "ae88c22f9ca831ad",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "2b38e5d032f55588",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "c11f02c7d592f865",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "7b92846b88b799cd",
+ "candidates": "0a86001ae911a77b"
+ }
+ ],
+ "visual_capture_bound_always/linux_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "40216332a214fafb",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 4,
+ "request": "f76646d52770d00e",
+ "candidates": "0a86001ae911a77b"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "7b92846b88b799cd",
+ "candidates": "0a86001ae911a77b"
+ }
+ ],
+ "visual_capture_bound_always/linux_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "7b92846b88b799cd",
+ "candidates": "0a86001ae911a77b"
+ }
+ ],
+ "visual_capture_bound_always/windows_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "40216332a214fafb",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 4,
+ "request": "8f69d7cab750cca9",
+ "candidates": "d11ea8f49c30445a"
+ }
+ ],
+ "visual_capture_bound_always/windows_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "40216332a214fafb",
+ "candidates": "d11ea8f49c30445a"
+ }
+ ],
+ "visual_capture_bound_always/windows_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "40216332a214fafb",
+ "candidates": "d11ea8f49c30445a"
+ }
+ ],
+ "visual_capture_bound_always/windows_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "0d32ec5c61a4a8b5",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 3,
+ "request": "40216332a214fafb",
+ "candidates": "d11ea8f49c30445a"
+ },
+ {
+ "step": 4,
+ "request": "8f69d7cab750cca9",
+ "candidates": "d11ea8f49c30445a"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_post_4255_regression_1": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1702e777270e55d9",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "13ad158a26367929",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "b3bd04ea455a1283",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_post_4255_regression_2": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1702e777270e55d9",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "13ad158a26367929",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "b3bd04ea455a1283",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_post_4255_regression_3": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1702e777270e55d9",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "13ad158a26367929",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "b3bd04ea455a1283",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_post_4255_regression_4": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1702e777270e55d9",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "13ad158a26367929",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "b3bd04ea455a1283",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_post_4255_regression_5": [
+ {
+ "step": 1,
+ "request": "b645dd31d80e6d4b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1702e777270e55d9",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "13ad158a26367929",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "b3bd04ea455a1283",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1dc3f220096db95a",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "566457a742192f26",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/linux_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1dc3f220096db95a",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "d5821263e5bced53",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 4,
+ "request": "a415ac151c2ee8a0",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1dc3f220096db95a",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "566457a742192f26",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/linux_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1dc3f220096db95a",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "566457a742192f26",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/windows_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1dc3f220096db95a",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "d5821263e5bced53",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 4,
+ "request": "1114de9dcd19be4c",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/windows_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1dc3f220096db95a",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "d5821263e5bced53",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/windows_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1dc3f220096db95a",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "d5821263e5bced53",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/windows_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "1dc3f220096db95a",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "d5821263e5bced53",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 4,
+ "request": "1114de9dcd19be4c",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_post_4255_regression_1": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "e6d4d6a95565d28c",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "20599cb6821eb742",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "8a03f8eedacfd33e",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_post_4255_regression_2": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "e6d4d6a95565d28c",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "20599cb6821eb742",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "8a03f8eedacfd33e",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_post_4255_regression_3": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "e6d4d6a95565d28c",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "20599cb6821eb742",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "8a03f8eedacfd33e",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_post_4255_regression_4": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "e6d4d6a95565d28c",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "20599cb6821eb742",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "8a03f8eedacfd33e",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_post_4255_regression_5": [
+ {
+ "step": 1,
+ "request": "8dfe7fa95bf30864",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "e6d4d6a95565d28c",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "20599cb6821eb742",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "8a03f8eedacfd33e",
+ "candidates": "70875f146d859b36"
+ }
+ ]
+ },
+ "single_requests": {
+ "page_other_value": {
+ "request": "900887082b50ecd6",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ "page_after_with_visual": {
+ "request": "9d507ebbaad75a8e",
+ "candidates": "d229b9dd4f676e88"
+ },
+ "page_after_no_button": {
+ "request": "77f2b9531e52e341",
+ "candidates": "daba6141ff108d4e"
+ },
+ "page_after_no_button_visual_path": {
+ "request": "2a0f4a3e90da5dab",
+ "candidates": "daba6141ff108d4e"
+ },
+ "no_form_refs": {
+ "request": "3905fd10ee1e16ae",
+ "candidates": "daba6141ff108d4e"
+ },
+ "visual_after_no_capture_bound_click": {
+ "request": "4a5f5196307c7a85",
+ "candidates": "daba6141ff108d4e"
+ },
+ "visual_after_without_submit_region": {
+ "request": "966403408348763b",
+ "candidates": "daba6141ff108d4e"
+ },
+ "visual_after_duplicate_submit_region": {
+ "request": "38460b37f9ee6b63",
+ "candidates": "daba6141ff108d4e"
+ },
+ "visual_after_foreground": {
+ "request": "3cd06681ec4e772d",
+ "candidates": "0a86001ae911a77b"
+ },
+ "visual_after_other_value": {
+ "request": "c6a54b6b1940bb00",
+ "candidates": "3c210f06a8903bc9"
+ }
+ },
+ "requests": {
+ "6b7824e66e1328bf": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "available"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "871e01954513c7d7": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "available"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "be0bdd31f3b11cbe": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "available"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "00f3ee95dc3813ad": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "available"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 3,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "7d7f17000d74bdb5": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "available"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "64ad866cbb7de3a0": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "available"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "55fa0c6279c5998f": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "available"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "7ed26ecebd151c79": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "available"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 3,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "c76f8a530fa915f0": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "visual_check_pending"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "0d32ec5c61a4a8b5": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "7b92846b88b799cd": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "submit-form",
+ "outcome": "Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "40216332a214fafb": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "f76646d52770d00e": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 3,
+ "selected_id": "submit-form",
+ "outcome": "Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "8f69d7cab750cca9": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 3,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "ae88c22f9ca831ad": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "visual_check_pending"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "2b38e5d032f55588": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "visual_check_pending"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "c11f02c7d592f865": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "visual_check_pending"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 3,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "b645dd31d80e6d4b": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "1702e777270e55d9": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "13ad158a26367929": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "b3bd04ea455a1283": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 3,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "8dfe7fa95bf30864": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "not_in_page_structure"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "1dc3f220096db95a": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "not_in_page_structure"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "566457a742192f26": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "not_in_page_structure"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "d5821263e5bced53": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "not_in_page_structure"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "a415ac151c2ee8a0": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "not_in_page_structure"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 3,
+ "selected_id": "reobserve",
+ "outcome": "Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "1114de9dcd19be4c": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "not_in_page_structure"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 3,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "e6d4d6a95565d28c": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "not_in_page_structure"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "20599cb6821eb742": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "not_in_page_structure"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "8a03f8eedacfd33e": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "empty",
+ "submit_button": "not_in_page_structure"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "step": 1,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ },
+ {
+ "step": 3,
+ "selected_id": "reobserve",
+ "outcome": "took no action and requested a fresh observation"
+ }
+ ]
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "900887082b50ecd6": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "contains_other_value",
+ "submit_button": "available"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "9d507ebbaad75a8e": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "available"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "77f2b9531e52e341": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "not_in_page_structure"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "2a0f4a3e90da5dab": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "visual_check_pending"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "3905fd10ee1e16ae": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": null,
+ "form": {
+ "verification_field": "not_found",
+ "submit_button": "not_in_page_structure"
+ },
+ "outline": null,
+ "visual": null
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "4a5f5196307c7a85": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "966403408348763b": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "not_found_visually"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ }
+ ]
+ }
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "38460b37f9ee6b63": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "not_found_visually"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ },
+ {
+ "id": "dup",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "3cd06681ec4e772d": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_required_token",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "c6a54b6b1940bb00": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:41605/"
+ },
+ "form": {
+ "verification_field": "contains_other_value",
+ "submit_button": "visual_only"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": {
+ "schema": "cua.visual_regions_v1",
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a",
+ "source": {
+ "kind": "window",
+ "pid": 2539,
+ "window_id": 52428804
+ },
+ "regions": [
+ {
+ "id": "text-9",
+ "kind": "text",
+ "text": "×",
+ "label": null,
+ "confidence": 0.8168784976005554,
+ "interactive": false,
+ "bounds": {
+ "x": 252,
+ "y": 10,
+ "width": 18,
+ "height": 19
+ }
+ },
+ {
+ "id": "text-11",
+ "kind": "text",
+ "text": "+",
+ "label": null,
+ "confidence": 0.8311772346496582,
+ "interactive": false,
+ "bounds": {
+ "x": 288,
+ "y": 13,
+ "width": 14,
+ "height": 14
+ }
+ },
+ {
+ "id": "icon-15",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8110984563827515,
+ "interactive": false,
+ "bounds": {
+ "x": 10,
+ "y": 45,
+ "width": 34,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-17",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8079566955566406,
+ "interactive": false,
+ "bounds": {
+ "x": 83,
+ "y": 46,
+ "width": 32,
+ "height": 36
+ }
+ },
+ {
+ "id": "icon-18",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.8043645024299622,
+ "interactive": false,
+ "bounds": {
+ "x": 47,
+ "y": 46,
+ "width": 31,
+ "height": 34
+ }
+ },
+ {
+ "id": "text-26",
+ "kind": "text",
+ "text": "←",
+ "label": null,
+ "confidence": 0.8558894395828247,
+ "interactive": false,
+ "bounds": {
+ "x": 17,
+ "y": 54,
+ "width": 20,
+ "height": 18
+ }
+ },
+ {
+ "id": "text-28",
+ "kind": "text",
+ "text": "Cua",
+ "label": null,
+ "confidence": 0.8023205995559692,
+ "interactive": false,
+ "bounds": {
+ "x": 175,
+ "y": 389,
+ "width": 118,
+ "height": 47
+ }
+ },
+ {
+ "id": "text-29",
+ "kind": "text",
+ "text": "TypeSafe Jev",
+ "label": null,
+ "confidence": 0.8917925357818604,
+ "interactive": false,
+ "bounds": {
+ "x": 471,
+ "y": 389,
+ "width": 302,
+ "height": 52
+ }
+ },
+ {
+ "id": "text-30",
+ "kind": "text",
+ "text": "Driver +",
+ "label": null,
+ "confidence": 0.865290641784668,
+ "interactive": false,
+ "bounds": {
+ "x": 270,
+ "y": 390,
+ "width": 207,
+ "height": 46
+ }
+ },
+ {
+ "id": "icon-33",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9488332271575928,
+ "interactive": false,
+ "bounds": {
+ "x": 637,
+ "y": 561,
+ "width": 132,
+ "height": 63
+ }
+ },
+ {
+ "id": "icon-34",
+ "kind": "icon",
+ "text": null,
+ "label": "icon-class-0",
+ "confidence": 0.9535441398620604,
+ "interactive": false,
+ "bounds": {
+ "x": 172,
+ "y": 562,
+ "width": 449,
+ "height": 65
+ }
+ },
+ {
+ "id": "text-35",
+ "kind": "text",
+ "text": "[verification token]",
+ "label": null,
+ "confidence": 0.8249707818031311,
+ "interactive": false,
+ "bounds": {
+ "x": 188,
+ "y": 575,
+ "width": 149,
+ "height": 35
+ }
+ },
+ {
+ "id": "text-36",
+ "kind": "text",
+ "text": "Submit",
+ "label": null,
+ "confidence": 0.8971455097198486,
+ "interactive": false,
+ "bounds": {
+ "x": 662,
+ "y": 575,
+ "width": 85,
+ "height": 32
+ }
+ }
+ ]
+ }
+ },
+ "history": []
+ },
+ "questions": {
+ "driver_action": {
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ }
+ },
+ "candidate_tables": {
+ "ec571c9800d7b4e8": [
+ {
+ "id": "type-verification-value",
+ "description": "Type the required verification token into the verification field, replacing its current contents.",
+ "tool": "browser_type",
+ "arguments": {
+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",
+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",
+ "ref": "p3:0",
+ "text": "jev-guide-live",
+ "replace": true
+ },
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "d229b9dd4f676e88": [
+ {
+ "id": "submit-form",
+ "description": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "tool": "browser_click",
+ "arguments": {
+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",
+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",
+ "ref": "p4:2",
+ "input_route": "dom_event"
+ },
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "70875f146d859b36": [
+ {
+ "id": "type-verification-value",
+ "description": "Type the required verification token into the verification field, replacing its current contents.",
+ "tool": "browser_type",
+ "arguments": {
+ "target_id": "bt-aeb40534-c16c-4da9-9567-aee3d2d4958a",
+ "tab_id": "tab-11edc8ea-1185-4d45-be26-6df2f7082d7e",
+ "ref": "p1:0",
+ "text": "jev-guide-live",
+ "replace": true
+ },
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "d11ea8f49c30445a": [
+ {
+ "id": "submit-form",
+ "description": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",
+ "tool": "click",
+ "arguments": {
+ "pid": 2539,
+ "window_id": 52428804,
+ "x": 704.5,
+ "y": 591.0,
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "delivery_mode": "background"
+ },
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a"
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "0a86001ae911a77b": [
+ {
+ "id": "submit-form-foreground",
+ "description": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",
+ "tool": "click",
+ "arguments": {
+ "pid": 2539,
+ "window_id": 52428804,
+ "x": 704.5,
+ "y": 591.0,
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "delivery_mode": "foreground"
+ },
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a"
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "daba6141ff108d4e": [
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "3c210f06a8903bc9": [
+ {
+ "id": "type-verification-value",
+ "description": "Type the required verification token into the verification field, replacing its current contents.",
+ "tool": "browser_type",
+ "arguments": {
+ "target_id": "bt-aeb40534-c16c-4da9-9567-aee3d2d4958a",
+ "tab_id": "tab-11edc8ea-1185-4d45-be26-6df2f7082d7e",
+ "ref": "p2:1",
+ "text": "jev-guide-live",
+ "replace": true
+ },
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ]
+ }
+}
diff --git a/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-typescript-v1.json b/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-typescript-v1.json
new file mode 100644
index 0000000000..a88c0db8b5
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/fixtures/jev-provider-request-golden-typescript-v1.json
@@ -0,0 +1,2189 @@
+{
+ "schema": "cua.jev_use_provider_request_golden_v1",
+ "language": "typescript",
+ "runs": {
+ "page/windows-page-1-python": [
+ {
+ "step": 1,
+ "request": "d144a9a07f2f3c01",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "7dca951dd791e8c7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "3826ca5aa3a4582f",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "60d9f833a0fb1977",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page/linux-page-3-python": [
+ {
+ "step": 1,
+ "request": "d144a9a07f2f3c01",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "7dca951dd791e8c7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "3826ca5aa3a4582f",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "60d9f833a0fb1977",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page/linux-page-7-typescript": [
+ {
+ "step": 1,
+ "request": "d144a9a07f2f3c01",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "7dca951dd791e8c7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "3826ca5aa3a4582f",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "60d9f833a0fb1977",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page/linux-page-2-python": [
+ {
+ "step": 1,
+ "request": "d144a9a07f2f3c01",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "7dca951dd791e8c7",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_auto/windows-page-1-python": [
+ {
+ "step": 1,
+ "request": "d144a9a07f2f3c01",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "7dca951dd791e8c7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "3826ca5aa3a4582f",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "60d9f833a0fb1977",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_auto/linux-page-3-python": [
+ {
+ "step": 1,
+ "request": "d144a9a07f2f3c01",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "7dca951dd791e8c7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "3826ca5aa3a4582f",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "60d9f833a0fb1977",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_auto/linux-page-7-typescript": [
+ {
+ "step": 1,
+ "request": "d144a9a07f2f3c01",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "7dca951dd791e8c7",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "3826ca5aa3a4582f",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "60d9f833a0fb1977",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_auto/linux-page-2-python": [
+ {
+ "step": 1,
+ "request": "d144a9a07f2f3c01",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "7dca951dd791e8c7",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_always/windows-page-1-python": [
+ {
+ "step": 1,
+ "request": "f05e1d864efb3c23",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "6cbb6199137b0925",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "13d9c8d27fdc73d1",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "5c30c0fb3c4b7442",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_always/linux-page-3-python": [
+ {
+ "step": 1,
+ "request": "f05e1d864efb3c23",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "6cbb6199137b0925",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "13d9c8d27fdc73d1",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "5c30c0fb3c4b7442",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_always/linux-page-7-typescript": [
+ {
+ "step": 1,
+ "request": "f05e1d864efb3c23",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "6cbb6199137b0925",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 3,
+ "request": "13d9c8d27fdc73d1",
+ "candidates": "d229b9dd4f676e88"
+ },
+ {
+ "step": 4,
+ "request": "5c30c0fb3c4b7442",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "page_capture_bound_always/linux-page-2-python": [
+ {
+ "step": 1,
+ "request": "f05e1d864efb3c23",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ {
+ "step": 2,
+ "request": "6cbb6199137b0925",
+ "candidates": "d229b9dd4f676e88"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "1b28206125d32608",
+ "candidates": "706d6b86ab8b3002"
+ }
+ ],
+ "visual_capture_bound_auto/linux_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "e9a218c29722d446",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 4,
+ "request": "8ab9a17f015b0fcc",
+ "candidates": "706d6b86ab8b3002"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "1b28206125d32608",
+ "candidates": "706d6b86ab8b3002"
+ }
+ ],
+ "visual_capture_bound_auto/linux_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "1b28206125d32608",
+ "candidates": "706d6b86ab8b3002"
+ }
+ ],
+ "visual_capture_bound_auto/windows_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "e9a218c29722d446",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 4,
+ "request": "695d64df7aea71d1",
+ "candidates": "366d1fa45767f12c"
+ }
+ ],
+ "visual_capture_bound_auto/windows_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "e9a218c29722d446",
+ "candidates": "366d1fa45767f12c"
+ }
+ ],
+ "visual_capture_bound_auto/windows_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "e9a218c29722d446",
+ "candidates": "366d1fa45767f12c"
+ }
+ ],
+ "visual_capture_bound_auto/windows_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "e9a218c29722d446",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 4,
+ "request": "695d64df7aea71d1",
+ "candidates": "366d1fa45767f12c"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_post_4255_regression_1": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "5423b86e68446d49",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "d5a8cd04d8624014",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "a68eda5180158762",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_post_4255_regression_2": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "5423b86e68446d49",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "d5a8cd04d8624014",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "a68eda5180158762",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_post_4255_regression_3": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "5423b86e68446d49",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "d5a8cd04d8624014",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "a68eda5180158762",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_post_4255_regression_4": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "5423b86e68446d49",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "d5a8cd04d8624014",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "a68eda5180158762",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_auto/linux_python_post_4255_regression_5": [
+ {
+ "step": 1,
+ "request": "e8d87e3e15e30453",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "5423b86e68446d49",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "d5a8cd04d8624014",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "a68eda5180158762",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "1b28206125d32608",
+ "candidates": "706d6b86ab8b3002"
+ }
+ ],
+ "visual_capture_bound_always/linux_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "e9a218c29722d446",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 4,
+ "request": "8ab9a17f015b0fcc",
+ "candidates": "706d6b86ab8b3002"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "1b28206125d32608",
+ "candidates": "706d6b86ab8b3002"
+ }
+ ],
+ "visual_capture_bound_always/linux_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "1b28206125d32608",
+ "candidates": "706d6b86ab8b3002"
+ }
+ ],
+ "visual_capture_bound_always/windows_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "e9a218c29722d446",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 4,
+ "request": "695d64df7aea71d1",
+ "candidates": "366d1fa45767f12c"
+ }
+ ],
+ "visual_capture_bound_always/windows_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "e9a218c29722d446",
+ "candidates": "366d1fa45767f12c"
+ }
+ ],
+ "visual_capture_bound_always/windows_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "e9a218c29722d446",
+ "candidates": "366d1fa45767f12c"
+ }
+ ],
+ "visual_capture_bound_always/windows_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "00bfc515ce7e492a",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 3,
+ "request": "e9a218c29722d446",
+ "candidates": "366d1fa45767f12c"
+ },
+ {
+ "step": 4,
+ "request": "695d64df7aea71d1",
+ "candidates": "366d1fa45767f12c"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_post_4255_regression_1": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "b4e5d58c3c0e8f93",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "e6ee50e7e55aee8d",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "cca37a534df52ebb",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_post_4255_regression_2": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "b4e5d58c3c0e8f93",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "e6ee50e7e55aee8d",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "cca37a534df52ebb",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_post_4255_regression_3": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "b4e5d58c3c0e8f93",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "e6ee50e7e55aee8d",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "cca37a534df52ebb",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_post_4255_regression_4": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "b4e5d58c3c0e8f93",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "e6ee50e7e55aee8d",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "cca37a534df52ebb",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_always/linux_python_post_4255_regression_5": [
+ {
+ "step": 1,
+ "request": "cc2331d3ee880e5b",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "b4e5d58c3c0e8f93",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "e6ee50e7e55aee8d",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "cca37a534df52ebb",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "947e79fa8942aeac",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "9f13ac1d2ad5d0bd",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/linux_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "947e79fa8942aeac",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "d00f64505137d76d",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 4,
+ "request": "91aed4f2a7b52459",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "947e79fa8942aeac",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "9f13ac1d2ad5d0bd",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/linux_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "947e79fa8942aeac",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "9f13ac1d2ad5d0bd",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/windows_python_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "947e79fa8942aeac",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "d00f64505137d76d",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 4,
+ "request": "a10e3de51047800d",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/windows_typescript_pre_4255_a": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "947e79fa8942aeac",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "d00f64505137d76d",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/windows_python_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "947e79fa8942aeac",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "d00f64505137d76d",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/windows_typescript_pre_4255_b": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "947e79fa8942aeac",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 3,
+ "request": "d00f64505137d76d",
+ "candidates": "daba6141ff108d4e"
+ },
+ {
+ "step": 4,
+ "request": "a10e3de51047800d",
+ "candidates": "daba6141ff108d4e"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_post_4255_regression_1": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "ef44e05f270244a2",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "4b51b7ad29961b42",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "08be10eed5bf3061",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_post_4255_regression_2": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "ef44e05f270244a2",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "4b51b7ad29961b42",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "08be10eed5bf3061",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_post_4255_regression_3": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "ef44e05f270244a2",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "4b51b7ad29961b42",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "08be10eed5bf3061",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_post_4255_regression_4": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "ef44e05f270244a2",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "4b51b7ad29961b42",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "08be10eed5bf3061",
+ "candidates": "70875f146d859b36"
+ }
+ ],
+ "visual_capture_bound_off/linux_python_post_4255_regression_5": [
+ {
+ "step": 1,
+ "request": "81a25fcd82f06cf4",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 2,
+ "request": "ef44e05f270244a2",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 3,
+ "request": "4b51b7ad29961b42",
+ "candidates": "70875f146d859b36"
+ },
+ {
+ "step": 4,
+ "request": "08be10eed5bf3061",
+ "candidates": "70875f146d859b36"
+ }
+ ]
+ },
+ "single_requests": {
+ "page_other_value": {
+ "request": "e1ad8e9440c7d080",
+ "candidates": "ec571c9800d7b4e8"
+ },
+ "page_after_with_visual": {
+ "request": "ebbcae05b138d8d2",
+ "candidates": "d229b9dd4f676e88"
+ },
+ "page_after_no_button": {
+ "request": "bc0dcf345fc4ff00",
+ "candidates": "daba6141ff108d4e"
+ },
+ "page_after_no_button_visual_path": {
+ "request": "77a2b7b29d13d90b",
+ "candidates": "daba6141ff108d4e"
+ },
+ "no_form_refs": {
+ "request": "74545c6ebe3990ae",
+ "candidates": "daba6141ff108d4e"
+ },
+ "visual_after_no_capture_bound_click": {
+ "request": "de5ff5949f9e4473",
+ "candidates": "daba6141ff108d4e"
+ },
+ "visual_after_without_submit_region": {
+ "request": "476a091a64f9f2ab",
+ "candidates": "daba6141ff108d4e"
+ },
+ "visual_after_duplicate_submit_region": {
+ "request": "540fdaf0e9d0b71e",
+ "candidates": "daba6141ff108d4e"
+ },
+ "visual_after_foreground": {
+ "request": "dc62deb5acc6dacc",
+ "candidates": "706d6b86ab8b3002"
+ },
+ "visual_after_other_value": {
+ "request": "7906009abc493b1e",
+ "candidates": "3c210f06a8903bc9"
+ }
+ },
+ "requests": {
+ "d144a9a07f2f3c01": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"available\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "7dca951dd791e8c7": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "3826ca5aa3a4582f": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "60d9f833a0fb1977": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "f05e1d864efb3c23": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"available\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "6cbb6199137b0925": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "13d9c8d27fdc73d1": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "5c30c0fb3c4b7442": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "e8d87e3e15e30453": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_check_pending\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "00bfc515ce7e492a": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "1b28206125d32608": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"submit-form\",\"outcome\":\"Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "e9a218c29722d446": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "8ab9a17f015b0fcc": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"submit-form\",\"outcome\":\"Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "695d64df7aea71d1": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "5423b86e68446d49": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_check_pending\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "d5a8cd04d8624014": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_check_pending\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "a68eda5180158762": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_check_pending\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "cc2331d3ee880e5b": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "b4e5d58c3c0e8f93": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "e6ee50e7e55aee8d": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "cca37a534df52ebb": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "81a25fcd82f06cf4": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"not_in_page_structure\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "947e79fa8942aeac": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "9f13ac1d2ad5d0bd": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "d00f64505137d76d": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "91aed4f2a7b52459": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"Driver refused background delivery (background_unavailable); no click happened and a foreground Submit candidate is offered next\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "a10e3de51047800d": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"type-verification-value\",\"outcome\":\"typed the required token into the verification field\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "ef44e05f270244a2": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"not_in_page_structure\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "4b51b7ad29961b42": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"not_in_page_structure\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "08be10eed5bf3061": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"empty\",\"submit_button\":\"not_in_page_structure\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - statictext \"Submit\"\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[{\"step\":1,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":2,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"},{\"step\":3,\"selected_id\":\"reobserve\",\"outcome\":\"took no action and requested a fresh observation\"}]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "e1ad8e9440c7d080": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"contains_other_value\",\"submit_button\":\"available\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "ebbcae05b138d8d2": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"available\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "bc0dcf345fc4ff00": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_in_page_structure\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "77a2b7b29d13d90b": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:8799/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_check_pending\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": "null"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "74545c6ebe3990ae": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "null",
+ "form": "{\"verification_field\":\"not_found\",\"submit_button\":\"not_in_page_structure\"}",
+ "outline": "",
+ "visual": "null"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "de5ff5949f9e4473": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "476a091a64f9f2ab": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_found_visually\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}}]}"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "540fdaf0e9d0b71e": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"not_found_visually\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}},{\"id\":\"dup\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "dc62deb5acc6dacc": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_required_token\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "submit-form-foreground": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ },
+ "7906009abc493b1e": {
+ "state": {
+ "goal": "Enter the required verification token into the verification field, then submit the form.",
+ "observation": {
+ "page": "{\"title\":\"about:blank\",\"url\":\"http://127.0.0.1:41605/\"}",
+ "form": "{\"verification_field\":\"contains_other_value\",\"submit_button\":\"visual_only\"}",
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - textbox \"verification value\": [verification token] [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - generic [editable=\"plaintext\"]\n - statictext \"Submit\"\n - status\n - statictext \"[verification token]\" [editable=\"plaintext\"]\n - statictext \"status=waiting\"",
+ "visual": "{\"schema\":\"cua.visual_regions_v1\",\"capture_id\":\"capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000\",\"screenshot_reference\":\"png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a\",\"source\":{\"kind\":\"window\",\"pid\":2539,\"window_id\":52428804},\"regions\":[{\"id\":\"text-9\",\"kind\":\"text\",\"text\":\"×\",\"label\":null,\"confidence\":0.8168784976005554,\"interactive\":false,\"bounds\":{\"x\":252,\"y\":10,\"width\":18,\"height\":19}},{\"id\":\"text-11\",\"kind\":\"text\",\"text\":\"+\",\"label\":null,\"confidence\":0.8311772346496582,\"interactive\":false,\"bounds\":{\"x\":288,\"y\":13,\"width\":14,\"height\":14}},{\"id\":\"icon-15\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8110984563827515,\"interactive\":false,\"bounds\":{\"x\":10,\"y\":45,\"width\":34,\"height\":36}},{\"id\":\"icon-17\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8079566955566406,\"interactive\":false,\"bounds\":{\"x\":83,\"y\":46,\"width\":32,\"height\":36}},{\"id\":\"icon-18\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.8043645024299622,\"interactive\":false,\"bounds\":{\"x\":47,\"y\":46,\"width\":31,\"height\":34}},{\"id\":\"text-26\",\"kind\":\"text\",\"text\":\"←\",\"label\":null,\"confidence\":0.8558894395828247,\"interactive\":false,\"bounds\":{\"x\":17,\"y\":54,\"width\":20,\"height\":18}},{\"id\":\"text-28\",\"kind\":\"text\",\"text\":\"Cua\",\"label\":null,\"confidence\":0.8023205995559692,\"interactive\":false,\"bounds\":{\"x\":175,\"y\":389,\"width\":118,\"height\":47}},{\"id\":\"text-29\",\"kind\":\"text\",\"text\":\"TypeSafe Jev\",\"label\":null,\"confidence\":0.8917925357818604,\"interactive\":false,\"bounds\":{\"x\":471,\"y\":389,\"width\":302,\"height\":52}},{\"id\":\"text-30\",\"kind\":\"text\",\"text\":\"Driver +\",\"label\":null,\"confidence\":0.865290641784668,\"interactive\":false,\"bounds\":{\"x\":270,\"y\":390,\"width\":207,\"height\":46}},{\"id\":\"icon-33\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9488332271575928,\"interactive\":false,\"bounds\":{\"x\":637,\"y\":561,\"width\":132,\"height\":63}},{\"id\":\"icon-34\",\"kind\":\"icon\",\"text\":null,\"label\":\"icon-class-0\",\"confidence\":0.9535441398620604,\"interactive\":false,\"bounds\":{\"x\":172,\"y\":562,\"width\":449,\"height\":65}},{\"id\":\"text-35\",\"kind\":\"text\",\"text\":\"[verification token]\",\"label\":null,\"confidence\":0.8249707818031311,\"interactive\":false,\"bounds\":{\"x\":188,\"y\":575,\"width\":149,\"height\":35}},{\"id\":\"text-36\",\"kind\":\"text\",\"text\":\"Submit\",\"label\":null,\"confidence\":0.8971455097198486,\"interactive\":false,\"bounds\":{\"x\":662,\"y\":575,\"width\":85,\"height\":32}}]}"
+ },
+ "history": "[]"
+ },
+ "questions": {
+ "driver_action": {
+ "type": "choice",
+ "instructions": "Which complete executable action should Cua Driver run next?",
+ "criteria": {
+ "type-verification-value": "Type the required verification token into the verification field, replacing its current contents.",
+ "reobserve": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "abstain": "Stop without acting if none of the proposed actions is safe for the observed state."
+ }
+ }
+ }
+ }
+ },
+ "candidate_tables": {
+ "ec571c9800d7b4e8": [
+ {
+ "id": "type-verification-value",
+ "description": "Type the required verification token into the verification field, replacing its current contents.",
+ "tool": "browser_type",
+ "arguments": {
+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",
+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",
+ "ref": "p3:0",
+ "text": "jev-guide-live",
+ "replace": true
+ },
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "d229b9dd4f676e88": [
+ {
+ "id": "submit-form",
+ "description": "Click the form's Submit button. The observed form state reports that the verification field already contains the required token, so the form is ready to submit.",
+ "tool": "browser_click",
+ "arguments": {
+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",
+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",
+ "ref": "p4:2",
+ "input_route": "dom_event"
+ },
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "70875f146d859b36": [
+ {
+ "id": "type-verification-value",
+ "description": "Type the required verification token into the verification field, replacing its current contents.",
+ "tool": "browser_type",
+ "arguments": {
+ "target_id": "bt-aeb40534-c16c-4da9-9567-aee3d2d4958a",
+ "tab_id": "tab-11edc8ea-1185-4d45-be26-6df2f7082d7e",
+ "ref": "p1:0",
+ "text": "jev-guide-live",
+ "replace": true
+ },
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "366d1fa45767f12c": [
+ {
+ "id": "submit-form",
+ "description": "Submit the form by clicking the unique validated visual Submit region. The observed form state reports that the verification field already contains the required token.",
+ "tool": "click",
+ "arguments": {
+ "pid": 2539,
+ "window_id": 52428804,
+ "x": 704.5,
+ "y": 591,
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "delivery_mode": "background"
+ },
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a"
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "706d6b86ab8b3002": [
+ {
+ "id": "submit-form-foreground",
+ "description": "Submit the form by clicking the unique validated visual Submit region with foreground delivery, which activates the browser window, because Driver refused background delivery for the previous visual click.",
+ "tool": "click",
+ "arguments": {
+ "pid": 2539,
+ "window_id": 52428804,
+ "x": 704.5,
+ "y": 591,
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "delivery_mode": "foreground"
+ },
+ "capture_id": "capture_8a22ffd72f02467f9ff0f0b6de791b6e_0000000000000000",
+ "screenshot_reference": "png-sha256:2be15022cfea6b1c45bf72902bd4db54075cec6748d1defec5ecd1d75cc9259a"
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "daba6141ff108d4e": [
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ],
+ "3c210f06a8903bc9": [
+ {
+ "id": "type-verification-value",
+ "description": "Type the required verification token into the verification field, replacing its current contents.",
+ "tool": "browser_type",
+ "arguments": {
+ "target_id": "bt-aeb40534-c16c-4da9-9567-aee3d2d4958a",
+ "tab_id": "tab-11edc8ea-1185-4d45-be26-6df2f7082d7e",
+ "ref": "p2:1",
+ "text": "jev-guide-live",
+ "replace": true
+ },
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "reobserve",
+ "description": "Take no action and obtain a fresh Driver observation, because the current observation is stale or contradicts the reported form state. A Submit control that is visual-only, or whose visual check is still pending, is not a reason to reobserve: the runner parses visual regions for Submit once no page-structure action remains.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ },
+ {
+ "id": "abstain",
+ "description": "Stop without acting if none of the proposed actions is safe for the observed state.",
+ "tool": null,
+ "arguments": {},
+ "capture_id": null,
+ "screenshot_reference": null
+ }
+ ]
+ }
+}
diff --git a/libs/cua-driver/examples/jev-use/python/core.py b/libs/cua-driver/examples/jev-use/python/core.py
index a4c1b4cd9e..b3c9781191 100644
--- a/libs/cua-driver/examples/jev-use/python/core.py
+++ b/libs/cua-driver/examples/jev-use/python/core.py
@@ -1,35 +1,57 @@
+"""Runner primitives shared by every jev-use task.
+
+Visual-region parsing, candidate validation, and the mock chooser live here.
+Candidate sources are in ``sources.py`` and task specs in ``tasks.py``; the
+names re-exported below keep existing imports from ``core`` working.
+"""
+
from __future__ import annotations
import math
from dataclasses import dataclass
-from types import MappingProxyType
from typing import Any, Literal, Mapping
-Outcome = Literal["verified", "refuted", "unknown", "abstained", "budget_exhausted"]
-VisualDelivery = Literal["background", "foreground"]
-
-SUBMIT_IDS = frozenset({"submit-form", "submit-form-foreground"})
-
-
-def _freeze(value: Any) -> Any:
- if isinstance(value, dict):
- return MappingProxyType({key: _freeze(item) for key, item in value.items()})
- if isinstance(value, list):
- return tuple(_freeze(item) for item in value)
- return value
-
-
-@dataclass(frozen=True)
-class Candidate:
- id: str
- description: str
- tool: str | None
- arguments: Mapping[str, Any]
- capture_id: str | None = None
- screenshot_reference: str | None = None
+from sources import Candidate
+from tasks import (
+ FIELD_NAME,
+ HISTORY_OUTCOMES,
+ REDACTED_TOKEN,
+ SUBMIT_IDS,
+ SUBMIT_NAME,
+ Outcome,
+ build_candidates,
+ classify,
+ form_state,
+ history_entry,
+ redact_token,
+ visual_submit_region,
+)
+
+__all__ = [
+ "FIELD_NAME",
+ "HISTORY_OUTCOMES",
+ "REDACTED_TOKEN",
+ "SUBMIT_IDS",
+ "SUBMIT_NAME",
+ "Candidate",
+ "Outcome",
+ "VisualDelivery",
+ "VisualObservation",
+ "VisualObservationError",
+ "VisualRegion",
+ "build_candidates",
+ "choose_mock",
+ "classify",
+ "form_state",
+ "has_executable_candidate",
+ "history_entry",
+ "parse_visual_regions",
+ "redact_token",
+ "validate_choice",
+ "visual_submit_region",
+]
- def __post_init__(self) -> None:
- object.__setattr__(self, "arguments", _freeze(dict(self.arguments)))
+VisualDelivery = Literal["background", "foreground"]
@dataclass(frozen=True)
@@ -213,228 +235,10 @@ def parse_visual_regions(
)
-FIELD_NAME = "verification value"
-SUBMIT_NAME = "Submit"
-REDACTED_TOKEN = "[verification token]"
-
-
-def _form_refs(snapshot: Mapping[str, Any]) -> tuple[Any, Any]:
- refs = snapshot.get("refs") or []
- field = next(
- (ref for ref in refs if ref.get("role") == "textbox" and ref.get("name") == FIELD_NAME),
- None,
- )
- button = next(
- (ref for ref in refs if ref.get("role") == "button" and ref.get("name") == SUBMIT_NAME),
- None,
- )
- return field, button
-
-
-def visual_submit_region(visual: VisualObservation | None) -> VisualRegion | None:
- """Return the unique validated visual Submit region, or ``None``."""
- if visual is None:
- return None
- matches = [
- region
- for region in visual.regions
- if region.confidence >= 0.8
- and _ascii_lower(region.text or region.label or "") == "submit"
- ]
- return matches[0] if len(matches) == 1 else None
-
-
-def form_state(
- snapshot: Mapping[str, Any],
- token: str,
- visual: VisualObservation | None = None,
- *,
- visual_path: bool = False,
-) -> dict[str, str]:
- """Summarize the form for the decision model without revealing the token.
-
- The raw field value never leaves the runner; the model receives only whether
- the field is empty, holds the required token, or holds something else.
-
- ``submit_button`` is ``available`` for a clickable page-structure ref. When
- the page structure has none and the capture-bound visual path is enabled
- (``visual_path``), it is ``visual_only`` if this observation holds a unique
- validated visual Submit region, ``visual_check_pending`` if no visual regions
- were parsed for this step yet (the runner parses them once no page-structure
- action remains), and ``not_found_visually`` otherwise. Without a visual path
- it is ``not_in_page_structure``.
- """
- field, button = _form_refs(snapshot)
- if field is None:
- field_state = "not_found"
- elif not field.get("value"):
- field_state = "empty"
- elif field.get("value") == token:
- field_state = "contains_required_token"
- else:
- field_state = "contains_other_value"
- if button is not None:
- submit_state = "available"
- elif not visual_path:
- submit_state = "not_in_page_structure"
- elif visual is None:
- submit_state = "visual_check_pending"
- elif visual_submit_region(visual) is not None:
- submit_state = "visual_only"
- else:
- submit_state = "not_found_visually"
- return {"verification_field": field_state, "submit_button": submit_state}
-
-
-def redact_token(value: Any, token: str) -> Any:
- """Replace every occurrence of the token in strings nested in ``value``."""
- if not token:
- return value
- if isinstance(value, str):
- return value.replace(token, REDACTED_TOKEN)
- if isinstance(value, Mapping):
- return {key: redact_token(item, token) for key, item in value.items()}
- if isinstance(value, (list, tuple)):
- return [redact_token(item, token) for item in value]
- return value
-
-
-HISTORY_OUTCOMES = {
- "type-verification-value": "typed the required token into the verification field",
- "submit-form": "clicked Submit; the submission was not yet confirmed",
- "submit-form-foreground": "clicked Submit in the foreground; the submission was not yet confirmed",
- "reobserve": "took no action and requested a fresh observation",
-}
-
-
-def history_entry(
- step: int, candidate_id: str, *, refusal: str | None = None
-) -> dict[str, Any]:
- """Build the compact decision-history item shown to the model.
-
- It records what each step did, not timings or model probabilities, so earlier
- choices do not become a signal to repeat themselves.
- """
- if refusal is not None:
- outcome = (
- f"Driver refused background delivery ({refusal}); no click happened and a "
- "foreground Submit candidate is offered next"
- )
- else:
- outcome = HISTORY_OUTCOMES.get(candidate_id, "completed")
- return {"step": step, "selected_id": candidate_id, "outcome": outcome}
-
-
-def _reserved_candidates() -> list[Candidate]:
- return [
- Candidate(
- "reobserve",
- "Take no action and obtain a fresh Driver observation, because the current "
- "observation is stale or contradicts the reported form state. A Submit control "
- "that is visual-only, or whose visual check is still pending, is not a reason to "
- "reobserve: the runner parses visual regions for Submit once no page-structure "
- "action remains.",
- None,
- {},
- ),
- Candidate(
- "abstain",
- "Stop without acting if none of the proposed actions is safe for the observed state.",
- None,
- {},
- ),
- ]
-
-
-def build_candidates(
- snapshot: dict[str, Any],
- token: str,
- visual: VisualObservation | None = None,
- *,
- capture_bound_click: bool = False,
- visual_delivery: VisualDelivery = "background",
-) -> list[Candidate]:
- """Build the closed candidate set for one decision.
-
- Page-structure refs always win. The capture-bound visual Submit is offered
- only when no Submit ref exists. ``visual_delivery="foreground"`` replaces the
- background visual click with a distinct ``submit-form-foreground`` candidate
- after Driver refused background delivery; the chooser must pick it explicitly.
- """
- common = {
- "target_id": snapshot["target_id"],
- "tab_id": snapshot["tab_id"],
- }
- field, button = _form_refs(snapshot)
- candidates: list[Candidate] = []
- if field and field.get("value") != token:
- candidates.append(
- Candidate(
- "type-verification-value",
- "Type the required verification token into the verification field, "
- "replacing its current contents.",
- "browser_type",
- {**common, "ref": field["ref"], "text": token, "replace": True},
- )
- )
- elif field and field.get("value") == token and button:
- candidates.append(
- Candidate(
- "submit-form",
- "Click the form's Submit button. The observed form state reports that the "
- "verification field already contains the required token, so the form is "
- "ready to submit.",
- "browser_click",
- {**common, "ref": button["ref"], "input_route": "dom_event"},
- )
- )
- elif (
- field
- and field.get("value") == token
- and visual
- and capture_bound_click
- ):
- region = visual_submit_region(visual)
- if region is not None:
- x, y = visual.screenshot_center(region)
- foreground = visual_delivery == "foreground"
- candidates.append(
- Candidate(
- "submit-form-foreground" if foreground else "submit-form",
- (
- "Submit the form by clicking the unique validated visual Submit "
- "region with foreground delivery, which activates the browser "
- "window, because Driver refused background delivery for the "
- "previous visual click."
- if foreground
- else "Submit the form by clicking the unique validated visual Submit "
- "region. The observed form state reports that the verification field "
- "already contains the required token."
- ),
- "click",
- {
- "pid": visual.pid,
- "window_id": visual.window_id,
- "x": x,
- "y": y,
- "capture_id": visual.capture_id,
- "delivery_mode": visual_delivery,
- },
- capture_id=visual.capture_id,
- screenshot_reference=visual.screenshot_reference,
- )
- )
- return candidates + _reserved_candidates()
-
-
def has_executable_candidate(candidates: list[Candidate]) -> bool:
return any(candidate.tool is not None for candidate in candidates)
-def _ascii_lower(value: str) -> str:
- return "".join(chr(ord(char) + 32) if "A" <= char <= "Z" else char for char in value)
-
-
def choose_mock(candidates: list[Candidate]) -> tuple[str | None, float, dict[str, float]]:
ids = {candidate.id for candidate in candidates}
selected = (
@@ -469,13 +273,3 @@ def validate_choice(
if candidate.capture_id is not None and candidate.capture_id != current_capture_id:
raise ValueError("provider selected a stale or capture-mismatched candidate")
return candidate
-
-
-def classify(submitted: str | None, token: str, *, steps: int, max_steps: int) -> Outcome:
- if submitted == token:
- return "verified"
- if submitted is not None:
- return "refuted"
- if steps >= max_steps:
- return "budget_exhausted"
- return "unknown"
diff --git a/libs/cua-driver/examples/jev-use/python/jev_adapter.py b/libs/cua-driver/examples/jev-use/python/jev_adapter.py
index 90db703b65..c263edf499 100644
--- a/libs/cua-driver/examples/jev-use/python/jev_adapter.py
+++ b/libs/cua-driver/examples/jev-use/python/jev_adapter.py
@@ -4,7 +4,8 @@ import math
from dataclasses import dataclass
from typing import Any, Mapping, Protocol
-from core import Candidate, VisualObservation, choose_mock, form_state, redact_token
+from core import Candidate, VisualObservation, choose_mock
+from tasks import FIXTURE_GOAL, FixtureFormTask, Task, TaskSources, fixture_sources
class TypeSafeClientLike(Protocol):
@@ -95,52 +96,66 @@ def visual_decision_state(visual: VisualObservation | None) -> dict[str, Any] |
}
-GOAL = "Enter the required verification token into the verification field, then submit the form."
+GOAL = FIXTURE_GOAL
-def decision_state(
- snapshot: Mapping[str, Any],
- visual: VisualObservation | None,
- history: list[dict[str, Any]],
- token: str,
- visual_path: bool = False,
+def task_decision_state(
+ task: Task, sources: TaskSources, history: list[dict[str, Any]]
) -> dict[str, Any]:
- """Build the compact, deterministic, token-redacted state sent to Jev.
+ """Build the compact, deterministic, secret-redacted state sent to Jev.
- ``form`` states the field and Submit status the runner verified from the page
- structure and, when ``visual_path`` is enabled, from the validated visual
- regions, so the model does not have to infer it from the outline. The token
- itself is replaced everywhere, including the outline and visual text.
+ ``form`` is the task's state summary, which states what the runner verified
+ from its candidate sources, so the model does not have to infer it from the
+ outline. Every secret task parameter is replaced everywhere, including the
+ outline and visual text.
"""
+ snapshot = sources.page.snapshot
+ visual = sources.visual.observation if sources.visual is not None else None
return {
- "goal": GOAL,
+ "goal": task.goal,
"observation": {
- "page": redact_token(snapshot.get("page"), token),
- "form": form_state(snapshot, token, visual, visual_path=visual_path),
- "outline": redact_token(snapshot.get("outline"), token),
- "visual": redact_token(visual_decision_state(visual), token),
+ "page": task.redact(snapshot.get("page")),
+ "form": task.state_summary(sources),
+ "outline": task.redact(snapshot.get("outline")),
+ "visual": task.redact(visual_decision_state(visual)),
},
"history": [dict(item) for item in history],
}
-def choose_with_typesafe(
- client: TypeSafeClientLike,
- candidates: list[Candidate],
- snapshot: dict[str, Any],
+def decision_state(
+ snapshot: Mapping[str, Any],
visual: VisualObservation | None,
history: list[dict[str, Any]],
token: str,
visual_path: bool = False,
+) -> dict[str, Any]:
+ """Build the fixture task's decision state; see ``task_decision_state``."""
+ return task_decision_state(
+ FixtureFormTask(token),
+ fixture_sources(snapshot, visual, visual_path=visual_path),
+ history,
+ )
+
+
+DRIVER_ACTION_INSTRUCTIONS = "Which complete executable action should Cua Driver run next?"
+
+
+def choose_for_task(
+ client: TypeSafeClientLike,
+ task: Task,
+ sources: TaskSources,
+ candidates: list[Candidate],
+ history: list[dict[str, Any]],
) -> tuple[str, float, dict[str, float]]:
from typesafe_sdk import Choice
criteria = _candidate_criteria(candidates)
response = client.system_one(
- state=decision_state(snapshot, visual, history, token, visual_path),
+ state=task_decision_state(task, sources, history),
questions={
"driver_action": Choice(
- instructions="Which complete executable action should Cua Driver run next?",
+ instructions=DRIVER_ACTION_INSTRUCTIONS,
criteria=criteria,
)
},
@@ -151,20 +166,59 @@ def choose_with_typesafe(
return answer.choice, answer.confidence, answer.probabilities
-def choose_live(
+def choose_with_typesafe(
+ client: TypeSafeClientLike,
candidates: list[Candidate],
snapshot: dict[str, Any],
visual: VisualObservation | None,
history: list[dict[str, Any]],
token: str,
visual_path: bool = False,
+) -> tuple[str, float, dict[str, float]]:
+ return choose_for_task(
+ client,
+ FixtureFormTask(token),
+ fixture_sources(snapshot, visual, visual_path=visual_path),
+ candidates,
+ history,
+ )
+
+
+def choose_live_for_task(
+ task: Task,
+ sources: TaskSources,
+ candidates: list[Candidate],
+ history: list[dict[str, Any]],
) -> tuple[str, float, dict[str, float]]:
from typesafe_sdk import TypeSafeClient
with TypeSafeClient() as client:
- return choose_with_typesafe(
- client, candidates, snapshot, visual, history, token, visual_path
- )
+ return choose_for_task(client, task, sources, candidates, history)
+
+
+def choose_mock_for_task(
+ _task: Task,
+ _sources: TaskSources,
+ candidates: list[Candidate],
+ _history: list[dict[str, Any]],
+) -> tuple[str | None, float, dict[str, float]]:
+ return choose_mock(candidates)
+
+
+def choose_live(
+ candidates: list[Candidate],
+ snapshot: dict[str, Any],
+ visual: VisualObservation | None,
+ history: list[dict[str, Any]],
+ token: str,
+ visual_path: bool = False,
+) -> tuple[str, float, dict[str, float]]:
+ return choose_live_for_task(
+ FixtureFormTask(token),
+ fixture_sources(snapshot, visual, visual_path=visual_path),
+ candidates,
+ history,
+ )
def choose_mock_adapter(
diff --git a/libs/cua-driver/examples/jev-use/python/run.py b/libs/cua-driver/examples/jev-use/python/run.py
index 1ec01b93b1..66f4edd454 100644
--- a/libs/cua-driver/examples/jev-use/python/run.py
+++ b/libs/cua-driver/examples/jev-use/python/run.py
@@ -9,31 +9,31 @@ import uuid
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
-from urllib.request import Request, urlopen
from mcp import ClientSession, StdioServerParameters
from mcp.client.stdio import stdio_client
from driver_env import driver_environment
from core import (
- SUBMIT_IDS,
Candidate,
VisualDelivery,
VisualObservation,
VisualObservationError,
- build_candidates,
- classify,
- history_entry,
has_executable_candidate,
parse_visual_regions,
validate_choice,
)
-from jev_adapter import choose_live, choose_mock_adapter
-
+from jev_adapter import choose_live_for_task, choose_mock_for_task
+from tasks import (
+ FixtureFormTask,
+ Task,
+ TaskSources,
+ fixture_sources,
+ fixture_state,
+ reset_fixture,
+)
-def fixture_state(fixture_url: str) -> dict[str, str | None]:
- with urlopen(f"{fixture_url.rstrip('/')}/state", timeout=2) as response:
- return json.loads(response.read())
+__all__ = ["fixture_state", "reset_fixture"]
def validate_fixture_url(value: str) -> str:
@@ -60,13 +60,6 @@ def select_tab_id(tabs: list[dict[str, Any]]) -> str:
return str(selected["tab_id"])
-def reset_fixture(fixture_url: str) -> None:
- request = Request(f"{fixture_url.rstrip('/')}/reset", method="POST", data=b"")
- with urlopen(request, timeout=2) as response:
- if response.status != 204:
- raise RuntimeError(f"fixture reset failed: HTTP {response.status}")
-
-
class DriverToolError(RuntimeError):
"""A Driver tool returned an error result, optionally with a stable error code."""
@@ -223,10 +216,10 @@ async def optional_visual_observation(
return visual
-async def candidates_for_step(
+async def task_candidates_for_step(
driver: Driver,
+ task: Task,
snapshot: dict[str, Any],
- token: str,
pid: int,
window_id: int,
available_tools: set[str],
@@ -234,35 +227,68 @@ async def candidates_for_step(
*,
visual_mode: str = "auto",
visual_delivery: VisualDelivery = "background",
-) -> tuple[list[Candidate], VisualObservation | None, dict[str, Any]]:
- """Build one step's candidates, parsing visual regions only when useful.
+) -> tuple[list[Candidate], TaskSources, dict[str, Any]]:
+ """Build one step's sources and candidates, parsing visual regions only when useful.
In ``auto`` mode the capture and parse run only when the page structure
offers no executable candidate, because only then can a visual region add
one. ``always`` restores the per-step parse; ``off`` never parses.
"""
- candidates = build_candidates(
+ # Whether a control missing from the page structure can still be found
+ # through a capture-bound visual region; reported in the task state.
+ visual_path = capture_bound_click and visual_mode != "off"
+ sources = fixture_sources(
snapshot,
- token,
None,
capture_bound_click=capture_bound_click,
visual_delivery=visual_delivery,
+ visual_path=visual_path,
)
+ candidates = task.candidates(sources)
if visual_mode == "off":
- return candidates, None, visual_status("skipped", reason="disabled")
+ return candidates, sources, visual_status("skipped", reason="disabled")
if visual_mode == "auto" and has_executable_candidate(candidates):
- return candidates, None, visual_status("skipped", reason="page_structure_candidate")
+ return candidates, sources, visual_status("skipped", reason="page_structure_candidate")
visual, record = await observe_visual(
driver, pid, window_id, available_tools, capture_bound_click
)
if visual is not None:
- candidates = build_candidates(
+ sources = fixture_sources(
snapshot,
- token,
visual,
capture_bound_click=capture_bound_click,
visual_delivery=visual_delivery,
+ visual_path=visual_path,
)
+ candidates = task.candidates(sources)
+ return candidates, sources, record
+
+
+async def candidates_for_step(
+ driver: Driver,
+ snapshot: dict[str, Any],
+ token: str,
+ pid: int,
+ window_id: int,
+ available_tools: set[str],
+ capture_bound_click: bool,
+ *,
+ visual_mode: str = "auto",
+ visual_delivery: VisualDelivery = "background",
+) -> tuple[list[Candidate], VisualObservation | None, dict[str, Any]]:
+ """Build one step's fixture-task candidates; see ``task_candidates_for_step``."""
+ candidates, sources, record = await task_candidates_for_step(
+ driver,
+ FixtureFormTask(token),
+ snapshot,
+ pid,
+ window_id,
+ available_tools,
+ capture_bound_click,
+ visual_mode=visual_mode,
+ visual_delivery=visual_delivery,
+ )
+ visual = sources.visual.observation if sources.visual is not None else None
return candidates, visual, record
@@ -289,6 +315,7 @@ def write_event(log_path: Path | None, event: dict[str, Any]) -> None:
async def run(args: argparse.Namespace) -> str:
token = args.token or f"jev-{uuid.uuid4().hex[:10]}"
+ task: Task = FixtureFormTask(token, args.fixture_url, args.max_steps)
label = f"jev-python-{uuid.uuid4().hex[:8]}"
# Compact what-happened record for the decision model. The full telemetry
# events (timings, probabilities) go only to the JSONL log.
@@ -297,7 +324,7 @@ async def run(args: argparse.Namespace) -> str:
log_path = Path(args.log) if args.log else None
if log_path:
log_path.write_text("", encoding="utf-8")
- reset_fixture(args.fixture_url)
+ task.reset()
params = StdioServerParameters(
command=os.getenv("CUA_DRIVER_BIN", "cua-driver"), args=["mcp"], env=driver_environment()
@@ -308,9 +335,6 @@ async def run(args: argparse.Namespace) -> str:
advertised_tools = (await session.list_tools()).tools
available_tools = {tool.name for tool in advertised_tools}
capture_bound_click = supports_capture_bound_click(advertised_tools)
- # Whether a Submit missing from the page structure can still be found
- # through a capture-bound visual region; reported in the form state.
- visual_path = capture_bound_click and args.visual_observation != "off"
driver = Driver(session, label)
prepared = await driver.call(
"browser_prepare",
@@ -328,9 +352,8 @@ async def run(args: argparse.Namespace) -> str:
{"target_id": target_id, "tab_id": tab_id, "url": args.fixture_url},
)
- for step in range(1, args.max_steps + 1):
- oracle = fixture_state(args.fixture_url)
- current = classify(oracle.get("submitted"), token, steps=step - 1, max_steps=args.max_steps)
+ for step in range(1, task.max_steps + 1):
+ current = task.classify(task.read_oracle(), steps=step - 1)
if current in {"verified", "refuted"}:
write_event(log_path, {"event": "outcome", "outcome": current, "token": token})
return current
@@ -344,10 +367,10 @@ async def run(args: argparse.Namespace) -> str:
"snapshot_format": "semantic_v2",
},
)
- candidates, visual, visual_record = await candidates_for_step(
+ candidates, sources, visual_record = await task_candidates_for_step(
driver,
+ task,
snapshot,
- token,
pid,
int(window["window_id"]),
available_tools,
@@ -362,13 +385,14 @@ async def run(args: argparse.Namespace) -> str:
)
return "abstained"
+ visual = sources.visual.observation if sources.visual is not None else None
if args.provider == "mock":
- choice, confidence, probabilities = choose_mock_adapter(
- candidates, snapshot, visual, history, token, visual_path
+ choice, confidence, probabilities = choose_mock_for_task(
+ task, sources, candidates, history
)
else:
choice, confidence, probabilities = await asyncio.to_thread(
- choose_live, candidates, snapshot, visual, history, token, visual_path
+ choose_live_for_task, task, sources, candidates, history
)
if choice is None:
return "abstained"
@@ -392,7 +416,7 @@ async def run(args: argparse.Namespace) -> str:
"tool": None,
"visual": visual_record,
}
- history.append(history_entry(step, candidate.id))
+ history.append(task.history_entry(step, candidate.id))
write_event(log_path, event)
continue
@@ -440,7 +464,7 @@ async def run(args: argparse.Namespace) -> str:
},
"visual": visual_record,
}
- history.append(history_entry(step, candidate.id, refusal=refusal))
+ history.append(task.history_entry(step, candidate.id, refusal=refusal))
write_event(log_path, event)
continue
write_event(
@@ -472,16 +496,13 @@ async def run(args: argparse.Namespace) -> str:
"delivery_mode": candidate.arguments.get("delivery_mode"),
"visual": visual_record,
}
- history.append(history_entry(step, candidate.id))
+ history.append(task.history_entry(step, candidate.id))
write_event(log_path, event)
if args.dry_run:
return "unknown"
- if candidate.id in SUBMIT_IDS:
+ if candidate.id in task.completion_candidate_ids:
for _ in range(20):
- oracle = fixture_state(args.fixture_url)
- outcome = classify(
- oracle.get("submitted"), token, steps=step, max_steps=args.max_steps
- )
+ outcome = task.classify(task.read_oracle(), steps=step)
if outcome in {"verified", "refuted"}:
write_event(
log_path, {"event": "outcome", "outcome": outcome, "token": token}
@@ -489,10 +510,7 @@ async def run(args: argparse.Namespace) -> str:
return outcome
await asyncio.sleep(0.1)
- oracle = fixture_state(args.fixture_url)
- outcome = classify(
- oracle.get("submitted"), token, steps=args.max_steps, max_steps=args.max_steps
- )
+ outcome = task.classify(task.read_oracle(), steps=task.max_steps)
write_event(log_path, {"event": "outcome", "outcome": outcome, "token": token})
return outcome
diff --git a/libs/cua-driver/examples/jev-use/python/sources.py b/libs/cua-driver/examples/jev-use/python/sources.py
new file mode 100644
index 0000000000..3fa2531bbb
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/python/sources.py
@@ -0,0 +1,197 @@
+"""Candidate sources: where a decision's executable candidates come from.
+
+A candidate source turns one Driver observation into addressable controls and
+builds fully specified candidates for them. The model only ever sees a
+candidate's ID and description; the tool name and exact arguments stay in the
+runner. A task (see ``tasks.py``) decides which controls matter and which
+candidate IDs and descriptions to offer; the source decides how to address a
+control and which Driver tool acts on it.
+
+Implemented sources:
+
+- ``BrowserSemanticSource`` reads a ``get_browser_state`` ``semantic_v2``
+ snapshot and acts through ``browser_type`` / ``browser_click`` on its refs.
+- ``VisualRegionSource`` reads a validated ``parse_visual_regions`` result and
+ acts through a capture-bound ``click`` on a region's center.
+
+Extension point (RFC #4268, Phase 1): a ``NativeAccessibilitySource`` will read
+``get_window_state`` elements, find controls by normalized role and label, and
+act through the snapshot's ``element_token``. It is deliberately not
+implemented here; it must satisfy ``CandidateSource`` and report
+``kind = "ax"``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Mapping, Protocol
+
+if TYPE_CHECKING:
+ from core import VisualDelivery, VisualObservation, VisualRegion
+
+# "ax" is reserved for the future NativeAccessibilitySource.
+SourceKind = Literal["page", "visual", "ax"]
+
+
+def _freeze(value: Any) -> Any:
+ if isinstance(value, dict):
+ return MappingProxyType({key: _freeze(item) for key, item in value.items()})
+ if isinstance(value, list):
+ return tuple(_freeze(item) for item in value)
+ return value
+
+
+def _ascii_lower(value: str) -> str:
+ return "".join(chr(ord(char) + 32) if "A" <= char <= "Z" else char for char in value)
+
+
+@dataclass(frozen=True)
+class Candidate:
+ id: str
+ description: str
+ tool: str | None
+ arguments: Mapping[str, Any]
+ capture_id: str | None = None
+ screenshot_reference: str | None = None
+
+ def __post_init__(self) -> None:
+ object.__setattr__(self, "arguments", _freeze(dict(self.arguments)))
+
+
+@dataclass(frozen=True)
+class Control:
+ """One addressable element a source found in its observation.
+
+ ``value`` is the element's current value when the source can read it (never
+ sent to a model; tasks summarize it). ``handle`` is source-private: a page
+ ref for the browser source, a ``VisualRegion`` for the visual source.
+ """
+
+ source: SourceKind
+ role: str
+ name: str
+ value: Any
+ handle: Any
+
+
+class CandidateSource(Protocol):
+ """The interface every candidate source implements.
+
+ ``find`` returns the unique control matching a role and accessible name, or
+ ``None``. ``click`` and ``type_text`` return a fully specified candidate for
+ that control, or ``None`` when this source cannot perform the action in the
+ current observation (for example a visual region cannot receive text, and a
+ visual click needs a capture-bound Driver ``click``).
+ """
+
+ kind: ClassVar[SourceKind]
+
+ def find(self, role: str, name: str) -> Control | None: ...
+
+ def click(self, control: Control, *, candidate_id: str, description: str) -> Candidate | None: ...
+
+ def type_text(
+ self, control: Control, text: str, *, candidate_id: str, description: str
+ ) -> Candidate | None: ...
+
+
+@dataclass(frozen=True)
+class BrowserSemanticSource:
+ """Controls from a ``get_browser_state`` snapshot, acted on through its refs."""
+
+ snapshot: Mapping[str, Any]
+ kind: ClassVar[SourceKind] = "page"
+
+ def require_target(self) -> dict[str, Any]:
+ """Return the browser target every page candidate addresses.
+
+ Raises ``KeyError`` when the snapshot has no target or tab.
+ """
+ return {"target_id": self.snapshot["target_id"], "tab_id": self.snapshot["tab_id"]}
+
+ def find(self, role: str, name: str) -> Control | None:
+ refs = self.snapshot.get("refs") or []
+ ref = next(
+ (item for item in refs if item.get("role") == role and item.get("name") == name),
+ None,
+ )
+ if ref is None:
+ return None
+ return Control("page", role, name, ref.get("value"), ref)
+
+ def click(self, control: Control, *, candidate_id: str, description: str) -> Candidate:
+ return Candidate(
+ candidate_id,
+ description,
+ "browser_click",
+ {**self.require_target(), "ref": control.handle["ref"], "input_route": "dom_event"},
+ )
+
+ def type_text(
+ self, control: Control, text: str, *, candidate_id: str, description: str
+ ) -> Candidate:
+ return Candidate(
+ candidate_id,
+ description,
+ "browser_type",
+ {**self.require_target(), "ref": control.handle["ref"], "text": text, "replace": True},
+ )
+
+
+@dataclass(frozen=True)
+class VisualRegionSource:
+ """Controls from validated OmniParser regions, clicked through their capture.
+
+ Regions carry no role, so ``find`` matches the visible text or icon label
+ (ASCII case-insensitive) of regions at or above ``MIN_CONFIDENCE`` and
+ returns a control only when exactly one region matches. ``click`` is offered
+ only when Driver advertises a capture-bound ``click`` (``capture_bound``); it
+ targets the region's screenshot-pixel center with the exact ``capture_id``
+ and the source's ``delivery`` mode.
+ """
+
+ observation: VisualObservation
+ delivery: VisualDelivery = "background"
+ capture_bound: bool = False
+ kind: ClassVar[SourceKind] = "visual"
+ MIN_CONFIDENCE: ClassVar[float] = 0.8
+
+ def find(self, role: str, name: str) -> Control | None:
+ wanted = _ascii_lower(name)
+ matches = [
+ region
+ for region in self.observation.regions
+ if region.confidence >= self.MIN_CONFIDENCE
+ and _ascii_lower(region.text or region.label or "") == wanted
+ ]
+ if len(matches) != 1:
+ return None
+ region: VisualRegion = matches[0]
+ return Control("visual", role, region.text or region.label or "", None, region)
+
+ def click(self, control: Control, *, candidate_id: str, description: str) -> Candidate | None:
+ if not self.capture_bound:
+ return None
+ visual = self.observation
+ x, y = visual.screenshot_center(control.handle)
+ return Candidate(
+ candidate_id,
+ description,
+ "click",
+ {
+ "pid": visual.pid,
+ "window_id": visual.window_id,
+ "x": x,
+ "y": y,
+ "capture_id": visual.capture_id,
+ "delivery_mode": self.delivery,
+ },
+ capture_id=visual.capture_id,
+ screenshot_reference=visual.screenshot_reference,
+ )
+
+ def type_text(
+ self, control: Control, text: str, *, candidate_id: str, description: str
+ ) -> None:
+ return None
diff --git a/libs/cua-driver/examples/jev-use/python/tasks.py b/libs/cua-driver/examples/jev-use/python/tasks.py
new file mode 100644
index 0000000000..5cb1bbe923
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/python/tasks.py
@@ -0,0 +1,382 @@
+"""Task specs: what one jev-use run is trying to achieve and how it is checked.
+
+A task spec owns everything that is specific to one application task: the goal
+shown to the model, its parameters (secrets are redacted from all model input),
+the step budget, the Driver action kinds it may use, the compact state summary
+the model sees, the candidate IDs and descriptions it offers, and the success
+oracle that verifies the outcome independently of the runner's own events.
+
+Candidate sources (``sources.py``) supply the controls and build the executable
+candidates. The runner stays task-agnostic: it observes, asks the task for
+candidates and a state summary, lets the model choose one supplied ID, acts,
+and asks the task's oracle whether the task is done.
+
+``FixtureFormTask`` is the one built-in task: type a secret verification token
+into the loopback fixture's form and submit it, verified through the fixture's
+``/state`` endpoint.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Literal, Mapping, Protocol
+from urllib.request import Request, urlopen
+
+from sources import BrowserSemanticSource, Candidate, VisualRegionSource
+
+if TYPE_CHECKING:
+ from core import VisualDelivery, VisualObservation, VisualRegion
+
+Outcome = Literal["verified", "refuted", "unknown", "abstained", "budget_exhausted"]
+
+REDACTED_TOKEN = "[verification token]"
+
+
+def redact_token(value: Any, token: str, placeholder: str = REDACTED_TOKEN) -> Any:
+ """Replace every occurrence of the token in strings nested in ``value``."""
+ if not token:
+ return value
+ if isinstance(value, str):
+ return value.replace(token, placeholder)
+ if isinstance(value, Mapping):
+ return {key: redact_token(item, token, placeholder) for key, item in value.items()}
+ if isinstance(value, (list, tuple)):
+ return [redact_token(item, token, placeholder) for item in value]
+ return value
+
+
+@dataclass(frozen=True)
+class TaskParameter:
+ """A named task input. A ``secret`` value is replaced by ``redaction``
+ everywhere in model input; it may still appear in executable arguments."""
+
+ name: str
+ value: str
+ secret: bool = False
+ redaction: str = REDACTED_TOKEN
+
+
+@dataclass(frozen=True)
+class TaskSources:
+ """The candidate sources available for one decision step.
+
+ ``visual`` is present only when this step parsed visual regions.
+ ``visual_path`` reports whether a capture-bound visual path exists at all,
+ so a task can say a control is still pending a visual check.
+ """
+
+ page: BrowserSemanticSource
+ visual: VisualRegionSource | None = None
+ visual_path: bool = False
+
+
+class Task(Protocol):
+ """The interface the runner uses for every task."""
+
+ id: str
+ goal: str
+ parameters: tuple[TaskParameter, ...]
+ max_steps: int
+ allowed_action_kinds: frozenset[str]
+ # Candidate IDs after which the runner polls the oracle for completion.
+ completion_candidate_ids: frozenset[str]
+
+ def redact(self, value: Any) -> Any: ...
+
+ def candidates(self, sources: TaskSources) -> list[Candidate]: ...
+
+ def state_summary(self, sources: TaskSources) -> dict[str, str]: ...
+
+ def history_entry(
+ self, step: int, candidate_id: str, *, refusal: str | None = None
+ ) -> dict[str, Any]: ...
+
+ def reset(self) -> None: ...
+
+ def read_oracle(self) -> Mapping[str, Any]: ...
+
+ def classify(self, oracle: Mapping[str, Any], *, steps: int) -> Outcome: ...
+
+
+# The built-in browser fixture task.
+
+FIXTURE_TASK_ID = "browser-fixture-form"
+FIXTURE_GOAL = (
+ "Enter the required verification token into the verification field, then submit the form."
+)
+DEFAULT_FIXTURE_URL = "http://127.0.0.1:8765/"
+DEFAULT_MAX_STEPS = 4
+FIELD_NAME = "verification value"
+SUBMIT_NAME = "Submit"
+SUBMIT_IDS = frozenset({"submit-form", "submit-form-foreground"})
+FIXTURE_ACTION_KINDS = frozenset({"browser_type", "browser_click", "click"})
+
+HISTORY_OUTCOMES = {
+ "type-verification-value": "typed the required token into the verification field",
+ "submit-form": "clicked Submit; the submission was not yet confirmed",
+ "submit-form-foreground": "clicked Submit in the foreground; the submission was not yet confirmed",
+ "reobserve": "took no action and requested a fresh observation",
+}
+
+
+def fixture_state(fixture_url: str) -> dict[str, str | None]:
+ with urlopen(f"{fixture_url.rstrip('/')}/state", timeout=2) as response:
+ return json.loads(response.read())
+
+
+def reset_fixture(fixture_url: str) -> None:
+ request = Request(f"{fixture_url.rstrip('/')}/reset", method="POST", data=b"")
+ with urlopen(request, timeout=2) as response:
+ if response.status != 204:
+ raise RuntimeError(f"fixture reset failed: HTTP {response.status}")
+
+
+def classify(submitted: str | None, token: str, *, steps: int, max_steps: int) -> Outcome:
+ if submitted == token:
+ return "verified"
+ if submitted is not None:
+ return "refuted"
+ if steps >= max_steps:
+ return "budget_exhausted"
+ return "unknown"
+
+
+def history_entry(
+ step: int, candidate_id: str, *, refusal: str | None = None
+) -> dict[str, Any]:
+ """Build the compact decision-history item shown to the model.
+
+ It records what each step did, not timings or model probabilities, so earlier
+ choices do not become a signal to repeat themselves.
+ """
+ if refusal is not None:
+ outcome = (
+ f"Driver refused background delivery ({refusal}); no click happened and a "
+ "foreground Submit candidate is offered next"
+ )
+ else:
+ outcome = HISTORY_OUTCOMES.get(candidate_id, "completed")
+ return {"step": step, "selected_id": candidate_id, "outcome": outcome}
+
+
+def _reserved_candidates() -> list[Candidate]:
+ return [
+ Candidate(
+ "reobserve",
+ "Take no action and obtain a fresh Driver observation, because the current "
+ "observation is stale or contradicts the reported form state. A Submit control "
+ "that is visual-only, or whose visual check is still pending, is not a reason to "
+ "reobserve: the runner parses visual regions for Submit once no page-structure "
+ "action remains.",
+ None,
+ {},
+ ),
+ Candidate(
+ "abstain",
+ "Stop without acting if none of the proposed actions is safe for the observed state.",
+ None,
+ {},
+ ),
+ ]
+
+
+@dataclass(frozen=True)
+class FixtureFormTask:
+ """Type the secret token into the fixture form and submit it.
+
+ The oracle is the fixture's ``/state`` endpoint: the task is verified only
+ when the fixture recorded exactly this token as submitted.
+ """
+
+ token: str
+ fixture_url: str = DEFAULT_FIXTURE_URL
+ max_steps: int = DEFAULT_MAX_STEPS
+ id: str = field(default=FIXTURE_TASK_ID, init=False)
+ goal: str = field(default=FIXTURE_GOAL, init=False)
+ allowed_action_kinds: frozenset[str] = field(default=FIXTURE_ACTION_KINDS, init=False)
+ completion_candidate_ids: frozenset[str] = field(default=SUBMIT_IDS, init=False)
+
+ @property
+ def parameters(self) -> tuple[TaskParameter, ...]:
+ return (TaskParameter("verification_token", self.token, secret=True),)
+
+ def redact(self, value: Any) -> Any:
+ for parameter in self.parameters:
+ if parameter.secret:
+ value = redact_token(value, parameter.value, parameter.redaction)
+ return value
+
+ def candidates(self, sources: TaskSources) -> list[Candidate]:
+ """Build the closed candidate set for one decision.
+
+ Page-structure refs always win. The capture-bound visual Submit is offered
+ only when no Submit ref exists. A visual source with ``foreground``
+ delivery yields a distinct ``submit-form-foreground`` candidate after
+ Driver refused background delivery; the chooser must pick it explicitly.
+ """
+ page = sources.page
+ # Every page candidate addresses the snapshot's target; reject a snapshot
+ # without one before offering anything.
+ page.require_target()
+ token = self.token
+ field_control = page.find("textbox", FIELD_NAME)
+ button = page.find("button", SUBMIT_NAME)
+ candidates: list[Candidate] = []
+ if field_control and field_control.value != token:
+ candidates.append(
+ page.type_text(
+ field_control,
+ token,
+ candidate_id="type-verification-value",
+ description="Type the required verification token into the verification "
+ "field, replacing its current contents.",
+ )
+ )
+ elif field_control and field_control.value == token and button:
+ candidates.append(
+ page.click(
+ button,
+ candidate_id="submit-form",
+ description="Click the form's Submit button. The observed form state "
+ "reports that the verification field already contains the required "
+ "token, so the form is ready to submit.",
+ )
+ )
+ elif field_control and field_control.value == token and sources.visual is not None:
+ visual = sources.visual
+ region = visual.find("button", SUBMIT_NAME)
+ if region is not None:
+ foreground = visual.delivery == "foreground"
+ candidate = visual.click(
+ region,
+ candidate_id="submit-form-foreground" if foreground else "submit-form",
+ description=(
+ "Submit the form by clicking the unique validated visual Submit "
+ "region with foreground delivery, which activates the browser "
+ "window, because Driver refused background delivery for the "
+ "previous visual click."
+ if foreground
+ else "Submit the form by clicking the unique validated visual Submit "
+ "region. The observed form state reports that the verification field "
+ "already contains the required token."
+ ),
+ )
+ if candidate is not None:
+ candidates.append(candidate)
+ for candidate in candidates:
+ if candidate.tool not in self.allowed_action_kinds:
+ raise ValueError(f"task {self.id} does not allow action kind {candidate.tool}")
+ return candidates + _reserved_candidates()
+
+ def state_summary(self, sources: TaskSources) -> dict[str, str]:
+ """Summarize the form for the decision model without revealing the token.
+
+ The raw field value never leaves the runner; the model receives only whether
+ the field is empty, holds the required token, or holds something else.
+
+ ``submit_button`` is ``available`` for a clickable page-structure ref. When
+ the page structure has none and the capture-bound visual path is enabled
+ (``visual_path``), it is ``visual_only`` if this observation holds a unique
+ validated visual Submit region, ``visual_check_pending`` if no visual regions
+ were parsed for this step yet (the runner parses them once no page-structure
+ action remains), and ``not_found_visually`` otherwise. Without a visual path
+ it is ``not_in_page_structure``.
+ """
+ field_control = sources.page.find("textbox", FIELD_NAME)
+ button = sources.page.find("button", SUBMIT_NAME)
+ if field_control is None:
+ field_state = "not_found"
+ elif not field_control.value:
+ field_state = "empty"
+ elif field_control.value == self.token:
+ field_state = "contains_required_token"
+ else:
+ field_state = "contains_other_value"
+ if button is not None:
+ submit_state = "available"
+ elif not sources.visual_path:
+ submit_state = "not_in_page_structure"
+ elif sources.visual is None:
+ submit_state = "visual_check_pending"
+ elif sources.visual.find("button", SUBMIT_NAME) is not None:
+ submit_state = "visual_only"
+ else:
+ submit_state = "not_found_visually"
+ return {"verification_field": field_state, "submit_button": submit_state}
+
+ def history_entry(
+ self, step: int, candidate_id: str, *, refusal: str | None = None
+ ) -> dict[str, Any]:
+ return history_entry(step, candidate_id, refusal=refusal)
+
+ def reset(self) -> None:
+ reset_fixture(self.fixture_url)
+
+ def read_oracle(self) -> Mapping[str, Any]:
+ return fixture_state(self.fixture_url)
+
+ def classify(self, oracle: Mapping[str, Any], *, steps: int) -> Outcome:
+ return classify(oracle.get("submitted"), self.token, steps=steps, max_steps=self.max_steps)
+
+
+def fixture_sources(
+ snapshot: Mapping[str, Any],
+ visual: VisualObservation | None = None,
+ *,
+ capture_bound_click: bool = False,
+ visual_delivery: VisualDelivery = "background",
+ visual_path: bool = False,
+) -> TaskSources:
+ """Build one step's sources from a browser snapshot and optional visual parse."""
+ return TaskSources(
+ BrowserSemanticSource(snapshot),
+ None
+ if visual is None
+ else VisualRegionSource(visual, visual_delivery, capture_bound_click),
+ visual_path,
+ )
+
+
+# Compatibility entry points. They keep the pre-task-spec signatures and
+# delegate to the built-in fixture task, so existing callers are unchanged.
+
+
+def visual_submit_region(visual: VisualObservation | None) -> VisualRegion | None:
+ """Return the unique validated visual Submit region, or ``None``."""
+ if visual is None:
+ return None
+ control = VisualRegionSource(visual).find("button", SUBMIT_NAME)
+ return control.handle if control is not None else None
+
+
+def form_state(
+ snapshot: Mapping[str, Any],
+ token: str,
+ visual: VisualObservation | None = None,
+ *,
+ visual_path: bool = False,
+) -> dict[str, str]:
+ """Summarize the fixture form; see ``FixtureFormTask.state_summary``."""
+ return FixtureFormTask(token).state_summary(
+ fixture_sources(snapshot, visual, visual_path=visual_path)
+ )
+
+
+def build_candidates(
+ snapshot: Mapping[str, Any],
+ token: str,
+ visual: VisualObservation | None = None,
+ *,
+ capture_bound_click: bool = False,
+ visual_delivery: VisualDelivery = "background",
+) -> list[Candidate]:
+ """Build the fixture candidate set; see ``FixtureFormTask.candidates``."""
+ return FixtureFormTask(token).candidates(
+ fixture_sources(
+ snapshot,
+ visual,
+ capture_bound_click=capture_bound_click,
+ visual_delivery=visual_delivery,
+ )
+ )
diff --git a/libs/cua-driver/examples/jev-use/python/tests/provider_request_golden.py b/libs/cua-driver/examples/jev-use/python/tests/provider_request_golden.py
new file mode 100644
index 0000000000..5f377fce7b
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/python/tests/provider_request_golden.py
@@ -0,0 +1,306 @@
+"""Record the exact TypeSafe Jev request payloads for the page and visual fixtures.
+
+The golden file ``fixtures/jev-provider-request-golden-python-v1.json`` was
+captured from this module on ``main`` before the candidate-source and task-spec
+refactor (RFC #4268 Phase 0). ``test_provider_request_golden.py`` replays the
+same scenarios and requires byte-identical payloads, so any change to the model
+input or to the candidate table is a test failure, not a silent drift.
+
+Regenerate only for an intentional contract change::
+
+ uv run python python/tests/provider_request_golden.py --write
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import hashlib
+import json
+import sys
+from dataclasses import replace
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any, Callable
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from core import ( # noqa: E402
+ SUBMIT_IDS,
+ build_candidates,
+ has_executable_candidate,
+ history_entry,
+ parse_visual_regions,
+ validate_choice,
+)
+from jev_adapter import choose_with_typesafe # noqa: E402
+
+ROOT = Path(__file__).resolve().parents[2]
+FIXTURES = ROOT / "fixtures"
+GOLDEN = FIXTURES / "jev-provider-request-golden-python-v1.json"
+PAGE = json.loads((FIXTURES / "jev-page-structure-replay-v1.json").read_text(encoding="utf-8"))
+VISUAL = json.loads((FIXTURES / "jev-visual-replay-v1.json").read_text(encoding="utf-8"))
+
+# build(snapshot, token, visual, capture_bound_click, visual_delivery) -> candidates
+Build = Callable[..., list]
+# choose(client, candidates, snapshot, visual, history, token, visual_path) -> choice tuple
+Choose = Callable[..., tuple]
+
+
+def observation(payload: dict[str, Any] | None = None):
+ payload = payload or VISUAL["visual_regions"]
+ source = payload["capture"]["source"]
+ return parse_visual_regions(
+ payload,
+ expected_capture_id=payload["capture"]["capture_id"],
+ expected_pid=source["pid"],
+ expected_window_id=source["window_id"],
+ )
+
+
+class RecordingClient:
+ """Answer with scripted IDs and keep every request exactly as sent.
+
+ A scripted ID absent from the offered criteria (for example a visual Submit
+ when visual parsing is off) is answered with ``reobserve``.
+ """
+
+ def __init__(self, choices: list[str]) -> None:
+ self.choices = list(choices)
+ self.requests: list[dict[str, Any]] = []
+
+ def system_one(self, **request: Any) -> Any:
+ self.requests.append(request)
+ choice = self.choices.pop(0)
+ if choice not in request["questions"]["driver_action"].criteria:
+ choice = "reobserve"
+ answer = SimpleNamespace(choice=choice, confidence=1.0, probabilities={choice: 1.0})
+ return SimpleNamespace(choices={"driver_action": answer})
+
+
+def wire_request(request: dict[str, Any]) -> dict[str, Any]:
+ question = request["questions"]["driver_action"]
+ return {
+ "state": request["state"],
+ "questions": {
+ "driver_action": {
+ "instructions": question.instructions,
+ "criteria": question.criteria,
+ }
+ },
+ }
+
+
+def wire_candidates(candidates: list) -> list[dict[str, Any]]:
+ return [
+ {
+ "id": candidate.id,
+ "description": candidate.description,
+ "tool": candidate.tool,
+ "arguments": json.loads(json.dumps(dict(candidate.arguments))),
+ "capture_id": candidate.capture_id,
+ "screenshot_reference": candidate.screenshot_reference,
+ }
+ for candidate in candidates
+ ]
+
+
+def replay(
+ build: Build,
+ choose: Choose,
+ fixture: dict[str, Any],
+ run: dict[str, Any],
+ *,
+ capture_bound_click: bool,
+ visual_mode: str,
+) -> list[dict[str, Any]]:
+ """Replay one recorded run the way the runner steps through it."""
+ token = fixture["token"]
+ before = fixture["snapshots"]["before_typing"]
+ after = fixture["snapshots"]["after_typing"]
+ visual_path = capture_bound_click and visual_mode != "off"
+ history: list[dict[str, Any]] = []
+ typed = False
+ delivery = "background"
+ records: list[dict[str, Any]] = []
+ client = RecordingClient([step["selected_id"] for step in run["steps"]])
+ for index, recorded in enumerate(run["steps"], start=1):
+ step = recorded.get("step", index)
+ snapshot = after if typed else before
+ candidates = build(snapshot, token, None, capture_bound_click, delivery)
+ visual = None
+ if visual_mode == "always" or (
+ visual_mode == "auto" and not has_executable_candidate(candidates)
+ ):
+ if capture_bound_click:
+ visual = observation()
+ candidates = build(snapshot, token, visual, capture_bound_click, delivery)
+ choice, _, _ = choose(client, candidates, snapshot, visual, history, token, visual_path)
+ candidate = validate_choice(
+ choice, candidates, current_capture_id=visual.capture_id if visual else None
+ )
+ records.append(
+ {
+ "step": step,
+ "request": wire_request(client.requests[-1]),
+ "candidates": wire_candidates(candidates),
+ }
+ )
+ refusal = recorded.get("action_error")
+ history.append(history_entry(step, candidate.id, refusal=refusal))
+ if candidate.id == "abstain":
+ break
+ if candidate.id == "type-verification-value":
+ typed = True
+ elif candidate.id in SUBMIT_IDS:
+ if refusal:
+ delivery = "foreground"
+ else:
+ break
+ return records
+
+
+def with_field_value(snapshot: dict[str, Any], value: str | None) -> dict[str, Any]:
+ changed = copy.deepcopy(snapshot)
+ for ref in changed["refs"]:
+ if ref.get("role") == "textbox" and ref.get("name") == "verification value":
+ ref["value"] = value
+ return changed
+
+
+def without_button(snapshot: dict[str, Any]) -> dict[str, Any]:
+ changed = copy.deepcopy(snapshot)
+ changed["refs"] = [ref for ref in changed["refs"] if ref.get("role") != "button"]
+ return changed
+
+
+def single_requests(build: Build, choose: Choose) -> dict[str, Any]:
+ """One request per distinct form and candidate state, with no history."""
+ page_token = PAGE["token"]
+ page_before = PAGE["snapshots"]["before_typing"]
+ page_after = PAGE["snapshots"]["after_typing"]
+ visual_token = VISUAL["token"]
+ visual_after = VISUAL["snapshots"]["after_typing"]
+ visual = observation()
+ no_submit = replace(visual, regions=tuple(r for r in visual.regions if r.text != "Submit"))
+ submit = next(r for r in visual.regions if r.text == "Submit")
+ duplicated = replace(visual, regions=visual.regions + (replace(submit, id="dup"),))
+ empty = {"target_id": "target", "tab_id": "tab", "refs": []}
+ cases = {
+ "page_other_value": (with_field_value(page_before, "other"), page_token, None, False, False, "background"),
+ "page_after_with_visual": (page_after, page_token, visual, True, True, "background"),
+ "page_after_no_button": (without_button(page_after), page_token, None, False, False, "background"),
+ "page_after_no_button_visual_path": (without_button(page_after), page_token, None, True, True, "background"),
+ "no_form_refs": (empty, page_token, None, False, False, "background"),
+ "visual_after_no_capture_bound_click": (visual_after, visual_token, visual, False, True, "background"),
+ "visual_after_without_submit_region": (visual_after, visual_token, no_submit, True, True, "background"),
+ "visual_after_duplicate_submit_region": (visual_after, visual_token, duplicated, True, True, "background"),
+ "visual_after_foreground": (visual_after, visual_token, visual, True, True, "foreground"),
+ "visual_after_other_value": (with_field_value(visual_after, "other"), visual_token, visual, True, True, "background"),
+ }
+ result: dict[str, Any] = {}
+ for name, (snapshot, token, current, cbc, visual_path, delivery) in cases.items():
+ candidates = build(snapshot, token, current, cbc, delivery)
+ client = RecordingClient(["abstain"])
+ choose(client, candidates, snapshot, current, [], token, visual_path)
+ result[name] = {
+ "request": wire_request(client.requests[-1]),
+ "candidates": wire_candidates(candidates),
+ }
+ return result
+
+
+def payloads(build: Build, choose: Choose) -> dict[str, Any]:
+ runs: dict[str, Any] = {}
+ modes = (
+ ("page", PAGE, False, "auto"),
+ ("page_capture_bound_auto", PAGE, True, "auto"),
+ ("page_capture_bound_always", PAGE, True, "always"),
+ ("visual_capture_bound_auto", VISUAL, True, "auto"),
+ ("visual_capture_bound_always", VISUAL, True, "always"),
+ ("visual_capture_bound_off", VISUAL, True, "off"),
+ )
+ for prefix, fixture, cbc, mode in modes:
+ for name, run in fixture["recorded_live_runs"].items():
+ runs[f"{prefix}/{name}"] = replay(
+ build, choose, fixture, run, capture_bound_click=cbc, visual_mode=mode
+ )
+ return deduplicate(
+ {
+ "schema": "cua.jev_use_provider_request_golden_v1",
+ "language": "python",
+ "runs": runs,
+ "single_requests": single_requests(build, choose),
+ }
+ )
+
+
+def _digest(value: Any) -> str:
+ return hashlib.sha256(
+ json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+ ).hexdigest()[:16]
+
+
+def deduplicate(result: dict[str, Any]) -> dict[str, Any]:
+ """Store each distinct request and candidate table once, keyed by digest.
+
+ Scenario records keep their order and reference the exact payloads, so the
+ golden stays reviewable without repeating the visual regions per step.
+ """
+ requests: dict[str, Any] = {}
+ tables: dict[str, Any] = {}
+
+ def reference(record: dict[str, Any]) -> dict[str, Any]:
+ request_id = _digest(record["request"])
+ table_id = _digest(record["candidates"])
+ requests.setdefault(request_id, record["request"])
+ tables.setdefault(table_id, record["candidates"])
+ refs = {"request": request_id, "candidates": table_id}
+ return {"step": record["step"], **refs} if "step" in record else refs
+
+ runs = {
+ name: [reference(record) for record in records]
+ for name, records in result["runs"].items()
+ }
+ singles = {name: reference(record) for name, record in result["single_requests"].items()}
+ return {
+ "schema": result["schema"],
+ "language": result["language"],
+ "runs": runs,
+ "single_requests": singles,
+ "requests": requests,
+ "candidate_tables": tables,
+ }
+
+
+def legacy_build(snapshot, token, visual, capture_bound_click, visual_delivery):
+ return build_candidates(
+ snapshot,
+ token,
+ visual,
+ capture_bound_click=capture_bound_click,
+ visual_delivery=visual_delivery,
+ )
+
+
+def legacy_choose(client, candidates, snapshot, visual, history, token, visual_path):
+ return choose_with_typesafe(client, candidates, snapshot, visual, history, token, visual_path)
+
+
+def encode(value: Any) -> str:
+ return json.dumps(value, indent=1, ensure_ascii=False) + "\n"
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--write", action="store_true")
+ args = parser.parse_args()
+ text = encode(payloads(legacy_build, legacy_choose))
+ if args.write:
+ GOLDEN.write_text(text, encoding="utf-8")
+ else:
+ sys.stdout.write(text)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_provider_request_golden.py b/libs/cua-driver/examples/jev-use/python/tests/test_provider_request_golden.py
new file mode 100644
index 0000000000..b9512f44f6
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/python/tests/test_provider_request_golden.py
@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from jev_adapter import choose_for_task # noqa: E402
+from provider_request_golden import ( # noqa: E402
+ GOLDEN,
+ encode,
+ legacy_build,
+ legacy_choose,
+ payloads,
+)
+from tasks import FixtureFormTask, fixture_sources # noqa: E402
+
+
+def task_build(snapshot, token, visual, capture_bound_click, visual_delivery):
+ return FixtureFormTask(token).candidates(
+ fixture_sources(
+ snapshot,
+ visual,
+ capture_bound_click=capture_bound_click,
+ visual_delivery=visual_delivery,
+ )
+ )
+
+
+def task_choose(client, candidates, snapshot, visual, history, token, visual_path):
+ return choose_for_task(
+ client,
+ FixtureFormTask(token),
+ fixture_sources(snapshot, visual, visual_path=visual_path),
+ candidates,
+ history,
+ )
+
+
+class ProviderRequestGoldenTest(unittest.TestCase):
+ """Provider requests and candidate tables match the capture from main.
+
+ The golden was recorded before the candidate-source and task-spec refactor
+ (RFC #4268 Phase 0), so equality proves the refactor sends byte-identical
+ model input and offers identical executable candidates.
+ """
+
+ maxDiff = None
+
+ def assert_golden(self, build, choose) -> None:
+ self.assertEqual(encode(payloads(build, choose)), GOLDEN.read_text(encoding="utf-8"))
+
+ def test_core_entry_points_send_byte_identical_requests(self) -> None:
+ self.assert_golden(legacy_build, legacy_choose)
+
+ def test_task_spec_and_sources_send_byte_identical_requests(self) -> None:
+ self.assert_golden(task_build, task_choose)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_sources_and_tasks.py b/libs/cua-driver/examples/jev-use/python/tests/test_sources_and_tasks.py
new file mode 100644
index 0000000000..89deecace1
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/python/tests/test_sources_and_tasks.py
@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import json
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from core import REDACTED_TOKEN, parse_visual_regions
+from sources import BrowserSemanticSource, CandidateSource, VisualRegionSource
+from tasks import (
+ FIXTURE_GOAL,
+ SUBMIT_IDS,
+ FixtureFormTask,
+ Task,
+ TaskSources,
+ fixture_sources,
+)
+
+FIXTURES = Path(__file__).resolve().parents[2] / "fixtures"
+PAGE = json.loads((FIXTURES / "jev-page-structure-replay-v1.json").read_text(encoding="utf-8"))
+VISUAL = json.loads((FIXTURES / "jev-visual-replay-v1.json").read_text(encoding="utf-8"))
+PAYLOAD = VISUAL["visual_regions"]
+
+
+def observation():
+ source = PAYLOAD["capture"]["source"]
+ return parse_visual_regions(
+ PAYLOAD,
+ expected_capture_id=PAYLOAD["capture"]["capture_id"],
+ expected_pid=source["pid"],
+ expected_window_id=source["window_id"],
+ )
+
+
+class CandidateSourceTest(unittest.TestCase):
+ def test_sources_implement_one_interface_with_distinct_kinds(self) -> None:
+ page: CandidateSource = BrowserSemanticSource(PAGE["snapshots"]["before_typing"])
+ visual: CandidateSource = VisualRegionSource(observation())
+ self.assertEqual((page.kind, visual.kind), ("page", "visual"))
+
+ def test_page_source_addresses_refs_with_exact_browser_arguments(self) -> None:
+ snapshot = PAGE["snapshots"]["after_typing"]
+ page = BrowserSemanticSource(snapshot)
+ button = page.find("button", "Submit")
+ self.assertIsNotNone(button)
+ self.assertEqual((button.source, button.handle["ref"]), ("page", "p4:2"))
+ candidate = page.click(button, candidate_id="c", description="d")
+ self.assertEqual(candidate.tool, "browser_click")
+ self.assertEqual(
+ dict(candidate.arguments),
+ {
+ "target_id": snapshot["target_id"],
+ "tab_id": snapshot["tab_id"],
+ "ref": "p4:2",
+ "input_route": "dom_event",
+ },
+ )
+ self.assertIsNone(page.find("button", "Cancel"))
+
+ def test_visual_source_clicks_only_capture_bound_and_never_types(self) -> None:
+ visual = observation()
+ source = VisualRegionSource(visual)
+ submit = source.find("button", "submit")
+ self.assertIsNotNone(submit)
+ self.assertIsNone(source.click(submit, candidate_id="c", description="d"))
+ self.assertIsNone(source.type_text(submit, "x", candidate_id="c", description="d"))
+ bound = VisualRegionSource(visual, "foreground", capture_bound=True)
+ candidate = bound.click(submit, candidate_id="c", description="d")
+ self.assertEqual(candidate.tool, "click")
+ self.assertEqual(candidate.capture_id, visual.capture_id)
+ self.assertEqual(candidate.arguments["delivery_mode"], "foreground")
+ self.assertEqual(
+ (candidate.arguments["x"], candidate.arguments["y"]),
+ visual.screenshot_center(submit.handle),
+ )
+
+
+class FixtureTaskSpecTest(unittest.TestCase):
+ def test_built_in_task_declares_its_spec(self) -> None:
+ task: Task = FixtureFormTask("secret-token", "http://127.0.0.1:9/", 3)
+ self.assertEqual(task.goal, FIXTURE_GOAL)
+ self.assertEqual(task.max_steps, 3)
+ self.assertEqual(task.completion_candidate_ids, SUBMIT_IDS)
+ self.assertEqual(task.allowed_action_kinds, {"browser_type", "browser_click", "click"})
+ [parameter] = task.parameters
+ self.assertTrue(parameter.secret)
+ self.assertEqual((parameter.value, parameter.redaction), ("secret-token", REDACTED_TOKEN))
+ self.assertEqual(
+ task.redact({"a": ["x secret-token y"]}), {"a": [f"x {REDACTED_TOKEN} y"]}
+ )
+
+ def test_oracle_classification_uses_the_task_budget(self) -> None:
+ task = FixtureFormTask("t", max_steps=2)
+ self.assertEqual(task.classify({"submitted": "t"}, steps=0), "verified")
+ self.assertEqual(task.classify({"submitted": "other"}, steps=0), "refuted")
+ self.assertEqual(task.classify({"submitted": None}, steps=1), "unknown")
+ self.assertEqual(task.classify({"submitted": None}, steps=2), "budget_exhausted")
+
+ def test_candidates_outside_the_allowed_action_kinds_are_refused(self) -> None:
+ task = FixtureFormTask(PAGE["token"])
+ object.__setattr__(task, "allowed_action_kinds", frozenset({"browser_type"}))
+ sources = fixture_sources(PAGE["snapshots"]["after_typing"])
+ with self.assertRaisesRegex(ValueError, "browser_click"):
+ task.candidates(sources)
+ before = task.candidates(fixture_sources(PAGE["snapshots"]["before_typing"]))
+ self.assertEqual(before[0].tool, "browser_type")
+
+ def test_state_summary_reads_the_step_sources(self) -> None:
+ task = FixtureFormTask(VISUAL["token"])
+ after = VISUAL["snapshots"]["after_typing"]
+ pending = TaskSources(BrowserSemanticSource(after), None, visual_path=True)
+ self.assertEqual(task.state_summary(pending)["submit_button"], "visual_check_pending")
+ parsed = TaskSources(
+ BrowserSemanticSource(after), VisualRegionSource(observation()), visual_path=True
+ )
+ self.assertEqual(
+ task.state_summary(parsed),
+ {"verification_field": "contains_required_token", "submit_button": "visual_only"},
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/libs/cua-driver/examples/jev-use/typescript/core.ts b/libs/cua-driver/examples/jev-use/typescript/core.ts
index 55d860917e..108cad6f30 100644
--- a/libs/cua-driver/examples/jev-use/typescript/core.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/core.ts
@@ -1,18 +1,33 @@
-export type Outcome = 'verified' | 'refuted' | 'unknown' | 'abstained' | 'budget_exhausted';
-export type VisualDelivery = 'background' | 'foreground';
-
-export const SUBMIT_IDS: ReadonlySet<string> = new Set(['submit-form', 'submit-form-foreground']);
+/**
+ * Runner primitives shared by every jev-use task: visual-region parsing,
+ * candidate validation, and the mock chooser. Candidate sources are in
+ * sources.ts and task specs in tasks.ts; the names re-exported below keep
+ * existing imports from core.ts working.
+ */
+import type { Candidate } from './sources.js';
+
+export type { Candidate } from './sources.js';
+export {
+ FIELD_NAME,
+ HISTORY_OUTCOMES,
+ REDACTED_TOKEN,
+ SUBMIT_IDS,
+ SUBMIT_NAME,
+ buildCandidates,
+ classify,
+ formState,
+ historyEntry,
+ redactToken,
+ visualSubmitRegion,
+ type FormState,
+ type HistoryEntry,
+ type Outcome,
+ type SubmitButtonState,
+} from './tasks.js';
-export type Candidate = Readonly<{
- id: string;
- description: string;
- tool: string | null;
- arguments: Readonly<Record<string, unknown>>;
- captureId?: string;
- screenshotReference?: string;
-}>;
+export type VisualDelivery = 'background' | 'foreground';
-type PageRef = {
+export type PageRef = {
role?: string;
name?: string | null;
ref?: string;
@@ -91,10 +106,6 @@ function pixelInt(value: unknown): number {
return Number(value);
}
-function immutableCandidate(candidate: Candidate): Candidate {
- return Object.freeze({ ...candidate, arguments: Object.freeze({ ...candidate.arguments }) });
-}
-
export function parseVisualRegions(
payload: unknown,
expectedCaptureId: string,
@@ -210,223 +221,10 @@ export function parseVisualRegions(
});
}
-export const REDACTED_TOKEN = '[verification token]';
-
-function formRefs(snapshot: BrowserSnapshot) {
- const refs = snapshot.refs ?? [];
- const field = refs.find(
- (item) => item.role === 'textbox' && item.name === 'verification value' && item.ref
- );
- const button = refs.find((item) => item.role === 'button' && item.name === 'Submit' && item.ref);
- return { field, button };
-}
-
-export type SubmitButtonState =
- | 'available'
- | 'visual_only'
- | 'visual_check_pending'
- | 'not_found_visually'
- | 'not_in_page_structure';
-
-export type FormState = Readonly<{
- verification_field: 'not_found' | 'empty' | 'contains_required_token' | 'contains_other_value';
- submit_button: SubmitButtonState;
-}>;
-
-/** Return the unique validated visual Submit region, or undefined. */
-export function visualSubmitRegion(visual?: VisualObservation) {
- if (!visual) return undefined;
- const matches = visual.regions.filter(
- (region) =>
- region.confidence >= 0.8 && asciiLower(region.text ?? region.label ?? '') === 'submit'
- );
- return matches.length === 1 ? matches[0] : undefined;
-}
-
-/**
- * Summarize the form for the decision model without revealing the token. The
- * raw field value never leaves the runner.
- *
- * submit_button is 'available' for a clickable page-structure ref. When the
- * page structure has none and the capture-bound visual path is enabled
- * (visualPath), it is 'visual_only' if this observation holds a unique
- * validated visual Submit region, 'visual_check_pending' if no visual regions
- * were parsed for this step yet (the runner parses them once no page-structure
- * action remains), and 'not_found_visually' otherwise. Without a visual path it
- * is 'not_in_page_structure'.
- */
-export function formState(
- snapshot: BrowserSnapshot,
- token: string,
- visual?: VisualObservation,
- visualPath = false
-): FormState {
- const { field, button } = formRefs(snapshot);
- const verificationField = !field
- ? 'not_found'
- : !field.value
- ? 'empty'
- : field.value === token
- ? 'contains_required_token'
- : 'contains_other_value';
- const submitButton: SubmitButtonState = button
- ? 'available'
- : !visualPath
- ? 'not_in_page_structure'
- : !visual
- ? 'visual_check_pending'
- : visualSubmitRegion(visual)
- ? 'visual_only'
- : 'not_found_visually';
- return { verification_field: verificationField, submit_button: submitButton };
-}
-
-/** Replace every occurrence of the token in strings nested in value. */
-export function redactToken(value: unknown, token: string): unknown {
- if (!token) return value;
- if (typeof value === 'string') return value.split(token).join(REDACTED_TOKEN);
- if (Array.isArray(value)) return value.map((item) => redactToken(item, token));
- if (value && typeof value === 'object') {
- return Object.fromEntries(
- Object.entries(value).map(([key, item]) => [key, redactToken(item, token)])
- );
- }
- return value;
-}
-
-export type HistoryEntry = Readonly<{ step: number; selected_id: string; outcome: string }>;
-
-const HISTORY_OUTCOMES: Readonly<Record<string, string>> = {
- 'type-verification-value': 'typed the required token into the verification field',
- 'submit-form': 'clicked Submit; the submission was not yet confirmed',
- 'submit-form-foreground':
- 'clicked Submit in the foreground; the submission was not yet confirmed',
- reobserve: 'took no action and requested a fresh observation',
-};
-
-/**
- * Build the compact decision-history item shown to the model. It records what
- * each step did, not timings or model probabilities, so earlier choices do not
- * become a signal to repeat themselves.
- */
-export function historyEntry(step: number, candidateId: string, refusal?: string): HistoryEntry {
- const outcome = refusal
- ? `Driver refused background delivery (${refusal}); no click happened and a ` +
- 'foreground Submit candidate is offered next'
- : (HISTORY_OUTCOMES[candidateId] ?? 'completed');
- return { step, selected_id: candidateId, outcome };
-}
-
-function reservedCandidates(): Candidate[] {
- return [
- immutableCandidate({
- id: 'reobserve',
- description:
- 'Take no action and obtain a fresh Driver observation, because the current ' +
- 'observation is stale or contradicts the reported form state. A Submit control ' +
- 'that is visual-only, or whose visual check is still pending, is not a reason to ' +
- 'reobserve: the runner parses visual regions for Submit once no page-structure ' +
- 'action remains.',
- tool: null,
- arguments: {},
- }),
- immutableCandidate({
- id: 'abstain',
- description: 'Stop without acting if none of the proposed actions is safe for the observed state.',
- tool: null,
- arguments: {},
- }),
- ];
-}
-
-/**
- * Build the closed candidate set for one decision. Page-structure refs always
- * win. The capture-bound visual Submit is offered only when no Submit ref
- * exists. visualDelivery 'foreground' replaces the background visual click with
- * a distinct submit-form-foreground candidate after Driver refused background
- * delivery; the chooser must pick it explicitly.
- */
-export function buildCandidates(
- snapshot: BrowserSnapshot,
- token: string,
- visual?: VisualObservation,
- captureBoundClick = false,
- visualDelivery: VisualDelivery = 'background'
-): Candidate[] {
- const common = { target_id: snapshot.target_id, tab_id: snapshot.tab_id };
- const { field, button } = formRefs(snapshot);
- const candidates: Candidate[] = [];
- if (field?.value !== token && field?.ref) {
- candidates.push(
- immutableCandidate({
- id: 'type-verification-value',
- description:
- 'Type the required verification token into the verification field, ' +
- 'replacing its current contents.',
- tool: 'browser_type',
- arguments: { ...common, ref: field.ref, text: token, replace: true },
- })
- );
- } else if (field?.value === token && button?.ref) {
- candidates.push(
- immutableCandidate({
- id: 'submit-form',
- description:
- "Click the form's Submit button. The observed form state reports that the " +
- 'verification field already contains the required token, so the form is ' +
- 'ready to submit.',
- tool: 'browser_click',
- arguments: { ...common, ref: button.ref, input_route: 'dom_event' },
- })
- );
- } else if (
- field?.value === token &&
- visual &&
- captureBoundClick
- ) {
- const region = visualSubmitRegion(visual);
- if (region) {
- const x = region.x + region.width / 2;
- const y = region.y + region.height / 2;
- const foreground = visualDelivery === 'foreground';
- candidates.push(
- immutableCandidate({
- id: foreground ? 'submit-form-foreground' : 'submit-form',
- description: foreground
- ? 'Submit the form by clicking the unique validated visual Submit region with ' +
- 'foreground delivery, which activates the browser window, because Driver ' +
- 'refused background delivery for the previous visual click.'
- : 'Submit the form by clicking the unique validated visual Submit region. ' +
- 'The observed form state reports that the verification field already ' +
- 'contains the required token.',
- tool: 'click',
- arguments: {
- pid: visual.pid,
- window_id: visual.windowId,
- x,
- y,
- capture_id: visual.captureId,
- delivery_mode: visualDelivery,
- },
- captureId: visual.captureId,
- screenshotReference: visual.screenshotReference,
- })
- );
- }
- }
- return [...candidates, ...reservedCandidates()];
-}
-
export function hasExecutableCandidate(candidates: readonly Candidate[]): boolean {
return candidates.some((candidate) => candidate.tool !== null);
}
-function asciiLower(value: string): string {
- return value.replace(/[A-Z]/g, (character) =>
- String.fromCharCode(character.charCodeAt(0) + 32)
- );
-}
-
export function chooseMock(candidates: Candidate[]) {
const ids = new Set(candidates.map((candidate) => candidate.id));
const selected = ids.has('type-verification-value')
@@ -462,15 +260,3 @@ export function validateChoice(
}
return candidate;
}
-
-export function classify(
- submitted: string | null,
- token: string,
- steps: number,
- maxSteps: number
-): Outcome {
- if (submitted === token) return 'verified';
- if (submitted !== null) return 'refuted';
- if (steps >= maxSteps) return 'budget_exhausted';
- return 'unknown';
-}
diff --git a/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts b/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts
index 5cec3b910c..482e20bd28 100644
--- a/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts
@@ -2,13 +2,12 @@ import { choice, TypeSafeClient } from '@typesafe-ai/sdk';
import {
chooseMock,
- formState,
- redactToken,
type BrowserSnapshot,
type HistoryEntry,
type Candidate,
type VisualObservation,
} from './core.js';
+import { FIXTURE_GOAL, FixtureFormTask, fixtureSources, type Task, type TaskSources } from './tasks.js';
type TypeSafeClientLike = Pick<TypeSafeClient, 'systemOne'>;
@@ -99,51 +98,63 @@ export function visualDecisionState(visual?: VisualObservation) {
};
}
-export const GOAL =
- 'Enter the required verification token into the verification field, then submit the form.';
+export const GOAL = FIXTURE_GOAL;
/**
- * Build the compact, deterministic, token-redacted state sent to Jev. `form`
- * states the field and Submit status the runner verified from the page
- * structure and, when visualPath is enabled, from the validated visual regions;
- * the token is replaced everywhere, including outline and visual text.
+ * Build the compact, deterministic, secret-redacted state sent to Jev. `form`
+ * is the task's state summary, which states what the runner verified from its
+ * candidate sources, so the model does not have to infer it from the outline.
+ * Every secret task parameter is replaced everywhere, including outline and
+ * visual text.
*/
-export function decisionState(
- snapshot: BrowserSnapshot,
- visual: VisualObservation | undefined,
- history: readonly HistoryEntry[],
- token: string,
- visualPath = false
+export function taskDecisionState(
+ task: Task,
+ sources: TaskSources,
+ history: readonly HistoryEntry[]
) {
+ const snapshot = sources.page.snapshot;
return {
- goal: GOAL,
+ goal: task.goal,
observation: {
- page: JSON.stringify(redactToken(snapshot.page ?? null, token)),
- form: JSON.stringify(formState(snapshot, token, visual, visualPath)),
- outline: redactToken(snapshot.outline ?? '', token) as string,
- visual: JSON.stringify(redactToken(visualDecisionState(visual), token)),
+ page: JSON.stringify(task.redact(snapshot.page ?? null)),
+ form: JSON.stringify(task.stateSummary(sources)),
+ outline: task.redact(snapshot.outline ?? '') as string,
+ visual: JSON.stringify(task.redact(visualDecisionState(sources.visual?.observation))),
},
history: JSON.stringify(history),
};
}
-export async function chooseWithTypeSafe(
- client: TypeSafeClientLike,
- candidates: Candidate[],
+/** Build the fixture task's decision state; see taskDecisionState. */
+export function decisionState(
snapshot: BrowserSnapshot,
visual: VisualObservation | undefined,
history: readonly HistoryEntry[],
token: string,
visualPath = false
+) {
+ return taskDecisionState(
+ new FixtureFormTask(token),
+ fixtureSources(snapshot, visual, false, 'background', visualPath),
+ history
+ );
+}
+
+export const DRIVER_ACTION_INSTRUCTIONS =
+ 'Which complete executable action should Cua Driver run next?';
+
+export async function chooseForTask(
+ client: TypeSafeClientLike,
+ task: Task,
+ sources: TaskSources,
+ candidates: Candidate[],
+ history: readonly HistoryEntry[]
) {
const criteria = candidateCriteria(candidates);
const response = await client.systemOne({
- state: decisionState(snapshot, visual, history, token, visualPath),
+ state: taskDecisionState(task, sources, history),
questions: {
- driver_action: choice(
- 'Which complete executable action should Cua Driver run next?',
- criteria
- ),
+ driver_action: choice(DRIVER_ACTION_INSTRUCTIONS, criteria),
},
});
const answer = response.answers.driver_action;
@@ -154,6 +165,42 @@ export async function chooseWithTypeSafe(
return answer;
}
+export function chooseWithTypeSafe(
+ client: TypeSafeClientLike,
+ candidates: Candidate[],
+ snapshot: BrowserSnapshot,
+ visual: VisualObservation | undefined,
+ history: readonly HistoryEntry[],
+ token: string,
+ visualPath = false
+) {
+ return chooseForTask(
+ client,
+ new FixtureFormTask(token),
+ fixtureSources(snapshot, visual, false, 'background', visualPath),
+ candidates,
+ history
+ );
+}
+
+export function chooseLiveForTask(
+ task: Task,
+ sources: TaskSources,
+ candidates: Candidate[],
+ history: readonly HistoryEntry[]
+) {
+ return chooseForTask(new TypeSafeClient(), task, sources, candidates, history);
+}
+
+export function chooseMockForTask(
+ _task: Task,
+ _sources: TaskSources,
+ candidates: Candidate[],
+ _history: readonly HistoryEntry[]
+) {
+ return chooseMock(candidates);
+}
+
export function chooseLive(
candidates: Candidate[],
snapshot: BrowserSnapshot,
@@ -162,14 +209,11 @@ export function chooseLive(
token: string,
visualPath = false
) {
- return chooseWithTypeSafe(
- new TypeSafeClient(),
+ return chooseLiveForTask(
+ new FixtureFormTask(token),
+ fixtureSources(snapshot, visual, false, 'background', visualPath),
candidates,
- snapshot,
- visual,
- history,
- token,
- visualPath
+ history
);
}
diff --git a/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.test.ts b/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.test.ts
new file mode 100644
index 0000000000..d848a94207
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.test.ts
@@ -0,0 +1,44 @@
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import test from 'node:test';
+
+import { chooseForTask } from './jev_adapter.js';
+import {
+ GOLDEN,
+ encode,
+ legacyBuild,
+ legacyChoose,
+ payloads,
+ type Build,
+ type Choose,
+} from './provider_request_golden.js';
+import { FixtureFormTask, fixtureSources } from './tasks.js';
+
+// The golden was recorded before the candidate-source and task-spec refactor
+// (RFC #4268 Phase 0), so equality proves the refactor sends byte-identical
+// model input and offers identical executable candidates.
+async function assertGolden(build: Build, choose: Choose) {
+ assert.equal(encode(await payloads(build, choose)), readFileSync(GOLDEN, 'utf8'));
+}
+
+const taskBuild: Build = (snapshot, token, visual, captureBoundClick, visualDelivery) =>
+ new FixtureFormTask(token).candidates(
+ fixtureSources(snapshot, visual, captureBoundClick, visualDelivery)
+ );
+
+const taskChoose: Choose = (client, candidates, snapshot, visual, history, token, visualPath) =>
+ chooseForTask(
+ client as never,
+ new FixtureFormTask(token),
+ fixtureSources(snapshot, visual, false, 'background', visualPath),
+ candidates,
+ history
+ );
+
+test('core entry points send byte-identical provider requests', async () => {
+ await assertGolden(legacyBuild, legacyChoose);
+});
+
+test('task spec and sources send byte-identical provider requests', async () => {
+ await assertGolden(taskBuild, taskChoose);
+});
diff --git a/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.ts b/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.ts
new file mode 100644
index 0000000000..a296578df4
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/typescript/provider_request_golden.ts
@@ -0,0 +1,303 @@
+/**
+ * Record the exact TypeSafe Jev request payloads for the page and visual
+ * fixtures.
+ *
+ * fixtures/jev-provider-request-golden-typescript-v1.json was captured from this
+ * module on main before the candidate-source and task-spec refactor (RFC #4268
+ * Phase 0). provider_request_golden.test.ts replays the same scenarios and
+ * requires byte-identical payloads, so any change to the model input or the
+ * candidate table is a test failure, not a silent drift.
+ *
+ * Regenerate only for an intentional contract change:
+ *
+ * npx tsx typescript/provider_request_golden.ts --write
+ */
+import { createHash } from 'node:crypto';
+import { readFileSync, writeFileSync } from 'node:fs';
+import process from 'node:process';
+import { pathToFileURL } from 'node:url';
+
+import {
+ SUBMIT_IDS,
+ buildCandidates,
+ hasExecutableCandidate,
+ historyEntry,
+ parseVisualRegions,
+ validateChoice,
+ type BrowserSnapshot,
+ type Candidate,
+ type HistoryEntry,
+ type VisualDelivery,
+ type VisualObservation,
+} from './core.js';
+import { chooseWithTypeSafe } from './jev_adapter.js';
+
+export const GOLDEN = new URL(
+ '../fixtures/jev-provider-request-golden-typescript-v1.json',
+ import.meta.url
+);
+
+const load = (name: string) =>
+ JSON.parse(readFileSync(new URL(`../fixtures/${name}`, import.meta.url), 'utf8'));
+const PAGE = load('jev-page-structure-replay-v1.json');
+const VISUAL = load('jev-visual-replay-v1.json');
+
+export type Build = (
+ snapshot: BrowserSnapshot,
+ token: string,
+ visual: VisualObservation | undefined,
+ captureBoundClick: boolean,
+ visualDelivery: VisualDelivery
+) => Candidate[];
+
+export type Choose = (
+ client: RecordingClient,
+ candidates: Candidate[],
+ snapshot: BrowserSnapshot,
+ visual: VisualObservation | undefined,
+ history: readonly HistoryEntry[],
+ token: string,
+ visualPath: boolean
+) => Promise<{ choice: string }>;
+
+type Fixture = {
+ token: string;
+ snapshots: { before_typing: BrowserSnapshot; after_typing: BrowserSnapshot };
+ recorded_live_runs: Record<string, { steps: RecordedStep[] }>;
+};
+
+type RecordedStep = { step?: number; selected_id: string; action_error?: string };
+
+type WireRecord = { step?: number; request: unknown; candidates: unknown };
+
+export function observation(payload = VISUAL.visual_regions): VisualObservation {
+ const source = payload.capture.source;
+ return parseVisualRegions(payload, payload.capture.capture_id, source.pid, source.window_id);
+}
+
+/**
+ * Answer with scripted IDs and keep every request exactly as sent. A scripted
+ * ID absent from the offered criteria (for example a visual Submit when visual
+ * parsing is off) is answered with reobserve.
+ */
+export class RecordingClient {
+ readonly requests: any[] = [];
+ private readonly choices: string[];
+
+ constructor(choices: string[]) {
+ this.choices = [...choices];
+ }
+
+ async systemOne(request: any) {
+ this.requests.push(request);
+ let selected = this.choices.shift()!;
+ if (!Object.hasOwn(request.questions.driver_action.criteria, selected)) selected = 'reobserve';
+ return {
+ answers: {
+ driver_action: {
+ type: 'choice' as const,
+ choice: selected,
+ confidence: 1,
+ probabilities: { [selected]: 1 },
+ },
+ },
+ };
+ }
+}
+
+function wireCandidates(candidates: readonly Candidate[]) {
+ return candidates.map((candidate) => ({
+ id: candidate.id,
+ description: candidate.description,
+ tool: candidate.tool,
+ arguments: JSON.parse(JSON.stringify(candidate.arguments)),
+ capture_id: candidate.captureId ?? null,
+ screenshot_reference: candidate.screenshotReference ?? null,
+ }));
+}
+
+function wireRequest(request: any) {
+ return JSON.parse(JSON.stringify(request));
+}
+
+async function replay(
+ build: Build,
+ choose: Choose,
+ fixture: Fixture,
+ run: { steps: RecordedStep[] },
+ captureBoundClick: boolean,
+ visualMode: 'auto' | 'always' | 'off'
+): Promise<WireRecord[]> {
+ const token = fixture.token;
+ const before = fixture.snapshots.before_typing;
+ const after = fixture.snapshots.after_typing;
+ const visualPath = captureBoundClick && visualMode !== 'off';
+ const history: HistoryEntry[] = [];
+ let typed = false;
+ let delivery: VisualDelivery = 'background';
+ const records: WireRecord[] = [];
+ const client = new RecordingClient(run.steps.map((step) => step.selected_id));
+ for (const [offset, recorded] of run.steps.entries()) {
+ const step = recorded.step ?? offset + 1;
+ const snapshot = typed ? after : before;
+ let candidates = build(snapshot, token, undefined, captureBoundClick, delivery);
+ let visual: VisualObservation | undefined;
+ if (
+ visualMode === 'always' ||
+ (visualMode === 'auto' && !hasExecutableCandidate(candidates))
+ ) {
+ if (captureBoundClick) {
+ visual = observation();
+ candidates = build(snapshot, token, visual, captureBoundClick, delivery);
+ }
+ }
+ const answer = await choose(client, candidates, snapshot, visual, history, token, visualPath);
+ const candidate = validateChoice(answer.choice, candidates, visual?.captureId);
+ records.push({
+ step,
+ request: wireRequest(client.requests.at(-1)),
+ candidates: wireCandidates(candidates),
+ });
+ const refusal = recorded.action_error;
+ history.push(historyEntry(step, candidate.id, refusal));
+ if (candidate.id === 'abstain') break;
+ if (candidate.id === 'type-verification-value') {
+ typed = true;
+ } else if (SUBMIT_IDS.has(candidate.id)) {
+ if (refusal) delivery = 'foreground';
+ else break;
+ }
+ }
+ return records;
+}
+
+function withFieldValue(snapshot: BrowserSnapshot, value: string | null): BrowserSnapshot {
+ const changed = structuredClone(snapshot);
+ for (const ref of changed.refs ?? []) {
+ if (ref.role === 'textbox' && ref.name === 'verification value') ref.value = value;
+ }
+ return changed;
+}
+
+function withoutButton(snapshot: BrowserSnapshot): BrowserSnapshot {
+ const changed = structuredClone(snapshot);
+ changed.refs = (changed.refs ?? []).filter((ref) => ref.role !== 'button');
+ return changed;
+}
+
+async function singleRequests(build: Build, choose: Choose) {
+ const pageToken: string = PAGE.token;
+ const pageBefore: BrowserSnapshot = PAGE.snapshots.before_typing;
+ const pageAfter: BrowserSnapshot = PAGE.snapshots.after_typing;
+ const visualToken: string = VISUAL.token;
+ const visualAfter: BrowserSnapshot = VISUAL.snapshots.after_typing;
+ const visual = observation();
+ const noSubmit = { ...visual, regions: visual.regions.filter((r) => r.text !== 'Submit') };
+ const submit = visual.regions.find((r) => r.text === 'Submit')!;
+ const duplicated = { ...visual, regions: [...visual.regions, { ...submit, id: 'dup' }] };
+ const empty: BrowserSnapshot = { target_id: 'target', tab_id: 'tab', refs: [] };
+ const cases: Record<
+ string,
+ [BrowserSnapshot, string, VisualObservation | undefined, boolean, boolean, VisualDelivery]
+ > = {
+ page_other_value: [withFieldValue(pageBefore, 'other'), pageToken, undefined, false, false, 'background'],
+ page_after_with_visual: [pageAfter, pageToken, visual, true, true, 'background'],
+ page_after_no_button: [withoutButton(pageAfter), pageToken, undefined, false, false, 'background'],
+ page_after_no_button_visual_path: [withoutButton(pageAfter), pageToken, undefined, true, true, 'background'],
+ no_form_refs: [empty, pageToken, undefined, false, false, 'background'],
+ visual_after_no_capture_bound_click: [visualAfter, visualToken, visual, false, true, 'background'],
+ visual_after_without_submit_region: [visualAfter, visualToken, noSubmit, true, true, 'background'],
+ visual_after_duplicate_submit_region: [visualAfter, visualToken, duplicated, true, true, 'background'],
+ visual_after_foreground: [visualAfter, visualToken, visual, true, true, 'foreground'],
+ visual_after_other_value: [withFieldValue(visualAfter, 'other'), visualToken, visual, true, true, 'background'],
+ };
+ const result: Record<string, WireRecord> = {};
+ for (const [name, [snapshot, token, current, cbc, visualPath, delivery]] of Object.entries(cases)) {
+ const candidates = build(snapshot, token, current, cbc, delivery);
+ const client = new RecordingClient(['abstain']);
+ await choose(client, candidates, snapshot, current, [], token, visualPath);
+ result[name] = {
+ request: wireRequest(client.requests.at(-1)),
+ candidates: wireCandidates(candidates),
+ };
+ }
+ return result;
+}
+
+function digest(value: unknown): string {
+ return createHash('sha256').update(JSON.stringify(value), 'utf8').digest('hex').slice(0, 16);
+}
+
+/**
+ * Store each distinct request and candidate table once, keyed by digest.
+ * Scenario records keep their order and reference the exact payloads.
+ */
+function deduplicate(
+ runs: Record<string, WireRecord[]>,
+ singles: Record<string, WireRecord>
+) {
+ const requests: Record<string, unknown> = {};
+ const tables: Record<string, unknown> = {};
+ const reference = (record: WireRecord) => {
+ const requestId = digest(record.request);
+ const tableId = digest(record.candidates);
+ requests[requestId] ??= record.request;
+ tables[tableId] ??= record.candidates;
+ const refs = { request: requestId, candidates: tableId };
+ return record.step === undefined ? refs : { step: record.step, ...refs };
+ };
+ return {
+ schema: 'cua.jev_use_provider_request_golden_v1',
+ language: 'typescript',
+ runs: Object.fromEntries(
+ Object.entries(runs).map(([name, records]) => [name, records.map(reference)])
+ ),
+ single_requests: Object.fromEntries(
+ Object.entries(singles).map(([name, record]) => [name, reference(record)])
+ ),
+ requests,
+ candidate_tables: tables,
+ };
+}
+
+export async function payloads(build: Build, choose: Choose) {
+ const modes: [string, Fixture, boolean, 'auto' | 'always' | 'off'][] = [
+ ['page', PAGE, false, 'auto'],
+ ['page_capture_bound_auto', PAGE, true, 'auto'],
+ ['page_capture_bound_always', PAGE, true, 'always'],
+ ['visual_capture_bound_auto', VISUAL, true, 'auto'],
+ ['visual_capture_bound_always', VISUAL, true, 'always'],
+ ['visual_capture_bound_off', VISUAL, true, 'off'],
+ ];
+ const runs: Record<string, WireRecord[]> = {};
+ for (const [prefix, fixture, cbc, mode] of modes) {
+ for (const [name, run] of Object.entries(fixture.recorded_live_runs)) {
+ runs[`${prefix}/${name}`] = await replay(build, choose, fixture, run, cbc, mode);
+ }
+ }
+ return deduplicate(runs, await singleRequests(build, choose));
+}
+
+export const legacyBuild: Build = (snapshot, token, visual, captureBoundClick, visualDelivery) =>
+ buildCandidates(snapshot, token, visual, captureBoundClick, visualDelivery);
+
+export const legacyChoose: Choose = (
+ client,
+ candidates,
+ snapshot,
+ visual,
+ history,
+ token,
+ visualPath
+) =>
+ chooseWithTypeSafe(client as never, candidates, snapshot, visual, history, token, visualPath);
+
+export function encode(value: unknown): string {
+ return `${JSON.stringify(value, null, 1)}\n`;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const text = encode(await payloads(legacyBuild, legacyChoose));
+ if (process.argv.includes('--write')) writeFileSync(GOLDEN, text, 'utf8');
+ else process.stdout.write(text);
+}
diff --git a/libs/cua-driver/examples/jev-use/typescript/run.ts b/libs/cua-driver/examples/jev-use/typescript/run.ts
index d34b97da92..853153b5d6 100644
--- a/libs/cua-driver/examples/jev-use/typescript/run.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/run.ts
@@ -6,12 +6,8 @@ import { pathToFileURL } from 'node:url';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
import {
- buildCandidates,
- classify,
hasExecutableCandidate,
- historyEntry,
parseVisualRegions,
- SUBMIT_IDS,
validateChoice,
VisualObservationError,
type BrowserSnapshot,
@@ -22,7 +18,8 @@ import {
type VisualObservation,
} from './core.js';
import { driverEnvironment } from './driver_env.js';
-import { chooseLive, chooseMockAdapter } from './jev_adapter.js';
+import { chooseLiveForTask, chooseMockForTask } from './jev_adapter.js';
+import { FixtureFormTask, fixtureSources, type Task, type TaskSources } from './tasks.js';
type VisualMode = 'auto' | 'always' | 'off';
@@ -248,29 +245,34 @@ export async function optionalVisualObservation(
}
/**
- * Build one step's candidates, parsing visual regions only when useful. In
- * auto mode the capture and parse run only when the page structure offers no
- * executable candidate, because only then can a visual region add one. always
- * restores the per-step parse; off never parses.
+ * Build one step's sources and candidates, parsing visual regions only when
+ * useful. In auto mode the capture and parse run only when the page structure
+ * offers no executable candidate, because only then can a visual region add
+ * one. always restores the per-step parse; off never parses.
*/
-export async function candidatesForStep(
+export async function taskCandidatesForStep(
driver: Driver,
+ task: Task,
snapshot: BrowserSnapshot,
- token: string,
pid: number,
windowId: number,
availableTools: ReadonlySet<string>,
captureBoundClick: boolean,
visualMode: VisualMode = 'auto',
visualDelivery: VisualDelivery = 'background'
-): Promise<{ candidates: Candidate[]; visual?: VisualObservation; status: VisualStatus }> {
- let candidates = buildCandidates(snapshot, token, undefined, captureBoundClick, visualDelivery);
+): Promise<{ candidates: Candidate[]; sources: TaskSources; status: VisualStatus }> {
+ // Whether a control missing from the page structure can still be found
+ // through a capture-bound visual region; reported in the task state.
+ const visualPath = captureBoundClick && visualMode !== 'off';
+ let sources = fixtureSources(snapshot, undefined, captureBoundClick, visualDelivery, visualPath);
+ let candidates = task.candidates(sources);
if (visualMode === 'off') {
- return { candidates, status: visualStatus('skipped', undefined, undefined, 'disabled') };
+ return { candidates, sources, status: visualStatus('skipped', undefined, undefined, 'disabled') };
}
if (visualMode === 'auto' && hasExecutableCandidate(candidates)) {
return {
candidates,
+ sources,
status: visualStatus('skipped', undefined, undefined, 'page_structure_candidate'),
};
}
@@ -282,20 +284,36 @@ export async function candidatesForStep(
captureBoundClick
);
if (visual) {
- candidates = buildCandidates(snapshot, token, visual, captureBoundClick, visualDelivery);
+ sources = fixtureSources(snapshot, visual, captureBoundClick, visualDelivery, visualPath);
+ candidates = task.candidates(sources);
}
- return { candidates, visual, status };
+ return { candidates, sources, status };
}
-async function fixtureState(fixtureUrl: string): Promise<{ submitted: string | null }> {
- const response = await fetch(new URL('state', fixtureUrl));
- if (!response.ok) throw new Error(`fixture state failed: HTTP ${response.status}`);
- return (await response.json()) as { submitted: string | null };
-}
-
-async function resetFixture(fixtureUrl: string): Promise<void> {
- const response = await fetch(new URL('reset', fixtureUrl), { method: 'POST' });
- if (response.status !== 204) throw new Error(`fixture reset failed: HTTP ${response.status}`);
+/** Build one step's fixture-task candidates; see taskCandidatesForStep. */
+export async function candidatesForStep(
+ driver: Driver,
+ snapshot: BrowserSnapshot,
+ token: string,
+ pid: number,
+ windowId: number,
+ availableTools: ReadonlySet<string>,
+ captureBoundClick: boolean,
+ visualMode: VisualMode = 'auto',
+ visualDelivery: VisualDelivery = 'background'
+): Promise<{ candidates: Candidate[]; visual?: VisualObservation; status: VisualStatus }> {
+ const { candidates, sources, status } = await taskCandidatesForStep(
+ driver,
+ new FixtureFormTask(token),
+ snapshot,
+ pid,
+ windowId,
+ availableTools,
+ captureBoundClick,
+ visualMode,
+ visualDelivery
+ );
+ return { candidates, visual: sources.visual?.observation, status };
}
async function waitForWindow(driver: Driver, pid: number) {
@@ -323,6 +341,7 @@ async function writeEvent(path: string | undefined, event: Record<string, unknow
async function run(args: Arguments): Promise<Outcome> {
const token = args.token ?? `jev-${randomUUID().replaceAll('-', '').slice(0, 10)}`;
+ const task: Task = new FixtureFormTask(token, args.fixtureUrl, args.maxSteps);
const transport = new StdioClientTransport({
command: process.env.CUA_DRIVER_BIN ?? 'cua-driver',
args: ['mcp'],
@@ -334,16 +353,13 @@ async function run(args: Arguments): Promise<Outcome> {
const history: HistoryEntry[] = [];
let visualDelivery: VisualDelivery = 'background';
if (args.log) await writeFile(args.log, '', 'utf8');
- await resetFixture(args.fixtureUrl);
+ await task.reset();
try {
await client.connect(transport);
const advertisedTools = (await client.listTools()).tools;
const availableTools = new Set(advertisedTools.map((tool) => tool.name));
const captureBoundClick = supportsCaptureBoundClick(advertisedTools);
- // Whether a Submit missing from the page structure can still be found through
- // a capture-bound visual region; reported in the form state.
- const visualPath = captureBoundClick && args.visualObservation !== 'off';
const driver = new Driver(client, `jev-typescript-${randomUUID().slice(0, 8)}`);
const prepared = await driver.call('browser_prepare', {
allow_launch: true,
@@ -363,9 +379,8 @@ async function run(args: Arguments): Promise<Outcome> {
url: args.fixtureUrl,
});
- for (let step = 1; step <= args.maxSteps; step += 1) {
- const oracle = await fixtureState(args.fixtureUrl);
- const current = classify(oracle.submitted, token, step - 1, args.maxSteps);
+ for (let step = 1; step <= task.maxSteps; step += 1) {
+ const current = task.classify(await task.readOracle(), step - 1);
if (current === 'verified' || current === 'refuted') {
await writeEvent(args.log, { event: 'outcome', outcome: current, token });
return current;
@@ -379,12 +394,12 @@ async function run(args: Arguments): Promise<Outcome> {
})) as BrowserSnapshot;
const {
candidates,
- visual,
+ sources,
status: visualRecord,
- } = await candidatesForStep(
+ } = await taskCandidatesForStep(
driver,
+ task,
snapshot,
- token,
pid,
Number(window.window_id),
availableTools,
@@ -401,10 +416,11 @@ async function run(args: Arguments): Promise<Outcome> {
});
return 'abstained';
}
+ const visual = sources.visual?.observation;
const answer =
args.provider === 'mock'
- ? chooseMockAdapter(candidates, snapshot, visual, history, token, visualPath)
- : await chooseLive(candidates, snapshot, visual, history, token, visualPath);
+ ? chooseMockForTask(task, sources, candidates, history)
+ : await chooseLiveForTask(task, sources, candidates, history);
if (!answer.choice) return 'abstained';
const candidate = validateChoice(answer.choice, candidates, visual?.captureId);
const decisionMs = Math.round((performance.now() - decisionStarted) * 100) / 100;
@@ -422,7 +438,7 @@ async function run(args: Arguments): Promise<Outcome> {
tool: null,
visual: visualRecord,
};
- history.push(historyEntry(step, candidate.id));
+ history.push(task.historyEntry(step, candidate.id));
await writeEvent(args.log, event);
continue;
}
@@ -466,7 +482,7 @@ async function run(args: Arguments): Promise<Outcome> {
escalation: { from: 'background', to: 'foreground', reason: refusal },
visual: visualRecord,
};
- history.push(historyEntry(step, candidate.id, refusal));
+ history.push(task.historyEntry(step, candidate.id, refusal));
await writeEvent(args.log, event);
continue;
}
@@ -496,13 +512,12 @@ async function run(args: Arguments): Promise<Outcome> {
delivery_mode: candidate.arguments.delivery_mode ?? null,
visual: visualRecord,
};
- history.push(historyEntry(step, candidate.id));
+ history.push(task.historyEntry(step, candidate.id));
await writeEvent(args.log, event);
if (args.dryRun) return 'unknown';
- if (SUBMIT_IDS.has(candidate.id)) {
+ if (task.completionCandidateIds.has(candidate.id)) {
for (let attempt = 0; attempt < 20; attempt += 1) {
- const oracle = await fixtureState(args.fixtureUrl);
- const outcome = classify(oracle.submitted, token, step, args.maxSteps);
+ const outcome = task.classify(await task.readOracle(), step);
if (outcome === 'verified' || outcome === 'refuted') {
await writeEvent(args.log, { event: 'outcome', outcome, token });
return outcome;
@@ -512,8 +527,7 @@ async function run(args: Arguments): Promise<Outcome> {
}
}
- const oracle = await fixtureState(args.fixtureUrl);
- const outcome = classify(oracle.submitted, token, args.maxSteps, args.maxSteps);
+ const outcome = task.classify(await task.readOracle(), task.maxSteps);
await writeEvent(args.log, { event: 'outcome', outcome, token });
return outcome;
} finally {
diff --git a/libs/cua-driver/examples/jev-use/typescript/sources.ts b/libs/cua-driver/examples/jev-use/typescript/sources.ts
new file mode 100644
index 0000000000..9e3c5b3e43
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/typescript/sources.ts
@@ -0,0 +1,195 @@
+/**
+ * Candidate sources: where a decision's executable candidates come from.
+ *
+ * A candidate source turns one Driver observation into addressable controls
+ * and builds fully specified candidates for them. The model only ever sees a
+ * candidate's ID and description; the tool name and exact arguments stay in
+ * the runner. A task (see tasks.ts) decides which controls matter and which
+ * candidate IDs and descriptions to offer; the source decides how to address a
+ * control and which Driver tool acts on it.
+ *
+ * Implemented sources:
+ *
+ * - BrowserSemanticSource reads a get_browser_state semantic_v2 snapshot and
+ * acts through browser_type / browser_click on its refs.
+ * - VisualRegionSource reads a validated parse_visual_regions result and acts
+ * through a capture-bound click on a region's center.
+ *
+ * Extension point (RFC #4268, Phase 1): a NativeAccessibilitySource will read
+ * get_window_state elements, find controls by normalized role and label, and
+ * act through the snapshot's element_token. It is deliberately not implemented
+ * here; it must implement CandidateSource with kind 'ax'.
+ */
+import type {
+ BrowserSnapshot,
+ PageRef,
+ VisualDelivery,
+ VisualObservation,
+ VisualRegion,
+} from './core.js';
+
+/** 'ax' is reserved for the future NativeAccessibilitySource. */
+export type SourceKind = 'page' | 'visual' | 'ax';
+
+export type Candidate = Readonly<{
+ id: string;
+ description: string;
+ tool: string | null;
+ arguments: Readonly<Record<string, unknown>>;
+ captureId?: string;
+ screenshotReference?: string;
+}>;
+
+export function immutableCandidate(candidate: Candidate): Candidate {
+ return Object.freeze({ ...candidate, arguments: Object.freeze({ ...candidate.arguments }) });
+}
+
+export function asciiLower(value: string): string {
+ return value.replace(/[A-Z]/g, (character) =>
+ String.fromCharCode(character.charCodeAt(0) + 32)
+ );
+}
+
+/**
+ * One addressable element a source found in its observation. value is the
+ * element's current value when the source can read it (never sent to a model;
+ * tasks summarize it). handle is source-private: a page ref for the browser
+ * source, a VisualRegion for the visual source.
+ */
+export type Control<Handle = unknown> = Readonly<{
+ source: SourceKind;
+ role: string;
+ name: string;
+ value: unknown;
+ handle: Handle;
+}>;
+
+/**
+ * The interface every candidate source implements. find returns the unique
+ * control matching a role and accessible name, or undefined. click and
+ * typeText return a fully specified candidate for that control, or undefined
+ * when this source cannot perform the action in the current observation (for
+ * example a visual region cannot receive text, and a visual click needs a
+ * capture-bound Driver click).
+ */
+export interface CandidateSource {
+ readonly kind: SourceKind;
+ find(role: string, name: string): Control | undefined;
+ click(control: Control, candidateId: string, description: string): Candidate | undefined;
+ typeText(
+ control: Control,
+ text: string,
+ candidateId: string,
+ description: string
+ ): Candidate | undefined;
+}
+
+/** Controls from a get_browser_state snapshot, acted on through its refs. */
+export class BrowserSemanticSource implements CandidateSource {
+ readonly kind = 'page' as const;
+
+ constructor(readonly snapshot: BrowserSnapshot) {}
+
+ /** The browser target every page candidate addresses. */
+ target() {
+ return { target_id: this.snapshot.target_id, tab_id: this.snapshot.tab_id };
+ }
+
+ find(role: string, name: string): Control<PageRef> | undefined {
+ const ref = (this.snapshot.refs ?? []).find(
+ (item) => item.role === role && item.name === name && item.ref
+ );
+ if (!ref) return undefined;
+ return { source: 'page', role, name, value: ref.value, handle: ref };
+ }
+
+ click(control: Control, candidateId: string, description: string): Candidate {
+ return immutableCandidate({
+ id: candidateId,
+ description,
+ tool: 'browser_click',
+ arguments: {
+ ...this.target(),
+ ref: (control.handle as PageRef).ref,
+ input_route: 'dom_event',
+ },
+ });
+ }
+
+ typeText(control: Control, text: string, candidateId: string, description: string): Candidate {
+ return immutableCandidate({
+ id: candidateId,
+ description,
+ tool: 'browser_type',
+ arguments: {
+ ...this.target(),
+ ref: (control.handle as PageRef).ref,
+ text,
+ replace: true,
+ },
+ });
+ }
+}
+
+/**
+ * Controls from validated OmniParser regions, clicked through their capture.
+ * Regions carry no role, so find matches the visible text or icon label (ASCII
+ * case-insensitive) of regions at or above MIN_CONFIDENCE and returns a
+ * control only when exactly one region matches. click is offered only when
+ * Driver advertises a capture-bound click (captureBound); it targets the
+ * region's screenshot-pixel center with the exact capture_id and the source's
+ * delivery mode.
+ */
+export class VisualRegionSource implements CandidateSource {
+ static readonly MIN_CONFIDENCE = 0.8;
+ readonly kind = 'visual' as const;
+
+ constructor(
+ readonly observation: VisualObservation,
+ readonly delivery: VisualDelivery = 'background',
+ readonly captureBound = false
+ ) {}
+
+ find(role: string, name: string): Control<VisualRegion> | undefined {
+ const wanted = asciiLower(name);
+ const matches = this.observation.regions.filter(
+ (region) =>
+ region.confidence >= VisualRegionSource.MIN_CONFIDENCE &&
+ asciiLower(region.text ?? region.label ?? '') === wanted
+ );
+ if (matches.length !== 1) return undefined;
+ const region = matches[0];
+ return {
+ source: 'visual',
+ role,
+ name: region.text ?? region.label ?? '',
+ value: undefined,
+ handle: region,
+ };
+ }
+
+ click(control: Control, candidateId: string, description: string): Candidate | undefined {
+ if (!this.captureBound) return undefined;
+ const visual = this.observation;
+ const region = control.handle as VisualRegion;
+ return immutableCandidate({
+ id: candidateId,
+ description,
+ tool: 'click',
+ arguments: {
+ pid: visual.pid,
+ window_id: visual.windowId,
+ x: region.x + region.width / 2,
+ y: region.y + region.height / 2,
+ capture_id: visual.captureId,
+ delivery_mode: this.delivery,
+ },
+ captureId: visual.captureId,
+ screenshotReference: visual.screenshotReference,
+ });
+ }
+
+ typeText(): undefined {
+ return undefined;
+ }
+}
diff --git a/libs/cua-driver/examples/jev-use/typescript/sources_and_tasks.test.ts b/libs/cua-driver/examples/jev-use/typescript/sources_and_tasks.test.ts
new file mode 100644
index 0000000000..4597a1b44b
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/typescript/sources_and_tasks.test.ts
@@ -0,0 +1,112 @@
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import test from 'node:test';
+
+import { REDACTED_TOKEN, parseVisualRegions, type VisualObservation } from './core.js';
+import { BrowserSemanticSource, VisualRegionSource, type CandidateSource } from './sources.js';
+import {
+ FIXTURE_GOAL,
+ FixtureFormTask,
+ SUBMIT_IDS,
+ fixtureSources,
+ type Task,
+ type TaskSources,
+} from './tasks.js';
+
+const load = (name: string) =>
+ JSON.parse(readFileSync(new URL(`../fixtures/${name}`, import.meta.url), 'utf8'));
+const PAGE = load('jev-page-structure-replay-v1.json');
+const VISUAL = load('jev-visual-replay-v1.json');
+const PAYLOAD = VISUAL.visual_regions;
+
+function observation(): VisualObservation {
+ const source = PAYLOAD.capture.source;
+ return parseVisualRegions(PAYLOAD, PAYLOAD.capture.capture_id, source.pid, source.window_id);
+}
+
+test('sources implement one interface with distinct kinds', () => {
+ const page: CandidateSource = new BrowserSemanticSource(PAGE.snapshots.before_typing);
+ const visual: CandidateSource = new VisualRegionSource(observation());
+ assert.deepEqual([page.kind, visual.kind], ['page', 'visual']);
+});
+
+test('page source addresses refs with exact browser arguments', () => {
+ const snapshot = PAGE.snapshots.after_typing;
+ const page = new BrowserSemanticSource(snapshot);
+ const button = page.find('button', 'Submit');
+ assert.ok(button);
+ assert.deepEqual([button.source, button.handle.ref], ['page', 'p4:2']);
+ const candidate = page.click(button, 'c', 'd');
+ assert.equal(candidate.tool, 'browser_click');
+ assert.deepEqual(candidate.arguments, {
+ target_id: snapshot.target_id,
+ tab_id: snapshot.tab_id,
+ ref: 'p4:2',
+ input_route: 'dom_event',
+ });
+ assert.equal(page.find('button', 'Cancel'), undefined);
+});
+
+test('visual source clicks only capture-bound and never types', () => {
+ const visual = observation();
+ const source = new VisualRegionSource(visual);
+ const submit = source.find('button', 'submit');
+ assert.ok(submit);
+ assert.equal(source.click(submit, 'c', 'd'), undefined);
+ assert.equal(source.typeText(), undefined);
+ const bound = new VisualRegionSource(visual, 'foreground', true);
+ const candidate = bound.click(submit, 'c', 'd');
+ assert.equal(candidate?.tool, 'click');
+ assert.equal(candidate?.captureId, visual.captureId);
+ assert.equal(candidate?.arguments.delivery_mode, 'foreground');
+ assert.deepEqual(
+ [candidate?.arguments.x, candidate?.arguments.y],
+ [submit.handle.x + submit.handle.width / 2, submit.handle.y + submit.handle.height / 2]
+ );
+});
+
+test('built-in task declares its spec', () => {
+ const task: Task = new FixtureFormTask('secret-token', 'http://127.0.0.1:9/', 3);
+ assert.equal(task.goal, FIXTURE_GOAL);
+ assert.equal(task.maxSteps, 3);
+ assert.equal(task.completionCandidateIds, SUBMIT_IDS);
+ assert.deepEqual([...task.allowedActionKinds].sort(), ['browser_click', 'browser_type', 'click']);
+ assert.equal(task.parameters.length, 1);
+ const [parameter] = task.parameters;
+ assert.equal(parameter.secret, true);
+ assert.deepEqual([parameter.value, parameter.redaction], ['secret-token', REDACTED_TOKEN]);
+ assert.deepEqual(task.redact({ a: ['x secret-token y'] }), { a: [`x ${REDACTED_TOKEN} y`] });
+});
+
+test('oracle classification uses the task budget', () => {
+ const task = new FixtureFormTask('t', undefined, 2);
+ assert.equal(task.classify({ submitted: 't' }, 0), 'verified');
+ assert.equal(task.classify({ submitted: 'other' }, 0), 'refuted');
+ assert.equal(task.classify({ submitted: null }, 1), 'unknown');
+ assert.equal(task.classify({ submitted: null }, 2), 'budget_exhausted');
+});
+
+test('candidates outside the allowed action kinds are refused', () => {
+ const task = new FixtureFormTask(PAGE.token);
+ (task as { allowedActionKinds: ReadonlySet<string> }).allowedActionKinds = new Set([
+ 'browser_type',
+ ]);
+ assert.throws(() => task.candidates(fixtureSources(PAGE.snapshots.after_typing)), /browser_click/);
+ assert.equal(task.candidates(fixtureSources(PAGE.snapshots.before_typing))[0].tool, 'browser_type');
+});
+
+test('state summary reads the step sources', () => {
+ const task = new FixtureFormTask(VISUAL.token);
+ const after = VISUAL.snapshots.after_typing;
+ const pending: TaskSources = { page: new BrowserSemanticSource(after), visualPath: true };
+ assert.equal(task.stateSummary(pending).submit_button, 'visual_check_pending');
+ const parsed: TaskSources = {
+ page: new BrowserSemanticSource(after),
+ visual: new VisualRegionSource(observation()),
+ visualPath: true,
+ };
+ assert.deepEqual(task.stateSummary(parsed), {
+ verification_field: 'contains_required_token',
+ submit_button: 'visual_only',
+ });
+});
diff --git a/libs/cua-driver/examples/jev-use/typescript/tasks.ts b/libs/cua-driver/examples/jev-use/typescript/tasks.ts
new file mode 100644
index 0000000000..29d7718e84
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/typescript/tasks.ts
@@ -0,0 +1,370 @@
+/**
+ * Task specs: what one jev-use run is trying to achieve and how it is checked.
+ *
+ * A task spec owns everything that is specific to one application task: the
+ * goal shown to the model, its parameters (secrets are redacted from all model
+ * input), the step budget, the Driver action kinds it may use, the compact
+ * state summary the model sees, the candidate IDs and descriptions it offers,
+ * and the success oracle that verifies the outcome independently of the
+ * runner's own events.
+ *
+ * Candidate sources (sources.ts) supply the controls and build the executable
+ * candidates. The runner stays task-agnostic: it observes, asks the task for
+ * candidates and a state summary, lets the model choose one supplied ID, acts,
+ * and asks the task's oracle whether the task is done.
+ *
+ * FixtureFormTask is the one built-in task: type a secret verification token
+ * into the loopback fixture's form and submit it, verified through the
+ * fixture's /state endpoint.
+ */
+import type { BrowserSnapshot, VisualDelivery, VisualObservation, VisualRegion } from './core.js';
+import {
+ BrowserSemanticSource,
+ immutableCandidate,
+ VisualRegionSource,
+ type Candidate,
+} from './sources.js';
+
+export type Outcome = 'verified' | 'refuted' | 'unknown' | 'abstained' | 'budget_exhausted';
+
+export const REDACTED_TOKEN = '[verification token]';
+
+/** Replace every occurrence of the token in strings nested in value. */
+export function redactToken(value: unknown, token: string, placeholder = REDACTED_TOKEN): unknown {
+ if (!token) return value;
+ if (typeof value === 'string') return value.split(token).join(placeholder);
+ if (Array.isArray(value)) return value.map((item) => redactToken(item, token, placeholder));
+ if (value && typeof value === 'object') {
+ return Object.fromEntries(
+ Object.entries(value).map(([key, item]) => [key, redactToken(item, token, placeholder)])
+ );
+ }
+ return value;
+}
+
+/**
+ * A named task input. A secret value is replaced by redaction everywhere in
+ * model input; it may still appear in executable arguments.
+ */
+export type TaskParameter = Readonly<{
+ name: string;
+ value: string;
+ secret: boolean;
+ redaction: string;
+}>;
+
+/**
+ * The candidate sources available for one decision step. visual is present
+ * only when this step parsed visual regions. visualPath reports whether a
+ * capture-bound visual path exists at all, so a task can say a control is still
+ * pending a visual check.
+ */
+export type TaskSources = Readonly<{
+ page: BrowserSemanticSource;
+ visual?: VisualRegionSource;
+ visualPath: boolean;
+}>;
+
+export type HistoryEntry = Readonly<{ step: number; selected_id: string; outcome: string }>;
+
+/** The interface the runner uses for every task. */
+export interface Task {
+ readonly id: string;
+ readonly goal: string;
+ readonly parameters: readonly TaskParameter[];
+ readonly maxSteps: number;
+ readonly allowedActionKinds: ReadonlySet<string>;
+ /** Candidate IDs after which the runner polls the oracle for completion. */
+ readonly completionCandidateIds: ReadonlySet<string>;
+ redact(value: unknown): unknown;
+ candidates(sources: TaskSources): Candidate[];
+ stateSummary(sources: TaskSources): Readonly<Record<string, string>>;
+ historyEntry(step: number, candidateId: string, refusal?: string): HistoryEntry;
+ reset(): Promise<void>;
+ readOracle(): Promise<Readonly<Record<string, unknown>>>;
+ classify(oracle: Readonly<Record<string, unknown>>, steps: number): Outcome;
+}
+
+// The built-in browser fixture task.
+
+export const FIXTURE_TASK_ID = 'browser-fixture-form';
+export const FIXTURE_GOAL =
+ 'Enter the required verification token into the verification field, then submit the form.';
+export const DEFAULT_FIXTURE_URL = 'http://127.0.0.1:8765/';
+export const DEFAULT_MAX_STEPS = 4;
+export const FIELD_NAME = 'verification value';
+export const SUBMIT_NAME = 'Submit';
+export const SUBMIT_IDS: ReadonlySet<string> = new Set(['submit-form', 'submit-form-foreground']);
+export const FIXTURE_ACTION_KINDS: ReadonlySet<string> = new Set([
+ 'browser_type',
+ 'browser_click',
+ 'click',
+]);
+
+export type SubmitButtonState =
+ | 'available'
+ | 'visual_only'
+ | 'visual_check_pending'
+ | 'not_found_visually'
+ | 'not_in_page_structure';
+
+export type FormState = Readonly<{
+ verification_field: 'not_found' | 'empty' | 'contains_required_token' | 'contains_other_value';
+ submit_button: SubmitButtonState;
+}>;
+
+export async function fixtureState(fixtureUrl: string): Promise<{ submitted: string | null }> {
+ const response = await fetch(new URL('state', fixtureUrl));
+ if (!response.ok) throw new Error(`fixture state failed: HTTP ${response.status}`);
+ return (await response.json()) as { submitted: string | null };
+}
+
+export async function resetFixture(fixtureUrl: string): Promise<void> {
+ const response = await fetch(new URL('reset', fixtureUrl), { method: 'POST' });
+ if (response.status !== 204) throw new Error(`fixture reset failed: HTTP ${response.status}`);
+}
+
+export function classify(
+ submitted: string | null,
+ token: string,
+ steps: number,
+ maxSteps: number
+): Outcome {
+ if (submitted === token) return 'verified';
+ if (submitted !== null) return 'refuted';
+ if (steps >= maxSteps) return 'budget_exhausted';
+ return 'unknown';
+}
+
+export const HISTORY_OUTCOMES: Readonly<Record<string, string>> = {
+ 'type-verification-value': 'typed the required token into the verification field',
+ 'submit-form': 'clicked Submit; the submission was not yet confirmed',
+ 'submit-form-foreground':
+ 'clicked Submit in the foreground; the submission was not yet confirmed',
+ reobserve: 'took no action and requested a fresh observation',
+};
+
+/**
+ * Build the compact decision-history item shown to the model. It records what
+ * each step did, not timings or model probabilities, so earlier choices do not
+ * become a signal to repeat themselves.
+ */
+export function historyEntry(step: number, candidateId: string, refusal?: string): HistoryEntry {
+ const outcome = refusal
+ ? `Driver refused background delivery (${refusal}); no click happened and a ` +
+ 'foreground Submit candidate is offered next'
+ : (HISTORY_OUTCOMES[candidateId] ?? 'completed');
+ return { step, selected_id: candidateId, outcome };
+}
+
+function reservedCandidates(): Candidate[] {
+ return [
+ immutableCandidate({
+ id: 'reobserve',
+ description:
+ 'Take no action and obtain a fresh Driver observation, because the current ' +
+ 'observation is stale or contradicts the reported form state. A Submit control ' +
+ 'that is visual-only, or whose visual check is still pending, is not a reason to ' +
+ 'reobserve: the runner parses visual regions for Submit once no page-structure ' +
+ 'action remains.',
+ tool: null,
+ arguments: {},
+ }),
+ immutableCandidate({
+ id: 'abstain',
+ description: 'Stop without acting if none of the proposed actions is safe for the observed state.',
+ tool: null,
+ arguments: {},
+ }),
+ ];
+}
+
+/**
+ * Type the secret token into the fixture form and submit it. The oracle is the
+ * fixture's /state endpoint: the task is verified only when the fixture
+ * recorded exactly this token as submitted.
+ */
+export class FixtureFormTask implements Task {
+ readonly id = FIXTURE_TASK_ID;
+ readonly goal = FIXTURE_GOAL;
+ readonly allowedActionKinds = FIXTURE_ACTION_KINDS;
+ readonly completionCandidateIds = SUBMIT_IDS;
+
+ constructor(
+ readonly token: string,
+ readonly fixtureUrl: string = DEFAULT_FIXTURE_URL,
+ readonly maxSteps: number = DEFAULT_MAX_STEPS
+ ) {}
+
+ get parameters(): readonly TaskParameter[] {
+ return [
+ { name: 'verification_token', value: this.token, secret: true, redaction: REDACTED_TOKEN },
+ ];
+ }
+
+ redact(value: unknown): unknown {
+ let result = value;
+ for (const parameter of this.parameters) {
+ if (parameter.secret) result = redactToken(result, parameter.value, parameter.redaction);
+ }
+ return result;
+ }
+
+ /**
+ * Build the closed candidate set for one decision. Page-structure refs always
+ * win. The capture-bound visual Submit is offered only when no Submit ref
+ * exists. A visual source with foreground delivery yields a distinct
+ * submit-form-foreground candidate after Driver refused background delivery;
+ * the chooser must pick it explicitly.
+ */
+ candidates(sources: TaskSources): Candidate[] {
+ const { page, visual } = sources;
+ const token = this.token;
+ const field = page.find('textbox', FIELD_NAME);
+ const button = page.find('button', SUBMIT_NAME);
+ const candidates: Candidate[] = [];
+ if (field && field.value !== token) {
+ candidates.push(
+ page.typeText(
+ field,
+ token,
+ 'type-verification-value',
+ 'Type the required verification token into the verification field, ' +
+ 'replacing its current contents.'
+ )
+ );
+ } else if (field && field.value === token && button) {
+ candidates.push(
+ page.click(
+ button,
+ 'submit-form',
+ "Click the form's Submit button. The observed form state reports that the " +
+ 'verification field already contains the required token, so the form is ' +
+ 'ready to submit.'
+ )
+ );
+ } else if (field && field.value === token && visual) {
+ const region = visual.find('button', SUBMIT_NAME);
+ if (region) {
+ const foreground = visual.delivery === 'foreground';
+ const candidate = visual.click(
+ region,
+ foreground ? 'submit-form-foreground' : 'submit-form',
+ foreground
+ ? 'Submit the form by clicking the unique validated visual Submit region with ' +
+ 'foreground delivery, which activates the browser window, because Driver ' +
+ 'refused background delivery for the previous visual click.'
+ : 'Submit the form by clicking the unique validated visual Submit region. ' +
+ 'The observed form state reports that the verification field already ' +
+ 'contains the required token.'
+ );
+ if (candidate) candidates.push(candidate);
+ }
+ }
+ for (const candidate of candidates) {
+ if (candidate.tool === null || !this.allowedActionKinds.has(candidate.tool)) {
+ throw new Error(`task ${this.id} does not allow action kind ${candidate.tool}`);
+ }
+ }
+ return [...candidates, ...reservedCandidates()];
+ }
+
+ /**
+ * Summarize the form for the decision model without revealing the token. The
+ * raw field value never leaves the runner.
+ *
+ * submit_button is 'available' for a clickable page-structure ref. When the
+ * page structure has none and the capture-bound visual path is enabled
+ * (visualPath), it is 'visual_only' if this observation holds a unique
+ * validated visual Submit region, 'visual_check_pending' if no visual regions
+ * were parsed for this step yet (the runner parses them once no page-structure
+ * action remains), and 'not_found_visually' otherwise. Without a visual path
+ * it is 'not_in_page_structure'.
+ */
+ stateSummary(sources: TaskSources): FormState {
+ const field = sources.page.find('textbox', FIELD_NAME);
+ const button = sources.page.find('button', SUBMIT_NAME);
+ const verificationField = !field
+ ? 'not_found'
+ : !field.value
+ ? 'empty'
+ : field.value === this.token
+ ? 'contains_required_token'
+ : 'contains_other_value';
+ const submitButton: SubmitButtonState = button
+ ? 'available'
+ : !sources.visualPath
+ ? 'not_in_page_structure'
+ : !sources.visual
+ ? 'visual_check_pending'
+ : sources.visual.find('button', SUBMIT_NAME)
+ ? 'visual_only'
+ : 'not_found_visually';
+ return { verification_field: verificationField, submit_button: submitButton };
+ }
+
+ historyEntry(step: number, candidateId: string, refusal?: string): HistoryEntry {
+ return historyEntry(step, candidateId, refusal);
+ }
+
+ reset(): Promise<void> {
+ return resetFixture(this.fixtureUrl);
+ }
+
+ readOracle(): Promise<{ submitted: string | null }> {
+ return fixtureState(this.fixtureUrl);
+ }
+
+ classify(oracle: Readonly<Record<string, unknown>>, steps: number): Outcome {
+ return classify(oracle.submitted as string | null, this.token, steps, this.maxSteps);
+ }
+}
+
+/** Build one step's sources from a browser snapshot and optional visual parse. */
+export function fixtureSources(
+ snapshot: BrowserSnapshot,
+ visual?: VisualObservation,
+ captureBoundClick = false,
+ visualDelivery: VisualDelivery = 'background',
+ visualPath = false
+): TaskSources {
+ return {
+ page: new BrowserSemanticSource(snapshot),
+ visual: visual ? new VisualRegionSource(visual, visualDelivery, captureBoundClick) : undefined,
+ visualPath,
+ };
+}
+
+// Compatibility entry points. They keep the pre-task-spec signatures and
+// delegate to the built-in fixture task, so existing callers are unchanged.
+
+/** Return the unique validated visual Submit region, or undefined. */
+export function visualSubmitRegion(visual?: VisualObservation): VisualRegion | undefined {
+ if (!visual) return undefined;
+ return new VisualRegionSource(visual).find('button', SUBMIT_NAME)?.handle;
+}
+
+/** Summarize the fixture form; see FixtureFormTask.stateSummary. */
+export function formState(
+ snapshot: BrowserSnapshot,
+ token: string,
+ visual?: VisualObservation,
+ visualPath = false
+): FormState {
+ return new FixtureFormTask(token).stateSummary(
+ fixtureSources(snapshot, visual, false, 'background', visualPath)
+ );
+}
+
+/** Build the fixture candidate set; see FixtureFormTask.candidates. */
+export function buildCandidates(
+ snapshot: BrowserSnapshot,
+ token: string,
+ visual?: VisualObservation,
+ captureBoundClick = false,
+ visualDelivery: VisualDelivery = 'background'
+): Candidate[] {
+ return new FixtureFormTask(token).candidates(
+ fixtureSources(snapshot, visual, captureBoundClick, visualDelivery)
+ );
+}