Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): give jev-use Jev the filled form state before submit (#4255)

Francesco Bonacci0e75660ce4c2
diff --git a/libs/cua-driver/examples/jev-use/README.md b/libs/cua-driver/examples/jev-use/README.md
index b3e6b4088d..0e1c2842e7 100644
--- a/libs/cua-driver/examples/jev-use/README.md
+++ b/libs/cua-driver/examples/jev-use/README.md
@@ -16,8 +16,14 @@ The TypeSafe-specific code lives in `python/jev_adapter.py` and
`typescript/jev_adapter.ts`, outside Driver. Both adapters send Jev the bounded
candidate IDs and descriptions plus a compact observation. When visual regions
are present, that observation includes their typed bounds and the exact
-`capture_id`; it never includes screenshot bytes or extension internals. A live
-answer must be one of the supplied IDs before the runner resolves it to the
+`capture_id`; it never includes screenshot bytes or extension internals. The
+observation also carries a `form` summary that the runner derives from the page
+structure. The summary reports whether the verification field is `empty`, holds
+the required token (`contains_required_token`), or holds `contains_other_value`,
+and whether a Submit button ref is `available`. The token itself is replaced
+with `[verification token]` in the page, outline, and visual text. `history` lists
+one `{step, selected_id, outcome}` item per earlier decision. Timings and model
+probabilities stay in the JSONL log only. A live answer must be one of the supplied IDs before the runner resolves it to the
original immutable action.
You can run the complete deterministic proof without credentials or network
diff --git a/libs/cua-driver/examples/jev-use/fixtures/jev-page-structure-replay-v1.json b/libs/cua-driver/examples/jev-use/fixtures/jev-page-structure-replay-v1.json
new file mode 100644
index 0000000000..6895d36042
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/fixtures/jev-page-structure-replay-v1.json
@@ -0,0 +1,299 @@
+{
+ "schema": "cua.jev_use_page_structure_replay_v1",
+ "description": "Page-structure (DOM) path fixture. Snapshots are real get_browser_state semantic_v2 results from the loopback fixture (captured with cua-driver on macOS); recorded_live_runs are the step choices TypeSafe Jev made in live Linux/Windows runs with released Cua Driver 0.30.1. pre_fix_submit_state is the exact state the runner sent Jev at the submit step before the filled-form-state fix.",
+ "token": "jev-guide-live",
+ "snapshots": {
+ "before_typing": {
+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",
+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",
+ "refs": [
+ {
+ "actions": ["type", "pointer"],
+ "frame": "main",
+ "name": "verification value",
+ "ref": "p3:0",
+ "role": "textbox",
+ "states": {
+ "editable": "plaintext",
+ "focusable": true,
+ "required": true
+ },
+ "value": null,
+ "visibility": "in_viewport"
+ },
+ {
+ "actions": ["click", "pointer"],
+ "frame": "main",
+ "name": "Submit",
+ "ref": "p3:1",
+ "role": "button",
+ "states": {
+ "focusable": true
+ },
+ "value": null,
+ "visibility": "in_viewport"
+ },
+ {
+ "actions": ["type"],
+ "frame": "main",
+ "name": null,
+ "ref": "p3:10",
+ "role": "generic",
+ "states": {
+ "editable": "plaintext"
+ },
+ "value": null,
+ "visibility": "unknown"
+ }
+ ]
+ },
+ "after_typing": {
+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",
+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": jev-guide-live [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"jev-guide-live\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "refs": [
+ {
+ "actions": ["type", "pointer"],
+ "frame": "main",
+ "name": "verification value",
+ "ref": "p4:1",
+ "role": "textbox",
+ "states": {
+ "editable": "plaintext",
+ "focusable": true,
+ "focused": true,
+ "required": true
+ },
+ "value": "jev-guide-live",
+ "visibility": "in_viewport"
+ },
+ {
+ "actions": ["click", "pointer"],
+ "frame": "main",
+ "name": "Submit",
+ "ref": "p4:2",
+ "role": "button",
+ "states": {
+ "focusable": true
+ },
+ "value": null,
+ "visibility": "in_viewport"
+ },
+ {
+ "actions": ["type"],
+ "frame": "main",
+ "name": null,
+ "ref": "p4:10",
+ "role": "generic",
+ "states": {
+ "editable": "plaintext"
+ },
+ "value": null,
+ "visibility": "unknown"
+ },
+ {
+ "actions": ["type"],
+ "frame": "main",
+ "name": "jev-guide-live",
+ "ref": "p4:11",
+ "role": "statictext",
+ "states": {
+ "editable": "plaintext"
+ },
+ "value": null,
+ "visibility": "unknown"
+ }
+ ]
+ }
+ },
+ "pre_fix_submit_state": {
+ "goal": "Enter the verification token, then submit the form.",
+ "observation": {
+ "page": {
+ "title": "about:blank",
+ "url": "http://127.0.0.1:8799/"
+ },
+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\": jev-guide-live [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"jev-guide-live\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",
+ "visual": null
+ },
+ "history": [
+ {
+ "event": "step",
+ "step": 1,
+ "candidate": "type-verification-value",
+ "confidence": 1.0,
+ "probabilities": {
+ "type-verification-value": 1.0,
+ "reobserve": 0.0,
+ "abstain": 0.0
+ },
+ "decision_ms": 41.43,
+ "action_ms": 1610.77,
+ "dry_run": false,
+ "tool": "browser_type",
+ "delivery_mode": null,
+ "visual": {
+ "status": "skipped",
+ "reason": "disabled"
+ }
+ }
+ ]
+ },
+ "recorded_live_runs": {
+ "windows-page-1-python": {
+ "steps": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "probabilities": {
+ "abstain": 0.11,
+ "reobserve": 0.06,
+ "type-verification-value": 0.83
+ }
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "probabilities": {
+ "abstain": 0.22,
+ "reobserve": 0.5,
+ "submit-form": 0.28
+ }
+ },
+ {
+ "step": 3,
+ "selected_id": "reobserve",
+ "probabilities": {
+ "abstain": 0.07,
+ "reobserve": 0.63,
+ "submit-form": 0.3
+ }
+ },
+ {
+ "step": 4,
+ "selected_id": "reobserve",
+ "probabilities": {
+ "abstain": 0.05,
+ "reobserve": 0.65,
+ "submit-form": 0.3
+ }
+ }
+ ],
+ "outcome": "budget_exhausted"
+ },
+ "linux-page-3-python": {
+ "steps": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "probabilities": {
+ "abstain": 0.11,
+ "reobserve": 0.07,
+ "type-verification-value": 0.82
+ }
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "probabilities": {
+ "abstain": 0.23,
+ "reobserve": 0.49,
+ "submit-form": 0.28
+ }
+ },
+ {
+ "step": 3,
+ "selected_id": "reobserve",
+ "probabilities": {
+ "abstain": 0.09,
+ "reobserve": 0.6,
+ "submit-form": 0.31
+ }
+ },
+ {
+ "step": 4,
+ "selected_id": "reobserve",
+ "probabilities": {
+ "abstain": 0.06,
+ "reobserve": 0.6,
+ "submit-form": 0.34
+ }
+ }
+ ],
+ "outcome": "budget_exhausted"
+ },
+ "linux-page-7-typescript": {
+ "steps": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "probabilities": {
+ "abstain": 0.1,
+ "reobserve": 0.08,
+ "type-verification-value": 0.82
+ }
+ },
+ {
+ "step": 2,
+ "selected_id": "reobserve",
+ "probabilities": {
+ "abstain": 0.23,
+ "reobserve": 0.53,
+ "submit-form": 0.24
+ }
+ },
+ {
+ "step": 3,
+ "selected_id": "reobserve",
+ "probabilities": {
+ "abstain": 0.11,
+ "reobserve": 0.75,
+ "submit-form": 0.14
+ }
+ },
+ {
+ "step": 4,
+ "selected_id": "submit-form",
+ "probabilities": {
+ "abstain": 0.03,
+ "reobserve": 0.48,
+ "submit-form": 0.49
+ }
+ }
+ ],
+ "outcome": "verified"
+ },
+ "linux-page-2-python": {
+ "steps": [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "probabilities": {
+ "abstain": 0.09,
+ "reobserve": 0.06,
+ "type-verification-value": 0.85
+ }
+ },
+ {
+ "step": 2,
+ "selected_id": "submit-form",
+ "probabilities": {
+ "abstain": 0.21,
+ "reobserve": 0.35,
+ "submit-form": 0.44
+ }
+ }
+ ],
+ "outcome": "verified"
+ }
+ }
+}
diff --git a/libs/cua-driver/examples/jev-use/python/core.py b/libs/cua-driver/examples/jev-use/python/core.py
index 72b7e62a82..a911bbe12e 100644
--- a/libs/cua-driver/examples/jev-use/python/core.py
+++ b/libs/cua-driver/examples/jev-use/python/core.py
@@ -213,11 +213,90 @@ def parse_visual_regions(
)
+FIELD_NAME = "verification value"
+SUBMIT_NAME = "Submit"
+REDACTED_TOKEN = "[verification token]"
+
+
+def _form_refs(snapshot: Mapping[str, Any]) -> tuple[Any, Any]:
+ refs = snapshot.get("refs") or []
+ field = next(
+ (ref for ref in refs if ref.get("role") == "textbox" and ref.get("name") == FIELD_NAME),
+ None,
+ )
+ button = next(
+ (ref for ref in refs if ref.get("role") == "button" and ref.get("name") == SUBMIT_NAME),
+ None,
+ )
+ return field, button
+
+
+def form_state(snapshot: Mapping[str, Any], token: str) -> dict[str, str]:
+ """Summarize the form for the decision model without revealing the token.
+
+ The raw field value never leaves the runner; the model receives only whether
+ the field is empty, holds the required token, or holds something else.
+ """
+ field, button = _form_refs(snapshot)
+ if field is None:
+ field_state = "not_found"
+ elif not field.get("value"):
+ field_state = "empty"
+ elif field.get("value") == token:
+ field_state = "contains_required_token"
+ else:
+ field_state = "contains_other_value"
+ return {
+ "verification_field": field_state,
+ "submit_button": "available" if button is not None else "not_in_page_structure",
+ }
+
+
+def redact_token(value: Any, token: str) -> Any:
+ """Replace every occurrence of the token in strings nested in ``value``."""
+ if not token:
+ return value
+ if isinstance(value, str):
+ return value.replace(token, REDACTED_TOKEN)
+ if isinstance(value, Mapping):
+ return {key: redact_token(item, token) for key, item in value.items()}
+ if isinstance(value, (list, tuple)):
+ return [redact_token(item, token) for item in value]
+ return value
+
+
+HISTORY_OUTCOMES = {
+ "type-verification-value": "typed the required token into the verification field",
+ "submit-form": "clicked Submit; the submission was not yet confirmed",
+ "submit-form-foreground": "clicked Submit in the foreground; the submission was not yet confirmed",
+ "reobserve": "took no action and requested a fresh observation",
+}
+
+
+def history_entry(
+ step: int, candidate_id: str, *, refusal: str | None = None
+) -> dict[str, Any]:
+ """Build the compact decision-history item shown to the model.
+
+ It records what each step did, not timings or model probabilities, so earlier
+ choices do not become a signal to repeat themselves.
+ """
+ if refusal is not None:
+ outcome = (
+ f"Driver refused background delivery ({refusal}); no click happened and a "
+ "foreground Submit candidate is offered next"
+ )
+ else:
+ outcome = HISTORY_OUTCOMES.get(candidate_id, "completed")
+ return {"step": step, "selected_id": candidate_id, "outcome": outcome}
+
+
def _reserved_candidates() -> list[Candidate]:
return [
Candidate(
"reobserve",
- "Discard this decision set and obtain a fresh Driver observation.",
+ "Take no action and obtain a fresh Driver observation, because the current "
+ "observation is stale, incomplete, or contradicts the reported form state.",
None,
{},
),
@@ -249,25 +328,14 @@ def build_candidates(
"target_id": snapshot["target_id"],
"tab_id": snapshot["tab_id"],
}
- refs = snapshot.get("refs", [])
- field = next(
- (
- ref
- for ref in refs
- if ref.get("role") == "textbox" and ref.get("name") == "verification value"
- ),
- None,
- )
- button = next(
- (ref for ref in refs if ref.get("role") == "button" and ref.get("name") == "Submit"),
- None,
- )
+ field, button = _form_refs(snapshot)
candidates: list[Candidate] = []
if field and field.get("value") != token:
candidates.append(
Candidate(
"type-verification-value",
- "Replace the verification field with the required token.",
+ "Type the required verification token into the verification field, "
+ "replacing its current contents.",
"browser_type",
{**common, "ref": field["ref"], "text": token, "replace": True},
)
@@ -276,7 +344,9 @@ def build_candidates(
candidates.append(
Candidate(
"submit-form",
- "Submit the form now that the verification field contains the token.",
+ "Click the form's Submit button. The observed form state reports that the "
+ "verification field already contains the required token, so the form is "
+ "ready to submit.",
"browser_click",
{**common, "ref": button["ref"], "input_route": "dom_event"},
)
@@ -305,7 +375,9 @@ def build_candidates(
"window, because Driver refused background delivery for the "
"previous visual click."
if foreground
- else "Submit the form using the unique validated visual Submit region."
+ else "Submit the form by clicking the unique validated visual Submit "
+ "region. The observed form state reports that the verification field "
+ "already contains the required token."
),
"click",
{
diff --git a/libs/cua-driver/examples/jev-use/python/jev_adapter.py b/libs/cua-driver/examples/jev-use/python/jev_adapter.py
index 0ab10202d2..2b67916874 100644
--- a/libs/cua-driver/examples/jev-use/python/jev_adapter.py
+++ b/libs/cua-driver/examples/jev-use/python/jev_adapter.py
@@ -4,7 +4,7 @@ import math
from dataclasses import dataclass
from typing import Any, Mapping, Protocol
-from core import Candidate, VisualObservation, choose_mock
+from core import Candidate, VisualObservation, choose_mock, form_state, redact_token
class TypeSafeClientLike(Protocol):
@@ -95,27 +95,46 @@ def visual_decision_state(visual: VisualObservation | None) -> dict[str, Any] |
}
+GOAL = "Enter the required verification token into the verification field, then submit the form."
+
+
+def decision_state(
+ snapshot: Mapping[str, Any],
+ visual: VisualObservation | None,
+ history: list[dict[str, Any]],
+ token: str,
+) -> dict[str, Any]:
+ """Build the compact, deterministic, token-redacted state sent to Jev.
+
+ ``form`` states the field and Submit status the runner verified from the page
+ structure, so the model does not have to infer it from the outline. The token
+ itself is replaced everywhere, including the outline and visual text.
+ """
+ return {
+ "goal": GOAL,
+ "observation": {
+ "page": redact_token(snapshot.get("page"), token),
+ "form": form_state(snapshot, token),
+ "outline": redact_token(snapshot.get("outline"), token),
+ "visual": redact_token(visual_decision_state(visual), token),
+ },
+ "history": [dict(item) for item in history],
+ }
+
+
def choose_with_typesafe(
client: TypeSafeClientLike,
candidates: list[Candidate],
snapshot: dict[str, Any],
visual: VisualObservation | None,
history: list[dict[str, Any]],
+ token: str,
) -> tuple[str, float, dict[str, float]]:
from typesafe_sdk import Choice
criteria = _candidate_criteria(candidates)
- state = {
- "goal": "Enter the verification token, then submit the form.",
- "observation": {
- "page": snapshot.get("page"),
- "outline": snapshot.get("outline"),
- "visual": visual_decision_state(visual),
- },
- "history": history,
- }
response = client.system_one(
- state=state,
+ state=decision_state(snapshot, visual, history, token),
questions={
"driver_action": Choice(
instructions="Which complete executable action should Cua Driver run next?",
@@ -134,11 +153,12 @@ def choose_live(
snapshot: dict[str, Any],
visual: VisualObservation | None,
history: list[dict[str, Any]],
+ token: str,
) -> tuple[str, float, dict[str, float]]:
from typesafe_sdk import TypeSafeClient
with TypeSafeClient() as client:
- return choose_with_typesafe(client, candidates, snapshot, visual, history)
+ return choose_with_typesafe(client, candidates, snapshot, visual, history, token)
def choose_mock_adapter(
@@ -146,5 +166,6 @@ def choose_mock_adapter(
_snapshot: dict[str, Any],
_visual: VisualObservation | None,
_history: list[dict[str, Any]],
+ _token: str,
) -> tuple[str | None, float, dict[str, float]]:
return choose_mock(candidates)
diff --git a/libs/cua-driver/examples/jev-use/python/run.py b/libs/cua-driver/examples/jev-use/python/run.py
index a6e2538484..b78b850cdc 100644
--- a/libs/cua-driver/examples/jev-use/python/run.py
+++ b/libs/cua-driver/examples/jev-use/python/run.py
@@ -23,6 +23,7 @@ from core import (
VisualObservationError,
build_candidates,
classify,
+ history_entry,
has_executable_candidate,
parse_visual_regions,
validate_choice,
@@ -289,6 +290,8 @@ def write_event(log_path: Path | None, event: dict[str, Any]) -> None:
async def run(args: argparse.Namespace) -> str:
token = args.token or f"jev-{uuid.uuid4().hex[:10]}"
label = f"jev-python-{uuid.uuid4().hex[:8]}"
+ # Compact what-happened record for the decision model. The full telemetry
+ # events (timings, probabilities) go only to the JSONL log.
history: list[dict[str, Any]] = []
visual_delivery: VisualDelivery = "background"
log_path = Path(args.log) if args.log else None
@@ -358,11 +361,11 @@ async def run(args: argparse.Namespace) -> str:
if args.provider == "mock":
choice, confidence, probabilities = choose_mock_adapter(
- candidates, snapshot, visual, history
+ candidates, snapshot, visual, history, token
)
else:
choice, confidence, probabilities = await asyncio.to_thread(
- choose_live, candidates, snapshot, visual, history
+ choose_live, candidates, snapshot, visual, history, token
)
if choice is None:
return "abstained"
@@ -386,7 +389,7 @@ async def run(args: argparse.Namespace) -> str:
"tool": None,
"visual": visual_record,
}
- history.append(event)
+ history.append(history_entry(step, candidate.id))
write_event(log_path, event)
continue
@@ -434,7 +437,7 @@ async def run(args: argparse.Namespace) -> str:
},
"visual": visual_record,
}
- history.append(event)
+ history.append(history_entry(step, candidate.id, refusal=refusal))
write_event(log_path, event)
continue
write_event(
@@ -466,7 +469,7 @@ async def run(args: argparse.Namespace) -> str:
"delivery_mode": candidate.arguments.get("delivery_mode"),
"visual": visual_record,
}
- history.append(event)
+ history.append(history_entry(step, candidate.id))
write_event(log_path, event)
if args.dry_run:
return "unknown"
diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_decision_state.py b/libs/cua-driver/examples/jev-use/python/tests/test_decision_state.py
new file mode 100644
index 0000000000..fbdac75e53
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/python/tests/test_decision_state.py
@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+import json
+import sys
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from core import (
+ REDACTED_TOKEN,
+ VisualObservation,
+ VisualRegion,
+ build_candidates,
+ form_state,
+ history_entry,
+ redact_token,
+ validate_choice,
+)
+from jev_adapter import choose_mock_adapter, choose_with_typesafe, decision_state
+
+FIXTURE = json.loads(
+ (Path(__file__).resolve().parents[2] / "fixtures/jev-page-structure-replay-v1.json").read_text(
+ encoding="utf-8"
+ )
+)
+TOKEN = FIXTURE["token"]
+BEFORE = FIXTURE["snapshots"]["before_typing"]
+AFTER = FIXTURE["snapshots"]["after_typing"]
+
+
+def with_field_value(value: str | None) -> dict:
+ snapshot = json.loads(json.dumps(BEFORE))
+ for ref in snapshot["refs"]:
+ if ref["role"] == "textbox" and ref["name"] == "verification value":
+ ref["value"] = value
+ return snapshot
+
+
+class ReplayClient:
+ """Return recorded live choices in order and keep every request."""
+
+ def __init__(self, recorded: list[dict]) -> None:
+ self.recorded = list(recorded)
+ self.requests: list[dict] = []
+
+ def system_one(self, **request):
+ self.requests.append(request)
+ step = self.recorded.pop(0)
+ answer = SimpleNamespace(
+ choice=step["selected_id"],
+ confidence=max(step["probabilities"].values()),
+ probabilities=step["probabilities"],
+ )
+ return SimpleNamespace(choices={"driver_action": answer})
+
+
+class FormStateTest(unittest.TestCase):
+ def test_form_state_reports_field_status_without_the_value(self) -> None:
+ self.assertEqual(
+ form_state(BEFORE, TOKEN),
+ {"verification_field": "empty", "submit_button": "available"},
+ )
+ self.assertEqual(form_state(AFTER, TOKEN)["verification_field"], "contains_required_token")
+ self.assertEqual(
+ form_state(with_field_value("something-else"), TOKEN)["verification_field"],
+ "contains_other_value",
+ )
+ no_button = {**AFTER, "refs": [ref for ref in AFTER["refs"] if ref["role"] != "button"]}
+ self.assertEqual(form_state(no_button, TOKEN)["submit_button"], "not_in_page_structure")
+ self.assertEqual(
+ form_state({"refs": []}, TOKEN),
+ {"verification_field": "not_found", "submit_button": "not_in_page_structure"},
+ )
+
+ def test_redaction_is_nested_and_deterministic(self) -> None:
+ value = {"a": [f"x {TOKEN} y", {"b": TOKEN}], "n": 3, "none": None}
+ redacted = redact_token(value, TOKEN)
+ self.assertEqual(
+ redacted,
+ {"a": [f"x {REDACTED_TOKEN} y", {"b": REDACTED_TOKEN}], "n": 3, "none": None},
+ )
+ self.assertEqual(redact_token(value, TOKEN), redacted)
+ self.assertEqual(redact_token("unchanged", ""), "unchanged")
+
+
+class SubmitStepObservationTest(unittest.TestCase):
+ def test_pre_fix_submit_state_hid_readiness_and_leaked_the_token(self) -> None:
+ # Pins the root-cause evidence: the old state had no form summary, sent the
+ # raw token in the outline, and fed raw telemetry back as history.
+ old = FIXTURE["pre_fix_submit_state"]
+ self.assertNotIn("form", old["observation"])
+ self.assertIn(TOKEN, old["observation"]["outline"])
+ self.assertIn("status=waiting", old["observation"]["outline"])
+ self.assertIn("probabilities", old["history"][0])
+
+ def test_after_typing_the_observation_states_the_filled_form(self) -> None:
+ history = [history_entry(1, "type-verification-value")]
+ state = decision_state(AFTER, None, history, TOKEN)
+ encoded = json.dumps(state, sort_keys=True)
+
+ self.assertNotIn(TOKEN, encoded)
+ self.assertEqual(
+ state["observation"]["form"],
+ {"verification_field": "contains_required_token", "submit_button": "available"},
+ )
+ self.assertIn(
+ f'textbox "verification value": {REDACTED_TOKEN}', state["observation"]["outline"]
+ )
+ self.assertEqual(
+ state["history"],
+ [
+ {
+ "step": 1,
+ "selected_id": "type-verification-value",
+ "outcome": "typed the required token into the verification field",
+ }
+ ],
+ )
+ self.assertEqual(
+ encoded, json.dumps(decision_state(AFTER, None, history, TOKEN), sort_keys=True)
+ )
+
+ candidates = build_candidates(AFTER, TOKEN)
+ criteria = {candidate.id: candidate.description for candidate in candidates}
+ self.assertEqual(list(criteria), ["submit-form", "reobserve", "abstain"])
+ self.assertIn("already contains the required token", criteria["submit-form"])
+ self.assertIn("stale, incomplete", criteria["reobserve"])
+ self.assertNotIn(TOKEN, json.dumps(criteria))
+
+ def test_visual_text_is_redacted_too(self) -> None:
+ region = VisualRegion("r1", "text", TOKEN, None, 0.9, False, 1, 1, 10, 10)
+ visual = VisualObservation("cap", "ref", 100, 100, 7, 9, 0.0, 0.0, 1.0, 1.0, (region,))
+ state = decision_state(AFTER, visual, [], TOKEN)
+ self.assertEqual(state["observation"]["visual"]["regions"][0]["text"], REDACTED_TOKEN)
+ self.assertNotIn(TOKEN, json.dumps(state))
+
+ def test_background_refusal_history_is_compact(self) -> None:
+ entry = history_entry(2, "submit-form", refusal="background_unsupported")
+ self.assertEqual(set(entry), {"step", "selected_id", "outcome"})
+ self.assertIn("background_unsupported", entry["outcome"])
+
+
+class ProviderPathTest(unittest.TestCase):
+ def test_mock_provider_types_then_submits_over_recorded_snapshots(self) -> None:
+ first = build_candidates(BEFORE, TOKEN)
+ self.assertEqual(
+ choose_mock_adapter(first, BEFORE, None, [], TOKEN)[0], "type-verification-value"
+ )
+ second = build_candidates(AFTER, TOKEN)
+ history = [history_entry(1, "type-verification-value")]
+ self.assertEqual(choose_mock_adapter(second, AFTER, None, history, TOKEN)[0], "submit-form")
+
+ def test_recorded_live_choices_replay_through_the_new_state(self) -> None:
+ for name, run in FIXTURE["recorded_live_runs"].items():
+ with self.subTest(run=name):
+ client = ReplayClient(run["steps"])
+ history: list[dict] = []
+ outcome = "budget_exhausted"
+ for step in range(1, 5):
+ snapshot = BEFORE if step == 1 else AFTER
+ candidates = build_candidates(snapshot, TOKEN)
+ recorded = run["steps"][step - 1]
+ # The recorded choice was made over the same bounded table.
+ self.assertEqual(
+ set(recorded["probabilities"]), {candidate.id for candidate in candidates}
+ )
+ choice, _, _ = choose_with_typesafe(
+ client, candidates, snapshot, None, history, TOKEN
+ )
+ candidate = validate_choice(choice, candidates)
+ request = client.requests[-1]
+ self.assertNotIn(TOKEN, json.dumps(request["state"], sort_keys=True))
+ self.assertEqual(
+ request["state"]["observation"]["form"]["verification_field"],
+ "empty" if step == 1 else "contains_required_token",
+ )
+ for item in request["state"]["history"]:
+ self.assertEqual(set(item), {"step", "selected_id", "outcome"})
+ self.assertIn("reobserve", request["questions"]["driver_action"].criteria)
+ self.assertIn("abstain", request["questions"]["driver_action"].criteria)
+ history.append(history_entry(step, candidate.id))
+ if candidate.id == "submit-form":
+ outcome = "verified"
+ break
+ self.assertEqual(outcome, run["outcome"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_live_adapter.py b/libs/cua-driver/examples/jev-use/python/tests/test_live_adapter.py
index 0a5b3e0c37..05e675f3d0 100644
--- a/libs/cua-driver/examples/jev-use/python/tests/test_live_adapter.py
+++ b/libs/cua-driver/examples/jev-use/python/tests/test_live_adapter.py
@@ -56,7 +56,7 @@ class LiveAdapterTest(unittest.TestCase):
)
candidates = build_candidates(snapshot, "expected", visual, capture_bound_click=True)
selected, confidence, probabilities = choose_with_typesafe(
- FakeClient(), candidates, snapshot, visual, []
+ FakeClient(), candidates, snapshot, visual, [], "expected"
)
self.assertEqual(selected, "submit-form")
@@ -70,6 +70,9 @@ class LiveAdapterTest(unittest.TestCase):
sent_visual = FakeClient.request["state"]["observation"]["visual"]
self.assertEqual(sent_visual["capture_id"], "capture-submit")
self.assertEqual(sent_visual["regions"][0]["id"], "submit-text")
+ sent_form = FakeClient.request["state"]["observation"]["form"]
+ self.assertEqual(sent_form["verification_field"], "contains_required_token")
+ self.assertNotIn("expected", json.dumps(FakeClient.request["state"]))
def test_live_adapter_rejects_id_outside_the_supplied_table(self) -> None:
snapshot = {"target_id": "target", "tab_id": "tab", "refs": []}
@@ -88,7 +91,7 @@ class LiveAdapterTest(unittest.TestCase):
)
with self.assertRaisesRegex(ValueError, "unknown candidate"):
- choose_with_typesafe(UnknownClient(), candidates, snapshot, None, [])
+ choose_with_typesafe(UnknownClient(), candidates, snapshot, None, [], "expected")
if __name__ == "__main__":
diff --git a/libs/cua-driver/examples/jev-use/typescript/core.test.ts b/libs/cua-driver/examples/jev-use/typescript/core.test.ts
index 30622a141e..dbc205e4a9 100644
--- a/libs/cua-driver/examples/jev-use/typescript/core.test.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/core.test.ts
@@ -311,7 +311,7 @@ test('live adapter sends one Choice keyed by executable candidate id', async ()
9
);
const candidates = buildCandidates(page, 'expected', visual, true);
- const answer = await chooseWithTypeSafe(client, candidates, page, visual, []);
+ const answer = await chooseWithTypeSafe(client, candidates, page, visual, [], 'expected');
assert.equal(answer.choice, 'submit-form');
assert.deepEqual(
@@ -321,6 +321,9 @@ test('live adapter sends one Choice keyed by executable candidate id', async ()
const sentVisual = JSON.parse(requestBody?.state.observation.visual);
assert.equal(sentVisual.capture_id, 'capture-submit');
assert.equal(sentVisual.regions[0].id, 'submit-text');
+ const sentForm = JSON.parse(requestBody?.state.observation.form);
+ assert.equal(sentForm.verification_field, 'contains_required_token');
+ assert.equal(JSON.stringify(requestBody?.state).includes('expected'), false);
});
test('live adapter rejects an id outside the supplied table', async () => {
@@ -339,7 +342,7 @@ test('live adapter rejects an id outside the supplied table', async () => {
const page = snapshot();
const candidates = buildCandidates(page, 'expected');
await assert.rejects(
- () => chooseWithTypeSafe(client as never, candidates, page, undefined, []),
+ () => chooseWithTypeSafe(client as never, candidates, page, undefined, [], 'expected'),
/unknown candidate/
);
});
diff --git a/libs/cua-driver/examples/jev-use/typescript/core.ts b/libs/cua-driver/examples/jev-use/typescript/core.ts
index 4dd6756467..cf05df3226 100644
--- a/libs/cua-driver/examples/jev-use/typescript/core.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/core.ts
@@ -210,11 +210,84 @@ export function parseVisualRegions(
});
}
+export const REDACTED_TOKEN = '[verification token]';
+
+function formRefs(snapshot: BrowserSnapshot) {
+ const refs = snapshot.refs ?? [];
+ const field = refs.find(
+ (item) => item.role === 'textbox' && item.name === 'verification value' && item.ref
+ );
+ const button = refs.find((item) => item.role === 'button' && item.name === 'Submit' && item.ref);
+ return { field, button };
+}
+
+export type FormState = Readonly<{
+ verification_field: 'not_found' | 'empty' | 'contains_required_token' | 'contains_other_value';
+ submit_button: 'available' | 'not_in_page_structure';
+}>;
+
+/**
+ * Summarize the form for the decision model without revealing the token. The
+ * raw field value never leaves the runner.
+ */
+export function formState(snapshot: BrowserSnapshot, token: string): FormState {
+ const { field, button } = formRefs(snapshot);
+ const verificationField = !field
+ ? 'not_found'
+ : !field.value
+ ? 'empty'
+ : field.value === token
+ ? 'contains_required_token'
+ : 'contains_other_value';
+ return {
+ verification_field: verificationField,
+ submit_button: button ? 'available' : 'not_in_page_structure',
+ };
+}
+
+/** Replace every occurrence of the token in strings nested in value. */
+export function redactToken(value: unknown, token: string): unknown {
+ if (!token) return value;
+ if (typeof value === 'string') return value.split(token).join(REDACTED_TOKEN);
+ if (Array.isArray(value)) return value.map((item) => redactToken(item, token));
+ if (value && typeof value === 'object') {
+ return Object.fromEntries(
+ Object.entries(value).map(([key, item]) => [key, redactToken(item, token)])
+ );
+ }
+ return value;
+}
+
+export type HistoryEntry = Readonly<{ step: number; selected_id: string; outcome: string }>;
+
+const HISTORY_OUTCOMES: Readonly<Record<string, string>> = {
+ 'type-verification-value': 'typed the required token into the verification field',
+ 'submit-form': 'clicked Submit; the submission was not yet confirmed',
+ 'submit-form-foreground':
+ 'clicked Submit in the foreground; the submission was not yet confirmed',
+ reobserve: 'took no action and requested a fresh observation',
+};
+
+/**
+ * Build the compact decision-history item shown to the model. It records what
+ * each step did, not timings or model probabilities, so earlier choices do not
+ * become a signal to repeat themselves.
+ */
+export function historyEntry(step: number, candidateId: string, refusal?: string): HistoryEntry {
+ const outcome = refusal
+ ? `Driver refused background delivery (${refusal}); no click happened and a ` +
+ 'foreground Submit candidate is offered next'
+ : (HISTORY_OUTCOMES[candidateId] ?? 'completed');
+ return { step, selected_id: candidateId, outcome };
+}
+
function reservedCandidates(): Candidate[] {
return [
immutableCandidate({
id: 'reobserve',
- description: 'Discard this decision set and obtain a fresh Driver observation.',
+ description:
+ 'Take no action and obtain a fresh Driver observation, because the current ' +
+ 'observation is stale, incomplete, or contradicts the reported form state.',
tool: null,
arguments: {},
}),
@@ -242,17 +315,15 @@ export function buildCandidates(
visualDelivery: VisualDelivery = 'background'
): Candidate[] {
const common = { target_id: snapshot.target_id, tab_id: snapshot.tab_id };
- const refs = snapshot.refs ?? [];
- const field = refs.find(
- (item) => item.role === 'textbox' && item.name === 'verification value' && item.ref
- );
- const button = refs.find((item) => item.role === 'button' && item.name === 'Submit' && item.ref);
+ const { field, button } = formRefs(snapshot);
const candidates: Candidate[] = [];
if (field?.value !== token && field?.ref) {
candidates.push(
immutableCandidate({
id: 'type-verification-value',
- description: 'Replace the verification field with the required token.',
+ description:
+ 'Type the required verification token into the verification field, ' +
+ 'replacing its current contents.',
tool: 'browser_type',
arguments: { ...common, ref: field.ref, text: token, replace: true },
})
@@ -261,7 +332,10 @@ export function buildCandidates(
candidates.push(
immutableCandidate({
id: 'submit-form',
- description: 'Submit the form now that the verification field contains the token.',
+ description:
+ "Click the form's Submit button. The observed form state reports that the " +
+ 'verification field already contains the required token, so the form is ' +
+ 'ready to submit.',
tool: 'browser_click',
arguments: { ...common, ref: button.ref, input_route: 'dom_event' },
})
@@ -288,7 +362,9 @@ export function buildCandidates(
? 'Submit the form by clicking the unique validated visual Submit region with ' +
'foreground delivery, which activates the browser window, because Driver ' +
'refused background delivery for the previous visual click.'
- : 'Submit the form using the unique validated visual Submit region.',
+ : 'Submit the form by clicking the unique validated visual Submit region. ' +
+ 'The observed form state reports that the verification field already ' +
+ 'contains the required token.',
tool: 'click',
arguments: {
pid: visual.pid,
diff --git a/libs/cua-driver/examples/jev-use/typescript/decision_state.test.ts b/libs/cua-driver/examples/jev-use/typescript/decision_state.test.ts
new file mode 100644
index 0000000000..2decc9d2a6
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/typescript/decision_state.test.ts
@@ -0,0 +1,212 @@
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import test from 'node:test';
+
+import {
+ buildCandidates,
+ formState,
+ historyEntry,
+ redactToken,
+ REDACTED_TOKEN,
+ validateChoice,
+ type BrowserSnapshot,
+ type HistoryEntry,
+ type VisualObservation,
+} from './core.js';
+import { chooseMockAdapter, chooseWithTypeSafe, decisionState } from './jev_adapter.js';
+
+const FIXTURE = JSON.parse(
+ readFileSync(new URL('../fixtures/jev-page-structure-replay-v1.json', import.meta.url), 'utf8')
+);
+const TOKEN: string = FIXTURE.token;
+const BEFORE: BrowserSnapshot = FIXTURE.snapshots.before_typing;
+const AFTER: BrowserSnapshot = FIXTURE.snapshots.after_typing;
+
+type RecordedStep = { step: number; selected_id: string; probabilities: Record<string, number> };
+
+function withFieldValue(value: string | null): BrowserSnapshot {
+ const snapshot = structuredClone(BEFORE);
+ for (const ref of snapshot.refs ?? []) {
+ if (ref.role === 'textbox' && ref.name === 'verification value') ref.value = value;
+ }
+ return snapshot;
+}
+
+function replayClient(recorded: RecordedStep[]) {
+ const queue = [...recorded];
+ const requests: any[] = [];
+ return {
+ requests,
+ systemOne: async (request: any) => {
+ requests.push(request);
+ const step = queue.shift()!;
+ return {
+ answers: {
+ driver_action: {
+ type: 'choice' as const,
+ choice: step.selected_id,
+ confidence: Math.max(...Object.values(step.probabilities)),
+ probabilities: step.probabilities,
+ },
+ },
+ };
+ },
+ };
+}
+
+test('form state reports field status without the value', () => {
+ assert.deepEqual(formState(BEFORE, TOKEN), {
+ verification_field: 'empty',
+ submit_button: 'available',
+ });
+ assert.equal(formState(AFTER, TOKEN).verification_field, 'contains_required_token');
+ assert.equal(
+ formState(withFieldValue('something-else'), TOKEN).verification_field,
+ 'contains_other_value'
+ );
+ const noButton = { ...AFTER, refs: (AFTER.refs ?? []).filter((ref) => ref.role !== 'button') };
+ assert.equal(formState(noButton, TOKEN).submit_button, 'not_in_page_structure');
+ assert.deepEqual(formState({ target_id: 't', tab_id: 't', refs: [] }, TOKEN), {
+ verification_field: 'not_found',
+ submit_button: 'not_in_page_structure',
+ });
+});
+
+test('redaction is nested and deterministic', () => {
+ const value = { a: [`x ${TOKEN} y`, { b: TOKEN }], n: 3, none: null };
+ assert.deepEqual(redactToken(value, TOKEN), {
+ a: [`x ${REDACTED_TOKEN} y`, { b: REDACTED_TOKEN }],
+ n: 3,
+ none: null,
+ });
+ assert.equal(redactToken('unchanged', ''), 'unchanged');
+});
+
+test('pre-fix submit state hid readiness and leaked the token', () => {
+ const old = FIXTURE.pre_fix_submit_state;
+ assert.equal('form' in old.observation, false);
+ assert.ok(old.observation.outline.includes(TOKEN));
+ assert.ok(old.observation.outline.includes('status=waiting'));
+ assert.ok('probabilities' in old.history[0]);
+});
+
+test('after typing the observation states the filled form', () => {
+ const history = [historyEntry(1, 'type-verification-value')];
+ const state = decisionState(AFTER, undefined, history, TOKEN);
+ const encoded = JSON.stringify(state);
+ assert.equal(encoded.includes(TOKEN), false);
+ assert.deepEqual(JSON.parse(state.observation.form), {
+ verification_field: 'contains_required_token',
+ submit_button: 'available',
+ });
+ assert.ok(state.observation.outline.includes(`textbox "verification value": ${REDACTED_TOKEN}`));
+ assert.deepEqual(JSON.parse(state.history), [
+ {
+ step: 1,
+ selected_id: 'type-verification-value',
+ outcome: 'typed the required token into the verification field',
+ },
+ ]);
+ assert.equal(JSON.stringify(decisionState(AFTER, undefined, history, TOKEN)), encoded);
+
+ const criteria = Object.fromEntries(
+ buildCandidates(AFTER, TOKEN).map((candidate) => [candidate.id, candidate.description])
+ );
+ assert.deepEqual(Object.keys(criteria), ['submit-form', 'reobserve', 'abstain']);
+ assert.ok(criteria['submit-form'].includes('already contains the required token'));
+ assert.ok(criteria.reobserve.includes('stale, incomplete'));
+ assert.equal(JSON.stringify(criteria).includes(TOKEN), false);
+});
+
+test('visual text is redacted too', () => {
+ const visual: VisualObservation = {
+ captureId: 'cap',
+ screenshotReference: 'ref',
+ screenshotWidth: 100,
+ screenshotHeight: 100,
+ pid: 7,
+ windowId: 9,
+ actionOriginX: 0,
+ actionOriginY: 0,
+ actionUnitsPerPixelX: 1,
+ actionUnitsPerPixelY: 1,
+ regions: [
+ {
+ id: 'r1',
+ kind: 'text',
+ text: TOKEN,
+ confidence: 0.9,
+ interactive: false,
+ x: 1,
+ y: 1,
+ width: 10,
+ height: 10,
+ },
+ ],
+ };
+ const state = decisionState(AFTER, visual, [], TOKEN);
+ assert.equal(JSON.parse(state.observation.visual).regions[0].text, REDACTED_TOKEN);
+ assert.equal(JSON.stringify(state).includes(TOKEN), false);
+});
+
+test('background refusal history is compact', () => {
+ const entry = historyEntry(2, 'submit-form', 'background_unsupported');
+ assert.deepEqual(Object.keys(entry).sort(), ['outcome', 'selected_id', 'step']);
+ assert.ok(entry.outcome.includes('background_unsupported'));
+});
+
+test('mock provider types then submits over recorded snapshots', () => {
+ const first = buildCandidates(BEFORE, TOKEN);
+ assert.equal(
+ chooseMockAdapter(first, BEFORE, undefined, [], TOKEN).choice,
+ 'type-verification-value'
+ );
+ const second = buildCandidates(AFTER, TOKEN);
+ const history = [historyEntry(1, 'type-verification-value')];
+ assert.equal(chooseMockAdapter(second, AFTER, undefined, history, TOKEN).choice, 'submit-form');
+});
+
+test('recorded live choices replay through the new state', async () => {
+ for (const [name, run] of Object.entries<any>(FIXTURE.recorded_live_runs)) {
+ const client = replayClient(run.steps);
+ const history: HistoryEntry[] = [];
+ let outcome = 'budget_exhausted';
+ for (let step = 1; step <= 4; step += 1) {
+ const snapshot = step === 1 ? BEFORE : AFTER;
+ const candidates = buildCandidates(snapshot, TOKEN);
+ const recorded: RecordedStep = run.steps[step - 1];
+ assert.deepEqual(
+ new Set(Object.keys(recorded.probabilities)),
+ new Set(candidates.map((candidate) => candidate.id)),
+ name
+ );
+ const answer = await chooseWithTypeSafe(
+ client as never,
+ candidates,
+ snapshot,
+ undefined,
+ history,
+ TOKEN
+ );
+ const candidate = validateChoice(answer.choice, candidates);
+ const request = client.requests.at(-1);
+ assert.equal(JSON.stringify(request.state).includes(TOKEN), false, name);
+ assert.equal(
+ JSON.parse(request.state.observation.form).verification_field,
+ step === 1 ? 'empty' : 'contains_required_token',
+ name
+ );
+ for (const item of JSON.parse(request.state.history)) {
+ assert.deepEqual(Object.keys(item).sort(), ['outcome', 'selected_id', 'step'], name);
+ }
+ const criteria = request.questions.driver_action.criteria;
+ assert.ok('reobserve' in criteria && 'abstain' in criteria, name);
+ history.push(historyEntry(step, candidate.id));
+ if (candidate.id === 'submit-form') {
+ outcome = 'verified';
+ break;
+ }
+ }
+ assert.equal(outcome, run.outcome, name);
+ }
+});
diff --git a/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts b/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts
index ad48bac3f5..1286ee20c2 100644
--- a/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts
@@ -2,7 +2,10 @@ import { choice, TypeSafeClient } from '@typesafe-ai/sdk';
import {
chooseMock,
+ formState,
+ redactToken,
type BrowserSnapshot,
+ type HistoryEntry,
type Candidate,
type VisualObservation,
} from './core.js';
@@ -96,24 +99,43 @@ export function visualDecisionState(visual?: VisualObservation) {
};
}
+export const GOAL =
+ 'Enter the required verification token into the verification field, then submit the form.';
+
+/**
+ * Build the compact, deterministic, token-redacted state sent to Jev. `form`
+ * states the field and Submit status the runner verified from the page
+ * structure; the token is replaced everywhere, including outline and visual text.
+ */
+export function decisionState(
+ snapshot: BrowserSnapshot,
+ visual: VisualObservation | undefined,
+ history: readonly HistoryEntry[],
+ token: string
+) {
+ return {
+ goal: GOAL,
+ observation: {
+ page: JSON.stringify(redactToken(snapshot.page ?? null, token)),
+ form: JSON.stringify(formState(snapshot, token)),
+ outline: redactToken(snapshot.outline ?? '', token) as string,
+ visual: JSON.stringify(redactToken(visualDecisionState(visual), token)),
+ },
+ history: JSON.stringify(history),
+ };
+}
+
export async function chooseWithTypeSafe(
client: TypeSafeClientLike,
candidates: Candidate[],
snapshot: BrowserSnapshot,
visual: VisualObservation | undefined,
- history: Record<string, unknown>[]
+ history: readonly HistoryEntry[],
+ token: string
) {
const criteria = candidateCriteria(candidates);
const response = await client.systemOne({
- state: {
- goal: 'Enter the verification token, then submit the form.',
- observation: {
- page: JSON.stringify(snapshot.page ?? null),
- outline: snapshot.outline ?? '',
- visual: JSON.stringify(visualDecisionState(visual)),
- },
- history: JSON.stringify(history),
- },
+ state: decisionState(snapshot, visual, history, token),
questions: {
driver_action: choice(
'Which complete executable action should Cua Driver run next?',
@@ -133,16 +155,18 @@ export function chooseLive(
candidates: Candidate[],
snapshot: BrowserSnapshot,
visual: VisualObservation | undefined,
- history: Record<string, unknown>[]
+ history: readonly HistoryEntry[],
+ token: string
) {
- return chooseWithTypeSafe(new TypeSafeClient(), candidates, snapshot, visual, history);
+ return chooseWithTypeSafe(new TypeSafeClient(), candidates, snapshot, visual, history, token);
}
export function chooseMockAdapter(
candidates: Candidate[],
_snapshot: BrowserSnapshot,
_visual: VisualObservation | undefined,
- _history: Record<string, unknown>[]
+ _history: readonly HistoryEntry[],
+ _token: string
) {
return chooseMock(candidates);
}
diff --git a/libs/cua-driver/examples/jev-use/typescript/run.ts b/libs/cua-driver/examples/jev-use/typescript/run.ts
index 8ad2b1bfb9..239ace676d 100644
--- a/libs/cua-driver/examples/jev-use/typescript/run.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/run.ts
@@ -9,12 +9,14 @@ import {
buildCandidates,
classify,
hasExecutableCandidate,
+ historyEntry,
parseVisualRegions,
SUBMIT_IDS,
validateChoice,
VisualObservationError,
type BrowserSnapshot,
type Candidate,
+ type HistoryEntry,
type VisualDelivery,
type Outcome,
type VisualObservation,
@@ -327,7 +329,9 @@ async function run(args: Arguments): Promise<Outcome> {
env: driverEnvironment(),
});
const client = new Client({ name: 'cua-driver-jev-use-example', version: '0.1.0' });
- const history: Record<string, unknown>[] = [];
+ // Compact what-happened record for the decision model. The full telemetry
+ // events (timings, probabilities) go only to the JSONL log.
+ const history: HistoryEntry[] = [];
let visualDelivery: VisualDelivery = 'background';
if (args.log) await writeFile(args.log, '', 'utf8');
await resetFixture(args.fixtureUrl);
@@ -396,8 +400,8 @@ async function run(args: Arguments): Promise<Outcome> {
}
const answer =
args.provider === 'mock'
- ? chooseMockAdapter(candidates, snapshot, visual, history)
- : await chooseLive(candidates, snapshot, visual, history);
+ ? chooseMockAdapter(candidates, snapshot, visual, history, token)
+ : await chooseLive(candidates, snapshot, visual, history, token);
if (!answer.choice) return 'abstained';
const candidate = validateChoice(answer.choice, candidates, visual?.captureId);
const decisionMs = Math.round((performance.now() - decisionStarted) * 100) / 100;
@@ -415,7 +419,7 @@ async function run(args: Arguments): Promise<Outcome> {
tool: null,
visual: visualRecord,
};
- history.push(event);
+ history.push(historyEntry(step, candidate.id));
await writeEvent(args.log, event);
continue;
}
@@ -459,7 +463,7 @@ async function run(args: Arguments): Promise<Outcome> {
escalation: { from: 'background', to: 'foreground', reason: refusal },
visual: visualRecord,
};
- history.push(event);
+ history.push(historyEntry(step, candidate.id, refusal));
await writeEvent(args.log, event);
continue;
}
@@ -489,7 +493,7 @@ async function run(args: Arguments): Promise<Outcome> {
delivery_mode: candidate.arguments.delivery_mode ?? null,
visual: visualRecord,
};
- history.push(event);
+ history.push(historyEntry(step, candidate.id));
await writeEvent(args.log, event);
if (args.dryRun) return 'unknown';
if (SUBMIT_IDS.has(candidate.id)) {