Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): give jev-use Jev the filled form state before submit (#4255)
Francesco Bonacci0e75660ce4c2
diff --git a/libs/cua-driver/examples/jev-use/README.md b/libs/cua-driver/examples/jev-use/README.mdindex b3e6b4088d..0e1c2842e7 100644--- a/libs/cua-driver/examples/jev-use/README.md+++ b/libs/cua-driver/examples/jev-use/README.md@@ -16,8 +16,14 @@ The TypeSafe-specific code lives in `python/jev_adapter.py` and`typescript/jev_adapter.ts`, outside Driver. Both adapters send Jev the boundedcandidate IDs and descriptions plus a compact observation. When visual regionsare present, that observation includes their typed bounds and the exact-`capture_id`; it never includes screenshot bytes or extension internals. A live-answer must be one of the supplied IDs before the runner resolves it to the+`capture_id`; it never includes screenshot bytes or extension internals. The+observation also carries a `form` summary that the runner derives from the page+structure. The summary reports whether the verification field is `empty`, holds+the required token (`contains_required_token`), or holds `contains_other_value`,+and whether a Submit button ref is `available`. The token itself is replaced+with `[verification token]` in the page, outline, and visual text. `history` lists+one `{step, selected_id, outcome}` item per earlier decision. Timings and model+probabilities stay in the JSONL log only. A live answer must be one of the supplied IDs before the runner resolves it to theoriginal immutable action.You can run the complete deterministic proof without credentials or networkdiff --git a/libs/cua-driver/examples/jev-use/fixtures/jev-page-structure-replay-v1.json b/libs/cua-driver/examples/jev-use/fixtures/jev-page-structure-replay-v1.jsonnew file mode 100644index 0000000000..6895d36042--- /dev/null+++ b/libs/cua-driver/examples/jev-use/fixtures/jev-page-structure-replay-v1.json@@ -0,0 +1,299 @@+{+ "schema": "cua.jev_use_page_structure_replay_v1",+ "description": "Page-structure (DOM) path fixture. Snapshots are real get_browser_state semantic_v2 results from the loopback fixture (captured with cua-driver on macOS); recorded_live_runs are the step choices TypeSafe Jev made in live Linux/Windows runs with released Cua Driver 0.30.1. pre_fix_submit_state is the exact state the runner sent Jev at the submit step before the filled-form-state fix.",+ "token": "jev-guide-live",+ "snapshots": {+ "before_typing": {+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\" [editable=\"plaintext\",focusable=true,required=true]\n - button \"Submit\" [focusable=true]\n - status\n - statictext \"status=waiting\"",+ "refs": [+ {+ "actions": ["type", "pointer"],+ "frame": "main",+ "name": "verification value",+ "ref": "p3:0",+ "role": "textbox",+ "states": {+ "editable": "plaintext",+ "focusable": true,+ "required": true+ },+ "value": null,+ "visibility": "in_viewport"+ },+ {+ "actions": ["click", "pointer"],+ "frame": "main",+ "name": "Submit",+ "ref": "p3:1",+ "role": "button",+ "states": {+ "focusable": true+ },+ "value": null,+ "visibility": "in_viewport"+ },+ {+ "actions": ["type"],+ "frame": "main",+ "name": null,+ "ref": "p3:10",+ "role": "generic",+ "states": {+ "editable": "plaintext"+ },+ "value": null,+ "visibility": "unknown"+ }+ ]+ },+ "after_typing": {+ "target_id": "bt-40b005f3-6f85-41d7-a7d5-47f158a19dc6",+ "tab_id": "tab-e66e7ea2-0fe8-4606-9495-c81518229489",+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - generic [editable=\"plaintext\"]\n - form\n - textbox \"verification value\": jev-guide-live [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"jev-guide-live\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "refs": [+ {+ "actions": ["type", "pointer"],+ "frame": "main",+ "name": "verification value",+ "ref": "p4:1",+ "role": "textbox",+ "states": {+ "editable": "plaintext",+ "focusable": true,+ "focused": true,+ "required": true+ },+ "value": "jev-guide-live",+ "visibility": "in_viewport"+ },+ {+ "actions": ["click", "pointer"],+ "frame": "main",+ "name": "Submit",+ "ref": "p4:2",+ "role": "button",+ "states": {+ "focusable": true+ },+ "value": null,+ "visibility": "in_viewport"+ },+ {+ "actions": ["type"],+ "frame": "main",+ "name": null,+ "ref": "p4:10",+ "role": "generic",+ "states": {+ "editable": "plaintext"+ },+ "value": null,+ "visibility": "unknown"+ },+ {+ "actions": ["type"],+ "frame": "main",+ "name": "jev-guide-live",+ "ref": "p4:11",+ "role": "statictext",+ "states": {+ "editable": "plaintext"+ },+ "value": null,+ "visibility": "unknown"+ }+ ]+ }+ },+ "pre_fix_submit_state": {+ "goal": "Enter the verification token, then submit the form.",+ "observation": {+ "page": {+ "title": "about:blank",+ "url": "http://127.0.0.1:8799/"+ },+ "outline": "- main\n - heading \"Cua Driver + TypeSafe Jev fixture\"\n - paragraph\n - statictext \"Enter the verification value and submit it.\"\n - form\n - generic [editable=\"plaintext\"]\n - textbox \"verification value\": jev-guide-live [editable=\"plaintext\",focusable=true,focused=true,required=true]\n - button \"Submit\" [focusable=true]\n - statictext \"jev-guide-live\" [editable=\"plaintext\"]\n - status\n - statictext \"status=waiting\"",+ "visual": null+ },+ "history": [+ {+ "event": "step",+ "step": 1,+ "candidate": "type-verification-value",+ "confidence": 1.0,+ "probabilities": {+ "type-verification-value": 1.0,+ "reobserve": 0.0,+ "abstain": 0.0+ },+ "decision_ms": 41.43,+ "action_ms": 1610.77,+ "dry_run": false,+ "tool": "browser_type",+ "delivery_mode": null,+ "visual": {+ "status": "skipped",+ "reason": "disabled"+ }+ }+ ]+ },+ "recorded_live_runs": {+ "windows-page-1-python": {+ "steps": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "probabilities": {+ "abstain": 0.11,+ "reobserve": 0.06,+ "type-verification-value": 0.83+ }+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "probabilities": {+ "abstain": 0.22,+ "reobserve": 0.5,+ "submit-form": 0.28+ }+ },+ {+ "step": 3,+ "selected_id": "reobserve",+ "probabilities": {+ "abstain": 0.07,+ "reobserve": 0.63,+ "submit-form": 0.3+ }+ },+ {+ "step": 4,+ "selected_id": "reobserve",+ "probabilities": {+ "abstain": 0.05,+ "reobserve": 0.65,+ "submit-form": 0.3+ }+ }+ ],+ "outcome": "budget_exhausted"+ },+ "linux-page-3-python": {+ "steps": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "probabilities": {+ "abstain": 0.11,+ "reobserve": 0.07,+ "type-verification-value": 0.82+ }+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "probabilities": {+ "abstain": 0.23,+ "reobserve": 0.49,+ "submit-form": 0.28+ }+ },+ {+ "step": 3,+ "selected_id": "reobserve",+ "probabilities": {+ "abstain": 0.09,+ "reobserve": 0.6,+ "submit-form": 0.31+ }+ },+ {+ "step": 4,+ "selected_id": "reobserve",+ "probabilities": {+ "abstain": 0.06,+ "reobserve": 0.6,+ "submit-form": 0.34+ }+ }+ ],+ "outcome": "budget_exhausted"+ },+ "linux-page-7-typescript": {+ "steps": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "probabilities": {+ "abstain": 0.1,+ "reobserve": 0.08,+ "type-verification-value": 0.82+ }+ },+ {+ "step": 2,+ "selected_id": "reobserve",+ "probabilities": {+ "abstain": 0.23,+ "reobserve": 0.53,+ "submit-form": 0.24+ }+ },+ {+ "step": 3,+ "selected_id": "reobserve",+ "probabilities": {+ "abstain": 0.11,+ "reobserve": 0.75,+ "submit-form": 0.14+ }+ },+ {+ "step": 4,+ "selected_id": "submit-form",+ "probabilities": {+ "abstain": 0.03,+ "reobserve": 0.48,+ "submit-form": 0.49+ }+ }+ ],+ "outcome": "verified"+ },+ "linux-page-2-python": {+ "steps": [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "probabilities": {+ "abstain": 0.09,+ "reobserve": 0.06,+ "type-verification-value": 0.85+ }+ },+ {+ "step": 2,+ "selected_id": "submit-form",+ "probabilities": {+ "abstain": 0.21,+ "reobserve": 0.35,+ "submit-form": 0.44+ }+ }+ ],+ "outcome": "verified"+ }+ }+}diff --git a/libs/cua-driver/examples/jev-use/python/core.py b/libs/cua-driver/examples/jev-use/python/core.pyindex 72b7e62a82..a911bbe12e 100644--- a/libs/cua-driver/examples/jev-use/python/core.py+++ b/libs/cua-driver/examples/jev-use/python/core.py@@ -213,11 +213,90 @@ def parse_visual_regions()+FIELD_NAME = "verification value"+SUBMIT_NAME = "Submit"+REDACTED_TOKEN = "[verification token]"+++def _form_refs(snapshot: Mapping[str, Any]) -> tuple[Any, Any]:+ refs = snapshot.get("refs") or []+ field = next(+ (ref for ref in refs if ref.get("role") == "textbox" and ref.get("name") == FIELD_NAME),+ None,+ )+ button = next(+ (ref for ref in refs if ref.get("role") == "button" and ref.get("name") == SUBMIT_NAME),+ None,+ )+ return field, button+++def form_state(snapshot: Mapping[str, Any], token: str) -> dict[str, str]:+ """Summarize the form for the decision model without revealing the token.++ The raw field value never leaves the runner; the model receives only whether+ the field is empty, holds the required token, or holds something else.+ """+ field, button = _form_refs(snapshot)+ if field is None:+ field_state = "not_found"+ elif not field.get("value"):+ field_state = "empty"+ elif field.get("value") == token:+ field_state = "contains_required_token"+ else:+ field_state = "contains_other_value"+ return {+ "verification_field": field_state,+ "submit_button": "available" if button is not None else "not_in_page_structure",+ }+++def redact_token(value: Any, token: str) -> Any:+ """Replace every occurrence of the token in strings nested in ``value``."""+ if not token:+ return value+ if isinstance(value, str):+ return value.replace(token, REDACTED_TOKEN)+ if isinstance(value, Mapping):+ return {key: redact_token(item, token) for key, item in value.items()}+ if isinstance(value, (list, tuple)):+ return [redact_token(item, token) for item in value]+ return value+++HISTORY_OUTCOMES = {+ "type-verification-value": "typed the required token into the verification field",+ "submit-form": "clicked Submit; the submission was not yet confirmed",+ "submit-form-foreground": "clicked Submit in the foreground; the submission was not yet confirmed",+ "reobserve": "took no action and requested a fresh observation",+}+++def history_entry(+ step: int, candidate_id: str, *, refusal: str | None = None+) -> dict[str, Any]:+ """Build the compact decision-history item shown to the model.++ It records what each step did, not timings or model probabilities, so earlier+ choices do not become a signal to repeat themselves.+ """+ if refusal is not None:+ outcome = (+ f"Driver refused background delivery ({refusal}); no click happened and a "+ "foreground Submit candidate is offered next"+ )+ else:+ outcome = HISTORY_OUTCOMES.get(candidate_id, "completed")+ return {"step": step, "selected_id": candidate_id, "outcome": outcome}++def _reserved_candidates() -> list[Candidate]:return [Candidate("reobserve",- "Discard this decision set and obtain a fresh Driver observation.",+ "Take no action and obtain a fresh Driver observation, because the current "+ "observation is stale, incomplete, or contradicts the reported form state.",None,{},),@@ -249,25 +328,14 @@ def build_candidates("target_id": snapshot["target_id"],"tab_id": snapshot["tab_id"],}- refs = snapshot.get("refs", [])- field = next(- (- ref- for ref in refs- if ref.get("role") == "textbox" and ref.get("name") == "verification value"- ),- None,- )- button = next(- (ref for ref in refs if ref.get("role") == "button" and ref.get("name") == "Submit"),- None,- )+ field, button = _form_refs(snapshot)candidates: list[Candidate] = []if field and field.get("value") != token:candidates.append(Candidate("type-verification-value",- "Replace the verification field with the required token.",+ "Type the required verification token into the verification field, "+ "replacing its current contents.","browser_type",{**common, "ref": field["ref"], "text": token, "replace": True},)@@ -276,7 +344,9 @@ def build_candidates(candidates.append(Candidate("submit-form",- "Submit the form now that the verification field contains the token.",+ "Click the form's Submit button. The observed form state reports that the "+ "verification field already contains the required token, so the form is "+ "ready to submit.","browser_click",{**common, "ref": button["ref"], "input_route": "dom_event"},)@@ -305,7 +375,9 @@ def build_candidates("window, because Driver refused background delivery for the ""previous visual click."if foreground- else "Submit the form using the unique validated visual Submit region."+ else "Submit the form by clicking the unique validated visual Submit "+ "region. The observed form state reports that the verification field "+ "already contains the required token."),"click",{diff --git a/libs/cua-driver/examples/jev-use/python/jev_adapter.py b/libs/cua-driver/examples/jev-use/python/jev_adapter.pyindex 0ab10202d2..2b67916874 100644--- a/libs/cua-driver/examples/jev-use/python/jev_adapter.py+++ b/libs/cua-driver/examples/jev-use/python/jev_adapter.py@@ -4,7 +4,7 @@ import mathfrom dataclasses import dataclassfrom typing import Any, Mapping, Protocol-from core import Candidate, VisualObservation, choose_mock+from core import Candidate, VisualObservation, choose_mock, form_state, redact_tokenclass TypeSafeClientLike(Protocol):@@ -95,27 +95,46 @@ def visual_decision_state(visual: VisualObservation | None) -> dict[str, Any] |}+GOAL = "Enter the required verification token into the verification field, then submit the form."+++def decision_state(+ snapshot: Mapping[str, Any],+ visual: VisualObservation | None,+ history: list[dict[str, Any]],+ token: str,+) -> dict[str, Any]:+ """Build the compact, deterministic, token-redacted state sent to Jev.++ ``form`` states the field and Submit status the runner verified from the page+ structure, so the model does not have to infer it from the outline. The token+ itself is replaced everywhere, including the outline and visual text.+ """+ return {+ "goal": GOAL,+ "observation": {+ "page": redact_token(snapshot.get("page"), token),+ "form": form_state(snapshot, token),+ "outline": redact_token(snapshot.get("outline"), token),+ "visual": redact_token(visual_decision_state(visual), token),+ },+ "history": [dict(item) for item in history],+ }++def choose_with_typesafe(client: TypeSafeClientLike,candidates: list[Candidate],snapshot: dict[str, Any],visual: VisualObservation | None,history: list[dict[str, Any]],+ token: str,) -> tuple[str, float, dict[str, float]]:from typesafe_sdk import Choicecriteria = _candidate_criteria(candidates)- state = {- "goal": "Enter the verification token, then submit the form.",- "observation": {- "page": snapshot.get("page"),- "outline": snapshot.get("outline"),- "visual": visual_decision_state(visual),- },- "history": history,- }response = client.system_one(- state=state,+ state=decision_state(snapshot, visual, history, token),questions={"driver_action": Choice(instructions="Which complete executable action should Cua Driver run next?",@@ -134,11 +153,12 @@ def choose_live(snapshot: dict[str, Any],visual: VisualObservation | None,history: list[dict[str, Any]],+ token: str,) -> tuple[str, float, dict[str, float]]:from typesafe_sdk import TypeSafeClientwith TypeSafeClient() as client:- return choose_with_typesafe(client, candidates, snapshot, visual, history)+ return choose_with_typesafe(client, candidates, snapshot, visual, history, token)def choose_mock_adapter(@@ -146,5 +166,6 @@ def choose_mock_adapter(_snapshot: dict[str, Any],_visual: VisualObservation | None,_history: list[dict[str, Any]],+ _token: str,) -> tuple[str | None, float, dict[str, float]]:return choose_mock(candidates)diff --git a/libs/cua-driver/examples/jev-use/python/run.py b/libs/cua-driver/examples/jev-use/python/run.pyindex a6e2538484..b78b850cdc 100644--- a/libs/cua-driver/examples/jev-use/python/run.py+++ b/libs/cua-driver/examples/jev-use/python/run.py@@ -23,6 +23,7 @@ from core import (VisualObservationError,build_candidates,classify,+ history_entry,has_executable_candidate,parse_visual_regions,validate_choice,@@ -289,6 +290,8 @@ def write_event(log_path: Path | None, event: dict[str, Any]) -> None:async def run(args: argparse.Namespace) -> str:token = args.token or f"jev-{uuid.uuid4().hex[:10]}"label = f"jev-python-{uuid.uuid4().hex[:8]}"+ # Compact what-happened record for the decision model. The full telemetry+ # events (timings, probabilities) go only to the JSONL log.history: list[dict[str, Any]] = []visual_delivery: VisualDelivery = "background"log_path = Path(args.log) if args.log else None@@ -358,11 +361,11 @@ async def run(args: argparse.Namespace) -> str:if args.provider == "mock":choice, confidence, probabilities = choose_mock_adapter(- candidates, snapshot, visual, history+ candidates, snapshot, visual, history, token)else:choice, confidence, probabilities = await asyncio.to_thread(- choose_live, candidates, snapshot, visual, history+ choose_live, candidates, snapshot, visual, history, token)if choice is None:return "abstained"@@ -386,7 +389,7 @@ async def run(args: argparse.Namespace) -> str:"tool": None,"visual": visual_record,}- history.append(event)+ history.append(history_entry(step, candidate.id))write_event(log_path, event)continue@@ -434,7 +437,7 @@ async def run(args: argparse.Namespace) -> str:},"visual": visual_record,}- history.append(event)+ history.append(history_entry(step, candidate.id, refusal=refusal))write_event(log_path, event)continuewrite_event(@@ -466,7 +469,7 @@ async def run(args: argparse.Namespace) -> str:"delivery_mode": candidate.arguments.get("delivery_mode"),"visual": visual_record,}- history.append(event)+ history.append(history_entry(step, candidate.id))write_event(log_path, event)if args.dry_run:return "unknown"diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_decision_state.py b/libs/cua-driver/examples/jev-use/python/tests/test_decision_state.pynew file mode 100644index 0000000000..fbdac75e53--- /dev/null+++ b/libs/cua-driver/examples/jev-use/python/tests/test_decision_state.py@@ -0,0 +1,191 @@+from __future__ import annotations++import json+import sys+import unittest+from pathlib import Path+from types import SimpleNamespace++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from core import (+ REDACTED_TOKEN,+ VisualObservation,+ VisualRegion,+ build_candidates,+ form_state,+ history_entry,+ redact_token,+ validate_choice,+)+from jev_adapter import choose_mock_adapter, choose_with_typesafe, decision_state++FIXTURE = json.loads(+ (Path(__file__).resolve().parents[2] / "fixtures/jev-page-structure-replay-v1.json").read_text(+ encoding="utf-8"+ )+)+TOKEN = FIXTURE["token"]+BEFORE = FIXTURE["snapshots"]["before_typing"]+AFTER = FIXTURE["snapshots"]["after_typing"]+++def with_field_value(value: str | None) -> dict:+ snapshot = json.loads(json.dumps(BEFORE))+ for ref in snapshot["refs"]:+ if ref["role"] == "textbox" and ref["name"] == "verification value":+ ref["value"] = value+ return snapshot+++class ReplayClient:+ """Return recorded live choices in order and keep every request."""++ def __init__(self, recorded: list[dict]) -> None:+ self.recorded = list(recorded)+ self.requests: list[dict] = []++ def system_one(self, **request):+ self.requests.append(request)+ step = self.recorded.pop(0)+ answer = SimpleNamespace(+ choice=step["selected_id"],+ confidence=max(step["probabilities"].values()),+ probabilities=step["probabilities"],+ )+ return SimpleNamespace(choices={"driver_action": answer})+++class FormStateTest(unittest.TestCase):+ def test_form_state_reports_field_status_without_the_value(self) -> None:+ self.assertEqual(+ form_state(BEFORE, TOKEN),+ {"verification_field": "empty", "submit_button": "available"},+ )+ self.assertEqual(form_state(AFTER, TOKEN)["verification_field"], "contains_required_token")+ self.assertEqual(+ form_state(with_field_value("something-else"), TOKEN)["verification_field"],+ "contains_other_value",+ )+ no_button = {**AFTER, "refs": [ref for ref in AFTER["refs"] if ref["role"] != "button"]}+ self.assertEqual(form_state(no_button, TOKEN)["submit_button"], "not_in_page_structure")+ self.assertEqual(+ form_state({"refs": []}, TOKEN),+ {"verification_field": "not_found", "submit_button": "not_in_page_structure"},+ )++ def test_redaction_is_nested_and_deterministic(self) -> None:+ value = {"a": [f"x {TOKEN} y", {"b": TOKEN}], "n": 3, "none": None}+ redacted = redact_token(value, TOKEN)+ self.assertEqual(+ redacted,+ {"a": [f"x {REDACTED_TOKEN} y", {"b": REDACTED_TOKEN}], "n": 3, "none": None},+ )+ self.assertEqual(redact_token(value, TOKEN), redacted)+ self.assertEqual(redact_token("unchanged", ""), "unchanged")+++class SubmitStepObservationTest(unittest.TestCase):+ def test_pre_fix_submit_state_hid_readiness_and_leaked_the_token(self) -> None:+ # Pins the root-cause evidence: the old state had no form summary, sent the+ # raw token in the outline, and fed raw telemetry back as history.+ old = FIXTURE["pre_fix_submit_state"]+ self.assertNotIn("form", old["observation"])+ self.assertIn(TOKEN, old["observation"]["outline"])+ self.assertIn("status=waiting", old["observation"]["outline"])+ self.assertIn("probabilities", old["history"][0])++ def test_after_typing_the_observation_states_the_filled_form(self) -> None:+ history = [history_entry(1, "type-verification-value")]+ state = decision_state(AFTER, None, history, TOKEN)+ encoded = json.dumps(state, sort_keys=True)++ self.assertNotIn(TOKEN, encoded)+ self.assertEqual(+ state["observation"]["form"],+ {"verification_field": "contains_required_token", "submit_button": "available"},+ )+ self.assertIn(+ f'textbox "verification value": {REDACTED_TOKEN}', state["observation"]["outline"]+ )+ self.assertEqual(+ state["history"],+ [+ {+ "step": 1,+ "selected_id": "type-verification-value",+ "outcome": "typed the required token into the verification field",+ }+ ],+ )+ self.assertEqual(+ encoded, json.dumps(decision_state(AFTER, None, history, TOKEN), sort_keys=True)+ )++ candidates = build_candidates(AFTER, TOKEN)+ criteria = {candidate.id: candidate.description for candidate in candidates}+ self.assertEqual(list(criteria), ["submit-form", "reobserve", "abstain"])+ self.assertIn("already contains the required token", criteria["submit-form"])+ self.assertIn("stale, incomplete", criteria["reobserve"])+ self.assertNotIn(TOKEN, json.dumps(criteria))++ def test_visual_text_is_redacted_too(self) -> None:+ region = VisualRegion("r1", "text", TOKEN, None, 0.9, False, 1, 1, 10, 10)+ visual = VisualObservation("cap", "ref", 100, 100, 7, 9, 0.0, 0.0, 1.0, 1.0, (region,))+ state = decision_state(AFTER, visual, [], TOKEN)+ self.assertEqual(state["observation"]["visual"]["regions"][0]["text"], REDACTED_TOKEN)+ self.assertNotIn(TOKEN, json.dumps(state))++ def test_background_refusal_history_is_compact(self) -> None:+ entry = history_entry(2, "submit-form", refusal="background_unsupported")+ self.assertEqual(set(entry), {"step", "selected_id", "outcome"})+ self.assertIn("background_unsupported", entry["outcome"])+++class ProviderPathTest(unittest.TestCase):+ def test_mock_provider_types_then_submits_over_recorded_snapshots(self) -> None:+ first = build_candidates(BEFORE, TOKEN)+ self.assertEqual(+ choose_mock_adapter(first, BEFORE, None, [], TOKEN)[0], "type-verification-value"+ )+ second = build_candidates(AFTER, TOKEN)+ history = [history_entry(1, "type-verification-value")]+ self.assertEqual(choose_mock_adapter(second, AFTER, None, history, TOKEN)[0], "submit-form")++ def test_recorded_live_choices_replay_through_the_new_state(self) -> None:+ for name, run in FIXTURE["recorded_live_runs"].items():+ with self.subTest(run=name):+ client = ReplayClient(run["steps"])+ history: list[dict] = []+ outcome = "budget_exhausted"+ for step in range(1, 5):+ snapshot = BEFORE if step == 1 else AFTER+ candidates = build_candidates(snapshot, TOKEN)+ recorded = run["steps"][step - 1]+ # The recorded choice was made over the same bounded table.+ self.assertEqual(+ set(recorded["probabilities"]), {candidate.id for candidate in candidates}+ )+ choice, _, _ = choose_with_typesafe(+ client, candidates, snapshot, None, history, TOKEN+ )+ candidate = validate_choice(choice, candidates)+ request = client.requests[-1]+ self.assertNotIn(TOKEN, json.dumps(request["state"], sort_keys=True))+ self.assertEqual(+ request["state"]["observation"]["form"]["verification_field"],+ "empty" if step == 1 else "contains_required_token",+ )+ for item in request["state"]["history"]:+ self.assertEqual(set(item), {"step", "selected_id", "outcome"})+ self.assertIn("reobserve", request["questions"]["driver_action"].criteria)+ self.assertIn("abstain", request["questions"]["driver_action"].criteria)+ history.append(history_entry(step, candidate.id))+ if candidate.id == "submit-form":+ outcome = "verified"+ break+ self.assertEqual(outcome, run["outcome"])+++if __name__ == "__main__":+ unittest.main()diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_live_adapter.py b/libs/cua-driver/examples/jev-use/python/tests/test_live_adapter.pyindex 0a5b3e0c37..05e675f3d0 100644--- a/libs/cua-driver/examples/jev-use/python/tests/test_live_adapter.py+++ b/libs/cua-driver/examples/jev-use/python/tests/test_live_adapter.py@@ -56,7 +56,7 @@ class LiveAdapterTest(unittest.TestCase):)candidates = build_candidates(snapshot, "expected", visual, capture_bound_click=True)selected, confidence, probabilities = choose_with_typesafe(- FakeClient(), candidates, snapshot, visual, []+ FakeClient(), candidates, snapshot, visual, [], "expected")self.assertEqual(selected, "submit-form")@@ -70,6 +70,9 @@ class LiveAdapterTest(unittest.TestCase):sent_visual = FakeClient.request["state"]["observation"]["visual"]self.assertEqual(sent_visual["capture_id"], "capture-submit")self.assertEqual(sent_visual["regions"][0]["id"], "submit-text")+ sent_form = FakeClient.request["state"]["observation"]["form"]+ self.assertEqual(sent_form["verification_field"], "contains_required_token")+ self.assertNotIn("expected", json.dumps(FakeClient.request["state"]))def test_live_adapter_rejects_id_outside_the_supplied_table(self) -> None:snapshot = {"target_id": "target", "tab_id": "tab", "refs": []}@@ -88,7 +91,7 @@ class LiveAdapterTest(unittest.TestCase):)with self.assertRaisesRegex(ValueError, "unknown candidate"):- choose_with_typesafe(UnknownClient(), candidates, snapshot, None, [])+ choose_with_typesafe(UnknownClient(), candidates, snapshot, None, [], "expected")if __name__ == "__main__":diff --git a/libs/cua-driver/examples/jev-use/typescript/core.test.ts b/libs/cua-driver/examples/jev-use/typescript/core.test.tsindex 30622a141e..dbc205e4a9 100644--- a/libs/cua-driver/examples/jev-use/typescript/core.test.ts+++ b/libs/cua-driver/examples/jev-use/typescript/core.test.ts@@ -311,7 +311,7 @@ test('live adapter sends one Choice keyed by executable candidate id', async ()9);const candidates = buildCandidates(page, 'expected', visual, true);- const answer = await chooseWithTypeSafe(client, candidates, page, visual, []);+ const answer = await chooseWithTypeSafe(client, candidates, page, visual, [], 'expected');assert.equal(answer.choice, 'submit-form');assert.deepEqual(@@ -321,6 +321,9 @@ test('live adapter sends one Choice keyed by executable candidate id', async ()const sentVisual = JSON.parse(requestBody?.state.observation.visual);assert.equal(sentVisual.capture_id, 'capture-submit');assert.equal(sentVisual.regions[0].id, 'submit-text');+ const sentForm = JSON.parse(requestBody?.state.observation.form);+ assert.equal(sentForm.verification_field, 'contains_required_token');+ assert.equal(JSON.stringify(requestBody?.state).includes('expected'), false);});test('live adapter rejects an id outside the supplied table', async () => {@@ -339,7 +342,7 @@ test('live adapter rejects an id outside the supplied table', async () => {const page = snapshot();const candidates = buildCandidates(page, 'expected');await assert.rejects(- () => chooseWithTypeSafe(client as never, candidates, page, undefined, []),+ () => chooseWithTypeSafe(client as never, candidates, page, undefined, [], 'expected'),/unknown candidate/);});diff --git a/libs/cua-driver/examples/jev-use/typescript/core.ts b/libs/cua-driver/examples/jev-use/typescript/core.tsindex 4dd6756467..cf05df3226 100644--- a/libs/cua-driver/examples/jev-use/typescript/core.ts+++ b/libs/cua-driver/examples/jev-use/typescript/core.ts@@ -210,11 +210,84 @@ export function parseVisualRegions(});}+export const REDACTED_TOKEN = '[verification token]';++function formRefs(snapshot: BrowserSnapshot) {+ const refs = snapshot.refs ?? [];+ const field = refs.find(+ (item) => item.role === 'textbox' && item.name === 'verification value' && item.ref+ );+ const button = refs.find((item) => item.role === 'button' && item.name === 'Submit' && item.ref);+ return { field, button };+}++export type FormState = Readonly<{+ verification_field: 'not_found' | 'empty' | 'contains_required_token' | 'contains_other_value';+ submit_button: 'available' | 'not_in_page_structure';+}>;++/**+ * Summarize the form for the decision model without revealing the token. The+ * raw field value never leaves the runner.+ */+export function formState(snapshot: BrowserSnapshot, token: string): FormState {+ const { field, button } = formRefs(snapshot);+ const verificationField = !field+ ? 'not_found'+ : !field.value+ ? 'empty'+ : field.value === token+ ? 'contains_required_token'+ : 'contains_other_value';+ return {+ verification_field: verificationField,+ submit_button: button ? 'available' : 'not_in_page_structure',+ };+}++/** Replace every occurrence of the token in strings nested in value. */+export function redactToken(value: unknown, token: string): unknown {+ if (!token) return value;+ if (typeof value === 'string') return value.split(token).join(REDACTED_TOKEN);+ if (Array.isArray(value)) return value.map((item) => redactToken(item, token));+ if (value && typeof value === 'object') {+ return Object.fromEntries(+ Object.entries(value).map(([key, item]) => [key, redactToken(item, token)])+ );+ }+ return value;+}++export type HistoryEntry = Readonly<{ step: number; selected_id: string; outcome: string }>;++const HISTORY_OUTCOMES: Readonly<Record<string, string>> = {+ 'type-verification-value': 'typed the required token into the verification field',+ 'submit-form': 'clicked Submit; the submission was not yet confirmed',+ 'submit-form-foreground':+ 'clicked Submit in the foreground; the submission was not yet confirmed',+ reobserve: 'took no action and requested a fresh observation',+};++/**+ * Build the compact decision-history item shown to the model. It records what+ * each step did, not timings or model probabilities, so earlier choices do not+ * become a signal to repeat themselves.+ */+export function historyEntry(step: number, candidateId: string, refusal?: string): HistoryEntry {+ const outcome = refusal+ ? `Driver refused background delivery (${refusal}); no click happened and a ` ++ 'foreground Submit candidate is offered next'+ : (HISTORY_OUTCOMES[candidateId] ?? 'completed');+ return { step, selected_id: candidateId, outcome };+}+function reservedCandidates(): Candidate[] {return [immutableCandidate({id: 'reobserve',- description: 'Discard this decision set and obtain a fresh Driver observation.',+ description:+ 'Take no action and obtain a fresh Driver observation, because the current ' ++ 'observation is stale, incomplete, or contradicts the reported form state.',tool: null,arguments: {},}),@@ -242,17 +315,15 @@ export function buildCandidates(visualDelivery: VisualDelivery = 'background'): Candidate[] {const common = { target_id: snapshot.target_id, tab_id: snapshot.tab_id };- const refs = snapshot.refs ?? [];- const field = refs.find(- (item) => item.role === 'textbox' && item.name === 'verification value' && item.ref- );- const button = refs.find((item) => item.role === 'button' && item.name === 'Submit' && item.ref);+ const { field, button } = formRefs(snapshot);const candidates: Candidate[] = [];if (field?.value !== token && field?.ref) {candidates.push(immutableCandidate({id: 'type-verification-value',- description: 'Replace the verification field with the required token.',+ description:+ 'Type the required verification token into the verification field, ' ++ 'replacing its current contents.',tool: 'browser_type',arguments: { ...common, ref: field.ref, text: token, replace: true },})@@ -261,7 +332,10 @@ export function buildCandidates(candidates.push(immutableCandidate({id: 'submit-form',- description: 'Submit the form now that the verification field contains the token.',+ description:+ "Click the form's Submit button. The observed form state reports that the " ++ 'verification field already contains the required token, so the form is ' ++ 'ready to submit.',tool: 'browser_click',arguments: { ...common, ref: button.ref, input_route: 'dom_event' },})@@ -288,7 +362,9 @@ export function buildCandidates(? 'Submit the form by clicking the unique validated visual Submit region with ' +'foreground delivery, which activates the browser window, because Driver ' +'refused background delivery for the previous visual click.'- : 'Submit the form using the unique validated visual Submit region.',+ : 'Submit the form by clicking the unique validated visual Submit region. ' ++ 'The observed form state reports that the verification field already ' ++ 'contains the required token.',tool: 'click',arguments: {pid: visual.pid,diff --git a/libs/cua-driver/examples/jev-use/typescript/decision_state.test.ts b/libs/cua-driver/examples/jev-use/typescript/decision_state.test.tsnew file mode 100644index 0000000000..2decc9d2a6--- /dev/null+++ b/libs/cua-driver/examples/jev-use/typescript/decision_state.test.ts@@ -0,0 +1,212 @@+import assert from 'node:assert/strict';+import { readFileSync } from 'node:fs';+import test from 'node:test';++import {+ buildCandidates,+ formState,+ historyEntry,+ redactToken,+ REDACTED_TOKEN,+ validateChoice,+ type BrowserSnapshot,+ type HistoryEntry,+ type VisualObservation,+} from './core.js';+import { chooseMockAdapter, chooseWithTypeSafe, decisionState } from './jev_adapter.js';++const FIXTURE = JSON.parse(+ readFileSync(new URL('../fixtures/jev-page-structure-replay-v1.json', import.meta.url), 'utf8')+);+const TOKEN: string = FIXTURE.token;+const BEFORE: BrowserSnapshot = FIXTURE.snapshots.before_typing;+const AFTER: BrowserSnapshot = FIXTURE.snapshots.after_typing;++type RecordedStep = { step: number; selected_id: string; probabilities: Record<string, number> };++function withFieldValue(value: string | null): BrowserSnapshot {+ const snapshot = structuredClone(BEFORE);+ for (const ref of snapshot.refs ?? []) {+ if (ref.role === 'textbox' && ref.name === 'verification value') ref.value = value;+ }+ return snapshot;+}++function replayClient(recorded: RecordedStep[]) {+ const queue = [...recorded];+ const requests: any[] = [];+ return {+ requests,+ systemOne: async (request: any) => {+ requests.push(request);+ const step = queue.shift()!;+ return {+ answers: {+ driver_action: {+ type: 'choice' as const,+ choice: step.selected_id,+ confidence: Math.max(...Object.values(step.probabilities)),+ probabilities: step.probabilities,+ },+ },+ };+ },+ };+}++test('form state reports field status without the value', () => {+ assert.deepEqual(formState(BEFORE, TOKEN), {+ verification_field: 'empty',+ submit_button: 'available',+ });+ assert.equal(formState(AFTER, TOKEN).verification_field, 'contains_required_token');+ assert.equal(+ formState(withFieldValue('something-else'), TOKEN).verification_field,+ 'contains_other_value'+ );+ const noButton = { ...AFTER, refs: (AFTER.refs ?? []).filter((ref) => ref.role !== 'button') };+ assert.equal(formState(noButton, TOKEN).submit_button, 'not_in_page_structure');+ assert.deepEqual(formState({ target_id: 't', tab_id: 't', refs: [] }, TOKEN), {+ verification_field: 'not_found',+ submit_button: 'not_in_page_structure',+ });+});++test('redaction is nested and deterministic', () => {+ const value = { a: [`x ${TOKEN} y`, { b: TOKEN }], n: 3, none: null };+ assert.deepEqual(redactToken(value, TOKEN), {+ a: [`x ${REDACTED_TOKEN} y`, { b: REDACTED_TOKEN }],+ n: 3,+ none: null,+ });+ assert.equal(redactToken('unchanged', ''), 'unchanged');+});++test('pre-fix submit state hid readiness and leaked the token', () => {+ const old = FIXTURE.pre_fix_submit_state;+ assert.equal('form' in old.observation, false);+ assert.ok(old.observation.outline.includes(TOKEN));+ assert.ok(old.observation.outline.includes('status=waiting'));+ assert.ok('probabilities' in old.history[0]);+});++test('after typing the observation states the filled form', () => {+ const history = [historyEntry(1, 'type-verification-value')];+ const state = decisionState(AFTER, undefined, history, TOKEN);+ const encoded = JSON.stringify(state);+ assert.equal(encoded.includes(TOKEN), false);+ assert.deepEqual(JSON.parse(state.observation.form), {+ verification_field: 'contains_required_token',+ submit_button: 'available',+ });+ assert.ok(state.observation.outline.includes(`textbox "verification value": ${REDACTED_TOKEN}`));+ assert.deepEqual(JSON.parse(state.history), [+ {+ step: 1,+ selected_id: 'type-verification-value',+ outcome: 'typed the required token into the verification field',+ },+ ]);+ assert.equal(JSON.stringify(decisionState(AFTER, undefined, history, TOKEN)), encoded);++ const criteria = Object.fromEntries(+ buildCandidates(AFTER, TOKEN).map((candidate) => [candidate.id, candidate.description])+ );+ assert.deepEqual(Object.keys(criteria), ['submit-form', 'reobserve', 'abstain']);+ assert.ok(criteria['submit-form'].includes('already contains the required token'));+ assert.ok(criteria.reobserve.includes('stale, incomplete'));+ assert.equal(JSON.stringify(criteria).includes(TOKEN), false);+});++test('visual text is redacted too', () => {+ const visual: VisualObservation = {+ captureId: 'cap',+ screenshotReference: 'ref',+ screenshotWidth: 100,+ screenshotHeight: 100,+ pid: 7,+ windowId: 9,+ actionOriginX: 0,+ actionOriginY: 0,+ actionUnitsPerPixelX: 1,+ actionUnitsPerPixelY: 1,+ regions: [+ {+ id: 'r1',+ kind: 'text',+ text: TOKEN,+ confidence: 0.9,+ interactive: false,+ x: 1,+ y: 1,+ width: 10,+ height: 10,+ },+ ],+ };+ const state = decisionState(AFTER, visual, [], TOKEN);+ assert.equal(JSON.parse(state.observation.visual).regions[0].text, REDACTED_TOKEN);+ assert.equal(JSON.stringify(state).includes(TOKEN), false);+});++test('background refusal history is compact', () => {+ const entry = historyEntry(2, 'submit-form', 'background_unsupported');+ assert.deepEqual(Object.keys(entry).sort(), ['outcome', 'selected_id', 'step']);+ assert.ok(entry.outcome.includes('background_unsupported'));+});++test('mock provider types then submits over recorded snapshots', () => {+ const first = buildCandidates(BEFORE, TOKEN);+ assert.equal(+ chooseMockAdapter(first, BEFORE, undefined, [], TOKEN).choice,+ 'type-verification-value'+ );+ const second = buildCandidates(AFTER, TOKEN);+ const history = [historyEntry(1, 'type-verification-value')];+ assert.equal(chooseMockAdapter(second, AFTER, undefined, history, TOKEN).choice, 'submit-form');+});++test('recorded live choices replay through the new state', async () => {+ for (const [name, run] of Object.entries<any>(FIXTURE.recorded_live_runs)) {+ const client = replayClient(run.steps);+ const history: HistoryEntry[] = [];+ let outcome = 'budget_exhausted';+ for (let step = 1; step <= 4; step += 1) {+ const snapshot = step === 1 ? BEFORE : AFTER;+ const candidates = buildCandidates(snapshot, TOKEN);+ const recorded: RecordedStep = run.steps[step - 1];+ assert.deepEqual(+ new Set(Object.keys(recorded.probabilities)),+ new Set(candidates.map((candidate) => candidate.id)),+ name+ );+ const answer = await chooseWithTypeSafe(+ client as never,+ candidates,+ snapshot,+ undefined,+ history,+ TOKEN+ );+ const candidate = validateChoice(answer.choice, candidates);+ const request = client.requests.at(-1);+ assert.equal(JSON.stringify(request.state).includes(TOKEN), false, name);+ assert.equal(+ JSON.parse(request.state.observation.form).verification_field,+ step === 1 ? 'empty' : 'contains_required_token',+ name+ );+ for (const item of JSON.parse(request.state.history)) {+ assert.deepEqual(Object.keys(item).sort(), ['outcome', 'selected_id', 'step'], name);+ }+ const criteria = request.questions.driver_action.criteria;+ assert.ok('reobserve' in criteria && 'abstain' in criteria, name);+ history.push(historyEntry(step, candidate.id));+ if (candidate.id === 'submit-form') {+ outcome = 'verified';+ break;+ }+ }+ assert.equal(outcome, run.outcome, name);+ }+});diff --git a/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts b/libs/cua-driver/examples/jev-use/typescript/jev_adapter.tsindex ad48bac3f5..1286ee20c2 100644--- a/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts+++ b/libs/cua-driver/examples/jev-use/typescript/jev_adapter.ts@@ -2,7 +2,10 @@ import { choice, TypeSafeClient } from '@typesafe-ai/sdk';import {chooseMock,+ formState,+ redactToken,type BrowserSnapshot,+ type HistoryEntry,type Candidate,type VisualObservation,} from './core.js';@@ -96,24 +99,43 @@ export function visualDecisionState(visual?: VisualObservation) {};}+export const GOAL =+ 'Enter the required verification token into the verification field, then submit the form.';++/**+ * Build the compact, deterministic, token-redacted state sent to Jev. `form`+ * states the field and Submit status the runner verified from the page+ * structure; the token is replaced everywhere, including outline and visual text.+ */+export function decisionState(+ snapshot: BrowserSnapshot,+ visual: VisualObservation | undefined,+ history: readonly HistoryEntry[],+ token: string+) {+ return {+ goal: GOAL,+ observation: {+ page: JSON.stringify(redactToken(snapshot.page ?? null, token)),+ form: JSON.stringify(formState(snapshot, token)),+ outline: redactToken(snapshot.outline ?? '', token) as string,+ visual: JSON.stringify(redactToken(visualDecisionState(visual), token)),+ },+ history: JSON.stringify(history),+ };+}+export async function chooseWithTypeSafe(client: TypeSafeClientLike,candidates: Candidate[],snapshot: BrowserSnapshot,visual: VisualObservation | undefined,- history: Record<string, unknown>[]+ history: readonly HistoryEntry[],+ token: string) {const criteria = candidateCriteria(candidates);const response = await client.systemOne({- state: {- goal: 'Enter the verification token, then submit the form.',- observation: {- page: JSON.stringify(snapshot.page ?? null),- outline: snapshot.outline ?? '',- visual: JSON.stringify(visualDecisionState(visual)),- },- history: JSON.stringify(history),- },+ state: decisionState(snapshot, visual, history, token),questions: {driver_action: choice('Which complete executable action should Cua Driver run next?',@@ -133,16 +155,18 @@ export function chooseLive(candidates: Candidate[],snapshot: BrowserSnapshot,visual: VisualObservation | undefined,- history: Record<string, unknown>[]+ history: readonly HistoryEntry[],+ token: string) {- return chooseWithTypeSafe(new TypeSafeClient(), candidates, snapshot, visual, history);+ return chooseWithTypeSafe(new TypeSafeClient(), candidates, snapshot, visual, history, token);}export function chooseMockAdapter(candidates: Candidate[],_snapshot: BrowserSnapshot,_visual: VisualObservation | undefined,- _history: Record<string, unknown>[]+ _history: readonly HistoryEntry[],+ _token: string) {return chooseMock(candidates);}diff --git a/libs/cua-driver/examples/jev-use/typescript/run.ts b/libs/cua-driver/examples/jev-use/typescript/run.tsindex 8ad2b1bfb9..239ace676d 100644--- a/libs/cua-driver/examples/jev-use/typescript/run.ts+++ b/libs/cua-driver/examples/jev-use/typescript/run.ts@@ -9,12 +9,14 @@ import {buildCandidates,classify,hasExecutableCandidate,+ historyEntry,parseVisualRegions,SUBMIT_IDS,validateChoice,VisualObservationError,type BrowserSnapshot,type Candidate,+ type HistoryEntry,type VisualDelivery,type Outcome,type VisualObservation,@@ -327,7 +329,9 @@ async function run(args: Arguments): Promise<Outcome> {env: driverEnvironment(),});const client = new Client({ name: 'cua-driver-jev-use-example', version: '0.1.0' });- const history: Record<string, unknown>[] = [];+ // Compact what-happened record for the decision model. The full telemetry+ // events (timings, probabilities) go only to the JSONL log.+ const history: HistoryEntry[] = [];let visualDelivery: VisualDelivery = 'background';if (args.log) await writeFile(args.log, '', 'utf8');await resetFixture(args.fixtureUrl);@@ -396,8 +400,8 @@ async function run(args: Arguments): Promise<Outcome> {}const answer =args.provider === 'mock'- ? chooseMockAdapter(candidates, snapshot, visual, history)- : await chooseLive(candidates, snapshot, visual, history);+ ? chooseMockAdapter(candidates, snapshot, visual, history, token)+ : await chooseLive(candidates, snapshot, visual, history, token);if (!answer.choice) return 'abstained';const candidate = validateChoice(answer.choice, candidates, visual?.captureId);const decisionMs = Math.round((performance.now() - decisionStarted) * 100) / 100;@@ -415,7 +419,7 @@ async function run(args: Arguments): Promise<Outcome> {tool: null,visual: visualRecord,};- history.push(event);+ history.push(historyEntry(step, candidate.id));await writeEvent(args.log, event);continue;}@@ -459,7 +463,7 @@ async function run(args: Arguments): Promise<Outcome> {escalation: { from: 'background', to: 'foreground', reason: refusal },visual: visualRecord,};- history.push(event);+ history.push(historyEntry(step, candidate.id, refusal));await writeEvent(args.log, event);continue;}@@ -489,7 +493,7 @@ async function run(args: Arguments): Promise<Outcome> {delivery_mode: candidate.arguments.delivery_mode ?? null,visual: visualRecord,};- history.push(event);+ history.push(historyEntry(step, candidate.id));await writeEvent(args.log, event);if (args.dryRun) return 'unknown';if (SUBMIT_IDS.has(candidate.id)) {