Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
test(computer): verify task event payloads (#36239)
Kelvin Amoaba219e99984631
diff --git a/config/benchmarks/computer-use/README.md b/config/benchmarks/computer-use/README.mdindex f58895cb13d..1b7c3a7c6d1 100644--- a/config/benchmarks/computer-use/README.md+++ b/config/benchmarks/computer-use/README.md@@ -30,7 +30,11 @@ Do not generalize these results to every application on a platform.- Strict selectors require a unique matching role and label. References are freshbefore the task and after every step. There is no label-only or role-only fallback.- Completion requires exact independent final state, including unchanged decoy text- and exactly one click/submit. Raw text is never cleaned. API errors, missing final+ and exactly one click/submit. Submission handlers capture the submitted text;+ Increment handlers capture the selected color at click time. Submit tasks require+ the marker in the submission, and choose-and-click requires Blue at the click.+ Final text/color alone cannot prove those events carried the intended values.+ Raw text is never cleaned. API errors, missing finalobservations and observed interference remain separate from task completion.- The observer starts before provider initialization and initial selector snapshot,and stays active through all steps, reads, shutdown and settling. No sentineldiff --git a/config/benchmarks/computer-use/electron/main.js b/config/benchmarks/computer-use/electron/main.jsindex 5da2090d86f..a060995c952 100644--- a/config/benchmarks/computer-use/electron/main.js+++ b/config/benchmarks/computer-use/electron/main.js@@ -42,17 +42,20 @@ app.whenReady().then(() => {const { ipcRenderer } = require('electron');const field = document.getElementById('field'), decoy = document.getElementById('decoy');const color = document.getElementById('color');- let count = 0, returns = 0, ack = null, seq = 0;- document.getElementById('button').onclick = () => count++;- field.addEventListener('keydown', event => { if (event.key === 'Enter') returns++ });+ let count = 0, returns = 0, ack = null, seq = 0, submittedText = null, clickedColor = null;+ document.getElementById('button').onclick = () => { count++; clickedColor = color.value };+ field.addEventListener('keydown', event => {+ if (event.key === 'Enter') { returns++; submittedText = field.value }+ });function report() {- ipcRenderer.send('st', { count, returns, ack, seq: ++seq, text: field.value, decoyText: decoy.value,+ ipcRenderer.send('st', { count, returns, submittedText, clickedColor, ack, seq: ++seq, text: field.value, decoyText: decoy.value,top: window.scrollY, combo: color.value, selStart: field.selectionStart,selLen: field.selectionEnd - field.selectionStart, check: false, menuHits: 0, dropOpen: false,logicalFocus: document.activeElement === field ? 'field' : document.activeElement === decoy ? 'decoy' : 'other' });}ipcRenderer.on('cmd', (_event, cmd) => {count = returns = 0;+ submittedText = clickedColor = null;field.value = cmd.task === 'replace_submit' ? 'hello' : '';field.setSelectionRange(field.value.length, field.value.length);decoy.value = 'untouched'; decoy.setSelectionRange(9, 9);diff --git a/config/benchmarks/computer-use/fixture_gtk.py b/config/benchmarks/computer-use/fixture_gtk.pyindex 978aad9fb0c..2e03e954fb1 100644--- a/config/benchmarks/computer-use/fixture_gtk.py+++ b/config/benchmarks/computer-use/fixture_gtk.py@@ -18,7 +18,7 @@ CMD = os.path.join(state_dir, "cmd")instance = os.environ["BENCH_INSTANCE"]ack = Nonesequence = 0-s = {"count": 0, "returns": 0, "menuHits": 0, "last": None}+s = {"count": 0, "returns": 0, "submittedText": None, "clickedColor": None, "menuHits": 0, "last": None}def named(widget, name):@@ -48,11 +48,21 @@ edit.set_submenu(sub)menubar.append(edit)box.pack_start(menubar, False, False, 0)+def increment(_button):+ s["count"] += 1+ s["clickedColor"] = combo.get_active_text()+++def submit(entry):+ s["returns"] += 1+ s["submittedText"] = entry.get_text()++button = Gtk.Button(label="Increment")-button.connect("clicked", lambda _b: s.__setitem__("count", s["count"] + 1))+button.connect("clicked", increment)check = Gtk.CheckButton(label="Check me")field = named(Gtk.Entry(), "field")-field.connect("activate", lambda _e: s.__setitem__("returns", s["returns"] + 1))+field.connect("activate", submit)decoy = named(Gtk.Entry(), "decoy")combo = named(Gtk.ComboBoxText(), "Color")for item in ("Red", "Green", "Blue"):@@ -74,6 +84,7 @@ def command(c):if c["instance"] != instance:returns["count"] = s["returns"] = s["menuHits"] = 0+ s["submittedText"] = s["clickedColor"] = Nonecheck.set_active(False)sub.popdown()combo.set_active(0)@@ -105,6 +116,7 @@ def tick():"instance": instance, "ack": ack, "seq": sequence, "decoyText": decoy.get_text(),"logicalFocus": "field" if win.get_focus() is field else "decoy" if win.get_focus() is decoy else "other","count": s["count"], "check": check.get_active(), "text": field.get_text(), "returns": s["returns"],+ "submittedText": s["submittedText"], "clickedColor": s["clickedColor"],"selStart": sel[0], "selLen": sel[1] - sel[0], "combo": combo.get_active_text(),"top": scroll.get_vadjustment().get_value(), "menuHits": s["menuHits"], "dropOpen": sub.get_visible(),"activeForm": win.is_active(), "fieldFocused": win.get_focus() is field, "pid": os.getpid(),diff --git a/config/benchmarks/computer-use/fixture_win.ps1 b/config/benchmarks/computer-use/fixture_win.ps1index f321fa5b6a6..42d45c2a9b9 100644--- a/config/benchmarks/computer-use/fixture_win.ps1+++ b/config/benchmarks/computer-use/fixture_win.ps1@@ -14,6 +14,8 @@ $script:ack = $null$script:sequence = 0$script:count = 0$script:returns = 0+$script:submittedText = $null+$script:clickedColor = $null$script:menuHits = 0$script:last = ""@@ -36,7 +38,7 @@ $form.Controls.Add($menu)$button = New-Object System.Windows.Forms.Button$button.Text = "Increment"$button.Location = New-Object System.Drawing.Point(20, 40)-$button.Add_Click({ $script:count++ })+$button.Add_Click({ $script:count++; $script:clickedColor = [string]$combo.SelectedItem })$form.Controls.Add($button)$check = New-Object System.Windows.Forms.CheckBox@@ -51,7 +53,9 @@ $field.Location = New-Object System.Drawing.Point(20, 120)$field.Width = 300$field.Add_KeyDown({param($sender, $e)- if ($e.KeyCode -eq [System.Windows.Forms.Keys]::Enter) { $script:returns++; $e.SuppressKeyPress = $true }+ if ($e.KeyCode -eq [System.Windows.Forms.Keys]::Enter) {+ $script:returns++; $script:submittedText = $sender.Text; $e.SuppressKeyPress = $true+ }})$form.Controls.Add($field)@@ -80,6 +84,7 @@ $form.Controls.Add($list)function Invoke-BenchCommand($c) {if ($c.instance -ne $instance) { return }$script:count = 0; $script:returns = 0; $script:menuHits = 0+ $script:submittedText = $null; $script:clickedColor = $null$check.Checked = $false$edit.HideDropDown()$combo.SelectedIndex = 0@@ -110,6 +115,7 @@ $timer.Add_Tick({decoyText = $decoy.TextlogicalFocus = $(if ($form.ActiveControl -eq $field) { 'field' } elseif ($form.ActiveControl -eq $decoy) { 'decoy' } else { 'other' })count = $script:count; check = $check.Checked; text = $field.Text; returns = $script:returns+ submittedText = $script:submittedText; clickedColor = $script:clickedColorselStart = $field.SelectionStart; selLen = $field.SelectionLength; combo = [string]$combo.SelectedItemtop = $list.TopIndex; menuHits = $script:menuHits; dropOpen = ($edit.DropDown.Visible -or $combo.DroppedDown)activeForm = ($null -ne [System.Windows.Forms.Form]::ActiveForm); fieldFocused = $field.Focuseddiff --git a/config/benchmarks/computer-use/test_trial_contract.py b/config/benchmarks/computer-use/test_trial_contract.pyindex 096dca0b41e..0b8aa890bc9 100644--- a/config/benchmarks/computer-use/test_trial_contract.py+++ b/config/benchmarks/computer-use/test_trial_contract.py@@ -15,10 +15,14 @@ class TaskContractTests(unittest.TestCase):state = {**initial_state("targeted_type"), "instance": "new", "pid": 19, "ack": "current"}self.assertTrue(prepared(state, "targeted_type", "new", 19, "current"))for key, value in (("ack", "previous"), ("instance", "old"), ("pid", 20),- ("logicalFocus", "field"), ("selLen", 1), ("top", 1), ("dropOpen", True)):+ ("logicalFocus", "field"), ("selLen", 1), ("top", 1), ("dropOpen", True),+ ("submittedText", "old"), ("clickedColor", "Blue")):with self.subTest(key=key):self.assertFalse(prepared({**state, key: value}, "targeted_type", "new", 19, "current"))self.assertFalse(acknowledged(state, "new", 19, "previous"))+ for key in ("submittedText", "clickedColor"):+ without_payload = {name: value for name, value in state.items() if name != key}+ self.assertFalse(prepared(without_payload, "targeted_type", "new", 19, "current"))def test_selectors_never_substitute_another_control(self):selector = (r"button", r"\bincrement\b")@@ -42,12 +46,22 @@ class TaskContractTests(unittest.TestCase):final = {**initial_state(task), key: 1}if task == "choose_and_click":final["combo"] = "Blue"+ payload, correct, wrong = "clickedColor", "Blue", "Red"else:final["text"] = MARKER+ payload, correct, wrong = "submittedText", MARKER, ""+ final[payload] = correctself.assertTrue(completed(final, task))+ for value in (wrong, None):+ with self.subTest(task=task, payload=value):+ self.assertFalse(completed({**final, payload: value}, task))+ self.assertFalse(completed({name: value for name, value in final.items() if name != payload}, task))for value in (0, 2):with self.subTest(task=task, value=value):self.assertFalse(completed({**final, key: value}, task))+ for color in ("Red", "Blue"):+ self.assertTrue(completed({**initial_state("click_interference"), "count": 1,+ "clickedColor": color}, "click_interference"))def test_observed_completion_is_independent_of_api_errors(self):row = {"prepared": True, "task": "targeted_type", "api_errors": ["timeout"]}diff --git a/config/benchmarks/computer-use/trial_contract.py b/config/benchmarks/computer-use/trial_contract.pyindex d7172625ec5..0e65acc1ab1 100644--- a/config/benchmarks/computer-use/trial_contract.py+++ b/config/benchmarks/computer-use/trial_contract.py@@ -40,6 +40,7 @@ def select_unique(elements, selector):def initial_state(task):text = "hello" if task == "replace_submit" else ""return {"text": text, "decoyText": DECOY, "count": 0, "returns": 0, "combo": "Red", "top": 0,+ "submittedText": None, "clickedColor": None,"check": False, "menuHits": 0, "dropOpen": False, "selStart": len(text), "selLen": 0,"logicalFocus": "decoy" if task == "targeted_type" else "field"}@@ -51,21 +52,23 @@ def acknowledged(state, instance, pid, nonce):def prepared(state, task, instance, pid, nonce):return acknowledged(state, instance, pid, nonce) and all(- state.get(key) == value for key, value in initial_state(task).items())+ key in state and state[key] == value for key, value in initial_state(task).items())def completed(state, task):expected = initial_state(task)- for key in ("selStart", "selLen", "logicalFocus"):+ for key in ("selStart", "selLen", "logicalFocus", "submittedText", "clickedColor"):del expected[key]if task in ("targeted_type", "type_submit", "replace_submit"):expected["text"] = MARKERif task in ("type_submit", "replace_submit"):expected["returns"] = 1+ expected["submittedText"] = MARKERif task in ("choose_and_click", "click_interference"):expected["count"] = 1if task == "choose_and_click":expected["combo"] = "Blue"+ expected["clickedColor"] = "Blue"if task == "scroll_interference":del expected["top"]if not isinstance(state.get("top"), (int, float)) or state["top"] <= 0: