Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
fix(computer): report macOS actions that changed nothing, read back wrong or hung (#36020)
Kelvin Amoabac97e6f16bc92
diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swiftindex e0c3609fa52..5f341d0c16a 100644--- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift+++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift@@ -159,6 +159,12 @@ struct Snapshot {let elements: [Int: ElementRecord]let truncated: Boollet maxDepthReached: Bool+ /// On-screen windows of the app (menus and sheets included), so a new or closed window counts as a change.+ let appWindowIds: [CGWindowID]++ func showsSameState(as other: Snapshot) -> Bool {+ treeText == other.treeText && windowBounds == other.windowBounds && appWindowIds == other.appWindowIds+ }func withoutScreenshotPayload() -> Snapshot {Snapshot(@@ -176,7 +182,8 @@ struct Snapshot {screenshotEngine: nil,elements: elements,truncated: truncated,- maxDepthReached: maxDepthReached+ maxDepthReached: maxDepthReached,+ appWindowIds: appWindowIds)}}@@ -209,6 +216,11 @@ final class Provider {private var snapshots: [String: Snapshot] = [:]private var snapshotEntries: [CachedSnapshotEntry] = []+ init() {+ // Why: the system-wide element sets the wait for every AX call in this process, so a hung app costs one bounded timeout whatever the macOS default.+ _ = AXUIElementSetMessagingTimeout(AXUIElementCreateSystemWide(), AccessibilityMessaging.timeoutSeconds)+ }+func handle(method: String, params: [String: JSONValue]) throws -> Any {switch method {case "handshake":@@ -246,8 +258,21 @@ final class Provider {var action = try runAction()// Why: only --restore-window activates the app or posts to the HID tap; every other route leaves the user's desktop alone.action["delivery"] = isForegroundRequest(params) ? "foreground" : "background"+ // Why: the action's currentSnapshot just cached this target's tree from before the action.+ let before = action["path"] as? String == "accessibility" && action["verification"] == nil+ ? try? cachedSnapshot(params: params)+ : nildo {- return try renderActionResult(action: action, snapshot: observe(params: params))+ let after = try observe(params: params)+ guard let before, after.showsSameState(as: before) else {+ return renderActionResult(action: action, snapshot: after)+ }+ if let changed = try observeLateChange(before: before, params: params) {+ return renderActionResult(action: action, snapshot: changed)+ }+ // Why: an AX call can succeed and do nothing, as AXPress does on many web controls (#31758).+ action["verification"] = unverifiedAction(reason: "suspected_noop")+ return renderActionResult(action: action, snapshot: after)} catch let error as ProviderError where (error.code == "window_not_found" || error.code == "window_stale") && hasRequestedWindowSelector(params) {var fallbackParams = paramsfallbackParams.removeValue(forKey: "windowId")@@ -259,6 +284,23 @@ final class Provider {}}+ /// Re-reads the target for about a second, since apps often apply an action on a later run-loop turn.+ /// Returns the observation to report once something changed, or nil if nothing did.+ private func observeLateChange(before: Snapshot, params: [String: JSONValue]) throws -> Snapshot? {+ let settleParams = params.merging(["restoreWindow": .bool(false)]) { _, replacement in replacement }+ let quietParams = settleParams.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }+ let deadline = Date().addingTimeInterval(1)+ while Date() < deadline {+ Thread.sleep(forTimeInterval: 0.2)+ let polled = try observe(params: quietParams)+ if !polled.showsSameState(as: before) {+ if params["noScreenshot"]?.bool == true { return polled }+ return try observe(params: settleParams)+ }+ }+ return nil+ }+private func observe(params: [String: JSONValue]) throws -> Snapshot {let query = try requiredString(params, "app")let windowId = try requestedWindowId(params)@@ -614,6 +656,12 @@ final class Provider {)}let appElement = AXUIElementCreateApplication(app.pid)+ var role: CFTypeRef?+ let probeStart = CFAbsoluteTimeGetCurrent()+ // Why: a hung app costs one timeout per AX read; walking it anyway held the request lock for every agent until the 60 s RPC timeout.+ if AccessibilityMessaging.timedOut(AXUIElementCopyAttributeValue(appElement, kAXRoleAttribute as CFString, &role), since: probeStart) {+ throw AccessibilityMessaging.appNotResponding(app.name)+ }enableManualAccessibilityIfNeeded(appElement, app: app)let windowCandidates = WindowCapture.candidates(pid: app.pid)let focused = try focusedWindow(@@ -643,6 +691,9 @@ final class Provider {compactBrowserTabs: app.isKnownBrowser)renderer.render(window)+ if renderer.appStoppedResponding {+ throw AccessibilityMessaging.appNotResponding(app.name)+ }let screenshot = includeScreenshot ? capture.screenshotPayload() : nillet screenshotStatus: ScreenshotStatus = if screenshot != nil {.captured@@ -668,7 +719,8 @@ final class Provider {screenshotEngine: capture.image?.engine,elements: renderer.records,truncated: renderer.truncated,- maxDepthReached: renderer.maxDepthReached+ maxDepthReached: renderer.maxDepthReached,+ appWindowIds: windowCandidates.map(\.windowId).sorted())}@@ -775,10 +827,10 @@ final class Provider {private func performClickAction(record: ElementRecord, mouseButton: MouseButtonSelection) throws -> String? {if mouseButton == .right {- return performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil+ return try performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil}for action in ["AXPress", "AXConfirm", "AXOpen"] {- if performAction(record.element, action) {+ if try performAction(record.element, action) {return action}}@@ -793,7 +845,7 @@ final class Provider {guard let action else {throw ProviderError.coded("action_not_supported", "'\(requested)' is not a valid secondary action for element \(record.index)")}- guard performAction(record.element, action) else {+ guard try performAction(record.element, action) else {throw ProviderError.coded("accessibility_error", "AXUIElementPerformAction(\(action)) failed")}return actionMetadata(path: "accessibility", actionName: action)@@ -809,6 +861,7 @@ final class Provider {let current = rawAttributeValue(record.element, kAXValueAttribute as String)let coercion = AttributeValueCoercion(existingValue: current, requested: expected)let result: AXError+ let writeStart = CFAbsoluteTimeGetCurrent()switch coercion.writeValue {case .string:result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, expected as CFString)@@ -819,6 +872,9 @@ final class Provider {case let .boolean(value):result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, value ? kCFBooleanTrue : kCFBooleanFalse)}+ if AccessibilityMessaging.timedOut(result, since: writeStart) {+ throw AccessibilityMessaging.unanswered("the value write")+ }guard result == .success else {throw ProviderError.coded("accessibility_error", "AXUIElementSetAttributeValue failed with \(result.rawValue)")}@@ -837,7 +893,7 @@ final class Provider {private func typeText(params: [String: JSONValue]) throws -> [String: Any] {let snapshot = try currentKeyboardSnapshot(params: params)let text = try requiredString(params, "text")- if let focused = focusedRecord(snapshot), let verification = TextInput.replaceSelection(focused.element, with: text) {+ if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) {return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)}try deliverKeys(try withTransportErrors { try BackgroundInputTransport.textEvents(text) }, snapshot: snapshot, params: params)@@ -879,7 +935,7 @@ final class Provider {private func pasteText(params: [String: JSONValue]) throws -> [String: Any] {let snapshot = try currentKeyboardSnapshot(params: params)let text = try requiredString(params, "text")- if let focused = focusedRecord(snapshot), let verification = TextInput.replaceSelection(focused.element, with: text) {+ if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) {return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)}let pasteKeys = try Input.keyEvents("cmd+v")@@ -903,7 +959,7 @@ final class Provider {if pages.rounded() == pages, let pageCount = boundedInteger(pages, as: Int.self),record.actions.contains(action) {for _ in 0..<max(1, pageCount) {- _ = performAction(record.element, action)+ _ = try performAction(record.element, action)}return actionMetadata(path: "accessibility", actionName: action)}@@ -1166,17 +1222,10 @@ private func focusedSystemWindow(systemWide: AXUIElement, app: AppDescriptor) ->}private func isTargetWindowFocused(_ snapshot: Snapshot) -> Bool {- guard let focusedWindow = focusedSystemWindow(systemWide: AXUIElementCreateSystemWide(), app: snapshot.app) else {- return false- }- if windowNumber(focusedWindow) == snapshot.windowId {- return true- }- guard let frame = absoluteFrame(focusedWindow) else {- return false- }- let intersection = frame.intersection(snapshot.windowBounds)- return !intersection.isNull && intersection.area >= min(frame.area, snapshot.windowBounds.area) * 0.75+ focusedSyntheticClickRecipient(+ targetPID: snapshot.app.pid,+ targetCandidates: { WindowCapture.candidates(pid: snapshot.app.pid) }+ ) == .focused(syntheticClickRecipient(pid: snapshot.app.pid, windowId: snapshot.windowId))}private enum AXElementProbe {@@ -1260,19 +1309,8 @@ private func focusedSyntheticClickRecipient(return .unavailable}- if let windowId = windowNumber(focusedWindow) {- return .focused(syntheticClickRecipient(pid: ownerPID, windowId: windowId))- }- guard ownerPID == targetPID else { return .unavailable }- guard let frame = absoluteFrame(focusedWindow),- let candidate = SyntheticMouseClickDelivery.uniqueWindowCandidate(- from: targetCandidates(),- matching: {- windowFramesMatch($0.bounds, frame)- }- )- else { return .unavailable }- return .focused(syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId))+ guard pidAttribute(focusedWindow) == ownerPID else { return .unavailable }+ return pointerRecipient(of: focusedWindow, targetPID: targetPID, targetCandidates: targetCandidates)}private func hitTestSyntheticClickRecipient(@@ -1308,13 +1346,13 @@ private func pointerRecipient(guard let ownerPID = pidAttribute(element), let window = containingWindow(element) else {return .unavailable}- var numberValue: CFTypeRef?- let numberStatus = AXUIElementCopyAttributeValue(window, "AXWindowNumber" as CFString, &numberValue)- if numberStatus == .success {- guard let number = numberValue as? NSNumber else { return .unavailable }- return .focused(syntheticClickRecipient(pid: ownerPID, windowId: CGWindowID(number.uint32Value)))+ do {+ if let windowID = try AccessibilityWindowIdentity.windowID(of: window) {+ return .focused(syntheticClickRecipient(pid: ownerPID, windowId: windowID))+ }+ } catch {+ return .unavailable}- guard numberStatus == .attributeUnsupported || numberStatus == .noValue else { return .unavailable }guard ownerPID == targetPID, let frame = absoluteFrame(window),let windows = copyArray(AXUIElementCreateApplication(ownerPID), kAXWindowsAttribute as String)else { return .unavailable }@@ -1723,10 +1761,7 @@ private func numberAttribute(_ element: AXUIElement, _ attribute: String) -> NSN}private func windowNumber(_ element: AXUIElement) -> CGWindowID? {- guard let number = numberAttribute(element, "AXWindowNumber") else {- return nil- }- return CGWindowID(number.uint32Value)+ try? AccessibilityWindowIdentity.windowID(of: element)}private func absoluteFrame(_ element: AXUIElement) -> CGRect? {@@ -1763,9 +1798,44 @@ private func actions(_ element: AXUIElement) -> [String] {return value as? [String] ?? []}-private func performAction(_ element: AXUIElement, _ action: String) -> Bool {- actions(element).contains(where: { $0.caseInsensitiveCompare(action) == .orderedSame }) &&- AXUIElementPerformAction(element, action as CFString) == .success+private enum AccessibilityMessaging {+ static let timeoutSeconds: Float = 3++ /// Why: AX also returns cannotComplete at once for some attributes; only a call that waited out the timeout means the app is not answering.+ static func timedOut(_ result: AXError, since start: CFAbsoluteTime) -> Bool {+ result == .cannotComplete && CFAbsoluteTimeGetCurrent() - start >= Double(timeoutSeconds) * 0.9+ }++ static func appNotResponding(_ appName: String?) -> ProviderError {+ .coded("action_timeout", "\(appName.map { "'\($0)'" } ?? "The app") is not answering accessibility requests (no reply within \(timeoutSeconds) s); it may be busy or hung. Wait until it responds, then retry.")+ }++ /// The request reached the app, which went quiet while handling it, so the effect may have happened.+ static func unanswered(_ request: String) -> ProviderError {+ .coded("action_timeout", "the app did not answer \(request) within \(timeoutSeconds) s; it may be busy or showing a dialog, and the action may already have taken effect. Run get-app-state to check before retrying.")+ }+}++private func performAction(_ element: AXUIElement, _ action: String) throws -> Bool {+ var names: CFArray?+ let start = CFAbsoluteTimeGetCurrent()+ let namesResult = AXUIElementCopyActionNames(element, &names)+ if AccessibilityMessaging.timedOut(namesResult, since: start) {+ throw AccessibilityMessaging.appNotResponding(nil)+ }+ guard namesResult == .success,+ let names = names as? [String],+ names.contains(where: { $0.caseInsensitiveCompare(action) == .orderedSame })+ else {+ return false+ }+ let pressStart = CFAbsoluteTimeGetCurrent()+ let result = AXUIElementPerformAction(element, action as CFString)+ // Why: falling through to another route after a delivered but unanswered press ran the action twice.+ if AccessibilityMessaging.timedOut(result, since: pressStart) {+ throw AccessibilityMessaging.unanswered(action)+ }+ return result == .success}private func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool {@@ -1889,6 +1959,8 @@ private final class TreeRenderer {static let maxNodes = 1200static let maxDepth = 64+ var appStoppedResponding: Bool { reader.appStoppedResponding }+init(windowBounds: CGRect, focused: AXUIElement?, compactBrowserTabs: Bool) {self.windowBounds = windowBoundsself.focused = focused@@ -1896,6 +1968,7 @@ private final class TreeRenderer {}func render(_ element: AXUIElement, depth: Int = 0, ancestors: [AXUIElement] = []) {+ guard !reader.appStoppedResponding else { return }guard nextIndex < Self.maxNodes else {truncated = truereturn@@ -2096,7 +2169,7 @@ private final class AXSnapshotReader {return actions}var value: CFArray?- let actions = AXUIElementCopyActionNames(element, &value) == .success ? value as? [String] ?? [] : []+ let actions = answered { AXUIElementCopyActionNames(element, &value) } == .success ? value as? [String] ?? [] : []cache.actions = actionsreturn actions}@@ -2107,11 +2180,24 @@ private final class AXSnapshotReader {return cached}var settable = DarwinBoolean(false)- let value = AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success && settable.boolValue+ let value = answered { AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) } == .success && settable.boolValuecache.settable[attribute] = valuereturn value}+ /// Set once the app waits out a messaging timeout; later reads return nothing instead of each waiting it out again.+ private(set) var appStoppedResponding = false++ private func answered(_ call: () -> AXError) -> AXError? {+ guard !appStoppedResponding else { return nil }+ let start = CFAbsoluteTimeGetCurrent()+ let result = call()+ if AccessibilityMessaging.timedOut(result, since: start) {+ appStoppedResponding = true+ }+ return result+ }+func frame(_ element: AXUIElement, windowBounds: CGRect) -> CGRect? {guard let absolute = absoluteFrame(element) else { return nil }return CGRect(@@ -2238,7 +2324,7 @@ private final class AXSnapshotReader {return nil}var value: CFTypeRef?- guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, let value else {+ guard answered({ AXUIElementCopyAttributeValue(element, attribute as CFString, &value) }) == .success, let value else {cache.attributes[attribute] = .missingreturn nil}@@ -2252,7 +2338,7 @@ private final class AXSnapshotReader {}cache.loadedAttributeNames = truevar value: CFArray?- guard AXUIElementCopyAttributeNames(cache.element, &value) == .success, let attributes = value as? [String] else {+ guard answered({ AXUIElementCopyAttributeNames(cache.element, &value) }) == .success, let attributes = value as? [String] else {return nil}cache.advertisedAttributes = Set(attributes)@@ -2813,7 +2899,7 @@ private enum Input {}private enum TextInput {- static func replaceSelection(_ element: AXUIElement, with text: String) -> [String: Any]? {+ static func replaceSelection(_ element: AXUIElement, with text: String) throws -> [String: Any]? {guard isSettable(element, kAXValueAttribute as String),let current = rawStringAttribute(element, kAXValueAttribute as String)else {@@ -2825,7 +2911,13 @@ private enum TextInput {let start = String.Index(utf16Offset: startOffset, in: current)let end = String.Index(utf16Offset: endOffset, in: current)let next = String(current[..<start]) + text + String(current[end...])- guard AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, next as CFString) == .success else {+ let writeStart = CFAbsoluteTimeGetCurrent()+ let result = AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, next as CFString)+ // Why: typing the text as keys after an unanswered write could enter it twice.+ if AccessibilityMessaging.timedOut(result, since: writeStart) {+ throw AccessibilityMessaging.unanswered("the text write")+ }+ guard result == .success else {return nil}setSelectedTextRange(element, CFRange(location: startOffset + text.utf16.count, length: 0))diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/AccessibilityWindowIdentity.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/AccessibilityWindowIdentity.swiftnew file mode 100644index 00000000000..444bd527bcc--- /dev/null+++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/AccessibilityWindowIdentity.swift@@ -0,0 +1,45 @@+import ApplicationServices+import CoreGraphics+import Darwin+import Foundation++public enum AccessibilityWindowIdentity {+ private enum Failure: Error {+ case unreadable+ }++ /// Nil permits geometry fallback; unreadable identity must refuse input.+ public static func windowID(of element: AXUIElement) throws -> CGWindowID? {+ var nativeAbsent = true+ if let getWindow {+ var windowID: CGWindowID = 0+ let status = getWindow(element, &windowID)+ if status == .success, windowID != 0 { return windowID }+ nativeAbsent = status == .attributeUnsupported || status == .noValue+ }++ var value: CFTypeRef?+ switch AXUIElementCopyAttributeValue(element, "AXWindowNumber" as CFString, &value) {+ case .success:+ guard let number = value as? NSNumber,+ CFGetTypeID(number) != CFBooleanGetTypeID(),+ let windowID = CGWindowID(exactly: number.doubleValue), windowID != 0+ else { throw Failure.unreadable }+ return windowID+ case .attributeUnsupported, .noValue:+ guard nativeAbsent else { throw Failure.unreadable }+ return nil+ default:+ throw Failure.unreadable+ }+ }++ private typealias GetWindow = @convention(c) (AXUIElement, UnsafeMutablePointer<CGWindowID>) -> AXError++ // AppKit and WebKit windows often omit AXWindowNumber.+ private static let getWindow: GetWindow? = {+ guard let symbol = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "_AXUIElementGetWindow") else { return nil }+ // SAFETY: HIServices exports AXError _AXUIElementGetWindow(AXUIElementRef, CGWindowID *).+ return unsafeBitCast(symbol, to: GetWindow.self)+ }()+}diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/BackgroundInputTransport.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/BackgroundInputTransport.swiftindex 5c49f131599..c66fe0402ca 100644--- a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/BackgroundInputTransport.swift+++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/BackgroundInputTransport.swift@@ -210,9 +210,8 @@ public enum BackgroundInputTransport {else { throw Failure.targetWindowNotKey }// SAFETY: the CFTypeID check above proves focused is an AXUIElement.let window = focused as! AXUIElement- var number: CFTypeRef?- guard AXUIElementCopyAttributeValue(window, "AXWindowNumber" as CFString, &number) == .success,- let number = number as? NSNumber, number.uint32Value == target.windowID+ guard let windowID = try? AccessibilityWindowIdentity.windowID(of: window),+ windowID == target.windowIDelse { throw Failure.targetWindowNotKey }try body()returndiff --git a/skill-guides/computer-use.md b/skill-guides/computer-use.mdindex af8c7bead13..7ec6d2d92b0 100644--- a/skill-guides/computer-use.md+++ b/skill-guides/computer-use.md@@ -87,6 +87,7 @@ printf '%s' "$TEXT" | ORCA computer set-value --app <app> --element-index <index- An action's verification is separate from whether its provider call succeeded:- `verified` means the changed value was read back.- `unverified (accessibility action unasserted)` means the accessibility call succeeded but no post-state assertion was made.+ - `unverified (suspected noop)` means the accessibility call succeeded but the window's tree and the app's windows were unchanged about a second later. Web and Electron controls often ignore it. Do not repeat the same element click; if the effect you expect would show in the window, retry once with `click --x --y` at the control from a fresh screenshot. If the effect lives elsewhere (clipboard, another app), check there first.- `unverified (synthetic input)` means input was fired into the void and is unverifiable.- Missing verification metadata is unverified, including responses from older runtimes.- Never report an unverified action as success. If it could have sent, submitted, bought, or deleted something, say the effect is unproven.@@ -164,6 +165,7 @@ Slack: the accessibility tree may be shallow while the screenshot contains usefu- `element_not_clickable`: the element has no actionable frame; use a parent/child element with a frame or choose window-local coordinates from the latest screenshot.- `invalid_argument`: fix the command flags; do not retry the same command unchanged.- `action_timeout`: inspect current state before retrying, then use a simpler semantic action or `--no-screenshot` if observation is slow.+- `verification_failed`: the write was read back and the element holds a different value, so the action failed. Run `get-app-state` to see which field changed. For editable text, click the intended field and confirm focus, select all (`cmd+a` on macOS, `ctrl+a` on Windows/Linux), then use `type-text` and inspect the resulting value instead of repeating `set-value`.- `screenshot_failed`: use `--no-screenshot` if tree state is enough; if the message names Screen Recording or screenshots permission, run `ORCA computer permissions --id screenshots --json`.- `accessibility_error`: run `ORCA computer capabilities --json`; if the message names Accessibility permission, run `ORCA computer permissions --id accessibility --json`.- Empty tree or no screenshot: app may have no visible window, be minimized, or need permissions.diff --git a/src/cli/bundled-skill-guides.ts b/src/cli/bundled-skill-guides.tsindex 5f83dffa691..3c38e17802b 100644--- a/src/cli/bundled-skill-guides.ts+++ b/src/cli/bundled-skill-guides.ts@@ -104,6 +104,7 @@ const COMPUTER_USE_MARKDOWN = ["- An action's verification is separate from whether its provider call succeeded:"," - `verified` means the changed value was read back."," - `unverified (accessibility action unasserted)` means the accessibility call succeeded but no post-state assertion was made.",+ " - `unverified (suspected noop)` means the accessibility call succeeded but the window's tree and the app's windows were unchanged about a second later. Web and Electron controls often ignore it. Do not repeat the same element click; if the effect you expect would show in the window, retry once with `click --x --y` at the control from a fresh screenshot. If the effect lives elsewhere (clipboard, another app), check there first."," - `unverified (synthetic input)` means input was fired into the void and is unverifiable."," - Missing verification metadata is unverified, including responses from older runtimes."," - Never report an unverified action as success. If it could have sent, submitted, bought, or deleted something, say the effect is unproven.",@@ -181,6 +182,7 @@ const COMPUTER_USE_MARKDOWN = ["- `element_not_clickable`: the element has no actionable frame; use a parent/child element with a frame or choose window-local coordinates from the latest screenshot.","- `invalid_argument`: fix the command flags; do not retry the same command unchanged.","- `action_timeout`: inspect current state before retrying, then use a simpler semantic action or `--no-screenshot` if observation is slow.",+ "- `verification_failed`: the write was read back and the element holds a different value, so the action failed. Run `get-app-state` to see which field changed. For editable text, click the intended field and confirm focus, select all (`cmd+a` on macOS, `ctrl+a` on Windows/Linux), then use `type-text` and inspect the resulting value instead of repeating `set-value`.","- `screenshot_failed`: use `--no-screenshot` if tree state is enough; if the message names Screen Recording or screenshots permission, run `ORCA computer permissions --id screenshots --json`.","- `accessibility_error`: run `ORCA computer capabilities --json`; if the message names Accessibility permission, run `ORCA computer permissions --id accessibility --json`.","- Empty tree or no screenshot: app may have no visible window, be minimized, or need permissions.",diff --git a/src/main/computer/computer-action-verification-normalization.ts b/src/main/computer/computer-action-verification-normalization.tsindex a1a510a5282..f3b896ec95a 100644--- a/src/main/computer/computer-action-verification-normalization.ts+++ b/src/main/computer/computer-action-verification-normalization.ts@@ -1,4 +1,5 @@import type { ComputerActionResult } from '../../shared/runtime-types'+import { RuntimeClientError } from './runtime-client-error'export function normalizeComputerActionResult(result: ComputerActionResult): ComputerActionResult {const action = result.action@@ -24,3 +25,21 @@ export function normalizeComputerActionResult(result: ComputerActionResult): Com}}}++// Why: a read-back showing another value proves the write failed; returned as ok, callers that check only the exit status took it as success (#33511).+export function rejectContradictedComputerAction(+ result: ComputerActionResult+): ComputerActionResult {+ const verification = result.action?.verification+ if (verification?.state !== 'unverified' || verification.reason !== 'value_mismatch') {+ return result+ }+ const actual =+ typeof verification.actualPreview === 'string'+ ? JSON.stringify(verification.actualPreview)+ : 'a different value'+ throw new RuntimeClientError(+ 'verification_failed',+ `set-value did not take effect: after the write the element reads ${actual}, not the requested value`+ )+}diff --git a/src/main/computer/sidecar-client.test.ts b/src/main/computer/sidecar-client.test.tsindex f6b01e82f21..9bb2d57a101 100644--- a/src/main/computer/sidecar-client.test.ts+++ b/src/main/computer/sidecar-client.test.ts@@ -285,6 +285,51 @@ describe('computer sidecar client', () => {})})+ it('fails set-value when the provider read the write back as a different value', async () => {+ const call = callComputerSidecarAction('setValue', {+ app: 'Safari',+ elementIndex: 15,+ value: 'pf1300-awllch'+ })+ const child = children[0]!+ const request = child.sent[0]!++ child.emit('message', {+ id: request.id,+ ok: true,+ result: {+ snapshot: {+ id: 'snap-1',+ app: { name: 'Safari', bundleId: 'com.apple.Safari', pid: 100 },+ window: { title: 'Form', id: 63, width: 800, height: 600 },+ coordinateSpace: 'window',+ treeText: 'tree',+ elementCount: 16,+ focusedElementId: null+ },+ screenshot: null,+ screenshotStatus: { state: 'skipped', reason: 'no_screenshot_flag' },+ action: {+ path: 'accessibility',+ actionName: 'AXSetValue',+ fallbackReason: null,+ targetWindowId: 63,+ verification: {+ state: 'unverified',+ reason: 'value_mismatch',+ expected: 'pf1300-awllch',+ actualPreview: 'orig-objective-1300'+ }+ }+ }+ })++ await expect(call).rejects.toMatchObject({+ code: 'verification_failed',+ message: expect.stringContaining('"orig-objective-1300"')+ })+ })+it('rejects oversized pasteText payloads before forking the sidecar', async () => {const secret = 'sidecar-secret-token'diff --git a/src/main/computer/sidecar-client.ts b/src/main/computer/sidecar-client.tsindex 57fd6689857..4a86a108632 100644--- a/src/main/computer/sidecar-client.ts+++ b/src/main/computer/sidecar-client.ts@@ -7,7 +7,10 @@ import type {ComputerProviderCapabilities,ComputerSnapshotResult} from '../../shared/runtime-types'-import { normalizeComputerActionResult } from './computer-action-verification-normalization'+import {+ normalizeComputerActionResult,+ rejectContradictedComputerAction+} from './computer-action-verification-normalization'import { isComputerSidecarDiagnostic, logComputerDiagnostic } from './computer-sidecar-diagnostics'import { validateComputerSidecarPasteText } from './computer-sidecar-paste-validation'import { RuntimeClientError } from './runtime-client-error'@@ -85,8 +88,10 @@ export async function callComputerSidecarAction(if (validation) {await validation}- return normalizeComputerActionResult(- (await getComputerSidecar().call(method, params)) as ComputerActionResult+ return rejectContradictedComputerAction(+ normalizeComputerActionResult(+ (await getComputerSidecar().call(method, params)) as ComputerActionResult+ ))}diff --git a/src/shared/computer-use-error-recovery.ts b/src/shared/computer-use-error-recovery.tsindex 86b7c18d262..43721c5d28b 100644--- a/src/shared/computer-use-error-recovery.ts+++ b/src/shared/computer-use-error-recovery.ts@@ -90,6 +90,11 @@ export function computerUseErrorRecoveryData('Run `orca computer get-app-state --app <app> --json` before retrying so you know whether the UI changed.','Retry with a simpler semantic action or `--no-screenshot` if observation is slow; do not repeat the same timed-out action blindly.')+ case 'verification_failed':+ return recoverWith(+ 'Run `orca computer get-app-state --app <app> --json` and check which field holds the value; another field may have changed instead.',+ 'Do not retry the same `set-value` unchanged. For editable text, click the field and confirm focus, select all (`cmd+a` on macOS, `ctrl+a` on Windows/Linux), then use `type-text` and inspect the resulting value.'+ )case 'screenshot_failed':return recoverWith('If the accessibility tree is sufficient, rerun the command with `--no-screenshot` instead of retrying the same screenshot capture.',diff --git a/src/shared/computer-use-runtime-types.ts b/src/shared/computer-use-runtime-types.tsindex e2e7cbebc4a..ff077347ae4 100644--- a/src/shared/computer-use-runtime-types.ts+++ b/src/shared/computer-use-runtime-types.ts@@ -14,6 +14,7 @@ export const COMPUTER_ERROR_CODES = {value_not_settable: 'value_not_settable',invalid_argument: 'invalid_argument',action_timeout: 'action_timeout',+ verification_failed: 'verification_failed',screenshot_failed: 'screenshot_failed',accessibility_error: 'accessibility_error'} as const@@ -112,6 +113,8 @@ export type ComputerActionVerification =| 'readback_unsupported'| 'window_changed'| 'value_mismatch'+ // The accessibility call succeeded but the window and the app's windows looked the same about a second later.+ | 'suspected_noop'expected?: string | nullactualPreview?: string | null}