👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
feat(click): 添加可配置点击实现
tisfeng0144360d8e34
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swiftindex 04cdd75..f5a21d0 100644--- a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift@@ -116,6 +116,7 @@ enum MacOSAppAgentProxy {let response = try client.request(["kind": "mcp","line": line,+ "environment": proxiedEnvironment(),])if let responseLine = response["response"] as? String {@@ -333,7 +334,11 @@ private final class AppAgentConnection: @unchecked Sendable {return ["ok": true]case "mcp":let line = request["line"] as? String ?? ""- if let response = server.handle(line: line) {+ let environment = request["environment"] as? [String: String] ?? [:]+ let response = AppAgentEnvironment.withOverrides(environment) {+ server.handle(line: line)+ }+ if let response {return ["response": response]}return ["response": NSNull()]diff --git a/apps/OpenComputerUseLinux/main.go b/apps/OpenComputerUseLinux/main.goindex 8888627..c0bc934 100644--- a/apps/OpenComputerUseLinux/main.go+++ b/apps/OpenComputerUseLinux/main.go@@ -22,6 +22,8 @@ import (var version = "0.2.1"+var clickMethodValues = []string{"auto", "accessibility", "app_post", "global"}+//go:embed runtime.pyvar linuxRuntimeScript string@@ -144,6 +146,7 @@ type linuxRequest struct {ToY *float64 `json:"to_y,omitempty"`ClickCount int `json:"click_count,omitempty"`MouseButton string `json:"mouse_button,omitempty"`+ ClickMethod string `json:"click_method,omitempty"`Action string `json:"action,omitempty"`Direction string `json:"direction,omitempty"`Pages float64 `json:"pages,omitempty"`@@ -202,6 +205,10 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {}return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth)case "click":+ clickMethod, err := parseClickMethod(optionalString(args, "click_method"))+ if err != nil {+ return textResult(err.Error(), true)+ }return s.click(requiredString(args, "app"),optionalElementIndex(args),@@ -209,6 +216,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {optionalFloat(args, "y"),intValue(optionalFloat(args, "click_count"), 1),defaultString(optionalString(args, "mouse_button"), "left"),+ clickMethod,)case "perform_secondary_action":return s.performSecondaryAction(@@ -277,13 +285,22 @@ func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, mareturn snapshot.result()}-func (s *service) click(app, elementIndex string, x, y *float64, clickCount int, mouseButton string) toolCallResult {+func (s *service) click(app, elementIndex string, x, y *float64, clickCount int, mouseButton, clickMethod string) toolCallResult {if app == "" {return textResult("Missing required argument: app", true)}if elementIndex == "" && (x == nil || y == nil) {return textResult("click requires either element_index or x/y", true)}+ if clickMethod == "accessibility" && elementIndex == "" {+ return textResult("click_method 'accessibility' requires element_index", true)+ }+ if clickMethod == "app_post" {+ return textResult("click_method 'app_post' is not supported on Linux", true)+ }+ if clickMethod == "global" && !globalPointerFallbacksEnabled() {+ return textResult("click_method 'global' requires OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1 because it may move the system pointer and change foreground focus", true)+ }snapshot := s.currentSnapshot(app)if snapshot == nil {return textResult("No app state is available for "+app+". Run get_app_state before action tools.", true)@@ -295,6 +312,7 @@ func (s *service) click(app, elementIndex string, x, y *float64, clickCount int,Y: y,ClickCount: clickCount,MouseButton: mouseButton,+ ClickMethod: clickMethod,WindowBounds: snapshot.WindowBounds,}if elementIndex != "" {@@ -1093,6 +1111,28 @@ func defaultString(value, fallback string) string {return value}+func parseClickMethod(value string) (string, error) {+ normalized := strings.ToLower(strings.TrimSpace(value))+ if normalized == "" {+ return "auto", nil+ }+ for _, candidate := range clickMethodValues {+ if normalized == candidate {+ return normalized, nil+ }+ }+ return "", fmt.Errorf("Invalid click_method %q. Expected one of: %s", value, strings.Join(clickMethodValues, ", "))+}++func globalPointerFallbacksEnabled() bool {+ switch strings.ToLower(strings.TrimSpace(os.Getenv("OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS"))) {+ case "1", "true", "yes", "on":+ return true+ default:+ return false+ }+}+func toolDefinitions() []toolDefinition {return []toolDefinition{{@@ -1106,6 +1146,7 @@ func toolDefinitions() []toolDefinition {"y": numberProperty("Y coordinate in screenshot pixel coordinates"),"click_count": integerProperty("Number of clicks. Defaults to 1"),"mouse_button": enumStringProperty("Mouse button to click. Defaults to left.", []string{"left", "right", "middle"}),+ "click_method": enumStringProperty("Click implementation: auto (default), accessibility, app_post, or global. Accessibility requires element_index. Linux supports global AT-SPI mouse synthesis and does not currently support app_post.", clickMethodValues),}, []string{"app"}),},{diff --git a/apps/OpenComputerUseLinux/main_test.go b/apps/OpenComputerUseLinux/main_test.goindex f4124cb..d3ccd78 100644--- a/apps/OpenComputerUseLinux/main_test.go+++ b/apps/OpenComputerUseLinux/main_test.go@@ -16,6 +16,59 @@ func TestToolDefinitionCount(t *testing.T) {}}+func TestClickMethodSchemaAndParser(t *testing.T) {+ tool := findToolDefinition(t, "click")+ properties := tool.InputSchema["properties"].(map[string]any)+ method := properties["click_method"].(map[string]any)+ values := method["enum"].([]string)+ if strings.Join(values, ",") != "auto,accessibility,app_post,global" {+ t.Fatalf("click_method enum = %#v", values)+ }++ for input, want := range map[string]string{+ "": "auto",+ " AUTO ": "auto",+ "Accessibility": "accessibility",+ "app_post": "app_post",+ "GLOBAL": "global",+ } {+ got, err := parseClickMethod(input)+ if err != nil {+ t.Fatalf("parseClickMethod(%q): %v", input, err)+ }+ if got != want {+ t.Fatalf("parseClickMethod(%q) = %q, want %q", input, got, want)+ }+ }++ for _, input := range []string{"physical", "targeted"} {+ if _, err := parseClickMethod(input); err == nil || !strings.Contains(err.Error(), "Expected one of: auto, accessibility, app_post, global") {+ t.Fatalf("parseClickMethod(%s) error = %v", input, err)+ }+ }+}++func TestLinuxClickMethodSafetyAndPlatformSupport(t *testing.T) {+ x, y := 10.0, 20.0+ service := newService()++ result := service.click("Text Editor", "", &x, &y, 1, "left", "app_post")+ if !result.IsError || result.Content[0].Text != "click_method 'app_post' is not supported on Linux" {+ t.Fatalf("app_post click result = %#v", result)+ }++ t.Setenv("OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS", "")+ result = service.click("Text Editor", "", &x, &y, 1, "left", "global")+ if !result.IsError || !strings.Contains(result.Content[0].Text, "requires OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1") {+ t.Fatalf("unauthorized global click result = %#v", result)+ }++ t.Setenv("OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS", "yes")+ if !globalPointerFallbacksEnabled() {+ t.Fatal("global pointer authorization should accept yes")+ }+}+func TestGetAppStateSchemaIncludesTextLimit(t *testing.T) {tool := findToolDefinition(t, "get_app_state")properties := tool.InputSchema["properties"].(map[string]any)diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.pyindex 493e9af..798bd8e 100644--- a/apps/OpenComputerUseLinux/runtime.py+++ b/apps/OpenComputerUseLinux/runtime.py@@ -786,10 +786,21 @@ def perform_operation(operation):element = find_element(app, element_record)if tool == "click":- handled = False- if element is not None and operation.get("mouse_button", "left") == "left":- handled = do_action_by_index(element, preferred_action_index(element))- if not handled:+ click_method = (operation.get("click_method") or "auto").lower()+ if click_method == "accessibility":+ if element is None:+ raise RuntimeError("click_method 'accessibility' requires element_index")+ if operation.get("mouse_button", "left") != "left":+ raise RuntimeError(+ "click_method 'accessibility' only supports mouse_button 'left'"+ )+ if not do_action_by_index(element, preferred_action_index(element)):+ raise RuntimeError(+ "click_method 'accessibility' could not click the requested element"+ )+ elif click_method == "app_post":+ raise RuntimeError("click_method 'app_post' is not supported on Linux")+ elif click_method == "global":x, y = screen_point(bounds,element_record,@@ -799,6 +810,25 @@ def perform_operation(operation):send_mouse_click(x, y, operation.get("mouse_button", "left"), operation.get("click_count", 1))+ elif click_method == "auto":+ handled = False+ if element is not None and operation.get("mouse_button", "left") == "left":+ handled = do_action_by_index(element, preferred_action_index(element))+ if not handled:+ x, y = screen_point(+ bounds,+ element_record,+ operation.get("x"),+ operation.get("y"),+ )+ send_mouse_click(+ x,+ y,+ operation.get("mouse_button", "left"),+ operation.get("click_count", 1),+ )+ else:+ raise RuntimeError("Invalid click_method '{}'".format(click_method))elif tool == "perform_secondary_action":invoke_secondary_action(element, operation.get("action", ""))elif tool == "scroll":diff --git a/apps/OpenComputerUseWindows/main.go b/apps/OpenComputerUseWindows/main.goindex 02f48fe..5bb1cf0 100644--- a/apps/OpenComputerUseWindows/main.go+++ b/apps/OpenComputerUseWindows/main.go@@ -19,6 +19,8 @@ import (var version = "0.2.1"+var clickMethodValues = []string{"auto", "accessibility", "app_post", "global"}+//go:embed runtime.ps1var windowsRuntimeScript string@@ -141,6 +143,7 @@ type psRequest struct {ToY *float64 `json:"to_y,omitempty"`ClickCount int `json:"click_count,omitempty"`MouseButton string `json:"mouse_button,omitempty"`+ ClickMethod string `json:"click_method,omitempty"`Action string `json:"action,omitempty"`Direction string `json:"direction,omitempty"`Pages float64 `json:"pages,omitempty"`@@ -199,6 +202,10 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {}return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth)case "click":+ clickMethod, err := parseClickMethod(optionalString(args, "click_method"))+ if err != nil {+ return textResult(err.Error(), true)+ }return s.click(requiredString(args, "app"),optionalElementIndex(args),@@ -206,6 +213,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {optionalFloat(args, "y"),intValue(optionalFloat(args, "click_count"), 1),defaultString(optionalString(args, "mouse_button"), "left"),+ clickMethod,)case "perform_secondary_action":return s.performSecondaryAction(@@ -274,13 +282,19 @@ func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, mareturn snapshot.result()}-func (s *service) click(app, elementIndex string, x, y *float64, clickCount int, mouseButton string) toolCallResult {+func (s *service) click(app, elementIndex string, x, y *float64, clickCount int, mouseButton, clickMethod string) toolCallResult {if app == "" {return textResult("Missing required argument: app", true)}if elementIndex == "" && (x == nil || y == nil) {return textResult("click requires either element_index or x/y", true)}+ if clickMethod == "accessibility" && elementIndex == "" {+ return textResult("click_method 'accessibility' requires element_index", true)+ }+ if clickMethod == "global" {+ return textResult("click_method 'global' is not supported on Windows", true)+ }snapshot := s.currentSnapshot(app)if snapshot == nil {return textResult("No app state is available for "+app+". Run get_app_state before action tools.", true)@@ -292,6 +306,7 @@ func (s *service) click(app, elementIndex string, x, y *float64, clickCount int,Y: y,ClickCount: clickCount,MouseButton: mouseButton,+ ClickMethod: clickMethod,WindowBounds: snapshot.WindowBounds,}if elementIndex != "" {@@ -676,6 +691,19 @@ func defaultString(value, fallback string) string {return value}+func parseClickMethod(value string) (string, error) {+ normalized := strings.ToLower(strings.TrimSpace(value))+ if normalized == "" {+ return "auto", nil+ }+ for _, candidate := range clickMethodValues {+ if normalized == candidate {+ return normalized, nil+ }+ }+ return "", fmt.Errorf("Invalid click_method %q. Expected one of: %s", value, strings.Join(clickMethodValues, ", "))+}+func toolDefinitions() []toolDefinition {return []toolDefinition{{@@ -689,6 +717,7 @@ func toolDefinitions() []toolDefinition {"y": numberProperty("Y coordinate in screenshot pixel coordinates"),"click_count": integerProperty("Number of clicks. Defaults to 1"),"mouse_button": enumStringProperty("Mouse button to click. Defaults to left.", []string{"left", "right", "middle"}),+ "click_method": enumStringProperty("Click implementation: auto (default), accessibility, app_post, or global. Accessibility requires element_index. Windows supports app_post through HWND messages and does not currently support global.", clickMethodValues),}, []string{"app"}),},{diff --git a/apps/OpenComputerUseWindows/main_test.go b/apps/OpenComputerUseWindows/main_test.goindex 816b610..2937941 100644--- a/apps/OpenComputerUseWindows/main_test.go+++ b/apps/OpenComputerUseWindows/main_test.go@@ -13,6 +13,46 @@ func TestToolDefinitionCount(t *testing.T) {}}+func TestClickMethodSchemaAndParser(t *testing.T) {+ tool := findToolDefinition(t, "click")+ properties := tool.InputSchema["properties"].(map[string]any)+ method := properties["click_method"].(map[string]any)+ values := method["enum"].([]string)+ if strings.Join(values, ",") != "auto,accessibility,app_post,global" {+ t.Fatalf("click_method enum = %#v", values)+ }++ for input, want := range map[string]string{+ "": "auto",+ " AUTO ": "auto",+ "Accessibility": "accessibility",+ "app_post": "app_post",+ "GLOBAL": "global",+ } {+ got, err := parseClickMethod(input)+ if err != nil {+ t.Fatalf("parseClickMethod(%q): %v", input, err)+ }+ if got != want {+ t.Fatalf("parseClickMethod(%q) = %q, want %q", input, got, want)+ }+ }++ for _, input := range []string{"physical", "targeted"} {+ if _, err := parseClickMethod(input); err == nil || !strings.Contains(err.Error(), "Expected one of: auto, accessibility, app_post, global") {+ t.Fatalf("parseClickMethod(%s) error = %v", input, err)+ }+ }+}++func TestWindowsRejectsUnsupportedGlobalClickBeforeSnapshotLookup(t *testing.T) {+ x, y := 10.0, 20.0+ result := newService().click("Notepad", "", &x, &y, 1, "left", "global")+ if !result.IsError || result.Content[0].Text != "click_method 'global' is not supported on Windows" {+ t.Fatalf("global click result = %#v", result)+ }+}+func TestGetAppStateSchemaIncludesTextLimit(t *testing.T) {tool := findToolDefinition(t, "get_app_state")properties := tool.InputSchema["properties"].(map[string]any)diff --git a/apps/OpenComputerUseWindows/runtime.ps1 b/apps/OpenComputerUseWindows/runtime.ps1index bc2f16f..9d95acf 100644--- a/apps/OpenComputerUseWindows/runtime.ps1+++ b/apps/OpenComputerUseWindows/runtime.ps1@@ -911,11 +911,18 @@ try {switch ($operation.tool) {"click" {- $handled = $false- if ($null -ne $element -and $operation.mouse_button -ne "right" -and $operation.mouse_button -ne "middle") {- $handled = Invoke-PreferredClick $element- }- if (-not $handled) {+ $clickMethod = [string]$operation.click_method+ if ([string]::IsNullOrWhiteSpace($clickMethod)) { $clickMethod = "auto" }++ if ($clickMethod -eq "accessibility") {+ if ($null -eq $element) { throw "click_method 'accessibility' requires element_index" }+ if ($operation.mouse_button -eq "right" -or $operation.mouse_button -eq "middle") {+ throw "click_method 'accessibility' does not support mouse_button '$($operation.mouse_button)'"+ }+ if (-not (Invoke-PreferredClick $element)) {+ throw "click_method 'accessibility' could not click the requested element"+ }+ } elseif ($clickMethod -eq "app_post") {if ($null -ne $operation.element -and $null -ne $operation.element.frame) {$point = Get-ScreenPoint $operation.element.frame $windowBounds} else {@@ -925,6 +932,26 @@ try {}}Send-MouseClick $hwnd $point.x $point.y $operation.mouse_button ([int]$operation.click_count)+ } elseif ($clickMethod -eq "global") {+ throw "click_method 'global' is not supported on Windows"+ } elseif ($clickMethod -eq "auto") {+ $handled = $false+ if ($null -ne $element -and $operation.mouse_button -ne "right" -and $operation.mouse_button -ne "middle") {+ $handled = Invoke-PreferredClick $element+ }+ if (-not $handled) {+ if ($null -ne $operation.element -and $null -ne $operation.element.frame) {+ $point = Get-ScreenPoint $operation.element.frame $windowBounds+ } else {+ $point = [pscustomobject]@{+ x = [int][math]::Round($windowBounds.x + [double]$operation.x)+ y = [int][math]::Round($windowBounds.y + [double]$operation.y)+ }+ }+ Send-MouseClick $hwnd $point.x $point.y $operation.mouse_button ([int]$operation.click_count)+ }+ } else {+ throw "Invalid click_method '$clickMethod'"}}"perform_secondary_action" {diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex 1de0431..e3005bf 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -40,6 +40,7 @@- `OpenComputerUse` 默认 app 模式会拉起 `PermissionOnboardingApp`。- app bundle 以 `LSUIElement` agent-style 形态运行,默认不在 Dock 暴露常驻图标,但仍可按需显示权限窗口。- 当用户从终端执行 macOS 版 `open-computer-use mcp`、`doctor`、`call`、`snapshot` 或 `list-apps` 时,CLI 会先通过 LaunchServices 启动同一个 `.app` bundle 的隐藏 app agent,并通过用户临时目录下的 Unix domain socket 转发请求;真正调用 Accessibility、ScreenCaptureKit 和动作 tools 的进程始终是 `Open Computer Use.app`,不是 iTerm / Terminal / Node launcher。+- CLI 与 MCP proxy 会把调用进程中 `OPEN_COMPUTER_USE_*` 前缀的环境变量随请求转发给 app agent,并只在该请求执行期间临时覆盖 agent 环境;这让 `click_method=global` 的进程级安全门和 debug 开关在 app-agent 架构下仍按调用方配置生效。- 主窗口负责渲染 `Accessibility` / `Screen & System Audio Recording` 两类权限卡片、`Allow` / `Done` 状态和 relaunch 后的状态收敛;当两项权限都已完成时会自动关闭,不再要求用户手动退出。- 辅助 drag panel 会跳转到对应的 `System Settings` 页面;点击 `Allow` 后,panel 会从主窗口里的按钮位置做一段 spring + curved frame 的入场,再落到 `System Settings` 内容区下沿。panel 默认保持在窗口右侧内容区下方居中并固定贴近窗口底边,不再依赖实时扫描权限页内部 `+ / -` 控件行;窗口层级上会显式排在当前 `System Settings` 窗口之上,避免被权限列表内容盖住,同时尽量减少对系统设置自身滚动区域的干扰。panel 内也补了显式返回按钮,允许用户中断当前 guidance、回到 onboarding 主窗口重新选择权限步骤。- 权限状态会合并 TCC 持久授权记录与当前 app 进程的 runtime preflight:TCC 中任一匹配 client 已授权即可视为 granted,避免 CLI 子进程与 GUI app 对授权状态看到不一致的结果;如果当前 `.app` 进程已经通过 `AXIsProcessTrusted()` / `CGPreflightScreenCaptureAccess()`,也会立即视为 granted,避免 stale 或不匹配的 TCC path 记录让 onboarding 浮层继续停留。正式 release 仍以 CI 打出来的 `Open Computer Use.app` 为准,而本地 debug/dev 打包现在显式命名为 `Open Computer Use (Dev).app`,并在 dev bundle 运行时优先认当前 dev 副本,避免系统设置里出现两个完全同名的条目。@@ -77,13 +78,15 @@- 动作型 tools 对普通 app 采用“非侵入优先,物理指针路径显式 opt-in”策略:- `perform_secondary_action` 只执行目标元素已经暴露出来的 AX action;无效 action 返回官方风格的 `... is not a valid secondary action for ...`,fixture 的 `Raise` 路径也不再为了测试去准备全局物理指针输入- `set_value` 会先用 `AXUIElementIsAttributeSettable(kAXValueAttribute)` 判断目标是否真的是可设置值元素,只有 settable 时才调用 `AXUIElementSetAttributeValue`;不可设置时返回官方风格的 non-settable 错误,不退到键盘输入、剪贴板或未公开的文本替换接口- - element-targeted `click` 的左键路径会先试原生列表的 `AXSelectedChildren` 选择,再试 `AXPress` / `AXConfirm` / `AXOpen` 这类真正语义化的激活动作;如果目标本身不可点,还会继续尝试其子孙 AX 元素(例如 Finder sidebar row 下面暴露 `AXOpen` 的 cell)和命中点附近的 AX hit-test 结果,最后才会退到 `postToPid` 定向鼠标事件。`AXRaise` / `kAXMainAttribute` / `kAXFocusedAttribute` 这类 activation-only fallback 只允许窗口级元素使用,避免普通静态文本或容器把“获得焦点”误报成“点击已处理”;`click_count > 1` 也会优先重复可用的 AX action+ - `click.click_method` 是开源版的可选扩展,支持 `auto`(默认)、`accessibility`、`app_post` 和 `global`。未传参数时继续使用原有自动路由;显式模式不会静默 fallback 到其他实现。`accessibility` 只接受 `element_index`;`app_post` / `global` 可以使用 `element_index` 的计算落点或原始 `x/y` 坐标。+ - element-targeted `click` 的 `auto` 左键路径会先试原生列表的 `AXSelectedChildren` 选择,再试 `AXPress` / `AXConfirm` / `AXOpen` 这类真正语义化的激活动作;如果目标本身不可点,还会继续尝试其子孙 AX 元素(例如 Finder sidebar row 下面暴露 `AXOpen` 的 cell)和命中点附近的 AX hit-test 结果,最后进入现有 non-AX 分支:未开启全局指针环境变量时使用 `postToPid` 定向鼠标事件,开启时直接使用全局 HID 事件。`AXRaise` / `kAXMainAttribute` / `kAXFocusedAttribute` 这类 activation-only fallback 只允许窗口级元素使用,避免普通静态文本或容器把“获得焦点”误报成“点击已处理”;`click_count > 1` 也会优先重复可用的 AX action。+ - 显式 `click_method=accessibility` 只执行上述 AX 语义动作;显式 `click_method=app_post` 绕过 AX 候选和后代扫描,直接通过 `CGEvent.postToPid` 向目标进程发送 `mouseMoved` / down / up;显式 `click_method=global` 同样绕过 AX,但通过 `.cghidEventTap` 发送系统级指针事件。`global` 还必须设置 `OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1`,否则在移动 visual cursor 或发送输入前拒绝请求。- 对 renderer 合成的 summary `text` 行,`click` 不再默认点击父容器中心;这类元素使用左侧安全锚点。对普通 row/container/text 的后代候选,也会过滤右侧紧凑 hover action(例如 Lark / Electron 会话列表里的“完成”勾),避免把主行点击误操作成 side action;Electron/Lark 这类 app 的 WebArea 合成文本会优先寻找紧邻的行级 `AXPress` 祖先并静默执行,避免用物理鼠标点行。浏览器 WebArea(例如 Chrome/GitHub)不走这条 Electron-scoped 行级祖先点击优化,仍保留通用 link/container 点击路径。命中点如果只反查到覆盖整页的 Electron/WebArea 级 AX 元素,不再继续扫描整个大容器的子孙候选,避免一次行点击被远处的可点击元素截走。- `AXUIElementCopyElementAtPosition` 做坐标命中,尽量把 coordinate click 反解成可操作 AX 元素- `CGEvent.postToPid` 定向发送键盘事件,避免为了 `type_text` / `press_key` 抢前台;`type_text` 会把文本按 Unicode extended grapheme cluster 聚合成小批量 `keyboardSetUnicodeString` 事件,避免中文标点、emoji / 代理对和组合字符被逐个 UTF-16 code unit 拆开后在 Electron 富文本输入框里乱序或变形。如果当前 focused element 的 `AXValue` 可设置,`type_text` 会优先按可编辑内容追加并写回 `AXValue`,这覆盖 Feishu / Electron 富文本输入框不可靠接收后台键盘事件的场景,并会过滤已知占位提示,避免把 placeholder 拼进草稿;如果当前 focused element 不是可编辑文本目标,`type_text` 会报错要求先 click 文本输入区或使用 `set_value`,不再把无效果的后台键盘投递当成成功;`press_key` 的 xdotool parser 覆盖官方 binary key table 里常见的 `BackSpace`、`Page_Up`、`Prior` / `Next`、`F1...F12` 和 `KP_*` alias- `scroll.pages` 对齐官方 `1.0.755` 的 `number` schema,支持小数页数;整数页且目标暴露 `AXScroll*ByPage` 时优先走 AX action,否则用 `CGEvent.postToPid` 向目标进程定向发送 scroll event- `drag` 仍是 coordinate-only API,但默认不再使用全局 `.cghidEventTap` mouse event;默认改为 `CGEvent.postToPid` 定向发送 mouse move / down / dragged / up 事件,避免移动用户真实硬件光标;这些 coordinate tool 的 `x/y` 先按 screenshot pixel 坐标解释,再依据截图像素尺寸与目标 window bounds 的比例映射回 window point / Quartz global 坐标,避免 Retina 窗口上把 2x 像素误当成 1x point 导致点击落到错误位置- - `click` / `scroll` / `drag` 默认不会走全局 `.cghidEventTap`,因此不会移动或抢占用户真实鼠标;`OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1` 只作为本地诊断开关保留,不作为正常 OCU/OBU 验证路径。默认路径不再为了 fallback 调用 `NSRunningApplication.activate`+ - `click` / `scroll` / `drag` 在环境变量未开启的默认配置下不会走全局 `.cghidEventTap`,因此不会移动或抢占用户真实鼠标;`OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1` 是全局指针能力的进程级安全门,显式 `click_method=global` 仍必须通过这层授权。默认路径不再为了 fallback 调用 `NSRunningApplication.activate`### 4. Fixture Bridge@@ -109,6 +112,7 @@- Go runtime 通过 `go:embed` 带上 `runtime.ps1`,执行 tool call 时临时落盘并调用 Windows PowerShell。PowerShell bridge 使用 `System.Windows.Automation` 做 app/window/element discovery、tree rendering、UIA pattern action、ValuePattern set value 和 ScrollPattern scroll;当目标 app 不暴露对应 pattern 时,fallback 到 `PostMessage` / `SendMessage` 形式的 Win32 window message。- Windows runtime 默认只连接已经运行的 app,不会在 `get_app_state` 找不到进程时自动 `Start-Process`,也不会默认允许 `SetFocus` secondary action;这两条前台抢占路径分别需要 `OPEN_COMPUTER_USE_WINDOWS_ALLOW_APP_LAUNCH=1` 和 `OPEN_COMPUTER_USE_WINDOWS_ALLOW_FOCUS_ACTIONS=1` 显式打开。`type_text` 默认优先对可写文本控件的 child HWND 发送 `EM_SETSEL` / `EM_REPLACESEL`,不再默认走可能触发前台激活的 UIA `ValuePattern.SetValue` fallback;需要旧行为时必须设置 `OPEN_COMPUTER_USE_WINDOWS_ALLOW_UIA_TEXT_FALLBACK=1`。UIA pattern 和 Win32 message fallback 本身仍是 best-effort:很多控件可以在后台响应,但 Windows 没有一套对所有 GUI toolkit 都等价于 macOS AX 的后台键鼠输入模型。- 这 9 个 tool 的协议面与 macOS 主线保持一致:`list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value`。其中 element-targeted action 会优先复用上一轮 `get_app_state` 的 runtime id / automation metadata,coordinate action 使用 screenshot/window-relative 坐标。+- Windows `click_method=accessibility` 映射到 UI Automation pattern,`app_post` 映射到 HWND `PostMessage`;第一版没有新增会移动系统指针的 `SendInput` 路径,因此 `global` 会明确返回 unsupported。`auto` 仍保持 UIA 优先、window message fallback 的现有行为。- Windows UI Automation 需要运行在已登录用户的桌面 session 里。通过 SSH 作为脱离桌面的后台进程运行时,PowerShell 可以启动并返回 JSON,但系统可能不给它暴露顶层窗口;这种情况下 `list_apps` 会是空,`get_app_state` 可能返回 `appNotFound(...)`。- 当前 Windows 侧仍是功能性第一版:没有 visual cursor overlay、没有 installer/onboarding、没有 code signing,也没有独立的 Windows smoke fixture。后续 TODO 记录在 `docs/exec-plans/active/20260422-windows-computer-use-runtime.md`。@@ -121,6 +125,7 @@- Linux runtime 需要运行在已登录桌面用户 session 里。缺少 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 或 display 环境时,Go runtime 会在启动 Python AT-SPI bridge 前尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户的 session bus、display / Wayland 值;纯 SSH tty 如果找不到已登录桌面 session,可以启动二进制,但不能直接 inspect 或操作 GUI session。- `get_app_state` 的 accessibility tree 在 GTK/GNOME app 上可能很深,Linux bridge 使用与 macOS / Windows 一致的 1200 节点、64 层默认 tree budget,并支持显式提高 `max_tree_nodes` / `max_tree_depth`。截图通过 GDK root window best-effort capture;GNOME Wayland 可能返回黑图,bridge 会检测全黑采样并省略 image block。- 这 9 个 tool 的协议面与 macOS / Windows 保持一致:`list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value`。其中 element-targeted action 会优先复用上一轮 `get_app_state` 的 runtime path metadata,coordinate action 使用 screenshot/window-relative 坐标。+- Linux `click_method=accessibility` 映射到 AT-SPI action,`global` 映射到 AT-SPI mouse synthesis 并要求全局指针环境变量;AT-SPI 没有等价的进程定向 mouse dispatch,因此 `app_post` 会明确返回 unsupported。`auto` 仍保持 AT-SPI action 优先、mouse synthesis fallback 的现有行为。- 当前 Linux 侧仍是功能性第一版:没有 visual cursor overlay、没有 installer/desktop entry,也没有独立 Linux fixture。后续 TODO 记录在 `docs/exec-plans/active/20260422-linux-computer-use-runtime.md`。## 关键边界diff --git a/docs/SECURITY.md b/docs/SECURITY.mdindex 24b8736..ed1624f 100644--- a/docs/SECURITY.md+++ b/docs/SECURITY.md@@ -27,6 +27,8 @@- `Open Computer Use.app` 的系统权限- 本地使用场景共同提供。+- `click_method=global` 是显式的系统级指针路径,可能移动真实鼠标、改变前台焦点或命中坐标处的其他窗口。调用参数本身不视为足够授权;macOS 和支持该模式的 Linux runtime 还要求进程环境中设置 `OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1`。未设置时必须在任何可见 cursor 移动或真实输入事件之前拒绝请求。+- `click_method=app_post` 与 `accessibility` 不允许静默切换到 `global`。这保证调用方选择的非侵入边界在失败时仍然成立。- 下一阶段应优先补:- session 级审批- 更清楚的敏感 app / 系统设置防护策略diff --git a/docs/exec-plans/completed/20260722-configurable-click-method.md b/docs/exec-plans/completed/20260722-configurable-click-method.mdnew file mode 100644index 0000000..8d1ec02--- /dev/null+++ b/docs/exec-plans/completed/20260722-configurable-click-method.md@@ -0,0 +1,61 @@+# Configurable click method++## 目标++为 `click` 增加向后兼容的 `click_method` 参数,让调用方可以显式选择 accessibility、app-posted mouse event 或 global pointer event,同时保证未传参数时的 `auto` 行为与当前版本一致。++## 范围++- 包含:+ - macOS `click` dispatcher、tool schema、service 路由和输入模拟安全边界。+ - Windows / Linux 同名协议参数、已有能力映射和 unsupported 错误。+ - Swift / Go 单元测试、架构、安全、skill usage 和 history 文档。+- 不包含:+ - 修改 `auto` 下的 AX 后代候选扫描策略。+ - 为 Windows 新增全局 `SendInput`,或为 Linux 新增进程定向鼠标后端。+ - 发布、打 tag 或推送远端。++## 背景++- 相关文档:`docs/ARCHITECTURE.md`、`docs/SECURITY.md`、`skills/open-computer-use/references/usage.md`。+- 相关代码路径:`ComputerUseService.click`、`InputSimulation.clickTargeted` / `clickGlobally`、`MacOSAppAgentProxy`、Windows UIA / `PostMessage` bridge、Linux AT-SPI bridge。+- 已知约束:默认行为不能变化;强制模式不能静默 fallback;全局指针仍需 `OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1`。++## 风险++- 风险:全局指针事件会移动真实鼠标、改变前台焦点或命中非目标窗口。+- 缓解方式:要求调用参数和环境变量双重显式授权,并在执行前拒绝未授权请求。+- 风险:跨平台底层能力不对称。+- 缓解方式:保持公共枚举一致,对平台不支持的模式返回稳定错误,不伪装成功或回退到其他实现。+- 风险:重构 `auto` 路径引入行为漂移。+- 缓解方式:保留现有分支和 fallback 函数,只在外围增加显式路由。++## 里程碑++1. 增加公共参数、解析与安全校验。+2. 接入三平台已有实现并补测试。+3. 更新文档、完成验证并记录 history。++## 验证方式++- 命令:`swift test`。+- 命令:`(cd apps/OpenComputerUseWindows && go test ./...)`。+- 命令:`(cd apps/OpenComputerUseLinux && go test ./...)`。+- 命令:`npm run package:skill`。+- 手工检查:确认 `tools/list` 暴露四个枚举值,默认仍为 `auto`。+- 观测检查:显式 `app_post` 不进入 AX 路由,显式 `global` 未授权时不产生鼠标事件。++## 进度记录++- [x] 确认范围、现有三平台实现和安全边界。+- [x] 完成协议与实现。+- [x] 完成测试与文档。+- [x] 完成验证并归档计划。++## 决策记录++- 2026-07-22:公共参数命名为 `click_method`,枚举为 `auto`、`accessibility`、`app_post`、`global`;`app_post` 表示向目标 app/window 投递事件,在 macOS 映射到 `postToPid`,在 Windows 映射到 HWND `PostMessage`。+- 2026-07-22:`accessibility` 要求 `element_index`;`app_post` / `global` 可使用 `element_index` 或 `x/y`。+- 2026-07-22:Windows 第一版不支持 `global`,Linux 第一版不支持 `app_post`,均返回显式错误。+- 2026-07-22:macOS CLI 与 MCP proxy 都随请求转发 `OPEN_COMPUTER_USE_*` 环境变量,确保真正执行输入的 app agent 能看到全局指针授权。+- 2026-07-22:`swift test`、Windows / Linux `go test ./...`、skill 打包、标准 9-tool smoke 和 visual cursor idle smoke 全部通过;未自动执行会移动真实鼠标的 live global click。diff --git a/docs/histories/2026-07/20260722-1718-add-configurable-click-method.md b/docs/histories/2026-07/20260722-1718-add-configurable-click-method.mdnew file mode 100644index 0000000..851bdc7--- /dev/null+++ b/docs/histories/2026-07/20260722-1718-add-configurable-click-method.md@@ -0,0 +1,41 @@+## [2026-07-22 17:18] | Task: add configurable click method++### 🤖 Execution Context+* **Agent ID**: `Codex`+* **Base Model**: `GPT-5.6`+* **Runtime**: `Codex desktop`++### 📥 User Query+> 为 `click` 增加可选实现方式,默认保持当前行为,显式选择时避免 AX 候选把坐标点击重定向到其他元素。++### 🛠 Changes Overview+**Scope:** macOS OpenComputerUseKit / app-agent proxy、Windows runtime、Linux runtime、tool schema、skill 与仓库文档++**Key Actions:**+- **[公共参数]**: 新增 `click_method=auto|accessibility|app_post|global`;未传参数继续走原有 `auto` 路由,显式模式失败时不静默 fallback。+- **[macOS 路由]**: `accessibility` 只执行 AX,`app_post` 绕过 AX 并使用 `CGEvent.postToPid`,`global` 绕过 AX 并使用 `.cghidEventTap`。+- **[安全门]**: `global` 继续要求 `OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1`,并让 CLI / MCP proxy 把受限前缀环境变量随请求传给 app agent。+- **[跨平台映射]**: Windows 将 `app_post` 映射到 HWND `PostMessage` 并拒绝 `global`;Linux 将 `global` 映射到 AT-SPI mouse synthesis 并拒绝 `app_post`。+- **[验证]**: Swift 全量测试、Windows / Linux Go 测试、skill 打包、标准 tool smoke 和 visual cursor idle smoke 全部通过。++### 🧠 Design Intent (Why)+坐标点击的自动 AX 路径可能从命中的容器扫描到不包含原坐标的可点击后代。新增显式实现选择后,调用方可以在保留默认兼容行为的同时,用 `app_post` 严格向目标应用发送原坐标鼠标事件;高风险的全局指针路径仍需调用参数与进程环境双重授权。++### 📁 Files Modified+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift`+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift`+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift`+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift`+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`+- `apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift`+- `apps/OpenComputerUseWindows/main.go`+- `apps/OpenComputerUseWindows/main_test.go`+- `apps/OpenComputerUseWindows/runtime.ps1`+- `apps/OpenComputerUseLinux/main.go`+- `apps/OpenComputerUseLinux/main_test.go`+- `apps/OpenComputerUseLinux/runtime.py`+- `docs/ARCHITECTURE.md`+- `docs/SECURITY.md`+- `docs/exec-plans/completed/20260722-configurable-click-method.md`+- `skills/open-computer-use/SKILL.md`+- `skills/open-computer-use/references/usage.md`diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swiftindex 310e6ae..4c1b659 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift@@ -8,6 +8,44 @@ struct VisualCursorTarget: Equatable {let window: CursorTargetWindow?}+public enum ClickMethod: String, CaseIterable, Sendable {+ case auto+ case accessibility+ case appPost = "app_post"+ case global+}++func parseClickMethod(_ rawValue: String?) throws -> ClickMethod {+ let normalized = rawValue?+ .trimmingCharacters(in: .whitespacesAndNewlines)+ .lowercased() ?? ClickMethod.auto.rawValue++ guard let method = ClickMethod(rawValue: normalized) else {+ let expected = ClickMethod.allCases.map(\.rawValue).joined(separator: ", ")+ throw ComputerUseError.message(+ "Invalid click_method '\(rawValue ?? "")'. Expected one of: \(expected)"+ )+ }++ return method+}++func validateClickMethod(+ _ method: ClickMethod,+ hasElementIndex: Bool,+ environment: [String: String]+) throws {+ if method == .accessibility, !hasElementIndex {+ throw ComputerUseError.message("click_method 'accessibility' requires element_index")+ }++ if method == .global, !globalPointerFallbacksEnabled(environment: environment) {+ throw ComputerUseError.message(+ "click_method 'global' requires OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1 because it may move the system pointer and change foreground focus"+ )+ }+}+struct VisualCursorScreenMapping: Equatable {let screenStateFrame: CGRectlet appKitFrame: CGRect@@ -368,10 +406,30 @@ public final class ComputerUseService {snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits), style: .fullState)}- public func click(app query: String, elementIndex: String?, x: Double?, y: Double?, clickCount: Int, mouseButton: String) throws -> ToolCallResult {+ public func click(+ app query: String,+ elementIndex: String?,+ x: Double?,+ y: Double?,+ clickCount: Int,+ mouseButton: String,+ clickMethod: ClickMethod = .auto+ ) throws -> ToolCallResult {+ try validateClickMethod(+ clickMethod,+ hasElementIndex: elementIndex != nil,+ environment: ProcessInfo.processInfo.environment+ )+let snapshot = try currentSnapshot(for: query)let button = MouseButtonKind(rawValue: mouseButton.lowercased()) ?? .leftif snapshot.mode == .fixture {+ guard clickMethod == .auto else {+ throw ComputerUseError.message(+ "click_method '\(clickMethod.rawValue)' is not supported for fixture apps"+ )+ }+let cursorTarget: VisualCursorTarget?if let elementIndex {let record = try lookupElement(snapshot: snapshot, index: elementIndex)@@ -409,15 +467,40 @@ public final class ComputerUseService {moveVisualCursor(to: cursorTarget)do {- if !(try performAXClickSequence(- on: record,- snapshot: snapshot,- button: button,- clickCount: clickCount,- includeNearbyHitTesting: true,- allowActivationFallback: true- )) {- try performNonAXClickFallback(+ switch clickMethod {+ case .auto:+ if !(try performAXClickSequence(+ on: record,+ snapshot: snapshot,+ button: button,+ clickCount: clickCount,+ includeNearbyHitTesting: true,+ allowActivationFallback: true+ )) {+ try performNonAXClickFallback(+ at: targetPoint,+ button: button,+ clickCount: clickCount,+ targetDescription: "element_index=\(elementIndex)",+ snapshot: snapshot+ )+ }+ case .accessibility:+ guard try performAXClickSequence(+ on: record,+ snapshot: snapshot,+ button: button,+ clickCount: clickCount,+ includeNearbyHitTesting: true,+ allowActivationFallback: true+ ) else {+ throw ComputerUseError.message(+ "click_method 'accessibility' could not click element_index=\(elementIndex)"+ )+ }+ case .appPost, .global:+ try performExplicitMouseClick(+ method: clickMethod,at: targetPoint,button: button,clickCount: clickCount,@@ -444,24 +527,38 @@ public final class ComputerUseService {moveVisualCursor(to: cursorTarget)do {- let candidates = try clickCandidates(at: point, in: snapshot)- var handled = false- for record in candidates {- if try performAXClickSequence(- on: record,- snapshot: snapshot,- button: button,- clickCount: clickCount,- includeNearbyHitTesting: false,- allowActivationFallback: false- ) {- handled = true- break+ switch clickMethod {+ case .auto:+ let candidates = try clickCandidates(at: point, in: snapshot)+ var handled = false+ for record in candidates {+ if try performAXClickSequence(+ on: record,+ snapshot: snapshot,+ button: button,+ clickCount: clickCount,+ includeNearbyHitTesting: false,+ allowActivationFallback: false+ ) {+ handled = true+ break+ }}- }- if !handled {- try performNonAXClickFallback(+ if !handled {+ try performNonAXClickFallback(+ at: targetPoint,+ button: button,+ clickCount: clickCount,+ targetDescription: "x=\(Int(screenshotPoint.x)) y=\(Int(screenshotPoint.y))",+ snapshot: snapshot+ )+ }+ case .accessibility:+ throw ComputerUseError.message("click_method 'accessibility' requires element_index")+ case .appPost, .global:+ try performExplicitMouseClick(+ method: clickMethod,at: targetPoint,button: button,clickCount: clickCount,@@ -1689,6 +1786,41 @@ public final class ComputerUseService {}}+ private func performExplicitMouseClick(+ method: ClickMethod,+ at point: CGPoint,+ button: MouseButtonKind,+ clickCount: Int,+ targetDescription: String,+ snapshot: AppSnapshot+ ) throws {+ let eventPoint = inputEventPoint(fromScreenStatePoint: point)++ switch method {+ case .appPost:+ debugClickDecision("requested=app_post executed=pid_post target=\(targetDescription)")+ try InputSimulation.clickTargeted(+ at: eventPoint,+ button: button,+ clickCount: clickCount,+ pid: snapshot.app.pid+ )+ case .global:+ guard globalPointerFallbacksEnabled(environment: ProcessInfo.processInfo.environment) else {+ throw ComputerUseError.message(+ "click_method 'global' requires OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1 because it may move the system pointer and change foreground focus"+ )+ }+ debugClickDecision("requested=global executed=global_hid target=\(targetDescription)")+ InputSimulation.prepareAppForGlobalPointerInput(snapshot.app)+ try InputSimulation.clickGlobally(at: eventPoint, button: button, clickCount: clickCount)+ case .auto, .accessibility:+ throw ComputerUseError.message(+ "click_method '\(method.rawValue)' is not a direct mouse event method"+ )+ }+ }+private func snapshotResult(for snapshot: AppSnapshot, style: SnapshotTextStyle) -> ToolCallResult {var content = [ToolResultContentItem.text(snapshot.renderedText(style: style))]if let screenshotPNGData = snapshot.screenshotPNGData {diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swiftindex 7d87a23..c7e5770 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift@@ -63,7 +63,8 @@ public final class ComputerUseToolDispatcher {x: optionalDouble("x", in: arguments),y: optionalDouble("y", in: arguments),clickCount: Int(optionalDouble("click_count", in: arguments) ?? 1),- mouseButton: optionalString("mouse_button", in: arguments) ?? "left"+ mouseButton: optionalString("mouse_button", in: arguments) ?? "left",+ clickMethod: try parseClickMethod(optionalString("click_method", in: arguments)))case "perform_secondary_action":return try service.performSecondaryAction(diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swiftindex fd41cfd..b7ba356 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift@@ -69,7 +69,7 @@ enum InputSimulation {static func clickTargeted(at point: CGPoint, button: MouseButtonKind, clickCount: Int, pid: pid_t) throws {guard let source = CGEventSource(stateID: .combinedSessionState) else {- throw ComputerUseError.message("Failed to create targeted event source.")+ throw ComputerUseError.message("Failed to create app-post event source.")}for _ in 0..<max(clickCount, 1) {diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swiftindex 7393438..4db494c 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift@@ -45,6 +45,10 @@ public enum ToolDefinitions {description: "Mouse button to click. Defaults to left.",enumValues: ["left", "right", "middle"]),+ "click_method": stringProperty(+ description: "Click implementation: auto (default), accessibility, app_post, or global. Accessibility requires element_index. app_post sends an event directly to the target app or window. Global may move the system pointer and requires OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1.",+ enumValues: ClickMethod.allCases.map(\.rawValue)+ ),],required: ["app"])diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swiftindex 0cc08d0..1ec1a09 100644--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift@@ -628,6 +628,10 @@ final class OpenComputerUseKitTests: XCTestCase {((tools["click"]?.inputSchema["properties"] as? [String: [String: Any]])?["mouse_button"]?["enum"] as? [String]) ?? [],["left", "right", "middle"])+ XCTAssertEqual(+ ((tools["click"]?.inputSchema["properties"] as? [String: [String: Any]])?["click_method"]?["enum"] as? [String]) ?? [],+ ["auto", "accessibility", "app_post", "global"]+ )let getAppStateSchema = tools["get_app_state"]?.inputSchemalet getAppStateProperties = getAppStateSchema?["properties"] as? [String: [String: Any]]XCTAssertNil(getAppStateProperties?["show_full_text"])@@ -1358,6 +1362,59 @@ final class OpenComputerUseKitTests: XCTestCase {XCTAssertFalse(globalPointerFallbacksEnabled(environment: ["OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS": "false"]))}+ func testClickMethodDefaultsToAutoAndNormalizesExplicitValues() throws {+ XCTAssertEqual(try parseClickMethod(nil), .auto)+ XCTAssertEqual(try parseClickMethod(" AUTO "), .auto)+ XCTAssertEqual(try parseClickMethod("Accessibility"), .accessibility)+ XCTAssertEqual(try parseClickMethod(" APP_POST "), .appPost)+ XCTAssertEqual(try parseClickMethod("GLOBAL"), .global)+ }++ func testClickMethodRejectsUnknownValues() {+ for value in ["physical", "targeted"] {+ XCTAssertThrowsError(try parseClickMethod(value)) { error in+ XCTAssertEqual(+ (error as? ComputerUseError)?.errorDescription,+ "Invalid click_method '\(value)'. Expected one of: auto, accessibility, app_post, global"+ )+ }+ }+ }++ func testAccessibilityClickMethodRequiresElementIndex() {+ XCTAssertThrowsError(+ try validateClickMethod(.accessibility, hasElementIndex: false, environment: [:])+ ) { error in+ XCTAssertEqual(+ (error as? ComputerUseError)?.errorDescription,+ "click_method 'accessibility' requires element_index"+ )+ }++ XCTAssertNoThrow(+ try validateClickMethod(.accessibility, hasElementIndex: true, environment: [:])+ )+ }++ func testGlobalClickMethodRequiresExplicitPointerAuthorization() {+ XCTAssertThrowsError(+ try validateClickMethod(.global, hasElementIndex: false, environment: [:])+ ) { error in+ XCTAssertEqual(+ (error as? ComputerUseError)?.errorDescription,+ "click_method 'global' requires OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1 because it may move the system pointer and change foreground focus"+ )+ }++ XCTAssertNoThrow(+ try validateClickMethod(+ .global,+ hasElementIndex: false,+ environment: ["OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS": "1"]+ )+ )+ }+func testSetValueAttributeGateMatchesOfficialSettableBoundary() throws {XCTAssertTrue(try setValueAttributeIsSettable(result: .success, settable: true, attribute: kAXValueAttribute))XCTAssertFalse(try setValueAttributeIsSettable(result: .success, settable: false, attribute: kAXValueAttribute))diff --git a/skills/open-computer-use/SKILL.md b/skills/open-computer-use/SKILL.mdindex 5f63853..014ef60 100644--- a/skills/open-computer-use/SKILL.md+++ b/skills/open-computer-use/SKILL.md@@ -36,7 +36,7 @@ It supports the same core tool surface across macOS, Linux, and Windows:- Do not assume Codex.app plugin helpers are available. Use the installed `open-computer-use` / `ocu` CLI or an explicit MCP config.- Always run `get_app_state` before using `element_index`; do not guess indexes across sessions or after large UI changes.- Prefer semantic actions and `set_value` for editable controls. Use coordinate `click`, `scroll`, and `drag` only when the element tree does not expose a safer target.-- On macOS, do not enable `OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1` unless the user explicitly wants diagnostic behavior that may move the real pointer.+- On macOS, do not enable `OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1` unless the user explicitly requested `click_method: "global"` or other diagnostic behavior that may move the real pointer.- On Windows and Linux, confirm the command is running inside the logged-in desktop session before assuming GUI automation is available.## Common CLI Actionsdiff --git a/skills/open-computer-use/references/usage.md b/skills/open-computer-use/references/usage.mdindex bd03ac5..6599e48 100644--- a/skills/open-computer-use/references/usage.md+++ b/skills/open-computer-use/references/usage.md@@ -105,11 +105,35 @@ open-computer-use snapshot --max-tree-nodes 3000 --max-tree-depth 96 "Google Chr- Re-run `get_app_state` after navigation, modal changes, page reloads, or failed actions.- Use coordinate actions only when the rendered tree does not expose the target as an element.+## Choosing a Click Method++`click_method` is optional. Omitting it uses `auto`, which preserves the platform's existing semantic-first behavior. Explicit methods never fall back to a different implementation:++- `accessibility`: only invoke the element's accessibility action and require `element_index`.+- `app_post`: bypass accessibility and post a mouse event directly to the target app/window without moving the system pointer. Supported on macOS and Windows.+- `global`: bypass accessibility and use the desktop's global pointer path. Supported on macOS and Linux, and requires `OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1` because it may move the real pointer or change foreground focus.++Use `app_post` for an exact blank-area or overlay click that must not be redirected to an accessibility descendant:++```sh+open-computer-use call click --args '{"app":"Google Chrome","x":875,"y":375,"click_method":"app_post"}'+```++Use `global` only after explicitly enabling the process-level safety gate:++```sh+OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1 open-computer-use call click --args '{"app":"Google Chrome","x":875,"y":375,"click_method":"global"}'+```++Keep the environment override scoped as narrowly as possible. While it remains enabled, the existing `auto` route may also choose the global pointer path after accessibility cannot handle a click.++Windows returns an unsupported error for `global`; Linux returns an unsupported error for `app_post`. An unsupported or failed explicit method does not fall back to `auto`.+## Platform Notes### macOS-The macOS runtime uses Accessibility, ScreenCaptureKit, and targeted input events. It normally avoids moving the user's real pointer. The visual cursor overlay is part of the Open Computer Use experience and can be disabled by the surrounding runtime only when needed.+The macOS runtime uses Accessibility, ScreenCaptureKit, and app-posted input events. It normally avoids moving the user's real pointer. The visual cursor overlay is part of the Open Computer Use experience and can be disabled by the surrounding runtime only when needed.### Windows