👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
fix(cli): accept numeric element_index arguments
tisfeng8e10d1ebb166
diff --git a/apps/OpenComputerUseLinux/main.go b/apps/OpenComputerUseLinux/main.goindex 590d735..49fb668 100644--- a/apps/OpenComputerUseLinux/main.go+++ b/apps/OpenComputerUseLinux/main.go@@ -8,6 +8,7 @@ import ("errors""fmt""io"+ "math""os""os/exec""path/filepath"@@ -176,7 +177,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {case "click":return s.click(requiredString(args, "app"),- optionalString(args, "element_index"),+ optionalElementIndex(args),optionalFloat(args, "x"),optionalFloat(args, "y"),intValue(optionalFloat(args, "click_count"), 1),@@ -185,14 +186,14 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {case "perform_secondary_action":return s.performSecondaryAction(requiredString(args, "app"),- requiredString(args, "element_index"),+ requiredElementIndex(args),requiredString(args, "action"),)case "scroll":return s.scroll(requiredString(args, "app"),requiredString(args, "direction"),- requiredString(args, "element_index"),+ requiredElementIndex(args),floatValue(optionalFloat(args, "pages"), 1),)case "drag":@@ -208,7 +209,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {case "press_key":return s.pressKey(requiredString(args, "app"), requiredString(args, "key"))case "set_value":- return s.setValue(requiredString(args, "app"), requiredString(args, "element_index"), requiredString(args, "value"))+ return s.setValue(requiredString(args, "app"), requiredElementIndex(args), requiredString(args, "value"))default:return textResult(fmt.Sprintf("unsupportedTool(%q)", name), true)}@@ -904,6 +905,42 @@ func optionalString(args map[string]any, key string) string {return value}+func requiredElementIndex(args map[string]any) string {+ return strings.TrimSpace(optionalElementIndex(args))+}++func optionalElementIndex(args map[string]any) string {+ return elementIndexString(args["element_index"])+}++func elementIndexString(value any) string {+ switch value := value.(type) {+ case string:+ return value+ case json.Number:+ if integer, err := value.Int64(); err == nil {+ return strconv.FormatInt(integer, 10)+ }+ if float, err := value.Float64(); err == nil {+ return integerElementIndexFloat(float)+ }+ case float64:+ return integerElementIndexFloat(value)+ case int:+ return strconv.Itoa(value)+ case int64:+ return strconv.FormatInt(value, 10)+ }+ return ""+}++func integerElementIndexFloat(value float64) string {+ if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {+ return ""+ }+ return strconv.FormatInt(int64(value), 10)+}+func requiredFloat(args map[string]any, key string) *float64 {return optionalFloat(args, key)}diff --git a/apps/OpenComputerUseLinux/main_test.go b/apps/OpenComputerUseLinux/main_test.goindex f320013..e85b3cf 100644--- a/apps/OpenComputerUseLinux/main_test.go+++ b/apps/OpenComputerUseLinux/main_test.go@@ -49,6 +49,22 @@ func TestReadArgumentsAcceptsJSONObject(t *testing.T) {}}+func TestElementIndexAcceptsStringAndJSONNumber(t *testing.T) {+ args, err := readArguments(`{"app":"Text Editor","element_index":0}`, "")+ if err != nil {+ t.Fatal(err)+ }+ if got := optionalElementIndex(args); got != "0" {+ t.Fatalf("numeric element_index = %q, want 0", got)+ }+ if got := optionalElementIndex(map[string]any{"element_index": "14"}); got != "14" {+ t.Fatalf("string element_index = %q, want 14", got)+ }+ if got := optionalElementIndex(map[string]any{"element_index": json.Number("1.5")}); got != "" {+ t.Fatalf("fractional element_index = %q, want empty", got)+ }+}+func TestMCPInitializeResponseContainsToolsCapability(t *testing.T) {request := map[string]any{"jsonrpc": "2.0",diff --git a/apps/OpenComputerUseWindows/main.go b/apps/OpenComputerUseWindows/main.goindex f94f7eb..a030ed0 100644--- a/apps/OpenComputerUseWindows/main.go+++ b/apps/OpenComputerUseWindows/main.go@@ -7,6 +7,7 @@ import ("errors""fmt""io"+ "math""os""os/exec""path/filepath"@@ -173,7 +174,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {case "click":return s.click(requiredString(args, "app"),- optionalString(args, "element_index"),+ optionalElementIndex(args),optionalFloat(args, "x"),optionalFloat(args, "y"),intValue(optionalFloat(args, "click_count"), 1),@@ -182,14 +183,14 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {case "perform_secondary_action":return s.performSecondaryAction(requiredString(args, "app"),- requiredString(args, "element_index"),+ requiredElementIndex(args),requiredString(args, "action"),)case "scroll":return s.scroll(requiredString(args, "app"),requiredString(args, "direction"),- requiredString(args, "element_index"),+ requiredElementIndex(args),floatValue(optionalFloat(args, "pages"), 1),)case "drag":@@ -205,7 +206,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {case "press_key":return s.pressKey(requiredString(args, "app"), requiredString(args, "key"))case "set_value":- return s.setValue(requiredString(args, "app"), requiredString(args, "element_index"), requiredString(args, "value"))+ return s.setValue(requiredString(args, "app"), requiredElementIndex(args), requiredString(args, "value"))default:return textResult(fmt.Sprintf("unsupportedTool(%q)", name), true)}@@ -487,6 +488,42 @@ func optionalString(args map[string]any, key string) string {return value}+func requiredElementIndex(args map[string]any) string {+ return strings.TrimSpace(optionalElementIndex(args))+}++func optionalElementIndex(args map[string]any) string {+ return elementIndexString(args["element_index"])+}++func elementIndexString(value any) string {+ switch value := value.(type) {+ case string:+ return value+ case json.Number:+ if integer, err := value.Int64(); err == nil {+ return strconv.FormatInt(integer, 10)+ }+ if float, err := value.Float64(); err == nil {+ return integerElementIndexFloat(float)+ }+ case float64:+ return integerElementIndexFloat(value)+ case int:+ return strconv.Itoa(value)+ case int64:+ return strconv.FormatInt(value, 10)+ }+ return ""+}++func integerElementIndexFloat(value float64) string {+ if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {+ return ""+ }+ return strconv.FormatInt(int64(value), 10)+}+func requiredFloat(args map[string]any, key string) *float64 {return optionalFloat(args, key)}diff --git a/apps/OpenComputerUseWindows/main_test.go b/apps/OpenComputerUseWindows/main_test.goindex d496836..1406fd2 100644--- a/apps/OpenComputerUseWindows/main_test.go+++ b/apps/OpenComputerUseWindows/main_test.go@@ -46,6 +46,22 @@ func TestReadArgumentsAcceptsJSONObject(t *testing.T) {}}+func TestElementIndexAcceptsStringAndJSONNumber(t *testing.T) {+ args, err := readArguments(`{"app":"Notepad","element_index":0}`, "")+ if err != nil {+ t.Fatal(err)+ }+ if got := optionalElementIndex(args); got != "0" {+ t.Fatalf("numeric element_index = %q, want 0", got)+ }+ if got := optionalElementIndex(map[string]any{"element_index": "14"}); got != "14" {+ t.Fatalf("string element_index = %q, want 14", got)+ }+ if got := optionalElementIndex(map[string]any{"element_index": json.Number("1.5")}); got != "" {+ t.Fatalf("fractional element_index = %q, want empty", got)+ }+}+func TestMCPInitializeResponseContainsToolsCapability(t *testing.T) {request := map[string]any{"jsonrpc": "2.0",diff --git a/docs/histories/2026-05/20260526-1126-fix-cli-element-index.md b/docs/histories/2026-05/20260526-1126-fix-cli-element-index.mdnew file mode 100644index 0000000..2f19eae--- /dev/null+++ b/docs/histories/2026-05/20260526-1126-fix-cli-element-index.md@@ -0,0 +1,28 @@+# 修复 CLI 数字型 element_index 参数++## 用户诉求++修复 Issue #18:`open-computer-use call click --args '{"app":"TextEdit","element_index":0}'` 会因为 CLI JSON 把 `0` 解析成数字,而 dispatcher 只接受字符串,最终误报 `click requires either element_index or x/y`。++## 主要改动++- Swift `ComputerUseToolDispatcher` 新增 `element_index` 专用规范化逻辑,允许 JSON 数字索引转成字符串索引,同时保持 tool schema 的 `string` 类型不变。+- `click`、`perform_secondary_action`、`scroll`、`set_value` 统一使用该专用读取逻辑。+- Windows 和 Linux Go runtime 同步接受数字型 `element_index`,保持三端 CLI 参数行为一致。+- skill 文档示例改成规范的字符串写法,避免继续引导用户传数字。+- 补充 Swift / Go 单元测试覆盖字符串索引、数字索引和小数索引拒绝路径。++## 设计动机++官方 tool surface 把 `element_index` 暴露为字符串,仓库也应继续保持这个 schema;但手写 CLI JSON 时用户自然会把索引写成数字。解析层把整数数字容错转换为字符串,可以兼容现有文档误导产生的命令,同时不扩大其它字符串参数的接受类型。++## 受影响文件++- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift`+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`+- `apps/OpenComputerUseWindows/main.go`+- `apps/OpenComputerUseWindows/main_test.go`+- `apps/OpenComputerUseLinux/main.go`+- `apps/OpenComputerUseLinux/main_test.go`+- `skills/open-computer-use/SKILL.md`+- `skills/open-computer-use/references/usage.md`diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swiftindex d327a07..4092273 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift@@ -1,5 +1,41 @@import Foundation+func normalizedElementIndexArgument(_ value: Any?) -> String? {+ if let string = value as? String {+ return string.isEmpty ? nil : string+ }++ if let integer = value as? Int {+ return String(integer)+ }++ if let number = value as? NSNumber {+ if CFGetTypeID(number as CFTypeRef) == CFBooleanGetTypeID() {+ return nil+ }++ return normalizedElementIndexNumber(number.doubleValue)+ }++ if let double = value as? Double {+ return normalizedElementIndexNumber(double)+ }++ return nil+}++private func normalizedElementIndexNumber(_ value: Double) -> String? {+ guard value.isFinite, value.rounded(.towardZero) == value else {+ return nil+ }++ guard value >= Double(Int.min), value <= Double(Int.max) else {+ return nil+ }++ return String(Int(value))+}+public final class ComputerUseToolDispatcher {private let service: ComputerUseService@@ -16,7 +52,7 @@ public final class ComputerUseToolDispatcher {case "click":return try service.click(app: requireString("app", in: arguments),- elementIndex: optionalString("element_index", in: arguments),+ elementIndex: optionalElementIndex(in: arguments),x: optionalDouble("x", in: arguments),y: optionalDouble("y", in: arguments),clickCount: Int(optionalDouble("click_count", in: arguments) ?? 1),@@ -25,14 +61,14 @@ public final class ComputerUseToolDispatcher {case "perform_secondary_action":return try service.performSecondaryAction(app: requireString("app", in: arguments),- elementIndex: requireString("element_index", in: arguments),+ elementIndex: requireElementIndex(in: arguments),action: requireString("action", in: arguments))case "scroll":return try service.scroll(app: requireString("app", in: arguments),direction: requireString("direction", in: arguments),- elementIndex: requireString("element_index", in: arguments),+ elementIndex: requireElementIndex(in: arguments),pages: optionalDouble("pages", in: arguments) ?? 1)case "drag":@@ -56,7 +92,7 @@ public final class ComputerUseToolDispatcher {case "set_value":return try service.setValue(app: requireString("app", in: arguments),- elementIndex: requireString("element_index", in: arguments),+ elementIndex: requireElementIndex(in: arguments),value: requireString("value", in: arguments))default:@@ -90,6 +126,18 @@ public final class ComputerUseToolDispatcher {arguments[key] as? String}+ private func requireElementIndex(in arguments: [String: Any]) throws -> String {+ guard let value = optionalElementIndex(in: arguments) else {+ throw ComputerUseError.missingArgument("element_index")+ }++ return value+ }++ private func optionalElementIndex(in arguments: [String: Any]) -> String? {+ normalizedElementIndexArgument(arguments["element_index"])+ }+private func requireDouble(_ key: String, in arguments: [String: Any]) throws -> Double {guard let value = optionalDouble(key, in: arguments) else {throw ComputerUseError.missingArgument(key)diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swiftindex 6f93147..505ed4a 100644--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift@@ -174,6 +174,30 @@ final class OpenComputerUseKitTests: XCTestCase {XCTAssertEqual((arguments["pages"] as? NSNumber)?.intValue, 2)}+ func testElementIndexAcceptsNumericToolArgument() throws {+ let arguments = try readOpenComputerUseToolArguments(+ json: #"{"app":"TextEdit","element_index":0}"#,+ file: nil+ )++ XCTAssertEqual(normalizedElementIndexArgument(arguments["element_index"]), "0")+ }++ func testElementIndexAcceptsNumericCallSequenceArgument() throws {+ let calls = try readOpenComputerUseCallSequence(+ json: #"[{"tool":"click","args":{"app":"TextEdit","element_index":0}}]"#,+ file: nil+ )++ XCTAssertEqual(normalizedElementIndexArgument(calls[0].arguments["element_index"]), "0")+ }++ func testElementIndexRejectsMissingEmptyAndFractionalArguments() {+ XCTAssertNil(normalizedElementIndexArgument(nil))+ XCTAssertNil(normalizedElementIndexArgument(""))+ XCTAssertNil(normalizedElementIndexArgument(1.5))+ }+func testReadToolArgumentsRejectsNonObject() {XCTAssertThrowsError(try readOpenComputerUseToolArguments(json: #"["TextEdit"]"#, file: nil)) { error inXCTAssertEqual(diff --git a/skills/open-computer-use/SKILL.md b/skills/open-computer-use/SKILL.mdindex de51720..d3929de 100644--- a/skills/open-computer-use/SKILL.md+++ b/skills/open-computer-use/SKILL.md@@ -41,7 +41,7 @@ open-computer-use -hopen-computer-use doctoropen-computer-use call list_appsopen-computer-use call get_app_state --args '{"app":"TextEdit"}'-open-computer-use call click --args '{"app":"TextEdit","element_index":0}'+open-computer-use call click --args '{"app":"TextEdit","element_index":"0"}'open-computer-use call type_text --args '{"app":"TextEdit","text":"Hello from Open Computer Use"}'```diff --git a/skills/open-computer-use/references/usage.md b/skills/open-computer-use/references/usage.mdindex b94e924..897b318 100644--- a/skills/open-computer-use/references/usage.md+++ b/skills/open-computer-use/references/usage.md@@ -46,7 +46,7 @@ Use `call` for one-off checks:```shopen-computer-use call list_appsopen-computer-use call get_app_state --args '{"app":"TextEdit"}'-open-computer-use call set_value --args '{"app":"TextEdit","element_index":1,"value":"Draft"}'+open-computer-use call set_value --args '{"app":"TextEdit","element_index":"1","value":"Draft"}'```Use `--calls` for short action sequences that need to reuse the same process state:@@ -54,7 +54,7 @@ Use `--calls` for short action sequences that need to reuse the same process sta```shopen-computer-use call --calls '[{"tool":"get_app_state","args":{"app":"TextEdit"}},- {"tool":"click","args":{"app":"TextEdit","element_index":1}},+ {"tool":"click","args":{"app":"TextEdit","element_index":"1"}},{"tool":"type_text","args":{"app":"TextEdit","text":"Hello"}}]'```