Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

fix(cli): accept numeric element_index arguments

tisfeng8e10d1ebb166
diff --git a/apps/OpenComputerUseLinux/main.go b/apps/OpenComputerUseLinux/main.go
index 590d735..49fb668 100644
--- a/apps/OpenComputerUseLinux/main.go
+++ b/apps/OpenComputerUseLinux/main.go
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"io"
+ "math"
"os"
"os/exec"
"path/filepath"
@@ -176,7 +177,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {
case "click":
return s.click(
requiredString(args, "app"),
- optionalString(args, "element_index"),
+ optionalElementIndex(args),
optionalFloat(args, "x"),
optionalFloat(args, "y"),
intValue(optionalFloat(args, "click_count"), 1),
@@ -185,14 +186,14 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {
case "perform_secondary_action":
return s.performSecondaryAction(
requiredString(args, "app"),
- requiredString(args, "element_index"),
+ requiredElementIndex(args),
requiredString(args, "action"),
)
case "scroll":
return s.scroll(
requiredString(args, "app"),
requiredString(args, "direction"),
- requiredString(args, "element_index"),
+ requiredElementIndex(args),
floatValue(optionalFloat(args, "pages"), 1),
)
case "drag":
@@ -208,7 +209,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {
case "press_key":
return s.pressKey(requiredString(args, "app"), requiredString(args, "key"))
case "set_value":
- return s.setValue(requiredString(args, "app"), requiredString(args, "element_index"), requiredString(args, "value"))
+ return s.setValue(requiredString(args, "app"), requiredElementIndex(args), requiredString(args, "value"))
default:
return textResult(fmt.Sprintf("unsupportedTool(%q)", name), true)
}
@@ -904,6 +905,42 @@ func optionalString(args map[string]any, key string) string {
return value
}
+func requiredElementIndex(args map[string]any) string {
+ return strings.TrimSpace(optionalElementIndex(args))
+}
+
+func optionalElementIndex(args map[string]any) string {
+ return elementIndexString(args["element_index"])
+}
+
+func elementIndexString(value any) string {
+ switch value := value.(type) {
+ case string:
+ return value
+ case json.Number:
+ if integer, err := value.Int64(); err == nil {
+ return strconv.FormatInt(integer, 10)
+ }
+ if float, err := value.Float64(); err == nil {
+ return integerElementIndexFloat(float)
+ }
+ case float64:
+ return integerElementIndexFloat(value)
+ case int:
+ return strconv.Itoa(value)
+ case int64:
+ return strconv.FormatInt(value, 10)
+ }
+ return ""
+}
+
+func integerElementIndexFloat(value float64) string {
+ if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {
+ return ""
+ }
+ return strconv.FormatInt(int64(value), 10)
+}
+
func requiredFloat(args map[string]any, key string) *float64 {
return optionalFloat(args, key)
}
diff --git a/apps/OpenComputerUseLinux/main_test.go b/apps/OpenComputerUseLinux/main_test.go
index f320013..e85b3cf 100644
--- a/apps/OpenComputerUseLinux/main_test.go
+++ b/apps/OpenComputerUseLinux/main_test.go
@@ -49,6 +49,22 @@ func TestReadArgumentsAcceptsJSONObject(t *testing.T) {
}
}
+func TestElementIndexAcceptsStringAndJSONNumber(t *testing.T) {
+ args, err := readArguments(`{"app":"Text Editor","element_index":0}`, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := optionalElementIndex(args); got != "0" {
+ t.Fatalf("numeric element_index = %q, want 0", got)
+ }
+ if got := optionalElementIndex(map[string]any{"element_index": "14"}); got != "14" {
+ t.Fatalf("string element_index = %q, want 14", got)
+ }
+ if got := optionalElementIndex(map[string]any{"element_index": json.Number("1.5")}); got != "" {
+ t.Fatalf("fractional element_index = %q, want empty", got)
+ }
+}
+
func TestMCPInitializeResponseContainsToolsCapability(t *testing.T) {
request := map[string]any{
"jsonrpc": "2.0",
diff --git a/apps/OpenComputerUseWindows/main.go b/apps/OpenComputerUseWindows/main.go
index f94f7eb..a030ed0 100644
--- a/apps/OpenComputerUseWindows/main.go
+++ b/apps/OpenComputerUseWindows/main.go
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"io"
+ "math"
"os"
"os/exec"
"path/filepath"
@@ -173,7 +174,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {
case "click":
return s.click(
requiredString(args, "app"),
- optionalString(args, "element_index"),
+ optionalElementIndex(args),
optionalFloat(args, "x"),
optionalFloat(args, "y"),
intValue(optionalFloat(args, "click_count"), 1),
@@ -182,14 +183,14 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {
case "perform_secondary_action":
return s.performSecondaryAction(
requiredString(args, "app"),
- requiredString(args, "element_index"),
+ requiredElementIndex(args),
requiredString(args, "action"),
)
case "scroll":
return s.scroll(
requiredString(args, "app"),
requiredString(args, "direction"),
- requiredString(args, "element_index"),
+ requiredElementIndex(args),
floatValue(optionalFloat(args, "pages"), 1),
)
case "drag":
@@ -205,7 +206,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {
case "press_key":
return s.pressKey(requiredString(args, "app"), requiredString(args, "key"))
case "set_value":
- return s.setValue(requiredString(args, "app"), requiredString(args, "element_index"), requiredString(args, "value"))
+ return s.setValue(requiredString(args, "app"), requiredElementIndex(args), requiredString(args, "value"))
default:
return textResult(fmt.Sprintf("unsupportedTool(%q)", name), true)
}
@@ -487,6 +488,42 @@ func optionalString(args map[string]any, key string) string {
return value
}
+func requiredElementIndex(args map[string]any) string {
+ return strings.TrimSpace(optionalElementIndex(args))
+}
+
+func optionalElementIndex(args map[string]any) string {
+ return elementIndexString(args["element_index"])
+}
+
+func elementIndexString(value any) string {
+ switch value := value.(type) {
+ case string:
+ return value
+ case json.Number:
+ if integer, err := value.Int64(); err == nil {
+ return strconv.FormatInt(integer, 10)
+ }
+ if float, err := value.Float64(); err == nil {
+ return integerElementIndexFloat(float)
+ }
+ case float64:
+ return integerElementIndexFloat(value)
+ case int:
+ return strconv.Itoa(value)
+ case int64:
+ return strconv.FormatInt(value, 10)
+ }
+ return ""
+}
+
+func integerElementIndexFloat(value float64) string {
+ if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {
+ return ""
+ }
+ return strconv.FormatInt(int64(value), 10)
+}
+
func requiredFloat(args map[string]any, key string) *float64 {
return optionalFloat(args, key)
}
diff --git a/apps/OpenComputerUseWindows/main_test.go b/apps/OpenComputerUseWindows/main_test.go
index d496836..1406fd2 100644
--- a/apps/OpenComputerUseWindows/main_test.go
+++ b/apps/OpenComputerUseWindows/main_test.go
@@ -46,6 +46,22 @@ func TestReadArgumentsAcceptsJSONObject(t *testing.T) {
}
}
+func TestElementIndexAcceptsStringAndJSONNumber(t *testing.T) {
+ args, err := readArguments(`{"app":"Notepad","element_index":0}`, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := optionalElementIndex(args); got != "0" {
+ t.Fatalf("numeric element_index = %q, want 0", got)
+ }
+ if got := optionalElementIndex(map[string]any{"element_index": "14"}); got != "14" {
+ t.Fatalf("string element_index = %q, want 14", got)
+ }
+ if got := optionalElementIndex(map[string]any{"element_index": json.Number("1.5")}); got != "" {
+ t.Fatalf("fractional element_index = %q, want empty", got)
+ }
+}
+
func TestMCPInitializeResponseContainsToolsCapability(t *testing.T) {
request := map[string]any{
"jsonrpc": "2.0",
diff --git a/docs/histories/2026-05/20260526-1126-fix-cli-element-index.md b/docs/histories/2026-05/20260526-1126-fix-cli-element-index.md
new file mode 100644
index 0000000..2f19eae
--- /dev/null
+++ b/docs/histories/2026-05/20260526-1126-fix-cli-element-index.md
@@ -0,0 +1,28 @@
+# 修复 CLI 数字型 element_index 参数
+
+## 用户诉求
+
+修复 Issue #18:`open-computer-use call click --args '{"app":"TextEdit","element_index":0}'` 会因为 CLI JSON 把 `0` 解析成数字,而 dispatcher 只接受字符串,最终误报 `click requires either element_index or x/y`。
+
+## 主要改动
+
+- Swift `ComputerUseToolDispatcher` 新增 `element_index` 专用规范化逻辑,允许 JSON 数字索引转成字符串索引,同时保持 tool schema 的 `string` 类型不变。
+- `click`、`perform_secondary_action`、`scroll`、`set_value` 统一使用该专用读取逻辑。
+- Windows 和 Linux Go runtime 同步接受数字型 `element_index`,保持三端 CLI 参数行为一致。
+- skill 文档示例改成规范的字符串写法,避免继续引导用户传数字。
+- 补充 Swift / Go 单元测试覆盖字符串索引、数字索引和小数索引拒绝路径。
+
+## 设计动机
+
+官方 tool surface 把 `element_index` 暴露为字符串,仓库也应继续保持这个 schema;但手写 CLI JSON 时用户自然会把索引写成数字。解析层把整数数字容错转换为字符串,可以兼容现有文档误导产生的命令,同时不扩大其它字符串参数的接受类型。
+
+## 受影响文件
+
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift`
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
+- `apps/OpenComputerUseWindows/main.go`
+- `apps/OpenComputerUseWindows/main_test.go`
+- `apps/OpenComputerUseLinux/main.go`
+- `apps/OpenComputerUseLinux/main_test.go`
+- `skills/open-computer-use/SKILL.md`
+- `skills/open-computer-use/references/usage.md`
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
index d327a07..4092273 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
@@ -1,5 +1,41 @@
import Foundation
+func normalizedElementIndexArgument(_ value: Any?) -> String? {
+ if let string = value as? String {
+ return string.isEmpty ? nil : string
+ }
+
+ if let integer = value as? Int {
+ return String(integer)
+ }
+
+ if let number = value as? NSNumber {
+ if CFGetTypeID(number as CFTypeRef) == CFBooleanGetTypeID() {
+ return nil
+ }
+
+ return normalizedElementIndexNumber(number.doubleValue)
+ }
+
+ if let double = value as? Double {
+ return normalizedElementIndexNumber(double)
+ }
+
+ return nil
+}
+
+private func normalizedElementIndexNumber(_ value: Double) -> String? {
+ guard value.isFinite, value.rounded(.towardZero) == value else {
+ return nil
+ }
+
+ guard value >= Double(Int.min), value <= Double(Int.max) else {
+ return nil
+ }
+
+ return String(Int(value))
+}
+
public final class ComputerUseToolDispatcher {
private let service: ComputerUseService
@@ -16,7 +52,7 @@ public final class ComputerUseToolDispatcher {
case "click":
return try service.click(
app: requireString("app", in: arguments),
- elementIndex: optionalString("element_index", in: arguments),
+ elementIndex: optionalElementIndex(in: arguments),
x: optionalDouble("x", in: arguments),
y: optionalDouble("y", in: arguments),
clickCount: Int(optionalDouble("click_count", in: arguments) ?? 1),
@@ -25,14 +61,14 @@ public final class ComputerUseToolDispatcher {
case "perform_secondary_action":
return try service.performSecondaryAction(
app: requireString("app", in: arguments),
- elementIndex: requireString("element_index", in: arguments),
+ elementIndex: requireElementIndex(in: arguments),
action: requireString("action", in: arguments)
)
case "scroll":
return try service.scroll(
app: requireString("app", in: arguments),
direction: requireString("direction", in: arguments),
- elementIndex: requireString("element_index", in: arguments),
+ elementIndex: requireElementIndex(in: arguments),
pages: optionalDouble("pages", in: arguments) ?? 1
)
case "drag":
@@ -56,7 +92,7 @@ public final class ComputerUseToolDispatcher {
case "set_value":
return try service.setValue(
app: requireString("app", in: arguments),
- elementIndex: requireString("element_index", in: arguments),
+ elementIndex: requireElementIndex(in: arguments),
value: requireString("value", in: arguments)
)
default:
@@ -90,6 +126,18 @@ public final class ComputerUseToolDispatcher {
arguments[key] as? String
}
+ private func requireElementIndex(in arguments: [String: Any]) throws -> String {
+ guard let value = optionalElementIndex(in: arguments) else {
+ throw ComputerUseError.missingArgument("element_index")
+ }
+
+ return value
+ }
+
+ private func optionalElementIndex(in arguments: [String: Any]) -> String? {
+ normalizedElementIndexArgument(arguments["element_index"])
+ }
+
private func requireDouble(_ key: String, in arguments: [String: Any]) throws -> Double {
guard let value = optionalDouble(key, in: arguments) else {
throw ComputerUseError.missingArgument(key)
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index 6f93147..505ed4a 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -174,6 +174,30 @@ final class OpenComputerUseKitTests: XCTestCase {
XCTAssertEqual((arguments["pages"] as? NSNumber)?.intValue, 2)
}
+ func testElementIndexAcceptsNumericToolArgument() throws {
+ let arguments = try readOpenComputerUseToolArguments(
+ json: #"{"app":"TextEdit","element_index":0}"#,
+ file: nil
+ )
+
+ XCTAssertEqual(normalizedElementIndexArgument(arguments["element_index"]), "0")
+ }
+
+ func testElementIndexAcceptsNumericCallSequenceArgument() throws {
+ let calls = try readOpenComputerUseCallSequence(
+ json: #"[{"tool":"click","args":{"app":"TextEdit","element_index":0}}]"#,
+ file: nil
+ )
+
+ XCTAssertEqual(normalizedElementIndexArgument(calls[0].arguments["element_index"]), "0")
+ }
+
+ func testElementIndexRejectsMissingEmptyAndFractionalArguments() {
+ XCTAssertNil(normalizedElementIndexArgument(nil))
+ XCTAssertNil(normalizedElementIndexArgument(""))
+ XCTAssertNil(normalizedElementIndexArgument(1.5))
+ }
+
func testReadToolArgumentsRejectsNonObject() {
XCTAssertThrowsError(try readOpenComputerUseToolArguments(json: #"["TextEdit"]"#, file: nil)) { error in
XCTAssertEqual(
diff --git a/skills/open-computer-use/SKILL.md b/skills/open-computer-use/SKILL.md
index de51720..d3929de 100644
--- a/skills/open-computer-use/SKILL.md
+++ b/skills/open-computer-use/SKILL.md
@@ -41,7 +41,7 @@ open-computer-use -h
open-computer-use doctor
open-computer-use call list_apps
open-computer-use call get_app_state --args '{"app":"TextEdit"}'
-open-computer-use call click --args '{"app":"TextEdit","element_index":0}'
+open-computer-use call click --args '{"app":"TextEdit","element_index":"0"}'
open-computer-use call type_text --args '{"app":"TextEdit","text":"Hello from Open Computer Use"}'
```
diff --git a/skills/open-computer-use/references/usage.md b/skills/open-computer-use/references/usage.md
index b94e924..897b318 100644
--- a/skills/open-computer-use/references/usage.md
+++ b/skills/open-computer-use/references/usage.md
@@ -46,7 +46,7 @@ Use `call` for one-off checks:
```sh
open-computer-use call list_apps
open-computer-use call get_app_state --args '{"app":"TextEdit"}'
-open-computer-use call set_value --args '{"app":"TextEdit","element_index":1,"value":"Draft"}'
+open-computer-use call set_value --args '{"app":"TextEdit","element_index":"1","value":"Draft"}'
```
Use `--calls` for short action sequences that need to reuse the same process state:
@@ -54,7 +54,7 @@ Use `--calls` for short action sequences that need to reuse the same process sta
```sh
open-computer-use call --calls '[
{"tool":"get_app_state","args":{"app":"TextEdit"}},
- {"tool":"click","args":{"app":"TextEdit","element_index":1}},
+ {"tool":"click","args":{"app":"TextEdit","element_index":"1"}},
{"tool":"type_text","args":{"app":"TextEdit","text":"Hello"}}
]'
```