Skip to content

πŸ‘Ύ Open Computer Use – Open-Source Alternative to Codex Computer Use

feat(snapshot): add full-text option with 500 character default

tisfengfaeca8cc4583
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift
index 6790d64..d51fa22 100644
--- a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift
+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift
@@ -384,9 +384,9 @@ private final class AppAgentConnection: @unchecked Sendable {
let service = ComputerUseService()
return CLIProxyResponse(stdout: (service.listApps().primaryText ?? "") + "\n", stderr: "", exitCode: EXIT_SUCCESS)
- case let .snapshot(app):
+ case let .snapshot(app, showFullText):
let service = ComputerUseService()
- let text = try service.getAppState(app: app).primaryText ?? ""
+ let text = try service.getAppState(app: app, showFullText: showFullText).primaryText ?? ""
return CLIProxyResponse(stdout: text + "\n", stderr: "", exitCode: EXIT_SUCCESS)
case let .call(invocation):
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
index 405633c..cef6d82 100644
--- a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
@@ -57,9 +57,9 @@ enum OpenComputerUseMain {
case .listApps:
let service = ComputerUseService()
print(service.listApps().primaryText ?? "")
- case let .snapshot(app):
+ case let .snapshot(app, showFullText):
let service = ComputerUseService()
- print(try service.getAppState(app: app).primaryText ?? "")
+ print(try service.getAppState(app: app, showFullText: showFullText).primaryText ?? "")
case let .call(invocation):
if VisualCursorSupport.isEnabled {
_ = NSApplication.shared.setActivationPolicy(.accessory)
diff --git a/apps/OpenComputerUseLinux/main.go b/apps/OpenComputerUseLinux/main.go
index 74fd458..ded83e2 100644
--- a/apps/OpenComputerUseLinux/main.go
+++ b/apps/OpenComputerUseLinux/main.go
@@ -151,6 +151,7 @@ type linuxRequest struct {
Key string `json:"key,omitempty"`
Value string `json:"value,omitempty"`
WindowBounds *frame `json:"windowBounds,omitempty"`
+ ShowFullText bool `json:"show_full_text,omitempty"`
}
type linuxResponse struct {
@@ -173,7 +174,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {
case "list_apps":
return s.listApps()
case "get_app_state":
- return s.getAppState(requiredString(args, "app"))
+ return s.getAppState(requiredString(args, "app"), optionalBool(args, "show_full_text"))
case "click":
return s.click(
requiredString(args, "app"),
@@ -229,11 +230,11 @@ func (s *service) listApps() toolCallResult {
return textResult(response.Text, false)
}
-func (s *service) getAppState(app string) toolCallResult {
+func (s *service) getAppState(app string, showFullText bool) toolCallResult {
if app == "" {
return textResult("Missing required argument: app", true)
}
- snapshot, result := s.refreshSnapshot(app, linuxRequest{Tool: "get_app_state", App: app})
+ snapshot, result := s.refreshSnapshot(app, linuxRequest{Tool: "get_app_state", App: app, ShowFullText: showFullText})
if result.IsError {
return result
}
@@ -961,6 +962,11 @@ func optionalFloat(args map[string]any, key string) *float64 {
return nil
}
+func optionalBool(args map[string]any, key string) bool {
+ value, _ := args[key].(bool)
+ return value
+}
+
func intValue(value *float64, fallback int) int {
if value == nil {
return fallback
@@ -1014,7 +1020,8 @@ func toolDefinitions() []toolDefinition {
Description: "Get the state of an already running app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.",
Annotations: readOnlyAnnotations(),
InputSchema: objectSchema(map[string]any{
- "app": stringProperty("App name or bundle identifier"),
+ "app": stringProperty("App name or bundle identifier"),
+ "show_full_text": booleanProperty("Return full accessibility text without the default 500 character truncation. Defaults to false."),
}, []string{"app"}),
},
{
@@ -1099,6 +1106,10 @@ func stringProperty(description string) map[string]any {
return map[string]any{"type": "string", "description": description}
}
+func booleanProperty(description string) map[string]any {
+ return map[string]any{"type": "boolean", "description": description}
+}
+
func enumStringProperty(description string, values []string) map[string]any {
property := stringProperty(description)
property["enum"] = values
@@ -1150,10 +1161,14 @@ func runCLI(args []string, stdout io.Writer) error {
fmt.Fprintln(stdout, result.Content[0].Text)
return nil
case "snapshot":
- if len(args) != 2 {
- return errors.New("snapshot requires an app name, process name, window title, or pid")
+ app, showFullText, err := parseSnapshotArgs(args[1:])
+ if err != nil {
+ return err
}
- result := newService().callTool("get_app_state", map[string]any{"app": args[1]})
+ result := newService().callTool("get_app_state", map[string]any{
+ "app": app,
+ "show_full_text": showFullText,
+ })
if result.IsError {
return errors.New(result.Content[0].Text)
}
@@ -1178,6 +1193,29 @@ func runCLI(args []string, stdout io.Writer) error {
}
}
+func parseSnapshotArgs(args []string) (string, bool, error) {
+ var app string
+ showFullText := false
+ for _, arg := range args {
+ switch arg {
+ case "--show-full-text":
+ showFullText = true
+ default:
+ if strings.HasPrefix(arg, "-") {
+ return "", false, fmt.Errorf("unknown snapshot option: %s", arg)
+ }
+ if app != "" {
+ return "", false, errors.New("snapshot accepts exactly one app name, process name, window title, or pid")
+ }
+ app = arg
+ }
+ }
+ if app == "" {
+ return "", false, errors.New("snapshot requires an app name, process name, window title, or pid")
+ }
+ return app, showFullText, nil
+}
+
func runCallCommand(args []string, svc *service) (any, bool, error) {
if len(args) == 0 {
return nil, false, errors.New("call requires a tool name or --calls/--calls-file")
@@ -1408,7 +1446,7 @@ func helpText(command string) string {
case "call":
return "Usage:\n open-computer-use call <tool> [--args '<json-object>']\n open-computer-use call --calls '<json-array>'\n\nThe JSON array form keeps all calls in one process so element_index state can be reused.\n"
case "snapshot":
- return "Usage:\n open-computer-use snapshot <app>\n\nPrint the current Linux AT-SPI snapshot for the target app.\n"
+ return "Usage:\n open-computer-use snapshot [--show-full-text] <app>\n\nPrint the current Linux AT-SPI snapshot for the target app.\n"
default:
return `Open Computer Use for Linux
diff --git a/apps/OpenComputerUseLinux/main_test.go b/apps/OpenComputerUseLinux/main_test.go
index e85b3cf..6e28938 100644
--- a/apps/OpenComputerUseLinux/main_test.go
+++ b/apps/OpenComputerUseLinux/main_test.go
@@ -16,6 +16,37 @@ func TestToolDefinitionCount(t *testing.T) {
}
}
+func TestGetAppStateSchemaIncludesShowFullText(t *testing.T) {
+ tool := findToolDefinition(t, "get_app_state")
+ properties := tool.InputSchema["properties"].(map[string]any)
+ showFullText := properties["show_full_text"].(map[string]any)
+ if got := showFullText["type"]; got != "boolean" {
+ t.Fatalf("show_full_text type = %v, want boolean", got)
+ }
+ required := tool.InputSchema["required"].([]string)
+ if len(required) != 1 || required[0] != "app" {
+ t.Fatalf("required = %#v, want [app]", required)
+ }
+}
+
+func TestParseSnapshotArgsSupportsShowFullText(t *testing.T) {
+ app, showFullText, err := parseSnapshotArgs([]string{"--show-full-text", "Text Editor"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if app != "Text Editor" || !showFullText {
+ t.Fatalf("parseSnapshotArgs = (%q, %v), want (Text Editor, true)", app, showFullText)
+ }
+
+ app, showFullText, err = parseSnapshotArgs([]string{"Text Editor"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if app != "Text Editor" || showFullText {
+ t.Fatalf("parseSnapshotArgs default = (%q, %v), want (Text Editor, false)", app, showFullText)
+ }
+}
+
func TestCallSequenceStopsAfterFirstToolError(t *testing.T) {
output, hasError, err := runCallCommand([]string{
"--calls",
@@ -105,6 +136,18 @@ func TestLinuxRuntimeDocumentsATSPIAndFallbackBoundary(t *testing.T) {
}
}
+func TestLinuxRuntimeTextLimitSupportsFullTextMode(t *testing.T) {
+ if !strings.Contains(linuxRuntimeScript, "TEXT_CHARACTER_LIMIT = 500") {
+ t.Fatal("Linux runtime should define the shared 500 character text limit")
+ }
+ if !strings.Contains(linuxRuntimeScript, "show_full_text=bool(operation.get(\"show_full_text\", False))") {
+ t.Fatal("Linux get_app_state should pass show_full_text into snapshot rendering")
+ }
+ if !strings.Contains(linuxRuntimeScript, "TEXT_CHARACTER_LIMIT + 1") {
+ t.Fatal("Linux default truncation should read one extra character so it can append ellipsis")
+ }
+}
+
func TestLinuxRuntimeEnvironmentDiscoversDesktopSession(t *testing.T) {
runtimeDir := shortTempDir(t)
listenUnixSocket(t, filepath.Join(runtimeDir, "bus"))
@@ -169,6 +212,17 @@ func listenUnixSocket(t *testing.T, path string) {
})
}
+func findToolDefinition(t *testing.T, name string) toolDefinition {
+ t.Helper()
+ for _, tool := range toolDefinitions() {
+ if tool.Name == name {
+ return tool
+ }
+ }
+ t.Fatalf("missing tool definition %q", name)
+ return toolDefinition{}
+}
+
func shortTempDir(t *testing.T) string {
t.Helper()
path, err := os.MkdirTemp("/tmp", "ocu-*")
diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.py
index 24bb994..a147234 100644
--- a/apps/OpenComputerUseLinux/runtime.py
+++ b/apps/OpenComputerUseLinux/runtime.py
@@ -26,6 +26,7 @@ from gi.repository import Atspi
MAX_ELEMENTS = 500
MAX_DEPTH = 64
+TEXT_CHARACTER_LIMIT = 500
def frame(x, y, width, height):
@@ -78,6 +79,15 @@ def node_name(node):
return str(safe(node.get_name, "") or "")
+def limit_text(value, show_full_text=False):
+ text = str(value or "")
+ if show_full_text:
+ return text
+ if len(text) > TEXT_CHARACTER_LIMIT:
+ return text[:TEXT_CHARACTER_LIMIT] + "..."
+ return text
+
+
def node_role(node):
return str(safe(node.get_role_name, "") or "")
@@ -206,7 +216,7 @@ def accessible_id(node):
return str(safe(node.get_accessible_id, "") or "")
-def text_value(node):
+def text_value(node, show_full_text=False):
if not bool(safe(node.is_text, False)):
return ""
text_iface = safe(node.get_text_iface)
@@ -215,10 +225,9 @@ def text_value(node):
count = int(safe(lambda: Atspi.Text.get_character_count(text_iface), 0) or 0)
if count <= 0:
return ""
- value = str(safe(lambda: Atspi.Text.get_text(text_iface, 0, min(count, 500)), "") or "")
- if count > 500:
- return value + "..."
- return value
+ end_offset = count if show_full_text else min(count, TEXT_CHARACTER_LIMIT + 1)
+ value = str(safe(lambda: Atspi.Text.get_text(text_iface, 0, end_offset), "") or "")
+ return limit_text(value, show_full_text=show_full_text)
def numeric_value(node):
@@ -231,29 +240,29 @@ def numeric_value(node):
return str(current)
-def element_value(node):
- return text_value(node) or numeric_value(node)
+def element_value(node, show_full_text=False):
+ return text_value(node, show_full_text=show_full_text) or numeric_value(node)
-def record_for(node, index, path, window_bounds):
+def record_for(node, index, path, window_bounds, show_full_text=False):
bounds = relative_frame(node, window_bounds)
role = node_role(node)
return {
"index": index,
"runtimeId": path[:],
"automationId": accessible_id(node),
- "name": node_name(node),
+ "name": limit_text(node_name(node), show_full_text=show_full_text),
"controlType": role,
"localizedControlType": role,
"className": str(safe(node.get_toolkit_name, "") or ""),
- "value": element_value(node),
+ "value": element_value(node, show_full_text=show_full_text),
"nativeWindowHandle": 0,
"frame": bounds,
"actions": action_names(node),
}
-def render_tree(root, window_bounds, root_path):
+def render_tree(root, window_bounds, root_path, show_full_text=False):
records = []
lines = []
@@ -261,7 +270,7 @@ def render_tree(root, window_bounds, root_path):
if len(records) >= MAX_ELEMENTS or depth > MAX_DEPTH or node is None:
return
index = len(records)
- record = record_for(node, index, path, window_bounds)
+ record = record_for(node, index, path, window_bounds, show_full_text=show_full_text)
records.append(record)
role = record["localizedControlType"] or record["controlType"] or "element"
@@ -352,7 +361,7 @@ def pixbuf_looks_black(pixbuf):
return False
-def focused_summary(app_pid):
+def focused_summary(app_pid, show_full_text=False):
try:
root = desktop()
for app in iter_apps():
@@ -365,13 +374,13 @@ def focused_summary(app_pid):
if focused is None:
return None
role = node_role(focused)
- name = node_name(focused)
+ name = limit_text(node_name(focused), show_full_text=show_full_text)
return (role + " " + name).strip()
except Exception:
return None
-def selected_text(app_pid):
+def selected_text(app_pid, show_full_text=False):
try:
for app in iter_apps():
if node_pid(app) != app_pid:
@@ -386,19 +395,23 @@ def selected_text(app_pid):
selections = safe(lambda: Atspi.Text.get_text_selections(text_iface), [])
if selections:
selection = selections[0]
- return Atspi.Text.get_text(
- text_iface, selection.start_offset, selection.end_offset
+ end_offset = selection.end_offset
+ if not show_full_text:
+ end_offset = min(end_offset, selection.start_offset + TEXT_CHARACTER_LIMIT + 1)
+ value = Atspi.Text.get_text(
+ text_iface, selection.start_offset, end_offset
)
+ return limit_text(value, show_full_text=show_full_text)
except Exception:
return None
return None
-def build_snapshot(query):
+def build_snapshot(query, show_full_text=False):
app = resolve_app(query)
window_index, window = main_window(app)
bounds = extents(window)
- records, lines = render_tree(window, bounds, [window_index])
+ records, lines = render_tree(window, bounds, [window_index], show_full_text=show_full_text)
pid = node_pid(app)
return {
"app": {
@@ -406,12 +419,12 @@ def build_snapshot(query):
"bundleIdentifier": node_name(app),
"pid": pid,
},
- "windowTitle": node_name(window),
+ "windowTitle": limit_text(node_name(window), show_full_text=show_full_text),
"windowBounds": bounds,
"screenshotPngBase64": capture_window_png(bounds),
"treeLines": lines,
- "focusedSummary": focused_summary(pid),
- "selectedText": selected_text(pid),
+ "focusedSummary": focused_summary(pid, show_full_text=show_full_text),
+ "selectedText": selected_text(pid, show_full_text=show_full_text),
"elements": records,
}
@@ -731,7 +744,13 @@ def perform_operation(operation):
if tool == "list_apps":
return {"ok": True, "text": list_apps_text()}
if tool == "get_app_state":
- return {"ok": True, "snapshot": build_snapshot(operation.get("app", ""))}
+ return {
+ "ok": True,
+ "snapshot": build_snapshot(
+ operation.get("app", ""),
+ show_full_text=bool(operation.get("show_full_text", False)),
+ ),
+ }
app = resolve_app(operation.get("app", ""))
_, window = main_window(app)
diff --git a/apps/OpenComputerUseWindows/main.go b/apps/OpenComputerUseWindows/main.go
index 7259e93..afa3001 100644
--- a/apps/OpenComputerUseWindows/main.go
+++ b/apps/OpenComputerUseWindows/main.go
@@ -148,6 +148,7 @@ type psRequest struct {
Key string `json:"key,omitempty"`
Value string `json:"value,omitempty"`
WindowBounds *frame `json:"windowBounds,omitempty"`
+ ShowFullText bool `json:"show_full_text,omitempty"`
}
type psResponse struct {
@@ -170,7 +171,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {
case "list_apps":
return s.listApps()
case "get_app_state":
- return s.getAppState(requiredString(args, "app"))
+ return s.getAppState(requiredString(args, "app"), optionalBool(args, "show_full_text"))
case "click":
return s.click(
requiredString(args, "app"),
@@ -226,11 +227,11 @@ func (s *service) listApps() toolCallResult {
return textResult(response.Text, false)
}
-func (s *service) getAppState(app string) toolCallResult {
+func (s *service) getAppState(app string, showFullText bool) toolCallResult {
if app == "" {
return textResult("Missing required argument: app", true)
}
- snapshot, result := s.refreshSnapshot(app, psRequest{Tool: "get_app_state", App: app})
+ snapshot, result := s.refreshSnapshot(app, psRequest{Tool: "get_app_state", App: app, ShowFullText: showFullText})
if result.IsError {
return result
}
@@ -544,6 +545,11 @@ func optionalFloat(args map[string]any, key string) *float64 {
return nil
}
+func optionalBool(args map[string]any, key string) bool {
+ value, _ := args[key].(bool)
+ return value
+}
+
func intValue(value *float64, fallback int) int {
if value == nil {
return fallback
@@ -597,7 +603,8 @@ func toolDefinitions() []toolDefinition {
Description: "Get the state of an already running app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.",
Annotations: readOnlyAnnotations(),
InputSchema: objectSchema(map[string]any{
- "app": stringProperty("App name or bundle identifier"),
+ "app": stringProperty("App name or bundle identifier"),
+ "show_full_text": booleanProperty("Return full accessibility text without the default 500 character truncation. Defaults to false."),
}, []string{"app"}),
},
{
@@ -682,6 +689,10 @@ func stringProperty(description string) map[string]any {
return map[string]any{"type": "string", "description": description}
}
+func booleanProperty(description string) map[string]any {
+ return map[string]any{"type": "boolean", "description": description}
+}
+
func enumStringProperty(description string, values []string) map[string]any {
property := stringProperty(description)
property["enum"] = values
@@ -733,10 +744,14 @@ func runCLI(args []string, stdout io.Writer) error {
fmt.Fprintln(stdout, result.Content[0].Text)
return nil
case "snapshot":
- if len(args) != 2 {
- return errors.New("snapshot requires an app name, process name, window title, or pid")
+ app, showFullText, err := parseSnapshotArgs(args[1:])
+ if err != nil {
+ return err
}
- result := newService().callTool("get_app_state", map[string]any{"app": args[1]})
+ result := newService().callTool("get_app_state", map[string]any{
+ "app": app,
+ "show_full_text": showFullText,
+ })
if result.IsError {
return errors.New(result.Content[0].Text)
}
@@ -761,6 +776,29 @@ func runCLI(args []string, stdout io.Writer) error {
}
}
+func parseSnapshotArgs(args []string) (string, bool, error) {
+ var app string
+ showFullText := false
+ for _, arg := range args {
+ switch arg {
+ case "--show-full-text":
+ showFullText = true
+ default:
+ if strings.HasPrefix(arg, "-") {
+ return "", false, fmt.Errorf("unknown snapshot option: %s", arg)
+ }
+ if app != "" {
+ return "", false, errors.New("snapshot accepts exactly one app name, process name, window title, or pid")
+ }
+ app = arg
+ }
+ }
+ if app == "" {
+ return "", false, errors.New("snapshot requires an app name, process name, window title, or pid")
+ }
+ return app, showFullText, nil
+}
+
func runCallCommand(args []string, svc *service) (any, bool, error) {
if len(args) == 0 {
return nil, false, errors.New("call requires a tool name or --calls/--calls-file")
@@ -991,7 +1029,7 @@ func helpText(command string) string {
case "call":
return "Usage:\n open-computer-use.exe call <tool> [--args '<json-object>']\n open-computer-use.exe call --calls '<json-array>'\n\nThe JSON array form keeps all calls in one process so element_index state can be reused.\n"
case "snapshot":
- return "Usage:\n open-computer-use.exe snapshot <app>\n\nPrint the current Windows UI Automation snapshot for the target app.\n"
+ return "Usage:\n open-computer-use.exe snapshot [--show-full-text] <app>\n\nPrint the current Windows UI Automation snapshot for the target app.\n"
default:
return `Open Computer Use for Windows
diff --git a/apps/OpenComputerUseWindows/main_test.go b/apps/OpenComputerUseWindows/main_test.go
index 4f2d622..a2adcf3 100644
--- a/apps/OpenComputerUseWindows/main_test.go
+++ b/apps/OpenComputerUseWindows/main_test.go
@@ -13,6 +13,37 @@ func TestToolDefinitionCount(t *testing.T) {
}
}
+func TestGetAppStateSchemaIncludesShowFullText(t *testing.T) {
+ tool := findToolDefinition(t, "get_app_state")
+ properties := tool.InputSchema["properties"].(map[string]any)
+ showFullText := properties["show_full_text"].(map[string]any)
+ if got := showFullText["type"]; got != "boolean" {
+ t.Fatalf("show_full_text type = %v, want boolean", got)
+ }
+ required := tool.InputSchema["required"].([]string)
+ if len(required) != 1 || required[0] != "app" {
+ t.Fatalf("required = %#v, want [app]", required)
+ }
+}
+
+func TestParseSnapshotArgsSupportsShowFullText(t *testing.T) {
+ app, showFullText, err := parseSnapshotArgs([]string{"--show-full-text", "Notepad"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if app != "Notepad" || !showFullText {
+ t.Fatalf("parseSnapshotArgs = (%q, %v), want (Notepad, true)", app, showFullText)
+ }
+
+ app, showFullText, err = parseSnapshotArgs([]string{"Notepad"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if app != "Notepad" || showFullText {
+ t.Fatalf("parseSnapshotArgs default = (%q, %v), want (Notepad, false)", app, showFullText)
+ }
+}
+
func TestCallSequenceStopsAfterFirstToolError(t *testing.T) {
output, hasError, err := runCallCommand([]string{
"--calls",
@@ -114,3 +145,26 @@ func TestUTF8EncodingInPowerShellScript(t *testing.T) {
t.Fatal("PowerShell script must set [Console]::OutputEncoding to UTF-8 for proper non-ASCII character handling")
}
}
+
+func TestWindowsRuntimeTextLimitSupportsFullTextMode(t *testing.T) {
+ if !strings.Contains(windowsRuntimeScript, "$TextCharacterLimit = 500") {
+ t.Fatal("Windows runtime should define the shared 500 character text limit")
+ }
+ if !strings.Contains(windowsRuntimeScript, "Build-Snapshot $operation.app ([bool]$operation.show_full_text)") {
+ t.Fatal("Windows get_app_state should pass show_full_text into snapshot rendering")
+ }
+ if !strings.Contains(windowsRuntimeScript, "$maxLength = if ($ShowFullText) { -1 } else { $script:TextCharacterLimit + 1 }") {
+ t.Fatal("Windows selected text should use full UIA text only in full-text mode")
+ }
+}
+
+func findToolDefinition(t *testing.T, name string) toolDefinition {
+ t.Helper()
+ for _, tool := range toolDefinitions() {
+ if tool.Name == name {
+ return tool
+ }
+ }
+ t.Fatalf("missing tool definition %q", name)
+ return toolDefinition{}
+}
diff --git a/apps/OpenComputerUseWindows/runtime.ps1 b/apps/OpenComputerUseWindows/runtime.ps1
index 63ca1ed..2a38ad3 100644
--- a/apps/OpenComputerUseWindows/runtime.ps1
+++ b/apps/OpenComputerUseWindows/runtime.ps1
@@ -4,6 +4,7 @@ param(
)
$ErrorActionPreference = "Stop"
+$TextCharacterLimit = 500
# Set output encoding to UTF-8 to properly handle non-ASCII characters
$OutputEncoding = [System.Text.Encoding]::UTF8
@@ -415,7 +416,20 @@ function Get-ElementControlTypeName($element) {
}
}
-function Get-ElementValue($element) {
+function Limit-Text([string]$Text, [bool]$ShowFullText = $false) {
+ if ($null -eq $Text) {
+ return ""
+ }
+ if ($ShowFullText) {
+ return $Text
+ }
+ if ($Text.Length -gt $script:TextCharacterLimit) {
+ return $Text.Substring(0, $script:TextCharacterLimit) + "..."
+ }
+ return $Text
+}
+
+function Get-ElementValue($element, [bool]$ShowFullText = $false) {
try {
$valuePattern = $element.GetCurrentPattern([Windows.Automation.ValuePattern]::Pattern)
$value = $valuePattern.Current.Value
@@ -423,16 +437,13 @@ function Get-ElementValue($element) {
return ""
}
$text = [string]$value
- if ($text.Length -gt 500) {
- return $text.Substring(0, 500)
- }
- return $text
+ return Limit-Text $text $ShowFullText
} catch {
return ""
}
}
-function Get-ElementRecord($element, [int]$index, $windowBounds) {
+function Get-ElementRecord($element, [int]$index, $windowBounds, [bool]$ShowFullText = $false) {
$frame = Get-ElementFrame $element $windowBounds
$runtimeId = @()
try { $runtimeId = @($element.GetRuntimeId()) } catch {}
@@ -440,11 +451,11 @@ function Get-ElementRecord($element, [int]$index, $windowBounds) {
index = $index
runtimeId = $runtimeId
automationId = Get-ElementString $element "AutomationId"
- name = Get-ElementString $element "Name"
+ name = Limit-Text (Get-ElementString $element "Name") $ShowFullText
controlType = Get-ElementControlTypeName $element
localizedControlType = Get-ElementString $element "LocalizedControlType"
className = Get-ElementString $element "ClassName"
- value = Get-ElementValue $element
+ value = Get-ElementValue $element $ShowFullText
nativeWindowHandle = Get-ElementInt64 $element "NativeWindowHandle"
frame = $frame
actions = @(Get-PatternNames $element)
@@ -461,7 +472,7 @@ function Get-ElementTitle($record) {
return ""
}
-function Render-Tree($element, $windowBounds) {
+function Render-Tree($element, $windowBounds, [bool]$ShowFullText = $false) {
$records = New-Object System.Collections.Generic.List[object]
$lines = New-Object System.Collections.Generic.List[string]
$visited = New-Object System.Collections.Generic.HashSet[string]
@@ -479,7 +490,7 @@ function Render-Tree($element, $windowBounds) {
$index = $script:nextIndex
$script:nextIndex++
- $record = Get-ElementRecord $node $index $script:windowBounds
+ $record = Get-ElementRecord $node $index $script:windowBounds $ShowFullText
$script:records.Add($record)
$role = $record.localizedControlType
@@ -544,12 +555,12 @@ function Capture-WindowPngBase64($bounds) {
}
}
-function Get-FocusedSummary($processId) {
+function Get-FocusedSummary($processId, [bool]$ShowFullText = $false) {
try {
$focused = [Windows.Automation.AutomationElement]::FocusedElement
if ($null -ne $focused -and $focused.Current.ProcessId -eq $processId) {
$role = $focused.Current.LocalizedControlType
- $name = $focused.Current.Name
+ $name = Limit-Text $focused.Current.Name $ShowFullText
if ([string]::IsNullOrWhiteSpace($name)) {
return $role
}
@@ -560,7 +571,7 @@ function Get-FocusedSummary($processId) {
return $null
}
-function Get-SelectedText($processId) {
+function Get-SelectedText($processId, [bool]$ShowFullText = $false) {
try {
$focused = [Windows.Automation.AutomationElement]::FocusedElement
if ($null -eq $focused -or $focused.Current.ProcessId -ne $processId) {
@@ -569,30 +580,31 @@ function Get-SelectedText($processId) {
$textPattern = $focused.GetCurrentPattern([Windows.Automation.TextPattern]::Pattern)
$selection = $textPattern.GetSelection()
if ($selection.Count -gt 0) {
- return $selection.Item(0).GetText(2048)
+ $maxLength = if ($ShowFullText) { -1 } else { $script:TextCharacterLimit + 1 }
+ return Limit-Text ($selection.Item(0).GetText($maxLength)) $ShowFullText
}
} catch {
}
return $null
}
-function Build-Snapshot([string]$query) {
+function Build-Snapshot([string]$query, [bool]$ShowFullText = $false) {
$process = Resolve-App $query
$element = Get-MainElement $process
$bounds = Get-WindowBounds $process $element
- $rendered = Render-Tree $element $bounds
+ $rendered = Render-Tree $element $bounds $ShowFullText
[pscustomobject]@{
app = [pscustomobject]@{
name = $process.ProcessName
bundleIdentifier = $process.ProcessName
pid = [int]$process.Id
}
- windowTitle = $process.MainWindowTitle
+ windowTitle = Limit-Text $process.MainWindowTitle $ShowFullText
windowBounds = $bounds
screenshotPngBase64 = Capture-WindowPngBase64 $bounds
treeLines = @($rendered.lines)
- focusedSummary = Get-FocusedSummary $process.Id
- selectedText = Get-SelectedText $process.Id
+ focusedSummary = Get-FocusedSummary $process.Id $ShowFullText
+ selectedText = Get-SelectedText $process.Id $ShowFullText
elements = @($rendered.records)
}
}
@@ -863,7 +875,7 @@ try {
if ($operation.tool -eq "list_apps") {
$response = [pscustomobject]@{ ok = $true; text = (List-Apps) }
} elseif ($operation.tool -eq "get_app_state") {
- $response = [pscustomobject]@{ ok = $true; snapshot = (Build-Snapshot $operation.app) }
+ $response = [pscustomobject]@{ ok = $true; snapshot = (Build-Snapshot $operation.app ([bool]$operation.show_full_text)) }
} else {
$process = Resolve-App $operation.app
$hwnd = [IntPtr]$process.MainWindowHandle
diff --git a/docs/histories/2026-06/20260605-1640-unify-snapshot-text-limit.md b/docs/histories/2026-06/20260605-1640-unify-snapshot-text-limit.md
new file mode 100644
index 0000000..85ddb2d
--- /dev/null
+++ b/docs/histories/2026-06/20260605-1640-unify-snapshot-text-limit.md
@@ -0,0 +1,27 @@
+## [2026-06-05 16:40] | Task: Unify snapshot text limit
+
+### Execution Context
+* **Agent ID**: `Codex`
+* **Base Model**: `GPT-5`
+* **Runtime**: `Codex desktop`
+
+### User Query
+> Unify the default snapshot text truncation across macOS, Linux, and Windows to 500 characters, append `...` when truncated, and add an explicit full-text parameter for `get_app_state` and `snapshot`.
+
+### Changes Overview
+**Scope:** macOS Swift renderer, Linux runtime, Windows runtime, CLI/tool schema, tests.
+
+**Key Actions:**
+- Added optional `show_full_text` / `--show-full-text` support for full accessibility text output.
+- Changed macOS snapshot text truncation from 160 to the shared 500 character default.
+- Unified Linux and Windows truncation markers so truncated text appends `...`.
+- Kept node count, tree depth, image, and action-result refresh protections unchanged.
+
+### Design Intent (Why)
+Default truncation protects snapshot size and preserves upstream-compatible behavior, while the explicit full-text option lets users inspect long semantic text such as chat messages without adding URL- or page-specific rendering rules.
+
+### Files Modified
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift`
+- `apps/OpenComputerUseLinux/runtime.py`
+- `apps/OpenComputerUseWindows/runtime.ps1`
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
index 51140ca..8b7b6d9 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
@@ -43,6 +43,7 @@ let screenshotCaptureTimeout: TimeInterval = 5
let screenshotResultMaxPNGBytes = 900_000
let screenshotResultMaxDimension: CGFloat = 1280
let screenshotResultMinScale: CGFloat = 0.25
+let snapshotTextDefaultCharacterLimit = 500
private let windowVisibilityRecoveryDelay: TimeInterval = 0.7
private let axWebAreaRole = "AXWebArea"
private let axContentsAttribute = "AXContents"
@@ -94,7 +95,7 @@ public enum SnapshotTextStyle {
}
enum SnapshotBuilder {
- static func build(for app: RunningAppDescriptor) throws -> AppSnapshot {
+ static func build(for app: RunningAppDescriptor, showFullText: Bool = false) throws -> AppSnapshot {
if app.name == FixtureBridge.appName, let fixtureState = try FixtureBridge.readState() {
return buildFixtureSnapshot(app: app, state: fixtureState)
}
@@ -142,7 +143,8 @@ enum SnapshotBuilder {
windowTitle: windowTitle,
windowCapture: windowCapture,
focusedApplication: focusedApplication,
- systemWide: systemWide
+ systemWide: systemWide,
+ showFullText: showFullText
)
}
@@ -153,13 +155,19 @@ enum SnapshotBuilder {
windowTitle: String?,
windowCapture: WindowCapture,
focusedApplication: AXUIElement?,
- systemWide: AXUIElement
+ systemWide: AXUIElement,
+ showFullText: Bool
) -> AppSnapshot {
let windowBounds = windowCapture.bounds
let screenshotPNGData = windowCapture.pngDataIfAvailable()
let focusedElement = preferredFocusedElement(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)
- let selectedText = focusedElement.flatMap(copySelectedText(_:))
- let context = RenderContext(windowBounds: windowBounds, focusedElement: focusedElement)
+ let textCharacterLimit = showFullText ? nil : snapshotTextDefaultCharacterLimit
+ let selectedText = focusedElement.flatMap { copySelectedText($0, characterLimit: textCharacterLimit) }
+ let context = RenderContext(
+ windowBounds: windowBounds,
+ focusedElement: focusedElement,
+ textCharacterLimit: textCharacterLimit
+ )
var renderer = TreeRenderer(context: context)
renderer.render(rootElement)
@@ -590,6 +598,7 @@ enum BlockingAsyncBridge {
private struct RenderContext {
let windowBounds: CGRect?
let focusedElement: AXUIElement?
+ let textCharacterLimit: Int?
}
private struct TreeRenderer {
@@ -620,21 +629,24 @@ private struct TreeRenderer {
let subrole = stringValue(of: root, attribute: kAXSubroleAttribute)
let baseRoleText = roleDescription(of: root, role: role, subrole: subrole)
let label = stringValue(of: root, attribute: kAXDescriptionAttribute)
+ .map { sanitizeText($0, characterLimit: context.textCharacterLimit) }
let help = stringValue(of: root, attribute: kAXHelpAttribute)
- let value = sanitizedValue(of: root)
+ .map { sanitizeText($0, characterLimit: context.textCharacterLimit) }
+ let value = sanitizedValue(of: root, characterLimit: context.textCharacterLimit)
let axIdentifier = displayIdentifier(stringValue(of: root, attribute: kAXIdentifierAttribute))
let traits = summarizeTraits(of: root)
let actions = copyActions(root) ?? []
let prettyActions = meaningfulActions(actions, role: role)
- let placeholder = placeholderValue(of: root)
+ let placeholder = placeholderValue(of: root, characterLimit: context.textCharacterLimit)
let webAreaDepth = webAreaDepth(role: role, ancestors: ancestors)
let localFrame = resolveLocalFrame(of: root, windowBounds: context.windowBounds)
- let rowTexts = role == kAXRowRole as String ? flattenedRowTexts(of: root) : []
+ let rowTexts = role == kAXRowRole as String ? flattenedRowTexts(of: root, characterLimit: context.textCharacterLimit) : []
let childElements = children(of: root)
let genericTextSummary = summarizedGenericText(
of: root,
role: role,
- childElements: childElements
+ childElements: childElements,
+ characterLimit: context.textCharacterLimit
)
let summaryImageChildren = genericTextSummary == nil ? [] : summaryImageDescendants(of: root)
let rendersSummaryAsChildren = shouldRenderGenericTextSummaryAsChildren(
@@ -647,9 +659,10 @@ private struct TreeRenderer {
label: label,
identifier: axIdentifier,
explicitValue: value,
- rowTexts: rowTexts
+ rowTexts: rowTexts,
+ characterLimit: context.textCharacterLimit
)
- let linkText = role == "AXLink" ? markdownLinkText(for: root, title: title, label: label, value: value) : nil
+ let linkText = role == "AXLink" ? markdownLinkText(for: root, title: title, label: label, value: value, characterLimit: context.textCharacterLimit) : nil
let displayTitle = linkText ?? title
let inlineRowSummary = outlineRowSummary(for: root, role: role)
let hidesChildren = shouldSuppressChildren(
@@ -696,7 +709,7 @@ private struct TreeRenderer {
let titleSegment = displayTitle.map { " \($0)" } ?? ""
let rowSummary = inlineRowSummary ?? (rendersSummaryAsChildren ? nil : genericTextSummary)
let rowSummarySegment = rowSummary.map { " \($0)" } ?? ""
- let labelSegment = formattedLabelSegment(label, title: displayTitle, linkText: linkText)
+ let labelSegment = formattedLabelSegment(label, title: displayTitle, linkText: linkText, characterLimit: context.textCharacterLimit)
let helpSegment = {
guard let help else {
return ""
@@ -706,7 +719,7 @@ private struct TreeRenderer {
}
return " Help: \(help)"
}()
- let urlSegment = formattedURLSegment(for: root, title: displayTitle, label: label)
+ let urlSegment = formattedURLSegment(for: root, title: displayTitle, label: label, characterLimit: context.textCharacterLimit)
let identifierSegment = displayIdentifierSegment(for: root, role: role, identifier: axIdentifier, title: displayTitle)
let rawValueSegment = formattedValueSegment(for: root, roleText: roleText, title: displayTitle, value: value)
let valueSegment = formattedValueSegmentWithSeparator(
@@ -993,13 +1006,13 @@ private func stringValue(of element: AXUIElement, attribute: String) -> String?
return nil
}
-private func copySelectedText(_ element: AXUIElement) -> String? {
+private func copySelectedText(_ element: AXUIElement, characterLimit: Int? = snapshotTextDefaultCharacterLimit) -> String? {
guard let value = stringValue(of: element, attribute: kAXSelectedTextAttribute) else {
return nil
}
- let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines)
- return trimmed.isEmpty ? nil : trimmed
+ let sanitized = sanitizeText(value, characterLimit: characterLimit)
+ return sanitized.isEmpty ? nil : sanitized
}
private func boolValue(of element: AXUIElement, attribute: String) -> Bool? {
@@ -1022,9 +1035,9 @@ private func isSettable(of element: AXUIElement, attribute: String) -> Bool {
return error == .success && settable.boolValue
}
-private func sanitizedValue(of element: AXUIElement) -> String? {
+private func sanitizedValue(of element: AXUIElement, characterLimit: Int? = snapshotTextDefaultCharacterLimit) -> String? {
if let string = stringValue(of: element, attribute: kAXValueAttribute) {
- let sanitized = sanitizeText(string)
+ let sanitized = sanitizeText(string, characterLimit: characterLimit)
return sanitized.isEmpty ? nil : sanitized
}
@@ -1043,10 +1056,10 @@ private func sanitizedValue(of element: AXUIElement) -> String? {
return nil
}
-private func placeholderValue(of element: AXUIElement) -> String? {
+private func placeholderValue(of element: AXUIElement, characterLimit: Int? = snapshotTextDefaultCharacterLimit) -> String? {
for attribute in ["AXPlaceholderValue", "AXPlaceholder"] {
if let string = stringValue(of: element, attribute: attribute) {
- let sanitized = sanitizeText(string)
+ let sanitized = sanitizeText(string, characterLimit: characterLimit)
if !sanitized.isEmpty {
return sanitized
}
@@ -1083,10 +1096,11 @@ private func preferredDisplayTitle(
label: String?,
identifier: String?,
explicitValue: String?,
- rowTexts: [String]
+ rowTexts: [String],
+ characterLimit: Int? = snapshotTextDefaultCharacterLimit
) -> String? {
if let title = stringValue(of: element, attribute: kAXTitleAttribute), !title.isEmpty {
- return sanitizeText(title)
+ return sanitizeText(title, characterLimit: characterLimit)
}
if role == kAXRowRole as String {
@@ -1098,18 +1112,18 @@ private func preferredDisplayTitle(
}
if (role == kAXButtonRole as String || role == kAXPopUpButtonRole as String), let label, !label.isEmpty {
- return sanitizeText(label)
+ return sanitizeText(label, characterLimit: characterLimit)
}
if role == kAXImageRole as String, let label, !label.isEmpty {
- return sanitizeText(label)
+ return sanitizeText(label, characterLimit: characterLimit)
}
if (role == kAXGroupRole as String || role == kAXUnknownRole as String || role == "AXWebArea"),
let label,
!label.isEmpty
{
- return sanitizeText(label)
+ return sanitizeText(label, characterLimit: characterLimit)
}
guard roleDescription(of: element, role: role, subrole: stringValue(of: element, attribute: kAXSubroleAttribute)) == "search text field" else {
@@ -1119,8 +1133,14 @@ private func preferredDisplayTitle(
return explicitValue
}
-private func markdownLinkText(for element: AXUIElement, title: String?, label: String?, value: String?) -> String? {
- guard let url = urlValue(of: element, attribute: kAXURLAttribute), !url.isEmpty else {
+private func markdownLinkText(
+ for element: AXUIElement,
+ title: String?,
+ label: String?,
+ value: String?,
+ characterLimit: Int? = snapshotTextDefaultCharacterLimit
+) -> String? {
+ guard let url = urlValue(of: element, attribute: kAXURLAttribute, characterLimit: characterLimit), !url.isEmpty else {
return nil
}
@@ -1129,7 +1149,7 @@ private func markdownLinkText(for element: AXUIElement, title: String?, label: S
guard let candidate else {
return nil
}
- let sanitized = sanitizeText(candidate)
+ let sanitized = sanitizeText(candidate, characterLimit: characterLimit)
return sanitized.isEmpty ? nil : sanitized
}
.first
@@ -1189,17 +1209,27 @@ private func formattedValueSegment(for element: AXUIElement, roleText: String, t
return " Value: \(value)"
}
-func formattedLabelSegment(_ label: String?, title: String?, linkText: String?) -> String {
+func formattedLabelSegment(
+ _ label: String?,
+ title: String?,
+ linkText: String?,
+ characterLimit: Int? = snapshotTextDefaultCharacterLimit
+) -> String {
guard let label, label != title else {
return ""
}
- let comparableLabel = markdownEscapedLinkText(sanitizeText(label))
+ let sanitizedLabel = sanitizeText(label, characterLimit: characterLimit)
+ guard !sanitizedLabel.isEmpty, sanitizedLabel != title else {
+ return ""
+ }
+
+ let comparableLabel = markdownEscapedLinkText(sanitizedLabel)
if let linkText, linkText.hasPrefix("[\(comparableLabel)](") {
return ""
}
- return " Description: \(label)"
+ return " Description: \(sanitizedLabel)"
}
private func formattedValueSegmentWithSeparator(_ valueSegment: String, precedingSegments: [String]) -> String {
@@ -1235,12 +1265,17 @@ private func shouldCommaSeparateActions(
|| segments.contains(where: { !$0.isEmpty })
}
-private func formattedURLSegment(for element: AXUIElement, title: String?, label: String?) -> String {
+private func formattedURLSegment(
+ for element: AXUIElement,
+ title: String?,
+ label: String?,
+ characterLimit: Int? = snapshotTextDefaultCharacterLimit
+) -> String {
guard stringValue(of: element, attribute: kAXRoleAttribute) == "AXWebArea" else {
return ""
}
- guard let url = urlValue(of: element, attribute: kAXURLAttribute), !url.isEmpty else {
+ guard let url = urlValue(of: element, attribute: kAXURLAttribute, characterLimit: characterLimit), !url.isEmpty else {
return ""
}
@@ -1251,18 +1286,22 @@ private func formattedURLSegment(for element: AXUIElement, title: String?, label
return ", URL: \(url)"
}
-private func urlValue(of element: AXUIElement, attribute: String) -> String? {
+private func urlValue(
+ of element: AXUIElement,
+ attribute: String,
+ characterLimit: Int? = snapshotTextDefaultCharacterLimit
+) -> String? {
guard let value = attributeValue(of: element, attribute: attribute) else {
return nil
}
if CFGetTypeID(value) == CFStringGetTypeID(), let string = value as? String {
- let sanitized = sanitizeText(string)
+ let sanitized = sanitizeText(string, characterLimit: characterLimit)
return sanitized.isEmpty ? nil : sanitized
}
if CFGetTypeID(value) == CFURLGetTypeID(), let url = value as? URL {
- let sanitized = sanitizeText(url.absoluteString)
+ let sanitized = sanitizeText(url.absoluteString, characterLimit: characterLimit)
return sanitized.isEmpty ? nil : sanitized
}
@@ -1394,7 +1433,8 @@ private func shouldSuppressChildren(
private func summarizedGenericText(
of element: AXUIElement,
role: String,
- childElements: [AXUIElement]
+ childElements: [AXUIElement],
+ characterLimit: Int? = snapshotTextDefaultCharacterLimit
) -> String? {
guard role == kAXGroupRole as String || role == kAXUnknownRole as String else {
return nil
@@ -1408,7 +1448,7 @@ private func summarizedGenericText(
return nil
}
- let texts = descendantTextsForSummary(of: element)
+ let texts = descendantTextsForSummary(of: element, characterLimit: characterLimit)
guard texts.count >= 2 else {
return nil
}
@@ -1417,7 +1457,7 @@ private func summarizedGenericText(
return nil
}
- let joined = sanitizeText(texts.joined(separator: " "))
+ let joined = sanitizeText(texts.joined(separator: " "), characterLimit: characterLimit)
.replacingOccurrences(of: " : ", with: " : ")
return joined.isEmpty ? nil : joined
}
@@ -1639,23 +1679,26 @@ private func splitCamelCase(_ value: String) -> String {
return result
}
-private func sanitizeText(_ value: String) -> String {
+func sanitizeText(_ value: String, characterLimit: Int? = snapshotTextDefaultCharacterLimit) -> String {
let collapsed = value
.replacingOccurrences(of: "\n", with: "\\n")
.trimmingCharacters(in: .whitespacesAndNewlines)
- if collapsed.count > 160 {
- return String(collapsed.prefix(160)) + "..."
+ if let characterLimit, characterLimit >= 0, collapsed.count > characterLimit {
+ return String(collapsed.prefix(characterLimit)) + "..."
}
return collapsed
}
-private func flattenedRowTexts(of element: AXUIElement) -> [String] {
+private func flattenedRowTexts(
+ of element: AXUIElement,
+ characterLimit: Int? = snapshotTextDefaultCharacterLimit
+) -> [String] {
let cells = copyArray(element, attribute: kAXChildrenAttribute) ?? []
let texts = cells
- .flatMap { descendantTexts(of: $0) }
- .map(sanitizeText)
+ .flatMap { descendantTexts(of: $0, characterLimit: characterLimit) }
+ .map { sanitizeText($0, characterLimit: characterLimit) }
.filter { !$0.isEmpty }
var unique: [String] = []
@@ -1669,7 +1712,11 @@ private func flattenedRowTexts(of element: AXUIElement) -> [String] {
return unique
}
-private func descendantTexts(of element: AXUIElement, depth: Int = 0) -> [String] {
+private func descendantTexts(
+ of element: AXUIElement,
+ depth: Int = 0,
+ characterLimit: Int? = snapshotTextDefaultCharacterLimit
+) -> [String] {
guard depth < 4 else {
return []
}
@@ -1677,54 +1724,61 @@ private func descendantTexts(of element: AXUIElement, depth: Int = 0) -> [String
var values: [String] = []
let role = stringValue(of: element, attribute: kAXRoleAttribute) ?? ""
if role == kAXStaticTextRole as String || role == kAXTextFieldRole as String {
- if let value = sanitizedValue(of: element) {
+ if let value = sanitizedValue(of: element, characterLimit: characterLimit) {
values.append(value)
} else if let title = stringValue(of: element, attribute: kAXTitleAttribute) {
- values.append(sanitizeText(title))
+ values.append(sanitizeText(title, characterLimit: characterLimit))
}
}
for child in copyArray(element, attribute: kAXChildrenAttribute) ?? [] {
- values.append(contentsOf: descendantTexts(of: child, depth: depth + 1))
+ values.append(contentsOf: descendantTexts(of: child, depth: depth + 1, characterLimit: characterLimit))
}
return values
}
-private func descendantTextsForSummary(of element: AXUIElement, depth: Int = 0) -> [String] {
+private func descendantTextsForSummary(
+ of element: AXUIElement,
+ depth: Int = 0,
+ characterLimit: Int? = snapshotTextDefaultCharacterLimit
+) -> [String] {
guard depth < 8 else {
return []
}
let role = stringValue(of: element, attribute: kAXRoleAttribute) ?? ""
- if role == "AXLink", let linkText = summaryTextForLink(element) {
+ if role == "AXLink", let linkText = summaryTextForLink(element, characterLimit: characterLimit) {
return [linkText]
}
if role == kAXStaticTextRole as String || role == kAXTextFieldRole as String {
- if let value = sanitizedValue(of: element), !value.isEmpty {
+ if let value = sanitizedValue(of: element, characterLimit: characterLimit), !value.isEmpty {
return [value]
}
if let title = stringValue(of: element, attribute: kAXTitleAttribute) {
- let sanitized = sanitizeText(title)
+ let sanitized = sanitizeText(title, characterLimit: characterLimit)
return sanitized.isEmpty ? [] : [sanitized]
}
}
return (copyArray(element, attribute: kAXChildrenAttribute) ?? [])
- .flatMap { descendantTextsForSummary(of: $0, depth: depth + 1) }
+ .flatMap { descendantTextsForSummary(of: $0, depth: depth + 1, characterLimit: characterLimit) }
}
-private func summaryTextForLink(_ element: AXUIElement) -> String? {
- guard let url = urlValue(of: element, attribute: kAXURLAttribute), !url.isEmpty else {
+private func summaryTextForLink(
+ _ element: AXUIElement,
+ characterLimit: Int? = snapshotTextDefaultCharacterLimit
+) -> String? {
+ guard let url = urlValue(of: element, attribute: kAXURLAttribute, characterLimit: characterLimit), !url.isEmpty else {
return nil
}
let childText = (copyArray(element, attribute: kAXChildrenAttribute) ?? [])
- .flatMap { descendantTextsForSummary(of: $0) }
+ .flatMap { descendantTextsForSummary(of: $0, characterLimit: characterLimit) }
.joined(separator: " ")
- let sanitized = sanitizeText(childText)
+ let sanitized = sanitizeText(childText, characterLimit: characterLimit)
guard !sanitized.isEmpty else {
return nil
}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
index f72efa9..c892773 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
@@ -360,8 +360,8 @@ public final class ComputerUseService {
)
}
- public func getAppState(app query: String) throws -> ToolCallResult {
- snapshotResult(for: try refreshSnapshot(for: query), style: .fullState)
+ public func getAppState(app query: String, showFullText: Bool = false) throws -> ToolCallResult {
+ snapshotResult(for: try refreshSnapshot(for: query, showFullText: showFullText), style: .fullState)
}
public func click(app query: String, elementIndex: String?, x: Double?, y: Double?, clickCount: Int, mouseButton: String) throws -> ToolCallResult {
@@ -655,9 +655,9 @@ public final class ComputerUseService {
}
@discardableResult
- private func refreshSnapshot(for query: String) throws -> AppSnapshot {
+ private func refreshSnapshot(for query: String, showFullText: Bool = false) throws -> AppSnapshot {
let app = try AppDiscovery.resolve(query)
- let snapshot = try SnapshotBuilder.build(for: app)
+ let snapshot = try SnapshotBuilder.build(for: app, showFullText: showFullText)
let keys = Set([
query.lowercased(),
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
index 4092273..892128f 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
@@ -48,7 +48,10 @@ public final class ComputerUseToolDispatcher {
case "list_apps":
return service.listApps()
case "get_app_state":
- return try service.getAppState(app: requireString("app", in: arguments))
+ return try service.getAppState(
+ app: requireString("app", in: arguments),
+ showFullText: optionalBool("show_full_text", in: arguments) ?? false
+ )
case "click":
return try service.click(
app: requireString("app", in: arguments),
@@ -126,6 +129,20 @@ public final class ComputerUseToolDispatcher {
arguments[key] as? String
}
+ private func optionalBool(_ key: String, in arguments: [String: Any]) -> Bool? {
+ if let bool = arguments[key] as? Bool {
+ return bool
+ }
+
+ if let number = arguments[key] as? NSNumber,
+ CFGetTypeID(number as CFTypeRef) == CFBooleanGetTypeID()
+ {
+ return number.boolValue
+ }
+
+ return nil
+ }
+
private func requireElementIndex(in arguments: [String: Any]) throws -> String {
guard let value = optionalElementIndex(in: arguments) else {
throw ComputerUseError.missingArgument("element_index")
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
index eb1307c..24d90da 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
@@ -5,7 +5,7 @@ public enum OpenComputerUseCLICommand: Equatable {
case mcp
case doctor
case listApps
- case snapshot(app: String)
+ case snapshot(app: String, showFullText: Bool = false)
case call(OpenComputerUseCallInvocation)
case turnEnded(payload: String?)
case help(command: String?)
@@ -148,11 +148,14 @@ public func openComputerUseHelpText(command: String? = nil) -> String {
case "snapshot":
return """
Usage:
- open-computer-use snapshot <app>
+ open-computer-use snapshot [--show-full-text] <app>
Arguments:
<app> App name or bundle identifier to inspect.
+ Options:
+ --show-full-text Do not truncate accessibility text to 500 characters.
+
Print the current accessibility snapshot for the target app.
"""
case "call":
@@ -261,20 +264,41 @@ private func parseTurnEnded(arguments: [String]) throws -> OpenComputerUseCLICom
}
private func parseSnapshot(arguments: [String]) throws -> OpenComputerUseCLICommand {
- if arguments.count == 1 {
- let value = arguments[0]
- if value == "-h" || value == "--help" {
- return .help(command: "snapshot")
- }
+ if arguments.isEmpty {
+ throw OpenComputerUseCLIError(message: "snapshot requires an app name or bundle identifier", helpCommand: "snapshot")
+ }
- return .snapshot(app: value)
+ if arguments.count == 1, let value = arguments.first, value == "-h" || value == "--help" {
+ return .help(command: "snapshot")
}
- if arguments.isEmpty {
+ var app: String?
+ var showFullText = false
+
+ for argument in arguments {
+ switch argument {
+ case "--show-full-text":
+ showFullText = true
+ case "-h", "--help":
+ throw OpenComputerUseCLIError(message: "snapshot help must be requested as `open-computer-use snapshot --help`", helpCommand: "snapshot")
+ default:
+ if argument.hasPrefix("-") {
+ throw OpenComputerUseCLIError(message: "Unknown snapshot option: \(argument)", helpCommand: "snapshot")
+ }
+
+ guard app == nil else {
+ throw OpenComputerUseCLIError(message: "snapshot accepts exactly one <app> argument", helpCommand: "snapshot")
+ }
+
+ app = argument
+ }
+ }
+
+ guard let app else {
throw OpenComputerUseCLIError(message: "snapshot requires an app name or bundle identifier", helpCommand: "snapshot")
}
- throw OpenComputerUseCLIError(message: "snapshot accepts exactly one <app> argument", helpCommand: "snapshot")
+ return .snapshot(app: app, showFullText: showFullText)
}
private func parseCall(arguments: [String]) throws -> OpenComputerUseCLICommand {
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
index cf67c95..e27e14f 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
@@ -71,6 +71,7 @@ public enum ToolDefinitions {
inputSchema: objectSchema(
properties: [
"app": stringProperty(description: "App name or bundle identifier"),
+ "show_full_text": booleanProperty(description: "Return full accessibility text without the default 500 character truncation. Defaults to false."),
],
required: ["app"]
)
@@ -191,6 +192,13 @@ private func stringProperty(description: String, enumValues: [String]? = nil) ->
return property
}
+private func booleanProperty(description: String) -> [String: Any] {
+ [
+ "type": "boolean",
+ "description": description,
+ ]
+}
+
private func integerProperty(description: String) -> [String: Any] {
[
"type": "integer",
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index 725d42d..ee8a614 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -75,6 +75,21 @@ final class OpenComputerUseKitTests: XCTestCase {
}
}
+ func testCLIRecognizesSnapshotFullTextFlag() throws {
+ XCTAssertEqual(
+ try parseOpenComputerUseCLI(arguments: ["snapshot", "TextEdit"]),
+ .snapshot(app: "TextEdit", showFullText: false)
+ )
+ XCTAssertEqual(
+ try parseOpenComputerUseCLI(arguments: ["snapshot", "--show-full-text", "TextEdit"]),
+ .snapshot(app: "TextEdit", showFullText: true)
+ )
+ XCTAssertEqual(
+ try parseOpenComputerUseCLI(arguments: ["snapshot", "TextEdit", "--show-full-text"]),
+ .snapshot(app: "TextEdit", showFullText: true)
+ )
+ }
+
func testCLIRejectsMixedCallSequenceInputs() {
XCTAssertThrowsError(try parseOpenComputerUseCLI(arguments: ["call", "list_apps", "--calls", "[]"])) { error in
XCTAssertEqual(
@@ -552,6 +567,10 @@ final class OpenComputerUseKitTests: XCTestCase {
((tools["click"]?.inputSchema["properties"] as? [String: [String: Any]])?["mouse_button"]?["enum"] as? [String]) ?? [],
["left", "right", "middle"]
)
+ let getAppStateSchema = tools["get_app_state"]?.inputSchema
+ let getAppStateProperties = getAppStateSchema?["properties"] as? [String: [String: Any]]
+ XCTAssertEqual(getAppStateProperties?["show_full_text"]?["type"] as? String, "boolean")
+ XCTAssertEqual(getAppStateSchema?["required"] as? [String], ["app"])
let scrollPages = (tools["scroll"]?.inputSchema["properties"] as? [String: [String: Any]])?["pages"]
XCTAssertEqual(scrollPages?["type"] as? String, "number")
XCTAssertEqual(
@@ -1041,6 +1060,16 @@ final class OpenComputerUseKitTests: XCTestCase {
)
}
+ func testSnapshotTextLimitDefaultsTo500AndSupportsFullText() {
+ let longText = String(repeating: "ε€™", count: snapshotTextDefaultCharacterLimit + 20)
+
+ XCTAssertEqual(
+ sanitizeText(longText),
+ String(longText.prefix(snapshotTextDefaultCharacterLimit)) + "..."
+ )
+ XCTAssertEqual(sanitizeText(longText, characterLimit: nil), longText)
+ }
+
func testAccessibilityRendererSuppressesDuplicateDescriptionForSameTextMarkdownLinks() {
XCTAssertEqual(
formattedLabelSegment(
@@ -1050,8 +1079,8 @@ final class OpenComputerUseKitTests: XCTestCase {
),
""
)
- let longURL = "https://example.com/docs?" + String(repeating: "query=value&", count: 20)
- let truncatedURL = String(longURL.prefix(160)) + "..."
+ let longURL = "https://example.com/docs?" + String(repeating: "query=value&", count: 60)
+ let truncatedURL = String(longURL.prefix(snapshotTextDefaultCharacterLimit)) + "..."
XCTAssertEqual(
formattedLabelSegment(
longURL,