Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

feat: add selective AX and screenshot collection across platforms (#101)

draplater31a7d7da5f28
diff --git a/.github/workflows/windows-observation.yml b/.github/workflows/windows-observation.yml
new file mode 100644
index 0000000..ef48f36
--- /dev/null
+++ b/.github/workflows/windows-observation.yml
@@ -0,0 +1,47 @@
+name: Windows observation
+
+on:
+ pull_request:
+ paths:
+ - 'apps/OpenComputerUseWindows/**'
+ - 'scripts/tests/windows-observation*'
+ - 'scripts/tests/fixtures/WindowsObservationFixture.ps1'
+ - '.github/workflows/windows-observation.yml'
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: windows-observation-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ windows-observation:
+ runs-on: windows-2025
+ timeout-minutes: 15
+ defaults:
+ run:
+ shell: pwsh
+ steps:
+ - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+ - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e
+ with:
+ node-version: '24'
+ - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c
+ with:
+ go-version: '1.26.1'
+ cache: false
+ - name: Windows Go tests and build
+ working-directory: apps/OpenComputerUseWindows
+ run: |
+ go test ./...
+ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+ go build -o "$env:RUNNER_TEMP/ocu-observation.exe" .
+ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+ - name: PowerShell collection branches
+ run: |
+ powershell.exe -NoProfile -NonInteractive -File ./scripts/tests/windows-observation.test.ps1 -RuntimePath apps/OpenComputerUseWindows/runtime.ps1
+ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+ - name: Real WPF and native MCP
+ run: ./scripts/tests/windows-observation-smoke.ps1 -NativePath "$env:RUNNER_TEMP/ocu-observation.exe"
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift
index 9c7deb7..94b8c5b 100644
--- a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift
+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift
@@ -523,9 +523,9 @@ private final class AppAgentConnection: @unchecked Sendable {
let service = ComputerUseService()
return CLIProxyResponse(stdout: (service.listApps().primaryText ?? "") + "\n", stderr: "", exitCode: EXIT_SUCCESS)
- case let .snapshot(app, textLimit, treeLimits):
+ case let .snapshot(app, textLimit, treeLimits, observation):
let service = ComputerUseService()
- let text = try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits).primaryText ?? ""
+ let text = try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits, observation: observation).textContent
return CLIProxyResponse(stdout: text + "\n", stderr: "", exitCode: EXIT_SUCCESS)
case let .call(invocation):
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
index fef1c19..7c4ce10 100644
--- a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
@@ -65,9 +65,9 @@ enum OpenComputerUseMain {
case .listApps:
let service = ComputerUseService()
print(service.listApps().primaryText ?? "")
- case let .snapshot(app, textLimit, treeLimits):
+ case let .snapshot(app, textLimit, treeLimits, observation):
let service = ComputerUseService()
- print(try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits).primaryText ?? "")
+ print(try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits, observation: observation).textContent)
case let .call(invocation):
if VisualCursorSupport.isEnabled {
_ = NSApplication.shared.setActivationPolicy(.accessory)
diff --git a/apps/OpenComputerUseLinux/main.go b/apps/OpenComputerUseLinux/main.go
index 28b68d8..30e98d2 100644
--- a/apps/OpenComputerUseLinux/main.go
+++ b/apps/OpenComputerUseLinux/main.go
@@ -88,6 +88,7 @@ type elementRecord struct {
}
type appSnapshot struct {
+ ScreenshotOmitted bool `json:"-"`
App appDescriptor `json:"app"`
WindowTitle string `json:"windowTitle,omitempty"`
WindowBounds *frame `json:"windowBounds,omitempty"`
@@ -116,6 +117,9 @@ func (s *appSnapshot) renderedText() string {
fmt.Sprintf("Window: %q, App: %s.", title, s.App.Name),
}
lines = append(lines, s.TreeLines...)
+ if s.ScreenshotOmitted {
+ lines = append(lines, "Screenshot omitted by include_screenshot=false")
+ }
if strings.TrimSpace(s.SelectedText) != "" {
lines = append(lines, "", fmt.Sprintf("Selected text: [%s]", s.SelectedText))
} else if strings.TrimSpace(s.FocusedSummary) != "" {
@@ -139,6 +143,9 @@ func (s *appSnapshot) result() toolCallResult {
}
type linuxRequest struct {
+ IncludeAX *bool `json:"include_ax,omitempty"`
+ IncludeScreenshot *bool `json:"include_screenshot,omitempty"`
+
Tool string `json:"tool"`
App string `json:"app,omitempty"`
Element *elementRecord `json:"element,omitempty"`
@@ -215,7 +222,18 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {
if windowPlacement != "keep" {
return textResult("window_placement '"+windowPlacement+"' is not supported on Linux", true)
}
- return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth)
+ includeAX, err := optionalBool(args, "include_ax")
+ if err != nil {
+ return textResult(err.Error(), true)
+ }
+ includeScreenshot, err := optionalBool(args, "include_screenshot")
+ if err != nil {
+ return textResult(err.Error(), true)
+ }
+ if includeAX != nil && !*includeAX && includeScreenshot != nil && !*includeScreenshot {
+ return textResult("At least one of include_ax or include_screenshot must be true", true)
+ }
+ return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth, includeAX, includeScreenshot)
case "click":
clickMethod, err := parseClickMethod(optionalString(args, "click_method"))
if err != nil {
@@ -284,11 +302,11 @@ func (s *service) listApps() toolCallResult {
return textResult(response.Text, false)
}
-func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int) toolCallResult {
+func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int, includeAX, includeScreenshot *bool) toolCallResult {
if app == "" {
return textResult("Missing required argument: app", true)
}
- request := linuxRequest{Tool: "get_app_state", App: app}
+ request := linuxRequest{Tool: "get_app_state", App: app, IncludeAX: includeAX, IncludeScreenshot: includeScreenshot}
if textLimit != nil {
request.TextLimit = textLimit.runtimeValue()
}
@@ -302,7 +320,11 @@ func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, ma
if result.IsError {
return result
}
- return snapshot.result()
+ result = snapshot.result()
+ if includeAX != nil && !*includeAX {
+ result.Content[0].Text += "\nAccessibility omitted; previous element indexes invalidated. Read AX before element-targeted actions."
+ }
+ return result
}
func (s *service) click(app, elementIndex string, x, y *float64, clickCount int, mouseButton, clickMethod string) toolCallResult {
@@ -328,6 +350,9 @@ func (s *service) click(app, elementIndex string, x, y *float64, clickCount int,
if snapshot == nil {
return textResult("No app state is available for "+app+". Run get_app_state before action tools.", true)
}
+ if elementIndex == "" && snapshot.ScreenshotOmitted {
+ return textResult("Screenshot omitted by include_screenshot=false; request a screenshot before screenshot-coordinate actions.", true)
+ }
request := linuxRequest{
Tool: "click",
App: app,
@@ -414,6 +439,9 @@ func (s *service) drag(app string, fromX, fromY, toX, toY *float64) toolCallResu
if snapshot == nil {
return textResult("No app state is available for "+app+". Run get_app_state before action tools.", true)
}
+ if snapshot.ScreenshotOmitted {
+ return textResult("Screenshot omitted by include_screenshot=false; request a screenshot before screenshot-coordinate actions.", true)
+ }
return s.actionResult(app, linuxRequest{Tool: "drag", App: app, FromX: fromX, FromY: fromY, ToX: toX, ToY: toY, WindowBounds: snapshot.WindowBounds})
}
@@ -512,6 +540,25 @@ func (s *service) refreshSnapshot(app string, request linuxRequest) (*appSnapsho
if response.Snapshot == nil {
return nil, textResult("Linux runtime did not return an app snapshot.", true)
}
+ if request.IncludeScreenshot != nil && !*request.IncludeScreenshot {
+ response.Snapshot.ScreenshotPNGBase64 = ""
+ response.Snapshot.ScreenshotOmitted = true
+ }
+ if request.IncludeAX != nil && !*request.IncludeAX {
+ // A screenshot cannot validate AX indexes, even if the window title matches.
+ // Invalidate every alias of the prior observation, including PID queries.
+ previous := s.currentSnapshot(app)
+ for alias, cached := range s.snapshots {
+ if cached == previous || cached.App.PID == response.Snapshot.App.PID ||
+ (previous != nil && cached.App.PID == previous.App.PID) {
+ delete(s.snapshots, alias)
+ }
+ }
+ response.Snapshot.Elements = nil
+ response.Snapshot.TreeLines = nil
+ response.Snapshot.FocusedSummary = ""
+ response.Snapshot.SelectedText = ""
+ }
s.rememberSnapshot(app, response.Snapshot)
return response.Snapshot, toolCallResult{}
}
@@ -1011,6 +1058,18 @@ func optionalString(args map[string]any, key string) string {
return value
}
+func optionalBool(args map[string]any, key string) (*bool, error) {
+ value, exists := args[key]
+ if !exists {
+ return nil, nil
+ }
+ boolean, ok := value.(bool)
+ if !ok {
+ return nil, fmt.Errorf("%s must be a boolean", key)
+ }
+ return &boolean, nil
+}
+
func requiredElementIndex(args map[string]any) string {
return strings.TrimSpace(optionalElementIndex(args))
}
@@ -1240,7 +1299,7 @@ func toolDefinitions() []toolDefinition {
},
{
Name: "get_app_state",
- Description: "Get the state of an already running app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.",
+ Description: "Get the state of an already running app's key window. By default returns a screenshot and accessibility tree; include_ax/include_screenshot can opt out of either collection. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.",
Annotations: readOnlyAnnotations(),
InputSchema: objectSchema(map[string]any{
"app": stringProperty("App name or bundle identifier"),
@@ -1248,6 +1307,9 @@ func toolDefinitions() []toolDefinition {
"max_tree_nodes": positiveIntegerProperty("Maximum accessibility tree nodes to render. Defaults to 1200."),
"max_tree_depth": positiveIntegerProperty("Maximum accessibility tree depth to render. Defaults to 64."),
"window_placement": enumStringProperty("keep (default) leaves the window where it is. agent_display and restore are macOS-only and not supported on Linux.", windowPlacementValues),
+
+ "include_ax": map[string]any{"type": "boolean", "default": true, "description": "Include accessibility tree, focus and selection. False skips AX collection and invalidates prior element indexes; coordinate actions remain available."},
+ "include_screenshot": map[string]any{"type": "boolean", "default": true, "description": "Capture and return a screenshot. False skips capture and PNG encoding."},
}, []string{"app"}),
},
{
@@ -1402,13 +1464,16 @@ func runCLI(args []string, stdout io.Writer) error {
fmt.Fprintln(stdout, result.Content[0].Text)
return nil
case "snapshot":
- app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(args[1:])
+ snapshotArgs, observation, err := parseSnapshotObservationArgs(args[1:])
if err != nil {
return err
}
- toolArgs := map[string]any{
- "app": app,
+ app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(snapshotArgs)
+ if err != nil {
+ return err
}
+ toolArgs := observation
+ toolArgs["app"] = app
if textLimit != nil {
toolArgs["text_limit"] = textLimit.runtimeValue()
}
@@ -1443,6 +1508,37 @@ func runCLI(args []string, stdout io.Writer) error {
}
}
+func parseSnapshotObservationArgs(args []string) ([]string, map[string]any, error) {
+ remaining := []string{}
+ options := map[string]any{}
+ for index := 0; index < len(args); index++ {
+ option := args[index]
+ if option != "--include-ax" && option != "--include-screenshot" {
+ remaining = append(remaining, option)
+ continue
+ }
+ index++
+ if index >= len(args) {
+ return nil, nil, fmt.Errorf("%s requires true or false", option)
+ }
+ var value bool
+ switch strings.ToLower(args[index]) {
+ case "true":
+ value = true
+ case "false":
+ value = false
+ default:
+ return nil, nil, fmt.Errorf("%s requires true or false", option)
+ }
+ key := strings.ReplaceAll(strings.TrimPrefix(option, "--"), "-", "_")
+ options[key] = value
+ }
+ if options["include_ax"] == false && options["include_screenshot"] == false {
+ return nil, nil, errors.New("At least one of include_ax or include_screenshot must be true")
+ }
+ return remaining, options, nil
+}
+
func parseSnapshotArgs(args []string) (string, *textLimit, *int, *int, error) {
var app string
var textLimit *textLimit
@@ -1746,7 +1842,7 @@ func helpText(command string) string {
case "call":
return "Usage:\n open-computer-use call <tool> [--args '<json-object>']\n open-computer-use call --calls '<json-array>'\n\nThe JSON array form keeps all calls in one process so element_index state can be reused.\n"
case "snapshot":
- return "Usage:\n open-computer-use snapshot [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>\n\nPrint the current Linux AT-SPI snapshot for the target app.\n"
+ return "Usage:\n open-computer-use snapshot [--include-ax true|false] [--include-screenshot true|false] [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>\n\nPrint the current Linux AT-SPI snapshot for the target app.\n"
default:
return `Open Computer Use for Linux
diff --git a/apps/OpenComputerUseLinux/main_test.go b/apps/OpenComputerUseLinux/main_test.go
index 13c4f7c..17fd6ae 100644
--- a/apps/OpenComputerUseLinux/main_test.go
+++ b/apps/OpenComputerUseLinux/main_test.go
@@ -3,9 +3,11 @@ package main
import (
"bytes"
"encoding/json"
+ "fmt"
"net"
"os"
"path/filepath"
+ "strconv"
"strings"
"testing"
)
@@ -118,6 +120,143 @@ func TestActionWithoutReadBackPropagatesBridgeErrors(t *testing.T) {
}
}
+func observationPythonResponse(t *testing.T, response string) string {
+ t.Helper()
+ dir := t.TempDir()
+ requestPath := filepath.Join(dir, "request.json")
+ script := "#!/bin/sh\ncat \"$2\" > \"$OCU_TEST_OBSERVATION_REQUEST\"\nprintf '%s\\n' \"$OCU_TEST_OBSERVATION_RESPONSE\"\n"
+ if err := os.WriteFile(filepath.Join(dir, "python3"), []byte(script), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
+ t.Setenv("OCU_TEST_OBSERVATION_REQUEST", requestPath)
+ t.Setenv("OCU_TEST_OBSERVATION_RESPONSE", response)
+ return requestPath
+}
+
+func TestSelectiveObservationRequestAndDefaultCompatibility(t *testing.T) {
+ for _, flags := range []map[string]any{{}, {"include_ax": false}, {"include_screenshot": false}, {"include_ax": true, "include_screenshot": true}} {
+ t.Run(fmt.Sprint(flags), func(t *testing.T) {
+ path := observationPythonResponse(t, `{"ok":true,"snapshot":{"app":{"name":"Editor","pid":42}}}`)
+ args := map[string]any{"app": "Editor"}
+ for key, value := range flags {
+ args[key] = value
+ }
+ if result := newService().callTool("get_app_state", args); result.IsError {
+ t.Fatalf("observation failed: %#v", result)
+ }
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var request map[string]any
+ if err := json.Unmarshal(data, &request); err != nil {
+ t.Fatal(err)
+ }
+ for _, key := range []string{"include_ax", "include_screenshot"} {
+ want, present := flags[key]
+ got, sent := request[key]
+ if present != sent || got != want {
+ t.Fatalf("%s serialized as %#v (%v), want %#v (%v)", key, got, sent, want, present)
+ }
+ }
+ })
+ }
+}
+
+func TestSelectiveObservationRejectsInvalidOptions(t *testing.T) {
+ for _, args := range []map[string]any{
+ {"app": "Editor", "include_ax": "false"},
+ {"app": "Editor", "include_screenshot": nil},
+ {"app": "Editor", "include_ax": false, "include_screenshot": false},
+ } {
+ if result := newService().callTool("get_app_state", args); !result.IsError {
+ t.Fatalf("invalid options accepted: %#v", args)
+ }
+ }
+}
+
+func TestSelectiveObservationSchemaKeepsFullDefaults(t *testing.T) {
+ tool := findToolDefinition(t, "get_app_state")
+ properties := tool.InputSchema["properties"].(map[string]any)
+ for _, name := range []string{"include_ax", "include_screenshot"} {
+ property := properties[name].(map[string]any)
+ if property["type"] != "boolean" || property["default"] != true {
+ t.Fatalf("%s schema = %#v", name, property)
+ }
+ }
+ if required := tool.InputSchema["required"].([]string); len(required) != 1 || required[0] != "app" {
+ t.Fatalf("observation options should be optional: %#v", required)
+ }
+}
+
+func TestScreenshotOnlyInvalidatesIndexesAcrossWindowAndProcessChanges(t *testing.T) {
+ for _, pid := range []int{42, 43} {
+ t.Run(strconv.Itoa(pid), func(t *testing.T) {
+ path := observationPythonResponse(t, fmt.Sprintf(`{"ok":true,"snapshot":{"app":{"name":"Editor","pid":%d},"windowTitle":"New Window","windowBounds":{"width":300,"height":200},"screenshotPngBase64":"cG5n"}}`, pid))
+ s := newService()
+ old := &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, WindowTitle: "Old Window", Elements: []elementRecord{{Index: 7}}}
+ s.rememberSnapshot("old-window-query", old)
+ s.snapshots["editor"] = old
+ other := &appSnapshot{App: appDescriptor{Name: "Other", PID: 99}}
+ s.rememberSnapshot("Other", other)
+ result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_ax": false})
+ if result.IsError || len(result.Content) != 2 || !strings.Contains(result.Content[0].Text, "indexes invalidated") {
+ t.Fatalf("screenshot-only result = %#v", result)
+ }
+ if s.currentSnapshot("old-window-query") != nil || s.currentSnapshot("Other") != other {
+ t.Fatal("old alias retained or unrelated app modified")
+ }
+ if pid != 42 && s.currentSnapshot("42") != nil {
+ t.Fatal("old PID alias retained across process change")
+ }
+ if _, err := lookupElement(s.currentSnapshot("Editor"), "7"); err == nil {
+ t.Fatal("stale index remains usable")
+ }
+ if result := s.setValue("Editor", "7", "bad"); !result.IsError {
+ t.Fatal("stale element action accepted")
+ }
+ x, y := 10.0, 20.0
+ if result := s.click("Editor", "", &x, &y, 1, "left", "auto"); result.IsError {
+ t.Fatalf("coordinate action unavailable: %#v", result)
+ }
+ data, err := os.ReadFile(path)
+ if err != nil || !strings.Contains(string(data), `"tool":"click"`) {
+ t.Fatalf("coordinate click not sent: %s, %v", data, err)
+ }
+ t.Setenv("OCU_TEST_OBSERVATION_RESPONSE", fmt.Sprintf(`{"ok":true,"snapshot":{"app":{"name":"Editor","pid":%d},"elements":[{"index":8}]}}`, pid))
+ if result := s.callTool("get_app_state", map[string]any{"app": "Editor"}); result.IsError {
+ t.Fatalf("AX refresh failed: %#v", result)
+ }
+ if _, err := lookupElement(s.currentSnapshot("Editor"), "8"); err != nil {
+ t.Fatal("explicit AX did not restore indexes")
+ }
+ })
+ }
+}
+
+func TestScreenshotOnlyInvalidatesOlderAliasesWhenProcessChanges(t *testing.T) {
+ observationPythonResponse(t, `{"ok":true,"snapshot":{"app":{"name":"Editor","pid":43},"windowBounds":{"width":300,"height":200},"screenshotPngBase64":"cG5n"}}`)
+ s := newService()
+ older := &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 6}}}
+ s.rememberSnapshot("older-query", older)
+ s.rememberSnapshot("Editor", &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 7}}})
+ unrelated := &appSnapshot{App: appDescriptor{Name: "Other", PID: 99}}
+ s.rememberSnapshot("Other", unrelated)
+ result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_ax": false})
+ if result.IsError {
+ t.Fatalf("screenshot-only observation failed: %#v", result)
+ }
+ for _, alias := range []string{"older-query", "42"} {
+ if s.currentSnapshot(alias) != nil {
+ t.Fatalf("stale AX alias %q survived process replacement", alias)
+ }
+ }
+ if s.currentSnapshot("Other") != unrelated {
+ t.Fatal("unrelated app observation invalidated")
+ }
+}
+
func TestClickMethodSchemaAndParser(t *testing.T) {
tool := findToolDefinition(t, "click")
properties := tool.InputSchema["properties"].(map[string]any)
diff --git a/apps/OpenComputerUseLinux/observation_cli_test.go b/apps/OpenComputerUseLinux/observation_cli_test.go
new file mode 100644
index 0000000..546f6b7
--- /dev/null
+++ b/apps/OpenComputerUseLinux/observation_cli_test.go
@@ -0,0 +1,29 @@
+package main
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestSnapshotObservationFlagsPreserveBudgetsAndBooleanTypes(t *testing.T) {
+ args, flags, err := parseSnapshotObservationArgs([]string{"--include-screenshot", "false", "--max-tree-nodes", "2000", "Editor"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ app, _, nodes, _, err := parseSnapshotArgs(args)
+ if err != nil || app != "Editor" || nodes == nil || *nodes != 2000 || flags["include_screenshot"] != false {
+ t.Fatalf("%v %v %v", args, flags, err)
+ }
+ for _, bad := range [][]string{{"--include-ax"}, {"--include-screenshot", "0"}, {"--include-ax", "no"}, {"--include-ax", "false", "--include-screenshot", "false"}} {
+ if _, _, err := parseSnapshotObservationArgs(bad); err == nil {
+ t.Fatalf("invalid flags accepted: %v", bad)
+ }
+ }
+ _, defaults, err := parseSnapshotObservationArgs([]string{"Editor"})
+ if err != nil || len(defaults) != 0 {
+ t.Fatalf("legacy call changed: %v %v", defaults, err)
+ }
+ if !strings.Contains(helpText("snapshot"), "--include-screenshot") {
+ t.Fatal("snapshot help omits observation flags")
+ }
+}
diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.py
index 3fc4c41..d5f72e0 100644
--- a/apps/OpenComputerUseLinux/runtime.py
+++ b/apps/OpenComputerUseLinux/runtime.py
@@ -442,21 +442,24 @@ def focus_details(window, text_limit=DEFAULT_TEXT_LIMIT, max_nodes=MAX_ELEMENTS,
return summary, selection, False
-def build_snapshot(query, text_limit=DEFAULT_TEXT_LIMIT, max_tree_nodes=MAX_ELEMENTS, max_tree_depth=MAX_DEPTH):
+def build_snapshot(query, text_limit=DEFAULT_TEXT_LIMIT, max_tree_nodes=MAX_ELEMENTS, max_tree_depth=MAX_DEPTH, include_ax=True, include_screenshot=True):
app = resolve_app(query)
window_index, window = main_window(app)
bounds = extents(window)
- records, lines = render_tree(
- window,
- bounds,
- [window_index],
- text_limit=text_limit,
- max_tree_nodes=max_tree_nodes,
- max_tree_depth=max_tree_depth,
- )
- focus, selection, focus_truncated = focus_details(window, text_limit, max_tree_nodes, max_tree_depth)
- if focus_truncated:
- lines.append("[Focus search incomplete: node/depth budget reached.]")
+ records, lines = [], []
+ focus, selection = None, None
+ if include_ax:
+ records, lines = render_tree(
+ window,
+ bounds,
+ [window_index],
+ text_limit=text_limit,
+ max_tree_nodes=max_tree_nodes,
+ max_tree_depth=max_tree_depth,
+ )
+ focus, selection, focus_truncated = focus_details(window, text_limit, max_tree_nodes, max_tree_depth)
+ if focus_truncated:
+ lines.append("[Focus search incomplete: node/depth budget reached.]")
pid = node_pid(app)
return {
"app": {
@@ -466,7 +469,7 @@ def build_snapshot(query, text_limit=DEFAULT_TEXT_LIMIT, max_tree_nodes=MAX_ELEM
},
"windowTitle": limit_text(node_name(window), text_limit=text_limit),
"windowBounds": bounds,
- "screenshotPngBase64": capture_window_png(bounds),
+ "screenshotPngBase64": capture_window_png(bounds) if include_screenshot else None,
"treeLines": lines,
"focusedSummary": focus,
"selectedText": selection,
@@ -803,6 +806,11 @@ def perform_operation(operation):
if tool == "list_apps":
return {"ok": True, "text": list_apps_text()}
if tool == "get_app_state":
+ for key in ("include_ax", "include_screenshot"):
+ if key in operation and not isinstance(operation[key], bool):
+ raise RuntimeError(key + " must be a boolean")
+ if not operation.get("include_ax", True) and not operation.get("include_screenshot", True):
+ raise RuntimeError("At least one of include_ax or include_screenshot must be true")
return {
"ok": True,
"snapshot": build_snapshot(
@@ -810,6 +818,8 @@ def perform_operation(operation):
text_limit=parse_text_limit(operation.get("text_limit"), DEFAULT_TEXT_LIMIT),
max_tree_nodes=positive_int(operation.get("max_tree_nodes"), MAX_ELEMENTS),
max_tree_depth=positive_int(operation.get("max_tree_depth"), MAX_DEPTH),
+ include_ax=operation.get("include_ax", True),
+ include_screenshot=operation.get("include_screenshot", True),
),
}
diff --git a/apps/OpenComputerUseLinux/runtime_test.py b/apps/OpenComputerUseLinux/runtime_test.py
index 05280b0..25cc814 100644
--- a/apps/OpenComputerUseLinux/runtime_test.py
+++ b/apps/OpenComputerUseLinux/runtime_test.py
@@ -172,6 +172,71 @@ class RuntimeKeyTests(unittest.TestCase):
self.assertEqual(self.events, [])
+class RuntimeSelectiveObservationTests(unittest.TestCase):
+ def setUp(self):
+ self.runtime = load_runtime()
+ self.app = object()
+ self.window = object()
+ self.patches = []
+ for name, value in {
+ "resolve_app": self.app,
+ "main_window": (0, self.window),
+ "extents": {"x": 1, "y": 2, "width": 300, "height": 200},
+ "node_pid": 42,
+ "node_name": "Editor",
+ "render_tree": ([{"index": 7}], ["tree"]),
+ "focus_details": ("focus", "selection", False),
+ "capture_window_png": "png",
+ }.items():
+ patch = mock.patch.object(self.runtime, name, return_value=value)
+ self.patches.append(patch)
+ patch.start()
+ self.addCleanup(patch.stop)
+
+ def test_defaults_still_collect_full_state(self):
+ state = self.runtime.build_snapshot("Editor")
+ self.assertEqual(state["elements"], [{"index": 7}])
+ self.assertEqual(state["screenshotPngBase64"], "png")
+ self.runtime.focus_details.assert_called_once()
+
+ def test_screenshot_only_skips_tree_focus_and_selection(self):
+ state = self.runtime.build_snapshot("Editor", include_ax=False)
+ self.assertEqual(state["screenshotPngBase64"], "png")
+ self.assertEqual(state["elements"], [])
+ self.assertEqual(state["treeLines"], [])
+ self.assertIsNone(state["focusedSummary"])
+ self.assertIsNone(state["selectedText"])
+ for name in ("render_tree", "focus_details"):
+ getattr(self.runtime, name).assert_not_called()
+
+ def test_ax_only_skips_capture(self):
+ state = self.runtime.build_snapshot("Editor", include_screenshot=False)
+ self.assertEqual(state["treeLines"], ["tree"])
+ self.assertIsNone(state["screenshotPngBase64"])
+ self.runtime.capture_window_png.assert_not_called()
+
+ def test_ax_only_preserves_bounded_focus_reporting(self):
+ self.runtime.focus_details.return_value = (None, None, True)
+ state = self.runtime.build_snapshot("Editor", text_limit=20, max_tree_nodes=2,
+ max_tree_depth=1, include_screenshot=False)
+ self.runtime.focus_details.assert_called_once_with(self.window, 20, 2, 1)
+ self.assertIn("[Focus search incomplete: node/depth budget reached.]", state["treeLines"])
+ self.runtime.capture_window_png.assert_not_called()
+
+ def test_tool_forwards_explicit_observation_flags(self):
+ state = self.runtime.perform_operation({"tool": "get_app_state", "app": "Editor", "include_ax": False})
+ self.assertTrue(state["ok"])
+ self.runtime.render_tree.assert_not_called()
+
+ def test_invalid_or_empty_observations_are_rejected(self):
+ for flags in ({"include_ax": "false"}, {"include_screenshot": None},
+ {"include_ax": 0}, {"include_ax": False, "include_screenshot": False}):
+ with self.subTest(flags=flags):
+ with self.assertRaises(RuntimeError):
+ self.runtime.perform_operation({"tool": "get_app_state", "app": "Editor", **flags})
+ self.runtime.resolve_app.assert_not_called()
+
+
class HiddenDetailsTests(unittest.TestCase):
def setUp(self):
self.runtime = load_runtime()
diff --git a/apps/OpenComputerUseWindows/main.go b/apps/OpenComputerUseWindows/main.go
index 47bef73..8b99ffb 100644
--- a/apps/OpenComputerUseWindows/main.go
+++ b/apps/OpenComputerUseWindows/main.go
@@ -93,6 +93,7 @@ type elementRecord struct {
}
type appSnapshot struct {
+ ScreenshotOmitted bool `json:"-"`
App appDescriptor `json:"app"`
WindowTitle string `json:"windowTitle,omitempty"`
WindowBounds *frame `json:"windowBounds,omitempty"`
@@ -121,7 +122,9 @@ func (s *appSnapshot) renderedText() string {
fmt.Sprintf("Window: %q, App: %s.", title, s.App.Name),
}
lines = append(lines, s.TreeLines...)
- if !s.hasUsableScreenshot() {
+ if s.ScreenshotOmitted {
+ lines = append(lines, "", "Screenshot omitted by include_screenshot=false")
+ } else if !s.hasUsableScreenshot() {
lines = append(lines, "", "Screenshot unavailable. Restore a visible, non-minimized, on-screen window before using screenshot-coordinate actions; element_index actions may still work.")
}
if strings.TrimSpace(s.SelectedText) != "" {
@@ -198,28 +201,30 @@ func validateScreenshotPNGBase64(encoded string) string {
}
type psRequest struct {
- Tool string `json:"tool"`
- App string `json:"app,omitempty"`
- Element *elementRecord `json:"element,omitempty"`
- X *float64 `json:"x,omitempty"`
- Y *float64 `json:"y,omitempty"`
- FromX *float64 `json:"from_x,omitempty"`
- FromY *float64 `json:"from_y,omitempty"`
- ToX *float64 `json:"to_x,omitempty"`
- ToY *float64 `json:"to_y,omitempty"`
- ClickCount int `json:"click_count,omitempty"`
- MouseButton string `json:"mouse_button,omitempty"`
- ClickMethod string `json:"click_method,omitempty"`
- Action string `json:"action,omitempty"`
- Direction string `json:"direction,omitempty"`
- Pages float64 `json:"pages,omitempty"`
- Text string `json:"text,omitempty"`
- Key string `json:"key,omitempty"`
- Value string `json:"value,omitempty"`
- WindowBounds *frame `json:"windowBounds,omitempty"`
- TextLimit any `json:"text_limit,omitempty"`
- MaxTreeNodes int `json:"max_tree_nodes,omitempty"`
- MaxTreeDepth int `json:"max_tree_depth,omitempty"`
+ IncludeAX *bool `json:"include_ax,omitempty"`
+ IncludeScreenshot *bool `json:"include_screenshot,omitempty"`
+ Tool string `json:"tool"`
+ App string `json:"app,omitempty"`
+ Element *elementRecord `json:"element,omitempty"`
+ X *float64 `json:"x,omitempty"`
+ Y *float64 `json:"y,omitempty"`
+ FromX *float64 `json:"from_x,omitempty"`
+ FromY *float64 `json:"from_y,omitempty"`
+ ToX *float64 `json:"to_x,omitempty"`
+ ToY *float64 `json:"to_y,omitempty"`
+ ClickCount int `json:"click_count,omitempty"`
+ MouseButton string `json:"mouse_button,omitempty"`
+ ClickMethod string `json:"click_method,omitempty"`
+ Action string `json:"action,omitempty"`
+ Direction string `json:"direction,omitempty"`
+ Pages float64 `json:"pages,omitempty"`
+ Text string `json:"text,omitempty"`
+ Key string `json:"key,omitempty"`
+ Value string `json:"value,omitempty"`
+ WindowBounds *frame `json:"windowBounds,omitempty"`
+ TextLimit any `json:"text_limit,omitempty"`
+ MaxTreeNodes int `json:"max_tree_nodes,omitempty"`
+ MaxTreeDepth int `json:"max_tree_depth,omitempty"`
}
type textLimit struct {
@@ -243,10 +248,11 @@ type psResponse struct {
type service struct {
snapshots map[string]*appSnapshot
+ runBridge func(psRequest) (*psResponse, error)
}
func newService() *service {
- return &service{snapshots: map[string]*appSnapshot{}}
+ return &service{snapshots: map[string]*appSnapshot{}, runBridge: runPowerShell}
}
func (s *service) callTool(name string, args map[string]any) toolCallResult {
@@ -273,7 +279,18 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {
if windowPlacement != "keep" {
return textResult("window_placement '"+windowPlacement+"' is not supported on Windows", true)
}
- return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth)
+ includeAX, err := optionalBool(args, "include_ax")
+ if err != nil {
+ return textResult(err.Error(), true)
+ }
+ includeScreenshot, err := optionalBool(args, "include_screenshot")
+ if err != nil {
+ return textResult(err.Error(), true)
+ }
+ if includeAX != nil && !*includeAX && includeScreenshot != nil && !*includeScreenshot {
+ return textResult("At least one of include_ax or include_screenshot must be true", true)
+ }
+ return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth, includeAX, includeScreenshot)
case "click":
clickMethod, err := parseClickMethod(optionalString(args, "click_method"))
if err != nil {
@@ -329,7 +346,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {
}
func (s *service) listApps() toolCallResult {
- response, err := runPowerShell(psRequest{Tool: "list_apps"})
+ response, err := s.runBridge(psRequest{Tool: "list_apps"})
if err != nil {
return textResult(err.Error(), true)
}
@@ -342,11 +359,11 @@ func (s *service) listApps() toolCallResult {
return textResult(response.Text, false)
}
-func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int) toolCallResult {
+func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int, includeAX, includeScreenshot *bool) toolCallResult {
if app == "" {
return textResult("Missing required argument: app", true)
}
- request := psRequest{Tool: "get_app_state", App: app}
+ request := psRequest{Tool: "get_app_state", App: app, IncludeAX: includeAX, IncludeScreenshot: includeScreenshot}
if textLimit != nil {
request.TextLimit = textLimit.runtimeValue()
}
@@ -360,7 +377,11 @@ func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, ma
if result.IsError {
return result
}
- return snapshot.result()
+ result = snapshot.result()
+ if includeAX != nil && !*includeAX {
+ result.Content[0].Text += "\nAccessibility omitted; previous element indexes invalidated. Read AX before element-targeted actions."
+ }
+ return result
}
func (s *service) click(app, elementIndex string, x, y *float64, clickCount int, mouseButton, clickMethod string) toolCallResult {
@@ -479,7 +500,7 @@ func (s *service) drag(app string, fromX, fromY, toX, toY *float64) toolCallResu
}
func screenshotCoordinatesUnavailableResult(app string) toolCallResult {
- return textResult("No usable screenshot is available for "+app+". Restore a visible, non-minimized, on-screen window and run get_app_state again before using screenshot coordinates.", true)
+ return textResult("No usable screenshot is available for "+app+". Run get_app_state with include_screenshot=true on a visible, non-minimized, on-screen window before using screenshot coordinates.", true)
}
func (s *service) typeText(app, text, keyMethod string) toolCallResult {
@@ -545,7 +566,7 @@ func (s *service) currentSnapshot(app string) *appSnapshot {
}
func (s *service) refreshSnapshot(app string, request psRequest) (*appSnapshot, toolCallResult) {
- response, err := runPowerShell(request)
+ response, err := s.runBridge(request)
if err != nil {
return nil, textResult(err.Error(), true)
}
@@ -555,6 +576,25 @@ func (s *service) refreshSnapshot(app string, request psRequest) (*appSnapshot,
if response.Snapshot == nil {
return nil, textResult("Windows runtime did not return an app snapshot.", true)
}
+ if request.IncludeScreenshot != nil && !*request.IncludeScreenshot {
+ response.Snapshot.ScreenshotPNGBase64 = ""
+ response.Snapshot.ScreenshotOmitted = true
+ }
+ if request.IncludeAX != nil && !*request.IncludeAX {
+ // A screenshot cannot validate AX indexes, even if the window title matches.
+ // Invalidate every alias of the prior observation, including PID queries.
+ previous := s.currentSnapshot(app)
+ for alias, cached := range s.snapshots {
+ if cached == previous || cached.App.PID == response.Snapshot.App.PID ||
+ (previous != nil && cached.App.PID == previous.App.PID) {
+ delete(s.snapshots, alias)
+ }
+ }
+ response.Snapshot.Elements = nil
+ response.Snapshot.TreeLines = nil
+ response.Snapshot.FocusedSummary = ""
+ response.Snapshot.SelectedText = ""
+ }
response.Snapshot.sanitizeScreenshot()
s.rememberSnapshot(app, response.Snapshot)
return response.Snapshot, toolCallResult{}
@@ -641,6 +681,18 @@ func optionalString(args map[string]any, key string) string {
return value
}
+func optionalBool(args map[string]any, key string) (*bool, error) {
+ value, exists := args[key]
+ if !exists {
+ return nil, nil
+ }
+ boolean, ok := value.(bool)
+ if !ok {
+ return nil, fmt.Errorf("%s must be a boolean", key)
+ }
+ return &boolean, nil
+}
+
func requiredElementIndex(args map[string]any) string {
return strings.TrimSpace(optionalElementIndex(args))
}
@@ -861,14 +913,16 @@ func toolDefinitions() []toolDefinition {
},
{
Name: "get_app_state",
- Description: "Get the state of an already running app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.",
+ Description: "Get the state of an already running app's key window. By default returns a screenshot and accessibility tree; include_ax/include_screenshot can opt out of either collection. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.",
Annotations: readOnlyAnnotations(),
InputSchema: objectSchema(map[string]any{
- "app": stringProperty("App name or bundle identifier"),
- "text_limit": textLimitProperty("Maximum text characters to return. Use \"max\" for full text. Defaults to 500."),
- "max_tree_nodes": positiveIntegerProperty("Maximum accessibility tree nodes to render. Defaults to 1200."),
- "max_tree_depth": positiveIntegerProperty("Maximum accessibility tree depth to render. Defaults to 64."),
- "window_placement": enumStringProperty("keep (default) leaves the window where it is. agent_display and restore are macOS-only and not supported on Windows.", windowPlacementValues),
+ "include_ax": map[string]any{"type": "boolean", "default": true, "description": "Include accessibility tree, focus and selection. False skips AX collection and invalidates prior element indexes; coordinate actions remain available."},
+ "include_screenshot": map[string]any{"type": "boolean", "default": true, "description": "Capture and return a screenshot. False skips capture and PNG encoding."},
+ "app": stringProperty("App name or bundle identifier"),
+ "text_limit": textLimitProperty("Maximum text characters to return. Use \"max\" for full text. Defaults to 500."),
+ "max_tree_nodes": positiveIntegerProperty("Maximum accessibility tree nodes to render. Defaults to 1200."),
+ "max_tree_depth": positiveIntegerProperty("Maximum accessibility tree depth to render. Defaults to 64."),
+ "window_placement": enumStringProperty("keep (default) leaves the window where it is. agent_display and restore are macOS-only and not supported on Windows.", windowPlacementValues),
}, []string{"app"}),
},
{
@@ -1023,13 +1077,16 @@ func runCLI(args []string, stdout io.Writer) error {
fmt.Fprintln(stdout, result.Content[0].Text)
return nil
case "snapshot":
- app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(args[1:])
+ snapshotArgs, observation, err := parseSnapshotObservationArgs(args[1:])
if err != nil {
return err
}
- toolArgs := map[string]any{
- "app": app,
+ app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(snapshotArgs)
+ if err != nil {
+ return err
}
+ toolArgs := observation
+ toolArgs["app"] = app
if textLimit != nil {
toolArgs["text_limit"] = textLimit.runtimeValue()
}
@@ -1064,6 +1121,37 @@ func runCLI(args []string, stdout io.Writer) error {
}
}
+func parseSnapshotObservationArgs(args []string) ([]string, map[string]any, error) {
+ remaining := []string{}
+ options := map[string]any{}
+ for index := 0; index < len(args); index++ {
+ option := args[index]
+ if option != "--include-ax" && option != "--include-screenshot" {
+ remaining = append(remaining, option)
+ continue
+ }
+ index++
+ if index >= len(args) {
+ return nil, nil, fmt.Errorf("%s requires true or false", option)
+ }
+ var value bool
+ switch strings.ToLower(args[index]) {
+ case "true":
+ value = true
+ case "false":
+ value = false
+ default:
+ return nil, nil, fmt.Errorf("%s requires true or false", option)
+ }
+ key := strings.ReplaceAll(strings.TrimPrefix(option, "--"), "-", "_")
+ options[key] = value
+ }
+ if options["include_ax"] == false && options["include_screenshot"] == false {
+ return nil, nil, errors.New("At least one of include_ax or include_screenshot must be true")
+ }
+ return remaining, options, nil
+}
+
func parseSnapshotArgs(args []string) (string, *textLimit, *int, *int, error) {
var app string
var textLimit *textLimit
@@ -1367,7 +1455,7 @@ func helpText(command string) string {
case "call":
return "Usage:\n open-computer-use.exe call <tool> [--args '<json-object>']\n open-computer-use.exe call --calls '<json-array>'\n\nThe JSON array form keeps all calls in one process so element_index state can be reused.\n"
case "snapshot":
- return "Usage:\n open-computer-use.exe snapshot [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>\n\nPrint the current Windows UI Automation snapshot for the target app.\n"
+ return "Usage:\n open-computer-use.exe snapshot [--include-ax true|false] [--include-screenshot true|false] [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>\n\nPrint the current Windows UI Automation snapshot for the target app.\n"
default:
return `Open Computer Use for Windows
diff --git a/apps/OpenComputerUseWindows/observation_cli_test.go b/apps/OpenComputerUseWindows/observation_cli_test.go
new file mode 100644
index 0000000..546f6b7
--- /dev/null
+++ b/apps/OpenComputerUseWindows/observation_cli_test.go
@@ -0,0 +1,29 @@
+package main
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestSnapshotObservationFlagsPreserveBudgetsAndBooleanTypes(t *testing.T) {
+ args, flags, err := parseSnapshotObservationArgs([]string{"--include-screenshot", "false", "--max-tree-nodes", "2000", "Editor"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ app, _, nodes, _, err := parseSnapshotArgs(args)
+ if err != nil || app != "Editor" || nodes == nil || *nodes != 2000 || flags["include_screenshot"] != false {
+ t.Fatalf("%v %v %v", args, flags, err)
+ }
+ for _, bad := range [][]string{{"--include-ax"}, {"--include-screenshot", "0"}, {"--include-ax", "no"}, {"--include-ax", "false", "--include-screenshot", "false"}} {
+ if _, _, err := parseSnapshotObservationArgs(bad); err == nil {
+ t.Fatalf("invalid flags accepted: %v", bad)
+ }
+ }
+ _, defaults, err := parseSnapshotObservationArgs([]string{"Editor"})
+ if err != nil || len(defaults) != 0 {
+ t.Fatalf("legacy call changed: %v %v", defaults, err)
+ }
+ if !strings.Contains(helpText("snapshot"), "--include-screenshot") {
+ t.Fatal("snapshot help omits observation flags")
+ }
+}
diff --git a/apps/OpenComputerUseWindows/observation_test.go b/apps/OpenComputerUseWindows/observation_test.go
new file mode 100644
index 0000000..e3cb946
--- /dev/null
+++ b/apps/OpenComputerUseWindows/observation_test.go
@@ -0,0 +1,115 @@
+package main
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+)
+
+func TestObservationRequestSerializationAndChannels(t *testing.T) {
+ for _, flags := range []map[string]any{{}, {"include_ax": false}, {"include_screenshot": false}, {"include_ax": true, "include_screenshot": true}} {
+ s := newService()
+ s.runBridge = func(request psRequest) (*psResponse, error) {
+ data, err := json.Marshal(request)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var sent map[string]any
+ if err := json.Unmarshal(data, &sent); err != nil {
+ t.Fatal(err)
+ }
+ for _, key := range []string{"include_ax", "include_screenshot"} {
+ want, present := flags[key]
+ got, exists := sent[key]
+ if present != exists || want != got {
+ t.Fatalf("%s serialized as %#v, want %#v", key, sent, flags)
+ }
+ }
+ return &psResponse{OK: true, Snapshot: &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42},
+ WindowBounds: &frame{Width: 300, Height: 200}, ScreenshotPNGBase64: pngBase64(t, 300, 200),
+ TreeLines: []string{"entry"}, FocusedSummary: "focus", SelectedText: "中文🙂", Elements: []elementRecord{{Index: 7}}}}, nil
+ }
+ args := map[string]any{"app": "Editor"}
+ for key, value := range flags {
+ args[key] = value
+ }
+ result := s.callTool("get_app_state", args)
+ if result.IsError {
+ t.Fatal(result)
+ }
+ if flags["include_screenshot"] == false {
+ if len(result.Content) != 1 || !strings.Contains(result.Content[0].Text, "include_screenshot=false") {
+ t.Fatal(result)
+ }
+ } else if len(result.Content) != 2 {
+ t.Fatal(result)
+ }
+ if flags["include_ax"] == false {
+ cached := s.currentSnapshot("Editor")
+ if len(cached.Elements) != 0 || len(cached.TreeLines) != 0 || cached.FocusedSummary != "" || cached.SelectedText != "" {
+ t.Fatal(cached)
+ }
+ if !strings.Contains(result.Content[0].Text, "indexes invalidated") {
+ t.Fatal(result)
+ }
+ }
+ }
+}
+
+func TestObservationRejectsInvalidOptionsBeforeBridge(t *testing.T) {
+ s := newService()
+ s.runBridge = func(psRequest) (*psResponse, error) { t.Fatal("invalid observation reached bridge"); return nil, nil }
+ for _, args := range []map[string]any{{"include_ax": "false"}, {"include_screenshot": nil}, {"include_ax": 0}, {"include_ax": false, "include_screenshot": false}} {
+ args["app"] = "Editor"
+ if result := s.callTool("get_app_state", args); !result.IsError {
+ t.Fatal(args)
+ }
+ }
+}
+
+func TestObservationInvalidatesOlderAliasesAcrossProcessReplacement(t *testing.T) {
+ s := newService()
+ s.rememberSnapshot("older-query", &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 6}}})
+ s.rememberSnapshot("Editor", &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 7}}})
+ other := &appSnapshot{App: appDescriptor{Name: "Other", PID: 99}}
+ s.rememberSnapshot("Other", other)
+ s.runBridge = func(request psRequest) (*psResponse, error) {
+ return &psResponse{OK: true, Snapshot: &appSnapshot{App: appDescriptor{Name: "Editor", PID: 43},
+ WindowBounds: &frame{Width: 300, Height: 200}, ScreenshotPNGBase64: pngBase64(t, 300, 200), Elements: []elementRecord{{Index: 8}}}}, nil
+ }
+ if result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_ax": false}); result.IsError {
+ t.Fatal(result)
+ }
+ if s.currentSnapshot("older-query") != nil || s.currentSnapshot("42") != nil || s.currentSnapshot("Other") != other {
+ t.Fatal("stale aliases retained or unrelated app invalidated")
+ }
+ if result := s.setValue("Editor", "7", "bad"); !result.IsError {
+ t.Fatal("stale element action accepted")
+ }
+ x, y := 10.0, 20.0
+ if result := s.click("Editor", "", &x, &y, 1, "left", "auto"); result.IsError {
+ t.Fatal("screenshot-only coordinate action failed", result)
+ }
+ if result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_screenshot": false}); result.IsError {
+ t.Fatal(result)
+ }
+ if _, err := lookupElement(s.currentSnapshot("Editor"), "8"); err != nil {
+ t.Fatal("AX refresh did not restore indexes", err)
+ }
+ if result := s.click("Editor", "", &x, &y, 1, "left", "auto"); !result.IsError {
+ t.Fatal("AX-only observation accepted screenshot coordinates")
+ }
+ if result := s.setValue("Editor", "8", "works"); result.IsError {
+ t.Fatal("AX-only element action failed", result)
+ }
+}
+
+func TestObservationSchemaDefaults(t *testing.T) {
+ props := findToolDefinition(t, "get_app_state").InputSchema["properties"].(map[string]any)
+ for _, key := range []string{"include_ax", "include_screenshot"} {
+ value := props[key].(map[string]any)
+ if value["type"] != "boolean" || value["default"] != true {
+ t.Fatal(value)
+ }
+ }
+}
diff --git a/apps/OpenComputerUseWindows/runtime.ps1 b/apps/OpenComputerUseWindows/runtime.ps1
index 3cf4df0..6cbe7ad 100644
--- a/apps/OpenComputerUseWindows/runtime.ps1
+++ b/apps/OpenComputerUseWindows/runtime.ps1
@@ -692,11 +692,38 @@ function Get-SelectedText($processId, $TextLimit = $script:DefaultTextLimit) {
return $null
}
-function Build-Snapshot([string]$query, $TextLimit = $script:DefaultTextLimit, [int]$MaxTreeNodes = $script:AccessibilityTreeMaxNodeCount, [int]$MaxTreeDepth = $script:AccessibilityTreeMaxDepth) {
+function Resolve-ObservationOptions($operation) {
+ $values = @{ include_ax = $true; include_screenshot = $true }
+ foreach ($key in @("include_ax", "include_screenshot")) {
+ $property = $operation.PSObject.Properties[$key]
+ if ($null -ne $property) {
+ if ($property.Value -isnot [bool]) { throw "$key must be a boolean" }
+ $values[$key] = $property.Value
+ }
+ }
+ if (-not $values.include_ax -and -not $values.include_screenshot) {
+ throw "At least one of include_ax or include_screenshot must be true"
+ }
+ return [pscustomobject]$values
+}
+
+function Build-Snapshot([string]$query, $TextLimit = $script:DefaultTextLimit, [int]$MaxTreeNodes = $script:AccessibilityTreeMaxNodeCount, [int]$MaxTreeDepth = $script:AccessibilityTreeMaxDepth, [bool]$IncludeAX = $true, [bool]$IncludeScreenshot = $true) {
+ if (-not $IncludeAX -and -not $IncludeScreenshot) {
+ throw "At least one of include_ax or include_screenshot must be true"
+ }
$process = Resolve-App $query
$element = Get-MainElement $process
$bounds = Get-WindowBounds $process $element
- $rendered = Render-Tree $element $bounds $TextLimit $MaxTreeNodes $MaxTreeDepth
+ $rendered = @{ lines = @(); records = @() }
+ $focus = $null
+ $selection = $null
+ $image = $null
+ if ($IncludeAX) {
+ $rendered = Render-Tree $element $bounds $TextLimit $MaxTreeNodes $MaxTreeDepth
+ $focus = Get-FocusedSummary $process.Id $TextLimit
+ $selection = Get-SelectedText $process.Id $TextLimit
+ }
+ if ($IncludeScreenshot) { $image = Capture-WindowPngBase64 $bounds }
[pscustomobject]@{
app = [pscustomobject]@{
name = $process.ProcessName
@@ -705,10 +732,10 @@ function Build-Snapshot([string]$query, $TextLimit = $script:DefaultTextLimit, [
}
windowTitle = Limit-Text $process.MainWindowTitle $TextLimit
windowBounds = $bounds
- screenshotPngBase64 = Capture-WindowPngBase64 $bounds
+ screenshotPngBase64 = $image
treeLines = @($rendered.lines)
- focusedSummary = Get-FocusedSummary $process.Id $TextLimit
- selectedText = Get-SelectedText $process.Id $TextLimit
+ focusedSummary = $focus
+ selectedText = $selection
elements = @($rendered.records)
}
}
@@ -982,7 +1009,8 @@ try {
if ($operation.tool -eq "list_apps") {
$response = [pscustomobject]@{ ok = $true; text = (List-Apps) }
} elseif ($operation.tool -eq "get_app_state") {
- $response = [pscustomobject]@{ ok = $true; snapshot = (Build-Snapshot $operation.app (Resolve-TextLimit $operation.text_limit) ([int]$operation.max_tree_nodes) ([int]$operation.max_tree_depth)) }
+ $observation = Resolve-ObservationOptions $operation
+ $response = [pscustomobject]@{ ok = $true; snapshot = (Build-Snapshot $operation.app (Resolve-TextLimit $operation.text_limit) ([int]$operation.max_tree_nodes) ([int]$operation.max_tree_depth) $observation.include_ax $observation.include_screenshot) }
} else {
$process = Resolve-App $operation.app
$hwnd = [IntPtr]$process.MainWindowHandle
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 8fc5d79..012785f 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -74,6 +74,7 @@
- `open-computer-use call <tool> --args '{...}'` 会直接输出 MCP-style JSON result;`open-computer-use call --calls '[...]'` / `--calls-file <path>` 会在同一进程里顺序执行 JSON 数组里的 tool calls,并复用同一个 `ComputerUseService` 内存态,因此 `get_app_state` 之后的 action tool 可以继续使用同一轮 snapshot 的 `element_index`。序列执行默认会在成功的相邻操作之间 sleep 1 秒,也可以用 `--sleep <seconds>` 覆盖;遇到 `isError=true` 的 tool result 后停止。
- 对真实 app 的 `get_app_state` / action tool 入口,当前只保留一层密码管理器 bundle denylist:bundle-id 直传时直接返回 safety denial;名称匹配时默认不解析到这些 app。终端、Chrome / Atlas 和系统组件不再属于内置阻止目标。
- 普通 app 的 element frame 当前按“窗口左上角为原点”的 window-relative 坐标输出,便于后续把 `element_index` 和截图坐标统一到同一套参考系。
+- 三个平台的显式 `get_app_state` 均支持 `include_ax` / `include_screenshot`,默认 true,至少保留一项。JS 对应 `includeAX` / `includeScreenshot`,CLI snapshot 提供同名连字符参数。关闭 AX 跳过树、焦点和选区,并使旧快照 / 查询索引及别名失效;关闭截图跳过捕获与编码,元素操作仍可用,截图坐标操作须重新获取图片。参数经 macOS app agent 转发;不改变动作回读,也不做自动图片去重。见 [按需观察](selective-observation.md)。
- macOS 额外提供 `query` 和 `select_text`,当前 native surface 为 11 个工具;Linux / Windows 仍为 9 个核心工具。JS `app.selectText(elementIndex, text, {prefix?, suffix?, selection?})` 复用快照或 query 索引,支持选中文字及前后光标。按字面文本和 UTF-16 定位、拒绝歧义;写入前及返回前检查会话,验证文本未变且实际选区等于请求,即使关闭完整回读也不能省略验证。失败通过 native `isError` / JS 异常传出;结果刷新使用 read-only recovery,不通过激活、键盘或真实指针补救。
- `click` / `set_value` 在执行真实动作前后,会额外驱动一层透明 `SoftwareCursorOverlay` window:两者的移动阶段现在共用一条 heading-driven 的官方风格 motion 内核,显式把“当前 cursor 朝向”和“最终 resting pose”一起喂给选路器,优先生成需要时先掉头、再沿车头方向推进的 C 形/单侧大弧轨迹;首次显示时按官方 binary 的 fresh state 从 AppKit 全局 `(0,0)` window origin 生成起点,后续动作继续复用上一帧 visible tip。真正显示出来的 cursor 不再直接等于 path sample,而是经过一层独立的 visual dynamics 状态,把 visible tip、velocity、angle 和 fog/offset 持续推进。`click` 结尾会衔接 click pulse 和更明显但仍然很小的 rotate wobble,`set_value` 则只做 settle / idle,不给 pulse;两者收尾后会在目标点继续保持 idle 状态,等待下一次动作时 tip 保持 anchored、只保留可感知的小角度摆动;只有连续 30 秒没有新动作时才做 cleanup,这样连续 tool call 不会反复从 fresh `(0,0)` 起步;如果宿主在任务 / turn 结束时发出 `turn-ended`,cursor 会立即消失并清掉本轮位置状态。
- overlay 用目标点相对窗口的锚点跟随窗口移动;行进、settle、点击反馈和 idle 都读取最新缓存帧,发现位置变化时直接重定位可见 tip 并清掉旧速度。窗口帧在后台串行读取,最多 30 Hz、同时最多一个请求,避免 AX 阻塞渲染线程;优先读取快照持有的 AX 窗口,仅在没有 AX 元素时使用窗口列表,读取失败保留最后有效帧。只保留当前窗口的追踪状态,reset / idle 超时后释放;这层跟随只调整显示位置,输入投递仍走既有路径。
@@ -127,7 +128,7 @@
- 这 9 个 tool 的协议面与 macOS 主线保持一致:`list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value`。其中 element-targeted action 会优先复用上一轮 `get_app_state` 的 runtime id / automation metadata,coordinate action 使用 screenshot/window-relative 坐标。
- Windows `click_method=accessibility` 映射到 UI Automation pattern,`app_post` 映射到 HWND `PostMessage`;macOS-only 的 `sky_click` 和没有实现的 `global` 都会在 snapshot lookup 前明确返回 unsupported。`type_text` / `press_key` 的 `key_method=sky_key` 同样在 snapshot lookup 前返回 unsupported。`auto` 仍保持 UIA 优先、window message fallback 的现有行为。
- Windows UI Automation 需要运行在已登录用户的桌面 session 里。通过 SSH 作为脱离桌面的后台进程运行时,PowerShell 可以启动并返回 JSON,但系统可能不给它暴露顶层窗口;这种情况下 `list_apps` 会是空,`get_app_state` 可能返回 `appNotFound(...)`。
-- 当前 Windows 侧仍是功能性第一版:没有 visual cursor overlay、没有 installer/onboarding、没有 code signing,也没有独立的 Windows smoke fixture。后续 TODO 记录在 `docs/exec-plans/active/20260422-windows-computer-use-runtime.md`。
+- Windows 已有独立 WPF 观察 fixture 和 Windows PR smoke,覆盖观察通道及截图边界。其余工具 / toolkit 覆盖,以及 visual cursor overlay、installer/onboarding、code signing 仍待补齐。后续 TODO 记录在 `docs/exec-plans/active/20260422-windows-computer-use-runtime.md`。
### 7. Linux Runtime
diff --git a/docs/CICD.md b/docs/CICD.md
index 589239c..3638317 100644
--- a/docs/CICD.md
+++ b/docs/CICD.md
@@ -19,6 +19,10 @@
所有 GitHub Actions 都已经 pin 到 commit SHA。后续升级 action 时,也要继续保持这个约束。
+## Windows 观察回归
+
+`.github/workflows/windows-observation.yml` 在相关 PR 上运行 Windows Go 测试、PowerShell 生产采集函数测试,以及独立 WPF / native MCP smoke,也支持手动触发。工作流仅需 `contents: read`,沿用固定 SHA 的 checkout / setup-node / setup-go,不发布制品。桌面 smoke 使用自己的窗口,验证采集开关、索引与 UIA 设值,以及正常 / 1x1 / 最小化 / cloaked 窗口的截图边界。复现命令见 [按需观察](selective-observation.md)。
+
## 推荐接入顺序
1. 保留 `ci.yml`,作为仓库的基础门禁。
diff --git a/docs/QUALITY_SCORE.md b/docs/QUALITY_SCORE.md
index 407622c..7002660 100644
--- a/docs/QUALITY_SCORE.md
+++ b/docs/QUALITY_SCORE.md
@@ -12,7 +12,7 @@
| 区域 | 评分 | 原因 | 下一步 |
| --- | --- | --- | --- |
| 产品面 | B | 已经有 Swift 本地 `computer-use` MCP server、默认 app 模式权限引导,以及共享的 9 个核心 tools;macOS 另有 query、select_text,JS 可调用。 | 继续收敛复杂 AX 场景下的 state rendering 细节、权限 UI 和更清晰的用户错误提示。 |
-| Windows runtime | C | 已新增独立 Go `.exe`,通过 Windows UI Automation + Win32 window message 暴露同样 9 个 tools、MCP server 和 `call --calls`;默认不再自动启动 app、执行 `SetFocus`,或让 `type_text` 走可能抢前台的 UIA text fallback;截图会过滤隐藏、最小化、cloaked、离屏和退化窗口,并在 Go 层复核 PNG,但整体仍是功能性第一版。 | 把当前实机 screenshot 边界用例收敛成仓库内 Windows fixture / 交互式 smoke,再补 installer/signing,以及更原生的 Go UIA 实现或更稳定的 bridge。 |
+| Windows runtime | C | 已新增独立 Go `.exe`,通过 Windows UI Automation + Win32 window message 暴露同样 9 个 tools、MCP server 和 `call --calls`;默认不再自动启动 app、执行 `SetFocus`,或让 `type_text` 走可能抢前台的 UIA text fallback;截图会过滤隐藏、最小化、cloaked、离屏和退化窗口,并在 Go 层复核 PNG,但整体仍是功能性第一版。 | 维护 WPF 观察 / 截图边界 fixture 和 Windows PR smoke,扩展更多工具与 toolkit;补 installer/signing,以及更原生的 Go UIA 实现或更稳定的 bridge。 |
| Linux runtime | C | 已新增独立 Go binary,通过 Python GI / AT-SPI2 暴露同样 9 个 tools、MCP server 和 `call --calls`;Ubuntu GNOME VM 已跑通 `list_apps`、MCP tools list 和 Text Editor 8-tool sequence,并已接入 npm bundled artifact 分发,但截图在 GNOME Wayland 下仍只能 best-effort,coordinate input 也不是通用后台模型。 | 扩展已有的隔离 GTK 动作回读 smoke,补完整工具覆盖、portal/compositor screenshot 路径,以及更原生的 Go D-Bus/libatspi bridge。 |
| macOS Locked Use | D | 已有签名 Broker / 插件、双进程保护、原生租约与恢复入口,离线和遮罩 rehearsal 有回归;安装恢复和无许可双保护释放已实测,独立无遮罩 Return 解锁 / 重锁已实测,受保护自动解锁 / AX / SCK / 重锁及隔离 legacy Keychain 已有一轮实测通过,完整会话认证状态 / 故障链路尚未通过实测;正常启用改为管理员 enable 与签名校验,普通 enable / 正常 MCP 的完整 GUI 闭环已实测。 | 物理覆盖已确认,继续完成 Data Protection / 会话认证状态、Secure Input / 崩溃 / 显示器故障回归及正式发布制品检查。 |
| 架构文档 | B | 顶层结构、fixture bridge、app 模式和验证路径已经落文档。 | 后续补 release artifact、code signing / notarization 和 host 集成方式。 |
diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.md
index 3bf1844..06cc2ad 100644
--- a/docs/RELIABILITY.md
+++ b/docs/RELIABILITY.md
@@ -19,7 +19,9 @@
- macOS 其他 Desktop 实机回归(需要第二个 Desktop,测试会短暂切换 Space 启动目标):`OPEN_COMPUTER_USE_RUN_CROSS_SPACE_LIVE_TEST=1 swift test --filter CrossSpaceLiveTests`
- Linux runtime:`(cd apps/OpenComputerUseLinux && go test ./...)`、`./scripts/build-open-computer-use-linux.sh --arch arm64`
- Linux 隐藏控件实测:`bash scripts/run-linux-read-back-smoke.sh /path/to/linux/open-computer-use --hidden-checks`,在私有 D-Bus / Xvfb 中验证真实 GTK 隐藏状态、属性读取、索引及页面切换;依赖与范围见 [测试说明](linux-hidden-details.md)。
+- Linux 按需观察实测:`bash scripts/run-linux-read-back-smoke.sh /path/to/linux/open-computer-use --observation-checks`,验证真实 GTK 采集通道、native MCP / Worker JS 输出、旧索引拒绝与恢复;另外需要 Node.js,见 [测试说明](selective-observation.md)。
- Windows runtime:`(cd apps/OpenComputerUseWindows && go test ./...)`、`./scripts/build-open-computer-use-windows.sh --arch amd64`;截图相关变更还要在 Windows 交互桌面验证正常窗口有 image、1x1/minimized/cloaked 窗口无 image。
+- Windows 观察实测:`pwsh -File scripts/tests/windows-observation-smoke.ps1 -NativePath /path/to/open-computer-use.exe`,使用独立 WPF 窗口走真实 UIA / native MCP,验证采集开关、索引与截图边界;专用 PR 工作流见 [CI 说明](CICD.md)。
- 本地诊断:
- `open-computer-use doctor`
- `open-computer-use snapshot <app>`
diff --git a/docs/exec-plans/completed/20261017-pr101-selective-observation.md b/docs/exec-plans/completed/20261017-pr101-selective-observation.md
new file mode 100644
index 0000000..cc440bb
--- /dev/null
+++ b/docs/exec-plans/completed/20261017-pr101-selective-observation.md
@@ -0,0 +1,42 @@
+# PR #101: cross-platform selective observation
+
+## Goal
+
+Integrate current main, review and fix selective AX/screenshot collection, extend it to macOS and Windows at the user's request, verify through CLI/native MCP/current JS, and push follow-up commits to the original PR. Do not merge.
+
+## Scope and decisions
+
+- Both channels default to true; at least one is required. Existing calls and action read-back behavior stay compatible.
+- AX-only skips capture/encoding; screenshot-only skips tree/focus/selection and invalidates old snapshot aliases, including PID replacement and macOS query indexes. Fresh AX restores element actions.
+- AX-only cannot validate screenshot coordinates: require a fresh image observation before coordinate click/drag.
+- Use disposable AppKit, GTK and WPF fixtures. macOS follows normal signing/session/permission admission; Linux uses private Xvfb/D-Bus on the existing container engine; Windows uses the hosted Actions desktop. No personal apps or local desktop VM.
+- Automatic unchanged-image suppression remains separate. It needs exact image comparison, first/reset/forced output and session/app/window scope; unchanged AX does not establish unchanged pixels.
+
+## Progress
+
+- [x] Merge main and retain existing bounded-focus, hidden-node and read-back behavior.
+- [x] Add macOS collection gates, schema/dispatcher/app-agent/CLI support and scoped cache/query invalidation.
+- [x] Add Windows Go/PowerShell collection gates, strict options, cache behavior and protocol tests.
+- [x] Expose matching snapshot flags on all platforms and reject AX-only coordinate input.
+- [x] Verify normal signed macOS release and real GTK through CLI/native MCP/Worker JS; run relevant suites/builds.
+- [x] Complete hosted real Windows WPF/UIA/MCP/CLI/Worker JS and screenshot-boundary regression.
+- [x] Synchronize final docs/history/PR, remotes and original PR branch; leave PR open.
+
+## Validation and findings
+
+- Main through `7cb8bb7` integrated, preserving the original contribution.
+- Older aliases after PID replacement initially retained stale indexes; tests now cover invalidation for both previous/new PIDs and preserve unrelated applications.
+- macOS CLI originally printed only its first text block; it now retains omission notes too.
+- Swift: 382 tests, 8 skipped, zero failures; selective-observation 8/8. Existing native tool and cursor smoke passed.
+- Normally signed macOS release + real AppKit AX/native MCP/CLI/Worker JS passed valid PNG, trace-confirmed collection gates, snapshot/query invalidation, coordinate refusal, real value update and recovery. Fixture cannot become key/main.
+- Linux: Go 31, Python 35 and real GTK/AT-SPI/Xvfb passed collection helper gates, PNG, Unicode selection, old name/PID index refusal, coordinate actions, AX recovery and CLI/Worker JS. Existing focus and hidden-detail checks passed.
+- Node: 44 passed. Windows Go: 29 passed. Production PowerShell function tests passed under PowerShell 7 and actual Windows PowerShell 5.1 with instrumented dependencies.
+- Linux/Windows ARM64/AMD64 builds and targeted vet passed; syntax/docs/whitespace/action-pinning checks passed.
+- WinForms fixture did not expose a settable ValuePattern; the positive UIA-write fixture was changed to WPF instead of broadening production input fallbacks. The real WPF/UIA/CLI/native MCP/Worker JS desktop smoke passed on Windows 2025.
+- Temporary Linux containers and build artifacts removed. No Wayland/other-toolkit performance claim or version bump.
+
+## Outcome
+
+All three platforms now provide the same explicit observation choices through CLI/native MCP/JS. Real desktop tests passed for AppKit, GTK and WPF; the Windows job additionally verified exact 1x1 Win32, minimized and DWM-cloaked screenshot suppression and image recovery. WPF DPI layout is avoided only for the exact one-pixel fixture; production screenshot thresholds and input fallbacks were unchanged.
+
+Windows desktop evidence: [Windows observation run](https://github.com/iFurySt/open-computer-use/actions/runs/39018965035) at `ae8ec2f`. Final follow-up commits are pushed to #101; the PR remains open for the maintainer to merge. Automatic image deduplication remains in the debt tracker.
diff --git a/docs/exec-plans/tech-debt-tracker.md b/docs/exec-plans/tech-debt-tracker.md
index a1055be..c3e424c 100644
--- a/docs/exec-plans/tech-debt-tracker.md
+++ b/docs/exec-plans/tech-debt-tracker.md
@@ -4,4 +4,5 @@
| 日期 | 区域 | 债务描述 | 为什么会存在 | 计划中的后续动作 |
| --- | --- | --- | --- | --- |
+| 2026-10-17 | 重复截图 | 请求截图时仍会返回未变化的画面。 | #101 已统一三个平台的显式采集开关;相同图像省略是独立能力。 | 评估按会话 / 应用 / 窗口隔离的精确图像去重;首帧、重置与强制返回必须明确,不能用 AX 未变推断画面未变,也不把去重当成跳过截图采集。 |
| 2026-04-17 | 普通 app AX snapshot | Finder 路径已经能拿到前台窗口子树并输出 window-relative frame,但当前还缺更多真实 app 回归样本,无法证明这套 rooting / traversal 对复杂 app 都稳定。 | 这一轮先把 Finder 这类真实 app 的坐标换算和窗口子树收敛好,再把 deterministic 回归继续留给 fixture。 | 增加 Safari / System Settings / Activity Monitor 等真实 app 样本验证,并继续收敛 `kAXMainWindowAttribute`、focused element parent chain 和多窗口回退策略。 |
diff --git a/docs/histories/2026-10/20261015-2023-linux-selective-observation.md b/docs/histories/2026-10/20261015-2023-linux-selective-observation.md
new file mode 100644
index 0000000..967ef8e
--- /dev/null
+++ b/docs/histories/2026-10/20261015-2023-linux-selective-observation.md
@@ -0,0 +1,66 @@
+## [2026-10-15 20:23] | Task: Add opt-in selective Linux observation
+
+### Execution Context
+- Agent: Codex
+- Runtime: Linux development environment
+
+### User Request
+Offer cheaper explicit observation as a compatibility-preserving option, following the existing lightweight app binding approach.
+
+### Changes
+- Add optional Linux `get_app_state` booleans `include_ax` and `include_screenshot`, both defaulting to true.
+- Skip AX tree, focus and selection collection for screenshot-only reads; skip capture/PNG encoding for AX-only reads.
+- Forward optional REPL `includeAX` and `includeScreenshot` without changing legacy calls.
+- Explicitly invalidate prior AX indexes and their aliases after screenshot-only reads, retain current window geometry for coordinate actions, and report the invalidation.
+- Add Python collection-path tests, Go protocol/cache/schema tests, Node compatibility tests, usage documentation and release notes.
+
+### Design Intent
+Changing `getScreenshot()` defaults would also change its implicit index refresh. Keep existing behavior and make selective observation opt-in. A screenshot cannot validate old indexes, including after a window/process switch, so clear them rather than guessing from matching titles. A fresh AX observation restores indexes.
+
+### Validation
+- Linux Go suite and Python bridge unit tests.
+- Node REPL regression tests, including unchanged legacy request arguments.
+- REPL tests: 23 passed. Full Node suite: 36/37 passed; the staged npm packaging test requires macOS `lipo` and fails identically on the unmodified base checkout on Linux.
+- Documentation and whitespace checks.
+- No live desktop timing, macOS runtime or Windows runtime validation is claimed. This change is independent of the action read-back fix and does not change input delivery or bridge process lifetime.
+
+### Files Modified
+- `apps/OpenComputerUseLinux/main.go`
+- `apps/OpenComputerUseLinux/main_test.go`
+- `apps/OpenComputerUseLinux/runtime.py`
+- `apps/OpenComputerUseLinux/runtime_test.py`
+- `scripts/node-repl/open-computer-use-repl.mjs`
+- `scripts/node-repl/open-computer-use-repl.test.mjs`
+- `skills/open-computer-use/references/usage.md`
+- `docs/ARCHITECTURE.md`
+- `docs/releases/feature-release-notes.md`
+
+## 2026-10-17:同步主线、修复别名遗漏并补实测
+
+用户要求处理 #101,基于最新 main 解决冲突、检查测试并推送,补真实 GTK;同时评估截图可选与未变化图片重复返回的关系。
+
+- 合入主线 `7cb8bb7`(含 #103),保留原贡献提交及两边的 Go / Python 测试、架构和发布记录。将条件 AX 采集接入现有一次有界 focus_details,保留焦点预算提示及隐藏节点选项。
+- 发现只取截图时,应用 PID 改变后较早观察的查询别名可能残留。新增独立测试先复现失败,再清理新旧 PID 对应的全部观察别名,同时保留其他应用缓存。
+- JS 在两种观察通道同时关闭时提前拒绝,不产生 native 请求;Persistent / Worker 两种会话均测试参数、错误及旧调用兼容性。
+- 扩展 `--observation-checks`:在真实 GTK 上包装生产采集函数,确认 AX-only 没有截图、image-only 没有树或焦点读取,完整模式同时采集;检查真实 PNG、中文 / emoji 选区、名称 / PID 别名下旧索引拒绝、坐标点击及 AX 刷新后的设值。独立 GTK 状态确认结果。
+- 新增 Worker JS → native MCP → GTK smoke,验证选择通道、图片输出、旧索引异常、AX 恢复及默认完整观察。环境为现有引擎上的临时 Debian 12 ARM64 容器、私有 Xvfb / D-Bus、GTK 3 / AT-SPI 2 和 Node 24,不启动桌面 VM。
+
+验证:Python 35 项、Linux Go 30 项、Node 44 项通过;Linux ARM64 / AMD64 构建、targeted vet、shell / Python / Node 语法检查通过;按需观察真实链路、原有默认动作回读、焦点 / Unicode 选区和隐藏详情 smoke 均通过。文档与 whitespace 检查通过。未运行 Wayland 或其他 toolkit,不宣称全任务提速,无版本 bump。
+
+需求评估:#101 提供 Linux 显式通道选择,不做截图去重,也未实现其他平台的通道开关。现有 macOS / Linux JS 动作已禁用自动回读,getAXState 不输出图片,但仍可能浪费 native 采集。未变化图片省略通常仍需截图后比较,主要节省传输和模型输入;后续的跨平台支持、会话 / 应用 / 窗口隔离、首帧 / 重置 / 强制返回和精确图像比较已记录到技术债追踪,不以 AX 未变作为画面未变的证据。
+
+验证结束后已删除临时容器和独立构建制品,未影响其他容器服务。
+
+## 2026-10-17:按用户要求统一三个平台
+
+用户明确要求扩展 macOS / Windows,而不只实现 Linux。本阶段取代前文的 Linux-only 范围;自动图片去重仍单独跟踪。
+
+- macOS 新增观察选项、严格布尔校验、schema / dispatcher / app-agent CLI 转发,截图关闭时跳过捕获和编码,AX 关闭时跳过树、选区、菜单及懒加载重试。fixture 同步行为。纯截图清理旧进程快照别名和 query 索引,旧别名不能触发隐式刷新继续使用旧编号。
+- Windows Go / PowerShell 支持同样的参数、默认和空观察拒绝;按通道跳过生产采集函数,防御性清理结果中的禁用通道及旧缓存。新增纯传输测试接口,不改变生产 Windows / PowerShell 宿主限制。
+- 三个平台的 snapshot CLI 新增 --include-ax / --include-screenshot true|false。只取 AX 后拒绝截图坐标输入,保留元素操作。macOS CLI 改为打印所有文本说明但不输出图片字节,修复实测发现的省略原因丢失。
+- macOS 正常签名 release + 自有 AppKit 窗口实测通过:CLI / MCP / Worker JS、PNG、timing 分支证据、普通 / query 索引失效、坐标拒绝与 AX 恢复;窗口不能成为 key/main,未修改系统准入或权限。
+- 本地检查:Swift 382 项(8 跳过、0 失败),观察专项 8 项;Linux Go 31 项、Python 35 项、Node 44 项;Windows Go 29 项与生产 PowerShell 函数分支测试通过;Linux / Windows 双架构构建及 targeted vet 通过。真实 GTK 观察、焦点、隐藏详情及 macOS 原生工具 / cursor smoke 通过。
+- 增加专门的 Windows Actions 检查与临时 WPF 夹具,真实 Windows 2025 桌面上的 Go / Windows PowerShell 5.1 / native MCP / CLI / Worker JS 全部通过。覆盖三个通道模式、旧索引拒绝、AX 恢复、真实 Unicode 设值和坐标拒绝;正常窗口返回 PNG,物理 1×1 Win32 小窗口、最小化及 DWM-cloaked 窗口不返回图片,恢复后重新返回 PNG。工作流使用仓库既有固定 SHA actions 和 contents:read,无发布步骤。
+- 初版 WinForms 控件不提供可写 ValuePattern,正向设值测试改用标准 WPF TextBox,没有扩大生产输入 fallback。夹具状态文件用无 BOM UTF-8 原子写入,避免 Windows PowerShell 空字符串与 null 的文件替换差异。精确 1×1 用同进程 Win32 窗口,避开 WPF 高 DPI 最小尺寸取整。
+- Windows [完整实测记录](https://github.com/iFurySt/open-computer-use/actions/runs/39018965035) 通过于 `ae8ec2f`。覆盖 AppKit / GTK / WPF,不宣称所有 toolkit、Wayland 或完整任务性能均已验证。
+- 文档同步为三平台使用方式、可复现测试命令、Windows CI 和当前覆盖范围;原 execution plan 完成归档,PR 保持开放,未执行合并。
diff --git a/docs/linux-selective-observation.md b/docs/linux-selective-observation.md
new file mode 100644
index 0000000..931b6be
--- /dev/null
+++ b/docs/linux-selective-observation.md
@@ -0,0 +1,3 @@
+# Linux 按需观察
+
+现已扩展到 macOS、Windows 和 Linux,完整参数、CLI / MCP 用法及测试见[按需观察](selective-observation.md)。
diff --git a/docs/references/js-repl.md b/docs/references/js-repl.md
index a4259c9..46c3aed 100644
--- a/docs/references/js-repl.md
+++ b/docs/references/js-repl.md
@@ -54,6 +54,15 @@ typeText(text, options?)
performSecondaryAction(elementIndex, action)
```
+On macOS, Windows and Linux, observation methods also accept optional `includeAX` and
+`includeScreenshot` booleans, both defaulting to true. Use
+`app.getAXState({includeScreenshot:false})` to skip native capture, or
+`app.getScreenshot({includeAX:false})` to skip the AX/focus walk. The latter
+invalidates previous element indexes; read AX again before element actions.
+Disabling both channels is rejected before a native call. These options do not
+deduplicate unchanged images. See [selective observation](../selective-observation.md)
+for collection semantics and real GTK/Worker validation.
+
On macOS, `selectText` selects exact text inside a text element. `prefix` and `suffix`
match the immediately adjacent text to disambiguate repeated occurrences. `selection`
is `text` (default), `cursor_before` or `cursor_after`. Snapshot and `query` indexes
diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.md
index c63ff06..cfa1d00 100644
--- a/docs/releases/feature-release-notes.md
+++ b/docs/releases/feature-release-notes.md
@@ -13,6 +13,7 @@
| 2026-10-15 | Linux 动作回读 | REPL 批量操作可省去每个动作后被丢弃的完整界面观察。 | Linux 开始支持现有的 `OPEN_COMPUTER_USE_ACTION_READ_BACK=0`,成功动作返回 `ok`,保留最后一次显式观察的索引缓存;默认原生 MCP 回读与显式 `get_app_state` 保持不变。 |
| 2026-10-15 | Linux 焦点与选区 | 大型界面减少重复查找,预算不足时明确提示,选中文字能正确返回。 | 复用快照已选窗口,一次查找生成焦点摘要与选区;按节点和深度预算停止读取,使用标准 AT-SPI 选区接口,并补隔离 GTK 验证。 |
| 2026-10-15 | Linux 隐藏控件详情 | 隐藏节点较多时,可减少不需要的属性读取。 | 新增默认关闭的 `OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN`;保留节点、索引和可见子节点,状态未知时完整读取,补充真实 GTK 标签页及 native MCP 回归。详见 [使用与验证](../linux-hidden-details.md)。 |
+| 2026-10-15 | 按需观察 | 只需控件信息时可省去截图,只需画面时可省去 AX 遍历。 | 统一 macOS / Windows / Linux 的 include_ax / include_screenshot、JS 和 CLI snapshot 参数,默认完整采集;纯截图清理旧索引,只取 AX 拒绝截图坐标输入,补真实 AppKit / GTK / WPF、Windows 截图边界与 PowerShell 分支验证。详见 [使用与验证](../selective-observation.md)。 |
| 2026-10-14 | 1.1.0 正式发布 | 用户可直接安装 Locked Use,在锁屏遮罩期间继续 GUI 操作,也可用定向查询和持久截图配置减少工具开销。 | 统一版本为 1.1.0;签名 universal macOS App 内嵌服务、Guardian、安装器和授权插件,普通 enable 请求权限,补齐冷启动握手隔离和发布制品检查;新增 app.query()、doctor --json 与 ocu config。 |
| 2026-10-10 | macOS Locked Use 验证版 | 可在已登录 Mac 锁屏时,通过全屏遮罩继续 GUI 自动化,并在结束或本地接管后安全重锁。 | 正式组件采用 OCU Guardian / OCULockService / OCULockInstaller / OCULockAuth 短名称,开发 App 保留 Dev 区分;新增签名 Broker / 授权插件、双进程显示器保护、一次性许可、租约排空 / 恢复及构建和实机验证脚本;管理员显式启用 validation profile,生产 backend 在完整验证前保持关闭。 |
| 2026-10-10 | macOS 定向 AX 查询 | 知道控件文案时,可以直接查询并操作,不必先获取截图和完整 AX tree。 | 新增 native `query` 和 JS `app.query()`,按文字、角色和窗口查找元素,返回完整性元数据;查询索引仅在同一会话内有效,操作前重新验证目标和位置。 |
diff --git a/docs/selective-observation.md b/docs/selective-observation.md
new file mode 100644
index 0000000..4b834a6
--- /dev/null
+++ b/docs/selective-observation.md
@@ -0,0 +1,82 @@
+# 按需观察
+
+macOS、Windows、Linux 的 `get_app_state` 都可以选择读取 AX 或截图。原生参数 `include_ax` / `include_screenshot` 对应 JS 的 `includeAX` / `includeScreenshot`,均默认 `true`,不能同时设为 `false`。
+
+| 需要什么 | JS 调用 | 实际采集 |
+| --- | --- | --- |
+| 只读控件和选区 | `await app.getAXState({includeScreenshot: false})` | AX、焦点、选区;不截图、不编码 |
+| 只看画面 | `await app.getScreenshot({includeAX: false})` | 截图;不遍历 AX、不查焦点和选区 |
+| 两者都要 | `await app.getAXStateAndScreenshot()` | 完整 AX 和截图 |
+
+```js
+var app = await cua.getApp("gedit", {initialState: false});
+await app.getAXState({includeScreenshot: false});
+```
+
+只取截图后,旧元素索引及查询别名会失效,包括应用重启前的索引。坐标操作仍可使用本次观察的窗口位置;元素操作要先重新读取 AX。窗口标题相同不代表旧索引仍然有效。
+
+不传参数的调用保持原行为。只取 AX 后可继续操作元素,但截图坐标点击 / 拖拽要先重新获取截图;只取截图后则要重新读取 AX 才能使用元素索引。macOS 的 query 索引也会随纯截图观察失效。
+
+## CLI 和 MCP
+
+JS 的 `ocu js` / `ocu repl` 和 MCP `js` 使用上面的调用。直接调用原生 CLI / MCP 时使用 snake_case:
+
+```sh
+ocu call get_app_state --args '{"app":"TextEdit","include_screenshot":false}'
+ocu snapshot --include-screenshot false TextEdit
+ocu snapshot --include-ax false TextEdit
+```
+
+原生 MCP `get_app_state` 接收同样的 `include_ax` / `include_screenshot` 布尔值。`snapshot` 两个参数都接受 `true` / `false`,不能同时关闭。`snapshot` 一直只打印文本;需要图片内容时使用 JS 或 `call` / MCP。
+
+这两个选项只控制本次显式观察,不改变动作回读。
+
+## 和重复截图的关系
+
+macOS / Linux 下的 JS 动作已默认关闭自动回读;`getAXState()` 本身也不向模型输出图片,但默认仍会在 native 层采集图片后丢弃。`includeScreenshot: false` 可以在三个平台连这部分采集和编码一起省掉。直接使用 native MCP 时,它也会省去返回的 image block。
+
+这里不做截图比较:请求了截图,即使画面相同也照常返回。“画面未变则省略图片”需要单独实现,通常仍要先截图再比较,主要减少传输和模型输入。后续可按会话、应用和窗口保存精确图像指纹,首次观察及状态重置时返回图片,并提供强制返回方式;不能仅凭 AX 没变化就认定画面没变化。
+
+## 测试
+
+```sh
+swift test
+python3 apps/OpenComputerUseLinux/runtime_test.py
+(cd apps/OpenComputerUseLinux && go test ./...)
+(cd apps/OpenComputerUseWindows && go test ./...)
+node --test scripts/node-repl/*.test.mjs
+```
+
+真实 GTK 测试复用 [私有 D-Bus / Xvfb 环境](linux-action-read-back.md),另外需要 Node.js:
+
+```sh
+(cd apps/OpenComputerUseLinux && go build -o /tmp/ocu-observation-test .)
+bash scripts/run-linux-read-back-smoke.sh /tmp/ocu-observation-test --observation-checks
+```
+
+脚本包装真实生产采集函数,确认关闭的通道没有被调用;再通过 native MCP 和 Worker JS 检查 PNG、选区、旧索引拒绝、坐标操作及 AX 恢复。GTK 独立导出状态确认动作结果。这是 X11 / GTK / AT-SPI 测试,不代表 Wayland 或完整任务性能测试。
+
+macOS 使用正常签名且已授权的 OCU 可执行文件验证真实 AppKit 窗口,无需修改准入配置:
+
+```sh
+node scripts/tests/macos-observation-smoke.mjs "/path/to/Open Computer Use.app/Contents/MacOS/OpenComputerUse"
+```
+
+它通过真实 AX / native MCP / CLI / Worker JS 验证三个采集模式、PNG、快照和 query 索引失效、坐标拒绝和元素恢复,并用现有 timing 日志确认跳过的采集分支未执行。窗口禁止成为 key/main window;脚本不向个人应用输入。
+
+Windows 的 PowerShell 采集分支可用以下命令单独验证:
+
+```powershell
+pwsh -NoProfile -File scripts/tests/windows-observation.test.ps1 -RuntimePath apps/OpenComputerUseWindows/runtime.ps1
+```
+
+这个测试解析生产脚本并执行其中的观察函数,使用带计数的 UIA / 截图依赖验证分支。它不等于 Windows 桌面实测。
+
+Windows 桌面测试需要已登录的交互会话、Windows PowerShell 和 Node.js:
+
+```powershell
+go -C apps/OpenComputerUseWindows build -o "$env:TEMP/ocu-observation.exe" .
+pwsh -File scripts/tests/windows-observation-smoke.ps1 -NativePath "$env:TEMP/ocu-observation.exe"
+```
+
+脚本创建自己的 WPF 窗口,通过真实 UIA / native MCP / CLI / Worker JS 检查通道选择、旧索引拒绝、AX 恢复、Unicode 设值和截图坐标拒绝;再用同进程的 Win32 小窗口验证物理 1×1 边界,并检查 WPF 最小化、cloaked 时不返回图片,恢复后重新返回 PNG。`.github/workflows/windows-observation.yml` 在 Windows runner 执行这组检查。WPF 的通过结果不代表所有 WinForms / WinUI / Electron 控件都支持相同 UIA pattern。
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
index 3a3db1e..2fbfc30 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
@@ -148,6 +148,7 @@ public struct AppSnapshot {
let elements: [Int: ElementRecord]
var screenshotNote: String? = nil
+ var observation: SnapshotObservationOptions = .defaults
public var renderedText: String {
renderedText(style: .fullState)
@@ -161,6 +162,9 @@ public struct AppSnapshot {
lines.append("App=\(appReference) (pid \(app.pid))")
lines.append("Window: \(quoted(displayTitle)), App: \(app.name).")
lines.append(contentsOf: treeLines)
+ if !observation.includeAX {
+ lines.append(SnapshotObservationOptions.accessibilityOmittedNote)
+ }
if let selectedText, !selectedText.isEmpty {
lines.append("")
@@ -184,10 +188,12 @@ enum SnapshotBuilder {
for app: RunningAppDescriptor,
textLimit: SnapshotTextLimit = .defaults,
treeLimits: AccessibilityTreeLimits = .defaults,
- recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation
+ recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation,
+ observation: SnapshotObservationOptions = .defaults
) throws -> AppSnapshot {
+ try observation.validate()
if app.name == FixtureBridge.appName, let fixtureState = try FixtureBridge.readState() {
- return buildFixtureSnapshot(app: app, state: fixtureState)
+ return buildFixtureSnapshot(app: app, state: fixtureState, observation: observation)
}
try requireUsableComputerUseSession()
@@ -198,8 +204,8 @@ enum SnapshotBuilder {
}
let appElement = AXUIElementCreateApplication(app.pid)
- let lazyWebAccessibility = enableBestEffortAccessibilityModes(appElement)
- || appHasLazyWebAccessibility(bundleURL: app.runningApplication.bundleURL)
+ let lazyWebAccessibility = observation.includeAX && (enableBestEffortAccessibilityModes(appElement)
+ || appHasLazyWebAccessibility(bundleURL: app.runningApplication.bundleURL))
let systemWide = AXUIElementCreateSystemWide()
var focusedApplication = copyElement(systemWide, attribute: kAXFocusedApplicationAttribute)
var focusedWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)
@@ -221,8 +227,8 @@ enum SnapshotBuilder {
// the tree, the capture and every element frame refer to the same window;
// the title/area heuristics remain the fallback.
var windowCapture = TimingLog.measure("snapshot.window_capture") {
- WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: rootWindow))
- ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle)
+ WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: rootWindow), capture: observation.includeScreenshot)
+ ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle, capture: observation.includeScreenshot)
}
if windowCapture == nil,
recoveryPolicy == .allowActivation,
@@ -231,8 +237,8 @@ enum SnapshotBuilder {
if let recoveredWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) {
rootWindow = recoveredWindow
windowTitle = stringValue(of: recoveredWindow, attribute: kAXTitleAttribute)
- windowCapture = WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: recoveredWindow))
- ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle)
+ windowCapture = WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: recoveredWindow), capture: observation.includeScreenshot)
+ ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle, capture: observation.includeScreenshot)
}
}
@@ -250,7 +256,8 @@ enum SnapshotBuilder {
systemWide: systemWide,
textLimit: textLimit,
treeLimits: treeLimits,
- lazyWebAccessibility: lazyWebAccessibility
+ lazyWebAccessibility: lazyWebAccessibility,
+ observation: observation
)
}
@@ -264,11 +271,21 @@ enum SnapshotBuilder {
systemWide: AXUIElement,
textLimit: SnapshotTextLimit,
treeLimits: AccessibilityTreeLimits,
- lazyWebAccessibility: Bool
+ lazyWebAccessibility: Bool,
+ observation: SnapshotObservationOptions
) -> AppSnapshot {
let windowBounds = windowCapture.bounds
- let screenshot = windowCapture.screenshotResult()
+ let screenshot = observation.includeScreenshot ? windowCapture.screenshotResult()
+ : ScreenshotEncodingResult(data: nil, note: SnapshotObservationOptions.screenshotOmittedNote)
let screenshotData = screenshot.data
+ guard observation.includeAX else {
+ return AppSnapshot(app: app, windowTitle: windowTitle, windowBounds: windowBounds,
+ targetWindowID: windowCapture.windowID, targetWindowLayer: windowCapture.layer,
+ screenshotData: screenshotData, mode: .accessibility, treeLines: [],
+ focusedSummary: nil, focusedElement: nil, selectedText: nil,
+ windowElement: rootElement, elements: [:], screenshotNote: screenshot.note,
+ observation: observation)
+ }
let focusedElement = preferredFocusedElement(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)
let selectedText = focusedElement.flatMap { copySelectedText($0, textLimit: textLimit) }
let context = RenderContext(
@@ -330,7 +347,8 @@ enum SnapshotBuilder {
selectedText: selectedText,
windowElement: rootElement,
elements: renderer.records,
- screenshotNote: screenshot.note
+ screenshotNote: screenshot.note,
+ observation: observation
)
}
@@ -444,14 +462,15 @@ enum SnapshotBuilder {
return copyElement(appElement, attribute: kAXFocusedUIElementAttribute)
}
- private static func buildFixtureSnapshot(app: RunningAppDescriptor, state: FixtureAppState) -> AppSnapshot {
+ static func buildFixtureSnapshot(app: RunningAppDescriptor, state: FixtureAppState,
+ observation: SnapshotObservationOptions = .defaults) -> AppSnapshot {
var lines: [String] = []
var records: [Int: ElementRecord] = [:]
let focusedIdentifier = state.focusedIdentifier
var focusedSummary: String?
- for element in state.elements.sorted(by: { $0.index < $1.index }) {
+ for element in (observation.includeAX ? state.elements.sorted(by: { $0.index < $1.index }) : []) {
let titleSegment = element.title.map { " \($0)" } ?? ""
let valueSegment = element.value.map { " Value: \($0)" } ?? ""
let actionsSegment = element.actions.isEmpty ? "" : " Secondary Actions: \(element.actions.joined(separator: ", "))"
@@ -485,9 +504,11 @@ enum SnapshotBuilder {
treeLines: lines,
focusedSummary: focusedSummary,
focusedElement: nil,
- selectedText: state.selectedText,
+ selectedText: observation.includeAX ? state.selectedText : nil,
windowElement: nil,
- elements: records
+ elements: records,
+ screenshotNote: observation.includeScreenshot ? nil : SnapshotObservationOptions.screenshotOmittedNote,
+ observation: observation
)
}
}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
index 76485aa..a74e823 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
@@ -541,6 +541,7 @@ func shouldPreferContainingWebRowAXClickCandidate(
public final class ComputerUseService {
private var snapshotsByApp: [String: AppSnapshot] = [:]
+ private var invalidatedSnapshotAliases: Set<String> = []
// Read per call: the app agent applies the caller's OPEN_COMPUTER_USE_* variables
// for the duration of each request.
private var actionReadBack: Bool { actionReadBackEnabled(environment: ProcessInfo.processInfo.environment) }
@@ -593,14 +594,16 @@ public final class ComputerUseService {
app query: String,
textLimit: SnapshotTextLimit = .defaults,
treeLimits: AccessibilityTreeLimits = .defaults,
- windowPlacement: WindowPlacement = .keep
+ windowPlacement: WindowPlacement = .keep,
+ observation: SnapshotObservationOptions = .defaults
) throws -> ToolCallResult {
+ try observation.validate()
guard windowPlacement != .keep else {
- return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits), style: .fullState)
+ return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, observation: observation), style: .fullState)
}
// Explicit placements never activate or raise; they only move the frame.
- let snapshot = try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly)
+ let snapshot = try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly, observation: observation)
guard snapshot.mode != .fixture else {
throw ComputerUseError.message("window_placement '\(windowPlacement.rawValue)' is not supported for fixture apps")
}
@@ -616,7 +619,7 @@ public final class ComputerUseService {
break
}
- return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly), style: .fullState)
+ return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly, observation: observation), style: .fullState)
}
public func click(
@@ -641,6 +644,9 @@ public final class ComputerUseService {
let snapshot = try snapshotForAction(app: query, elementIndex: elementIndex)
let button = MouseButtonKind(rawValue: mouseButton.lowercased()) ?? .left
+ if elementIndex == nil, !snapshot.observation.includeScreenshot {
+ throw ComputerUseError.stateUnavailable(SnapshotObservationOptions.screenshotOmittedNote + "; request a screenshot before screenshot-coordinate actions")
+ }
if snapshot.mode == .fixture {
guard clickMethod == .auto else {
throw ComputerUseError.message(
@@ -878,6 +884,9 @@ public final class ComputerUseService {
public func drag(app query: String, fromX: Double, fromY: Double, toX: Double, toY: Double) throws -> ToolCallResult {
let snapshot = try currentSnapshot(for: query)
+ guard snapshot.observation.includeScreenshot else {
+ throw ComputerUseError.stateUnavailable(SnapshotObservationOptions.screenshotOmittedNote + "; request a screenshot before screenshot-coordinate actions")
+ }
if snapshot.mode == .fixture {
try FixtureBridge.post(FixtureCommand(kind: "drag", identifier: "fixture-drag-pad", x: fromX, y: fromY, toX: toX, toY: toY))
Thread.sleep(forTimeInterval: 0.15)
@@ -1185,7 +1194,7 @@ public final class ComputerUseService {
return dict
}
- private func currentSnapshot(for query: String) throws -> AppSnapshot {
+ func currentSnapshot(for query: String) throws -> AppSnapshot {
if let snapshot = snapshotsByApp[query.lowercased()] {
if snapshot.mode != .fixture {
do {
@@ -1198,6 +1207,9 @@ public final class ComputerUseService {
return snapshot
}
+ if invalidatedSnapshotAliases.contains(query.lowercased()) {
+ throw ComputerUseError.stateUnavailable(SnapshotObservationOptions.accessibilityOmittedNote)
+ }
return try refreshSnapshot(for: query)
}
@@ -1206,27 +1218,47 @@ public final class ComputerUseService {
for query: String,
textLimit: SnapshotTextLimit = .defaults,
treeLimits: AccessibilityTreeLimits = .defaults,
- recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation
+ recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation,
+ observation: SnapshotObservationOptions = .defaults
) throws -> AppSnapshot {
let app = try AppDiscovery.resolve(query)
let snapshot = try SnapshotBuilder.build(
for: app,
textLimit: textLimit,
treeLimits: treeLimits,
- recoveryPolicy: recoveryPolicy
+ recoveryPolicy: recoveryPolicy,
+ observation: observation
)
+ rememberSnapshot(snapshot, for: query)
+ return snapshot
+ }
+
+ func rememberSnapshot(_ snapshot: AppSnapshot, for query: String) {
+ let app = snapshot.app
+ if !snapshot.observation.includeAX {
+ var pids: Set<Int32> = [app.pid]
+ if let previous = snapshotsByApp[query.lowercased()] { pids.insert(previous.app.pid) }
+ let aliases = snapshotsByApp.compactMap { pids.contains($0.value.app.pid) ? $0.key : nil }
+ for alias in aliases {
+ snapshotsByApp.removeValue(forKey: alias)
+ invalidatedSnapshotAliases.insert(alias)
+ }
+ targetedElements.removeAll(forPIDs: pids)
+ }
+
let keys = Set([
query.lowercased(),
+ String(app.pid),
app.name.lowercased(),
(app.bundleIdentifier ?? "").lowercased(),
].filter { !$0.isEmpty })
for key in keys {
snapshotsByApp[key] = snapshot
+ invalidatedSnapshotAliases.remove(key)
}
- return snapshot
}
private func lookupElement(snapshot: AppSnapshot, index: String) throws -> ElementRecord {
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
index e104de9..cfa984c 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
@@ -67,7 +67,11 @@ public final class ComputerUseToolDispatcher {
maxNodeCount: try optionalPositiveInt("max_tree_nodes", in: arguments),
maxDepth: try optionalPositiveInt("max_tree_depth", in: arguments)
),
- windowPlacement: try parseWindowPlacement(optionalString("window_placement", in: arguments))
+ windowPlacement: try parseWindowPlacement(optionalString("window_placement", in: arguments)),
+ observation: SnapshotObservationOptions(
+ includeAX: try strictObservationBool("include_ax", in: arguments),
+ includeScreenshot: try strictObservationBool("include_screenshot", in: arguments)
+ )
)
case "click":
return try service.click(
@@ -154,6 +158,14 @@ public final class ComputerUseToolDispatcher {
return value
}
+ private func strictObservationBool(_ key: String, in arguments: [String: Any]) throws -> Bool {
+ guard let value = arguments[key] else { return true }
+ guard let number = value as? NSNumber, CFGetTypeID(number) == CFBooleanGetTypeID() else {
+ throw ComputerUseError.invalidArguments("\(key) must be a boolean")
+ }
+ return number.boolValue
+ }
+
private func optionalBool(_ key: String, in arguments: [String: Any]) -> Bool? {
switch arguments[key] {
case let value as Bool: return value
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
index 0eeb6fa..60dd85b 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
@@ -7,7 +7,7 @@ public enum OpenComputerUseCLICommand: Equatable {
case lockedUseStatus(json: Bool)
case lockedUseManagement(action: String, validation: Bool)
case listApps
- case snapshot(app: String, textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults)
+ case snapshot(app: String, textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults, observation: SnapshotObservationOptions = .defaults)
case call(OpenComputerUseCallInvocation)
case turnEnded(payload: String?)
case help(command: String?)
@@ -175,12 +175,14 @@ public func openComputerUseHelpText(command: String? = nil) -> String {
case "snapshot":
return """
Usage:
- open-computer-use snapshot [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>
+ open-computer-use snapshot [--include-ax true|false] [--include-screenshot true|false] [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>
Arguments:
<app> App name or bundle identifier to inspect.
Options:
+ --include-ax Include AX/focus/selection (true by default). False invalidates prior indexes.
+ --include-screenshot Include screenshot capture (true by default). At least one channel must be true.
--text-limit Override the default 500 character text limit. Use `max` for full text.
--max-tree-nodes Override the default 1200 node accessibility tree budget.
--max-tree-depth Override the default 64 level accessibility tree depth.
@@ -301,6 +303,7 @@ private func parseSnapshot(arguments: [String]) throws -> OpenComputerUseCLIComm
return .help(command: "snapshot")
}
+ var observation = SnapshotObservationOptions.defaults
var app: String?
var textLimit = SnapshotTextLimit.defaults
var maxTreeNodes: Int?
@@ -310,6 +313,13 @@ private func parseSnapshot(arguments: [String]) throws -> OpenComputerUseCLIComm
while index < arguments.count {
let argument = arguments[index]
switch argument {
+ case "--include-ax", "--include-screenshot":
+ index += 1
+ guard index < arguments.count, let value = Bool(arguments[index].lowercased()) else {
+ throw OpenComputerUseCLIError(message: "\(argument) requires true or false", helpCommand: "snapshot")
+ }
+ if argument == "--include-ax" { observation.includeAX = value }
+ else { observation.includeScreenshot = value }
case "--text-limit":
index += 1
guard index < arguments.count else {
@@ -348,13 +358,17 @@ private func parseSnapshot(arguments: [String]) throws -> OpenComputerUseCLIComm
throw OpenComputerUseCLIError(message: "snapshot requires an app name or bundle identifier", helpCommand: "snapshot")
}
+ guard observation.includeAX || observation.includeScreenshot else {
+ throw OpenComputerUseCLIError(message: "At least one of include_ax or include_screenshot must be true", helpCommand: "snapshot")
+ }
return .snapshot(
app: app,
textLimit: textLimit,
treeLimits: AccessibilityTreeLimits.defaults.replacing(
maxNodeCount: maxTreeNodes,
maxDepth: maxTreeDepth
- )
+ ),
+ observation: observation
)
}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SnapshotObservationOptions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SnapshotObservationOptions.swift
new file mode 100644
index 0000000..942d826
--- /dev/null
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SnapshotObservationOptions.swift
@@ -0,0 +1,21 @@
+import Foundation
+
+public struct SnapshotObservationOptions: Equatable, Sendable {
+ public var includeAX: Bool
+ public var includeScreenshot: Bool
+
+ public static let defaults = SnapshotObservationOptions()
+ public static let accessibilityOmittedNote = "Accessibility omitted; previous element indexes invalidated. Read AX before element-targeted actions."
+ public static let screenshotOmittedNote = "Screenshot omitted by include_screenshot=false"
+
+ public init(includeAX: Bool = true, includeScreenshot: Bool = true) {
+ self.includeAX = includeAX
+ self.includeScreenshot = includeScreenshot
+ }
+
+ public func validate() throws {
+ guard includeAX || includeScreenshot else {
+ throw ComputerUseError.invalidArguments("At least one of include_ax or include_screenshot must be true")
+ }
+ }
+}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swift
index b58381d..c8a6cff 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swift
@@ -60,6 +60,10 @@ final class TargetedElementRegistry<Value> {
}
func remove(_ index: Int) { entries.removeValue(forKey: index) }
+ func removeAll(forPIDs pids: Set<Int32>) {
+ entries = entries.filter { !pids.contains($0.value.app.pid) }
+ order.removeAll { entries[$0] == nil }
+ }
func clear() { entries.removeAll(); order.removeAll() }
}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
index e53da12..d6c0fd3 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
@@ -94,6 +94,8 @@ public enum ToolDefinitions {
annotations: defaultAnnotations(),
inputSchema: objectSchema(
properties: [
+ "include_ax": ["type": "boolean", "default": true, "description": "Include accessibility tree, focus and selection. False skips collection and invalidates previous element indexes; read AX again before element actions."],
+ "include_screenshot": ["type": "boolean", "default": true, "description": "Capture and return a screenshot. False skips capture and encoding; screenshot-coordinate actions require a fresh screenshot."],
"app": stringProperty(description: "App name or bundle identifier"),
"text_limit": textLimitProperty(description: "Maximum text characters to return. Use \"max\" for full text. Defaults to 500."),
"max_tree_nodes": positiveIntegerProperty(description: "Maximum accessibility tree nodes to render. Defaults to 1200."),
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift
index d1f4737..37276e6 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift
@@ -43,6 +43,13 @@ public struct ToolCallResult: @unchecked Sendable {
content.first(where: { $0.dictionary["type"] as? String == "text" })?.dictionary["text"] as? String
}
+ /// Plain-text CLI output includes observation notes but never image bytes.
+ public var textContent: String {
+ content.compactMap { item in
+ item.dictionary["type"] as? String == "text" ? item.dictionary["text"] as? String : nil
+ }.joined(separator: "\n")
+ }
+
public var asDictionary: [String: Any] {
[
"content": content.map(\.dictionary),
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/SelectiveObservationTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/SelectiveObservationTests.swift
new file mode 100644
index 0000000..e196be7
--- /dev/null
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/SelectiveObservationTests.swift
@@ -0,0 +1,114 @@
+import AppKit
+import XCTest
+@testable import OpenComputerUseKit
+
+final class SelectiveObservationTests: XCTestCase {
+ private func app(pid: Int32 = 42, name: String = "ObservationFixture") -> RunningAppDescriptor {
+ .init(name: name, bundleIdentifier: "test.\(name)", pid: pid, runningApplication: .current)
+ }
+
+ private func snapshot(pid: Int32 = 42, name: String = "ObservationFixture",
+ options: SnapshotObservationOptions = .defaults) -> AppSnapshot {
+ let state = FixtureAppState(windowTitle: "test", windowBounds: .init(rect: CGRect(x: 0, y: 0, width: 300, height: 200)),
+ focusedIdentifier: "input", elements: [
+ .init(identifier: "input", index: 7, role: "text", title: "Input", value: "中文🙂", actions: [],
+ frame: .init(rect: CGRect(x: 10, y: 10, width: 100, height: 30)))
+ ], selectedText: "中文🙂")
+ return SnapshotBuilder.buildFixtureSnapshot(app: app(pid: pid, name: name), state: state, observation: options)
+ }
+
+ func testEmptyObservationFailsBeforeResolvingApp() {
+ XCTAssertThrowsError(try ComputerUseService().getAppState(app: "missing-test-app",
+ observation: .init(includeAX: false, includeScreenshot: false))) {
+ XCTAssertTrue(String(describing: $0).contains("At least one"))
+ }
+ }
+
+ func testDispatcherRequiresActualJSONBooleans() {
+ let dispatcher = ComputerUseToolDispatcher()
+ for key in ["include_ax", "include_screenshot"] {
+ for value: Any in ["false", 0, 1, NSNull(), [], [:]] {
+ XCTAssertThrowsError(try dispatcher.callTool(name: "get_app_state", arguments: ["app": "missing-test-app", key: value])) {
+ XCTAssertTrue(String(describing: $0).contains("\(key) must be a boolean"))
+ }
+ }
+ }
+ }
+
+ func testSchemaHasMatchingFullDefaults() throws {
+ let tool = try XCTUnwrap(ToolDefinitions.all.first { $0.name == "get_app_state" })
+ let properties = try XCTUnwrap(tool.inputSchema["properties"] as? [String: Any])
+ for key in ["include_ax", "include_screenshot"] {
+ let value = try XCTUnwrap(properties[key] as? [String: Any])
+ XCTAssertEqual(value["type"] as? String, "boolean")
+ XCTAssertEqual(value["default"] as? Bool, true)
+ }
+ }
+
+ func testCLIObservationOptionsAndDefaultCompatibility() throws {
+ XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["snapshot", "TextEdit"]), .snapshot(app: "TextEdit"))
+ XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["snapshot", "--include-screenshot", "false", "TextEdit"]),
+ .snapshot(app: "TextEdit", observation: .init(includeScreenshot: false)))
+ XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["snapshot", "TextEdit", "--include-ax", "FALSE"]),
+ .snapshot(app: "TextEdit", observation: .init(includeAX: false)))
+ for args in [["--include-ax"], ["--include-screenshot", "0"], ["--include-ax", "no"],
+ ["--include-ax", "false", "--include-screenshot", "false"]] {
+ XCTAssertThrowsError(try parseOpenComputerUseCLI(arguments: ["snapshot", "TextEdit"] + args))
+ }
+ }
+
+ func testFixtureChannelsPreserveOrOmitAXAndCoordinateGuidance() {
+ let full = snapshot()
+ XCTAssertEqual(full.selectedText, "中文🙂")
+ XCTAssertEqual(full.elements.count, 1)
+ let image = snapshot(options: .init(includeAX: false))
+ XCTAssertTrue(image.elements.isEmpty)
+ XCTAssertTrue(image.treeLines.isEmpty)
+ XCTAssertNil(image.focusedSummary)
+ XCTAssertNil(image.selectedText)
+ XCTAssertTrue(image.renderedText.contains("indexes invalidated"))
+ let ax = snapshot(options: .init(includeScreenshot: false))
+ XCTAssertEqual(ax.elements.count, 1)
+ XCTAssertEqual(ax.selectedText, "中文🙂")
+ XCTAssertNil(ax.screenshotData)
+ XCTAssertEqual(ax.screenshotNote, SnapshotObservationOptions.screenshotOmittedNote)
+ }
+
+ func testScreenshotOnlyInvalidatesOlderAliasesAcrossPIDChangeAndAXRestoresIndexes() throws {
+ let service = ComputerUseService()
+ service.rememberSnapshot(snapshot(), for: "older-query")
+ service.rememberSnapshot(snapshot(), for: "ObservationFixture")
+ service.rememberSnapshot(snapshot(pid: 99, name: "Other"), for: "Other")
+ service.rememberSnapshot(snapshot(pid: 43, options: .init(includeAX: false)), for: "ObservationFixture")
+ XCTAssertThrowsError(try service.currentSnapshot(for: "older-query"))
+ XCTAssertThrowsError(try service.currentSnapshot(for: "42"))
+ XCTAssertTrue(try service.currentSnapshot(for: "ObservationFixture").elements.isEmpty)
+ XCTAssertEqual(try service.currentSnapshot(for: "Other").elements.count, 1)
+ service.rememberSnapshot(snapshot(pid: 43, options: .init(includeScreenshot: false)), for: "ObservationFixture")
+ XCTAssertEqual(try service.currentSnapshot(for: "ObservationFixture").elements.count, 1)
+ }
+
+ func testQueryIndexInvalidationIsScopedAndNeverReissuesIDs() throws {
+ let registry = TargetedElementRegistry<String>()
+ let first = TargetedAppIdentity(pid: 42, bundleIdentifier: "a", launchedAt: nil)
+ let other = TargetedAppIdentity(pid: 99, bundleIdentifier: "b", launchedAt: nil)
+ let a = registry.insert(app: first) { _ in "a" }
+ let b = registry.insert(app: other) { _ in "b" }
+ registry.removeAll(forPIDs: [42])
+ XCTAssertThrowsError(try registry.resolve(a, app: first))
+ XCTAssertEqual(try registry.resolve(b, app: other), "b")
+ XCTAssertGreaterThan(registry.insert(app: first) { _ in "new" }, b)
+ }
+
+ func testAXOnlyRejectsCoordinateInputBeforeAnyFixtureCommand() {
+ let service = ComputerUseService()
+ service.rememberSnapshot(snapshot(options: .init(includeScreenshot: false)), for: "fixture")
+ XCTAssertThrowsError(try service.click(app: "fixture", elementIndex: nil, x: 10, y: 10,
+ clickCount: 1, mouseButton: "left")) {
+ XCTAssertTrue(String(describing: $0).contains("include_screenshot=false"))
+ }
+ XCTAssertThrowsError(try service.drag(app: "fixture", fromX: 1, fromY: 1, toX: 2, toY: 2)) {
+ XCTAssertTrue(String(describing: $0).contains("include_screenshot=false"))
+ }
+ }
+}
diff --git a/scripts/node-repl/open-computer-use-repl.mjs b/scripts/node-repl/open-computer-use-repl.mjs
index 89b798c..5e5c780 100755
--- a/scripts/node-repl/open-computer-use-repl.mjs
+++ b/scripts/node-repl/open-computer-use-repl.mjs
@@ -41,6 +41,7 @@ The runtime exposes an asynchronous app-bound API:
- \`await app.getAXState({ emit?, textLimit?, maxTreeNodes?, maxTreeDepth?, windowPlacement? })\`
- \`await app.getScreenshot({ emit? })\`
- \`await app.getAXStateAndScreenshot(options?)\`
+- Optional observation on macOS, Windows and Linux: \`app.getScreenshot({ includeAX: false })\` skips AX/focus/selection and invalidates prior element indexes; refresh AX before element actions. \`app.getAXState({ includeScreenshot: false })\` skips capture; screenshot-coordinate actions require a fresh screenshot. Omitted options retain full native observation.
- \`await app.click(elementIndexOrPoint, { mouseButton?, clickCount?, clickMethod? })\`
- \`await app.scroll(elementIndex, direction, pages?)\`
- \`await app.drag([fromX, fromY], [toX, toY])\`
@@ -258,6 +259,15 @@ function optionsToSnapshotArgs(options = {}) {
if (options.maxTreeNodes !== undefined) args.max_tree_nodes = options.maxTreeNodes;
if (options.maxTreeDepth !== undefined) args.max_tree_depth = options.maxTreeDepth;
if (options.windowPlacement !== undefined) args.window_placement = options.windowPlacement;
+ for (const [option, argument] of [["includeAX", "include_ax"], ["includeScreenshot", "include_screenshot"]]) {
+ if (options[option] !== undefined) {
+ if (typeof options[option] !== "boolean") throw new Error(`${option} must be a boolean`);
+ args[argument] = options[option];
+ }
+ }
+ if (args.include_ax === false && args.include_screenshot === false) {
+ throw new Error("At least one of includeAX or includeScreenshot must be true");
+ }
return args;
}
diff --git a/scripts/node-repl/open-computer-use-repl.test.mjs b/scripts/node-repl/open-computer-use-repl.test.mjs
index f343bdc..5c7aaad 100644
--- a/scripts/node-repl/open-computer-use-repl.test.mjs
+++ b/scripts/node-repl/open-computer-use-repl.test.mjs
@@ -102,6 +102,42 @@ test("app-bound API composes actions and state in one js call", async () => {
assert.ok(result.content.some(item => item.type === "text" && item.text === "state:Text"));
});
+for (const Session of [PersistentJavaScriptSession, WorkerJavaScriptSession]) {
+ test(`${Session.name}: selective observation is explicit and legacy calls keep their arguments`, async t => {
+ const native = mockNative();
+ const session = new Session({ native });
+ t.after(() => session.close?.());
+ const result = await session.run(`
+ var selectiveApp = await cua.getApp("Text", { initialState: false });
+ await selectiveApp.getScreenshot({ includeAX: false, emit: false });
+ await selectiveApp.getAXState({ includeScreenshot: false, emit: false });
+ await selectiveApp.getScreenshot({ emit: false });
+ await selectiveApp.getAXState({ emit: false });
+ `);
+ assert.equal(result.isError, false);
+ assert.deepEqual(native.calls.map(call => call.arguments), [
+ { app: "Text", include_ax: false, text_limit: 1 },
+ { app: "Text", include_screenshot: false },
+ { app: "Text", text_limit: 1 },
+ { app: "Text" },
+ ]);
+ const count = native.calls.length;
+ for (const code of [
+ 'await selectiveApp.getScreenshot({ includeAX: "false" })',
+ 'await selectiveApp.getAXState({ includeScreenshot: null })',
+ ]) {
+ const invalid = await session.run(code);
+ assert.equal(invalid.isError, true);
+ assert.match(invalid.content.at(-1).text, /must be a boolean/);
+ }
+ assert.equal(native.calls.length, count);
+ const empty = await session.run('await selectiveApp.getAXStateAndScreenshot({ includeAX: false, includeScreenshot: false })');
+ assert.equal(empty.isError, true);
+ assert.match(empty.content.at(-1).text, /At least one/);
+ assert.equal(native.calls.length, count);
+ });
+}
+
test("app-bound API forwards background-operation options", async () => {
const native = mockNative();
const session = new PersistentJavaScriptSession({ native });
diff --git a/scripts/run-linux-read-back-smoke.sh b/scripts/run-linux-read-back-smoke.sh
index 8a9507f..608c637 100755
--- a/scripts/run-linux-read-back-smoke.sh
+++ b/scripts/run-linux-read-back-smoke.sh
@@ -3,7 +3,7 @@ set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
if [[ "$(uname -s)" != Linux || $# -lt 1 ]]; then
- echo "Usage (Linux): bash scripts/run-linux-read-back-smoke.sh /path/to/open-computer-use [--focus-checks | --hidden-checks]" >&2
+ echo "Usage (Linux): bash scripts/run-linux-read-back-smoke.sh /path/to/open-computer-use [--focus-checks | --hidden-checks | --observation-checks]" >&2
exit 2
fi
diff --git a/scripts/tests/fixtures/MacOSObservationFixture.swift b/scripts/tests/fixtures/MacOSObservationFixture.swift
new file mode 100644
index 0000000..d5b5d2c
--- /dev/null
+++ b/scripts/tests/fixtures/MacOSObservationFixture.swift
@@ -0,0 +1,25 @@
+import AppKit
+
+final class ObservationWindow: NSWindow {
+ override var canBecomeKey: Bool { false }
+ override var canBecomeMain: Bool { false }
+}
+
+let app = NSApplication.shared
+app.setActivationPolicy(.accessory)
+let window = ObservationWindow(contentRect: NSRect(x: 200, y: 200, width: 420, height: 180),
+ styleMask: [.titled, .closable], backing: .buffered, defer: false)
+window.title = "OCU Observation Fixture"
+window.isReleasedWhenClosed = false
+let input = NSTextField(frame: NSRect(x: 20, y: 75, width: 380, height: 30))
+input.stringValue = "initial-中文🙂"
+input.setAccessibilityLabel("Read-back input")
+window.contentView?.addSubview(input)
+window.orderFront(nil)
+let statePath = CommandLine.arguments[1]
+let timer = Timer.scheduledTimer(withTimeInterval: 0.03, repeats: true) { _ in
+ let data = try! JSONSerialization.data(withJSONObject: ["text": input.stringValue])
+ try? data.write(to: URL(fileURLWithPath: statePath), options: .atomic)
+}
+Timer.scheduledTimer(withTimeInterval: 120, repeats: false) { _ in app.terminate(nil) }
+app.run()
diff --git a/scripts/tests/fixtures/WindowsObservationFixture.ps1 b/scripts/tests/fixtures/WindowsObservationFixture.ps1
new file mode 100644
index 0000000..9d30f81
--- /dev/null
+++ b/scripts/tests/fixtures/WindowsObservationFixture.ps1
@@ -0,0 +1,105 @@
+param([Parameter(Mandatory = $true)][string]$StatePath)
+$ErrorActionPreference = "Stop"
+Add-Type -AssemblyName PresentationFramework
+Add-Type -TypeDefinition @"
+using System;
+using System.Runtime.InteropServices;
+public static class ObservationDwm {
+ [StructLayout(LayoutKind.Sequential)]
+ public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
+ [DllImport("user32.dll")]
+ public static extern bool GetWindowRect(IntPtr hwnd, out RECT rect);
+ [DllImport("user32.dll", CharSet = CharSet.Unicode)]
+ private static extern IntPtr CreateWindowEx(uint extendedStyle, string windowClass, string title, uint style,
+ int x, int y, int width, int height, IntPtr parent, IntPtr menu, IntPtr instance, IntPtr parameter);
+ [DllImport("user32.dll")]
+ public static extern bool DestroyWindow(IntPtr hwnd);
+ public static IntPtr CreateTinyWindow() {
+ return CreateWindowEx(0, "STATIC", "OCU Windows Observation Fixture", 0x90000000u,
+ 80, 80, 1, 1, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero);
+ }
+ [DllImport("dwmapi.dll")]
+ public static extern int DwmSetWindowAttribute(IntPtr hwnd, int attr, ref int value, int size);
+ [DllImport("dwmapi.dll")]
+ public static extern int DwmGetWindowAttribute(IntPtr hwnd, int attr, out int value, int size);
+}
+"@
+$window = New-Object System.Windows.Window
+$window.Title = "OCU Windows Observation Fixture"
+$window.ShowInTaskbar = $true
+$window.Left = 80
+$window.Top = 80
+$window.Width = 600
+$window.Height = 280
+$inputBox = New-Object System.Windows.Controls.TextBox
+[System.Windows.Automation.AutomationProperties]::SetName($inputBox, "Observation input")
+$inputBox.Margin = New-Object System.Windows.Thickness(20)
+$inputBox.Height = 30
+$inputBox.VerticalAlignment = "Center"
+$inputBox.Text = "initial-value"
+$window.Content = $inputBox
+$state = @{ sequence = 0; text = ""; width = 0; height = 0; minimized = $false; cloakHResult = 0; cloaked = 0 }
+$stateEncoding = New-Object System.Text.UTF8Encoding($false)
+$tinyWindow = @{ handle = [IntPtr]::Zero }
+$commandPath = $StatePath + ".command"
+$timer = New-Object System.Windows.Threading.DispatcherTimer
+$timer.Interval = [TimeSpan]::FromMilliseconds(50)
+$timer.Add_Tick({
+ $hwnd = (New-Object System.Windows.Interop.WindowInteropHelper($window)).Handle
+ if (Test-Path -LiteralPath $commandPath) {
+ try {
+ $command = [System.IO.File]::ReadAllText($commandPath) | ConvertFrom-Json
+ if ($command.sequence -gt $state.sequence) {
+ switch ($command.kind) {
+ "tiny" {
+ # WPF rounds a one-DIP window to multiple physical
+ # pixels on high-DPI desktops. Use an owned Win32
+ # fixture window for the exact physical 1x1 case.
+ $window.Hide()
+ $tinyWindow.handle = [ObservationDwm]::CreateTinyWindow()
+ }
+ "minimize" { $window.WindowState = "Minimized" }
+ "cloak" {
+ $value = 1
+ $state.cloakHResult = [ObservationDwm]::DwmSetWindowAttribute($hwnd, 13, [ref]$value, 4)
+ }
+ "restore" {
+ $value = 0
+ [void][ObservationDwm]::DwmSetWindowAttribute($hwnd, 13, [ref]$value, 4)
+ if ($tinyWindow.handle -ne [IntPtr]::Zero) {
+ [void][ObservationDwm]::DestroyWindow($tinyWindow.handle)
+ $tinyWindow.handle = [IntPtr]::Zero
+ }
+ $window.WindowState = "Normal"
+ $window.Width = 600
+ $window.Height = 280
+ $window.Show()
+ }
+ }
+ $state.sequence = $command.sequence
+ }
+ } catch { }
+ }
+ $hwnd = if ($tinyWindow.handle -ne [IntPtr]::Zero) { $tinyWindow.handle }
+ else { (New-Object System.Windows.Interop.WindowInteropHelper($window)).Handle }
+ $cloaked = 0
+ [void][ObservationDwm]::DwmGetWindowAttribute($hwnd, 14, [ref]$cloaked, 4)
+ $state.cloaked = $cloaked
+ $state.text = $inputBox.Text
+ $rect = New-Object ObservationDwm+RECT
+ [void][ObservationDwm]::GetWindowRect($hwnd, [ref]$rect)
+ $state.width = $rect.Right - $rect.Left
+ $state.height = $rect.Bottom - $rect.Top
+ $state.minimized = $window.WindowState -eq "Minimized"
+ $temporary = $StatePath + ".tmp"
+ [System.IO.File]::WriteAllText($temporary, ($state | ConvertTo-Json -Compress), $stateEncoding)
+ try {
+ if (Test-Path -LiteralPath $StatePath) { [System.IO.File]::Replace($temporary, $StatePath, ($StatePath + ".bak")) }
+ else { [System.IO.File]::Move($temporary, $StatePath) }
+ } catch [System.IO.IOException] { }
+})
+$timer.Start()
+$window.Add_Closed({ $timer.Stop() })
+$application = New-Object System.Windows.Application
+$application.ShutdownMode = "OnMainWindowClose"
+[void]$application.Run($window)
diff --git a/scripts/tests/linux-observation-js-smoke.mjs b/scripts/tests/linux-observation-js-smoke.mjs
new file mode 100644
index 0000000..d106c33
--- /dev/null
+++ b/scripts/tests/linux-observation-js-smoke.mjs
@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+// Runs inside the private GTK smoke session; Python owns fixture lifecycle.
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { JsonLinePeer, WorkerJavaScriptSession } from "../node-repl/open-computer-use-repl.mjs";
+
+const [nativePath, appName, stateFile] = process.argv.slice(2);
+assert.ok(nativePath && appName && stateFile, "Run through --observation-checks");
+const native = new JsonLinePeer({ command: nativePath, args: ["mcp"],
+ env: { ...process.env, OPEN_COMPUTER_USE_ACTION_READ_BACK: "0" } });
+let session;
+async function run(code, expectedError) {
+ const result = await session.run(code);
+ const text = result.content.filter(item => item.type === "text").map(item => item.text).join("\n");
+ if (expectedError) {
+ assert.equal(result.isError, true, text);
+ assert.match(text, expectedError);
+ } else assert.equal(result.isError, false, text);
+ return result;
+}
+try {
+ await native.initialize();
+ const request = native.request.bind(native);
+ native.request = async (method, params, timeout) => {
+ const result = await request(method, params, timeout);
+ if (method === "tools/call" && params.name === "get_app_state" && !result.isError) {
+ const images = result.content.filter(item => item.type === "image");
+ assert.equal(images.length, params.arguments.include_screenshot === false ? 0 : 1);
+ }
+ return result;
+ };
+ session = new WorkerJavaScriptSession({ native });
+ const ax = await run(`var app = await cua.getApp(${JSON.stringify(appName)}, {initialState:false});
+ var ax = await app.getAXState({includeScreenshot:false});
+ var oldIndex = ax.match(/^\\s*(\\d+)\\s+[^\\n]*Read-back input(?:\\s|$)/m)[1];`);
+ assert.equal(ax.content.some(item => item.type === "image"), false);
+ const shot = await run(`var screenshot = await app.getScreenshot({includeAX:false});
+ if(screenshot.subarray(0,8).toString('hex') !== '89504e470d0a1a0a') throw Error('Invalid PNG');`);
+ assert.equal(shot.content.filter(item => item.type === "image").length, 1);
+ await run(`await app.setValue(oldIndex, "must-not-write");`, /unknown element_index/);
+ await run(`var fresh = await app.getAXState({includeScreenshot:false,emit:false});
+ var index = fresh.match(/^\\s*(\\d+)\\s+[^\\n]*Read-back input(?:\\s|$)/m)[1];
+ await app.setValue(index, "worker-live-中文🙂");`);
+ const deadline = Date.now() + 5000;
+ while (JSON.parse(readFileSync(stateFile, "utf8")).text !== "worker-live-中文🙂") {
+ assert.ok(Date.now() < deadline, "GTK value did not update");
+ await new Promise(resolve => setTimeout(resolve, 30));
+ }
+ const full = await run(`await app.getAXStateAndScreenshot();`);
+ assert.equal(full.content.filter(item => item.type === "image").length, 1);
+ assert.ok(full.content.some(item => item.type === "text" && item.text.includes("worker-live-中文🙂")));
+ console.log("Worker JS → native MCP → GTK: selective collection, PNG output, stale-index error and AX recovery passed.");
+} finally {
+ await session?.close();
+ native.close();
+}
diff --git a/scripts/tests/linux-read-back-smoke.py b/scripts/tests/linux-read-back-smoke.py
index cebad77..b6181e8 100644
--- a/scripts/tests/linux-read-back-smoke.py
+++ b/scripts/tests/linux-read-back-smoke.py
@@ -1,7 +1,8 @@
#!/usr/bin/env python3
-"""Exercise native Linux MCP actions against an isolated real GTK fixture."""
+"""Exercise native Linux MCP observation and actions against an isolated GTK fixture."""
import argparse
+import base64
from contextlib import ExitStack
import importlib.util
import json
@@ -10,6 +11,7 @@ from pathlib import Path
import re
import select
import subprocess
+import struct
import sys
import tempfile
import time
@@ -285,14 +287,98 @@ def check_hidden(client, native, name, state_file, stderr):
print("Native MCP: default/opt-in output, stable indexes, page changes and full detail restoration passed.")
-def smoke(native, focus_checks=False, hidden_checks=False):
+def check_observation(client, native, name, state_file, stderr):
+ wait_for(lambda: json.loads(state_file.read_text())["selected"] == "中文🙂", "fixture selection")
+ runtime_path = Path(__file__).resolve().parents[2] / "apps/OpenComputerUseLinux/runtime.py"
+ spec = importlib.util.spec_from_file_location("observation_smoke_runtime", runtime_path)
+ runtime = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(runtime)
+
+ def png(data):
+ raw = base64.b64decode(data, validate=True)
+ assert raw[:8] == b"\x89PNG\r\n\x1a\n" and raw[12:16] == b"IHDR", raw[:24]
+ width, height = struct.unpack(">II", raw[16:24])
+ assert width > 0 and height > 0, (width, height)
+
+ def snapshot(**flags):
+ with ExitStack() as stack:
+ calls = {key: stack.enter_context(mock.patch.object(runtime, key, wraps=getattr(runtime, key)))
+ for key in ("render_tree", "focus_details", "capture_window_png")}
+ result = runtime.build_snapshot(name, **flags)
+ return result, {key: query.call_count for key, query in calls.items()}
+
+ full, counts = snapshot()
+ assert counts == {"render_tree": 1, "focus_details": 1, "capture_window_png": 1}, counts
+ png(full["screenshotPngBase64"])
+ assert full["selectedText"] == "中文🙂", full
+ ax, counts = snapshot(include_screenshot=False)
+ assert counts == {"render_tree": 1, "focus_details": 1, "capture_window_png": 0}, counts
+ assert ax["screenshotPngBase64"] is None and ax["elements"] == full["elements"], ax
+ assert ax["selectedText"] == "中文🙂", ax
+ image, counts = snapshot(include_ax=False)
+ assert counts == {"render_tree": 0, "focus_details": 0, "capture_window_png": 1}, counts
+ png(image["screenshotPngBase64"])
+ assert image["elements"] == [] and image["treeLines"] == [], image
+ assert image["focusedSummary"] is None and image["selectedText"] is None, image
+ assert image["windowBounds"] == full["windowBounds"], image
+ print("Real GTK collection: full, AX-only (no capture), image-only (no tree/focus) passed.")
+
+ ax = client.call("get_app_state", name, include_screenshot=False)
+ expect_snapshot(ax)
+ assert all(item["type"] != "image" for item in ax["content"]), ax
+ assert "Selected text: [中文🙂]" in text(ax), text(ax)
+ refused = client.call("click", name, x=10, y=10)
+ assert refused.get("isError") and "include_screenshot=false" in text(refused), refused
+ # The standalone snapshot and JSON call entry points must preserve flags.
+ cli = subprocess.run([native, "snapshot", "--include-screenshot", "false", name],
+ check=True, capture_output=True, text=True, timeout=40)
+ assert "Read-back input" in cli.stdout and "include_screenshot=false" in cli.stdout, cli.stdout
+ cli = subprocess.run([native, "call", "get_app_state", "--args",
+ json.dumps({"app": name, "include_screenshot": False})],
+ check=True, capture_output=True, text=True, timeout=40)
+ assert all(item["type"] != "image" for item in json.loads(cli.stdout)["content"]), cli.stdout
+ old_index = element_index(ax, "Read-back input")
+ invalid = client.call("get_app_state", name, include_ax=False, include_screenshot=False)
+ assert invalid.get("isError"), invalid
+ image = client.call("get_app_state", name, include_ax=False)
+ assert not image.get("isError"), image
+ images = [item for item in image["content"] if item["type"] == "image"]
+ assert len(images) == 1, image
+ png(images[0]["data"])
+ assert "indexes invalidated" in text(image), text(image)
+ for omitted in ("Read-back input", "The focused UI element", "Selected text:"):
+ assert omitted not in text(image), text(image)
+ for alias in (name, str(full["app"]["pid"])):
+ result = client.call("set_value", alias, element_index=old_index, value="must-not-write")
+ assert result.get("isError"), result
+ assert json.loads(state_file.read_text())["text"] == "prefix-中文🙂-suffix"
+
+ button = next(row for row in full["elements"] if row["name"] == "Increment")
+ bounds = button["frame"]
+ result = client.call("click", name, x=bounds["x"] + bounds["width"] / 2,
+ y=bounds["y"] + bounds["height"] / 2)
+ assert not result.get("isError"), result
+ wait_for(lambda: json.loads(state_file.read_text())["count"] == 1, "coordinate click after image-only read")
+ refreshed = client.call("get_app_state", name, include_screenshot=False)
+ expect_snapshot(refreshed)
+ result = client.call("set_value", name, element_index=element_index(refreshed, "Read-back input"),
+ value="selective-中文🙂")
+ assert not result.get("isError"), result
+ wait_for(lambda: json.loads(state_file.read_text())["text"] == "selective-中文🙂", "AX index recovery")
+ print("Native MCP: valid PNG, omitted channels, stale name/PID indexes refused, coordinate action and AX recovery passed.")
+
+ subprocess.run(["node", str(Path(__file__).with_name("linux-observation-js-smoke.mjs")),
+ native, name, str(state_file)], check=True, timeout=30)
+
+
+def smoke(native, focus_checks=False, hidden_checks=False, observation_checks=False):
with tempfile.TemporaryDirectory(prefix="ocu-read-back-smoke-") as directory:
directory = Path(directory)
state_file = directory / "state.json"
name = "OCUReadBackFixture" + str(os.getpid())
with (directory / "stderr.log").open("w+") as stderr:
arguments = [sys.executable, __file__, "--fixture", name, "--state-file", str(state_file)]
- if focus_checks:
+ if focus_checks or observation_checks:
arguments.append("--focus-checks")
if hidden_checks:
arguments.append("--hidden-checks")
@@ -308,6 +394,9 @@ def smoke(native, focus_checks=False, hidden_checks=False):
if focus_checks:
check_focus(client, name, state_file)
return
+ if observation_checks:
+ check_observation(client, native, name, state_file, stderr)
+ return
if hidden_checks:
check_hidden(client, native, name, state_file, stderr)
observed = client.call("get_app_state", name)
@@ -381,6 +470,7 @@ if __name__ == "__main__":
checks = parser.add_mutually_exclusive_group()
checks.add_argument("--focus-checks", action="store_true")
checks.add_argument("--hidden-checks", action="store_true")
+ checks.add_argument("--observation-checks", action="store_true")
parser.add_argument("--state-file", type=Path)
args = parser.parse_args()
if args.fixture:
@@ -388,4 +478,4 @@ if __name__ == "__main__":
else:
if not args.native:
parser.error("--native is required")
- smoke(str(Path(args.native).resolve()), args.focus_checks, args.hidden_checks)
+ smoke(str(Path(args.native).resolve()), args.focus_checks, args.hidden_checks, args.observation_checks)
diff --git a/scripts/tests/macos-observation-smoke.mjs b/scripts/tests/macos-observation-smoke.mjs
new file mode 100644
index 0000000..73df8fe
--- /dev/null
+++ b/scripts/tests/macos-observation-smoke.mjs
@@ -0,0 +1,111 @@
+#!/usr/bin/env node
+import assert from "node:assert/strict";
+import { spawn, spawnSync } from "node:child_process";
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { JsonLinePeer, WorkerJavaScriptSession } from "../node-repl/open-computer-use-repl.mjs";
+
+assert.equal(process.platform, "darwin");
+assert.ok(process.argv[2], "Pass the normally signed OCU app executable with existing permissions/admission");
+const nativePath = path.resolve(process.argv[2]);
+const directory = mkdtempSync(path.join(tmpdir(), "ocu-observation-"));
+const fixturePath = path.join(directory, "OCUObservationFixture");
+const stateFile = path.join(directory, "state.json");
+const appName = "OCUObservationFixture";
+const env = { ...process.env, OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY: "1",
+ OPEN_COMPUTER_USE_VISUAL_CURSOR: "0", OPEN_COMPUTER_USE_ACTION_READ_BACK: "0",
+ OPEN_COMPUTER_USE_DEBUG_TIMING: "1", OPEN_COMPUTER_USE_CONFIG_FILE: path.join(directory, "config.json") };
+for (const key of Object.keys(env)) if (key.startsWith("OPEN_COMPUTER_USE_IMAGE_")) delete env[key];
+writeFileSync(env.OPEN_COMPUTER_USE_CONFIG_FILE, "{}");
+let fixture, native, session;
+const text = result => result.content.filter(item => item.type === "text").map(item => item.text).join("\n");
+const images = result => result.content.filter(item => item.type === "image");
+function indexOf(state) {
+ const match = state.match(/^\s*(\d+)\s+[^\n]*Read-back input(?:[\s,]|$)/m);
+ assert.ok(match, state);
+ return match[1];
+}
+async function waitFor(test) {
+ const deadline = Date.now() + 8000;
+ while (!test()) { assert.ok(Date.now() < deadline, "Fixture timeout"); await new Promise(r => setTimeout(r, 30)); }
+}
+async function call(name, args, expectedError) {
+ const result = await native.request("tools/call", { name, arguments: { app: appName, ...args } });
+ assert.equal(!!result.isError, !!expectedError, text(result));
+ if (expectedError) assert.match(text(result), expectedError);
+ return result;
+}
+async function observe(args) {
+ native.stderr = "";
+ const result = await call("get_app_state", args);
+ await new Promise(r => setTimeout(r, 50));
+ if (args.include_screenshot === false) {
+ assert.equal(images(result).length, 0);
+ assert.doesNotMatch(native.stderr, /timing snapshot\.capture_(?:hw|sck) /);
+ assert.match(native.stderr, /timing snapshot\.tree_walk /);
+ } else {
+ assert.equal(images(result).length, 1, text(result));
+ assert.equal(Buffer.from(images(result)[0].data, "base64").subarray(0, 8).toString("hex"), "89504e470d0a1a0a");
+ assert.match(native.stderr, /timing snapshot\.capture_(?:hw|sck) /);
+ }
+ if (args.include_ax === false) {
+ assert.doesNotMatch(native.stderr, /timing snapshot\.tree_walk /);
+ assert.ok(!text(result).includes("Read-back input"));
+ assert.match(text(result), /indexes invalidated/);
+ }
+ return result;
+}
+try {
+ const source = fileURLToPath(new URL("./fixtures/MacOSObservationFixture.swift", import.meta.url));
+ const build = spawnSync("swiftc", [source, "-o", fixturePath], { encoding: "utf8" });
+ assert.equal(build.status, 0, build.stderr);
+ fixture = spawn(fixturePath, [stateFile], { stdio: ["ignore", "ignore", "ignore"] });
+ await waitFor(() => existsSync(stateFile));
+ native = new JsonLinePeer({ command: nativePath, args: ["mcp"], env });
+ await native.initialize();
+ const full = await observe({});
+ const oldIndex = indexOf(text(full));
+ const queried = JSON.parse(text(await call("query", { role: "AXTextField" })));
+ assert.equal(queried.matches.length, 1);
+ await observe({ include_ax: false });
+ await call("set_value", { element_index: oldIndex, value: "bad" }, /unknown element_index/);
+ await call("set_value", { element_index: queried.matches[0].index, value: "bad" }, /expired|no longer available/);
+ const ax = await observe({ include_screenshot: false });
+ await call("click", { x: 10, y: 10 }, /include_screenshot=false/);
+ await call("set_value", { element_index: indexOf(text(ax)), value: "native-live-中文🙂" });
+ await waitFor(() => JSON.parse(readFileSync(stateFile, "utf8")).text === "native-live-中文🙂");
+ for (const command of [["snapshot", "--include-screenshot", "false", appName],
+ ["call", "get_app_state", "--args", JSON.stringify({ app: appName, include_screenshot: false })]]) {
+ const cli = spawnSync(nativePath, command, { env, encoding: "utf8", timeout: 15000 });
+ assert.equal(cli.status, 0, cli.stderr);
+ assert.match(cli.stdout, /include_screenshot=false/);
+ if (command[0] === "call") assert.equal(images(JSON.parse(cli.stdout)).length, 0);
+ }
+ session = new WorkerJavaScriptSession({ native });
+ let result = await session.run(`var app = await cua.getApp(${JSON.stringify(appName)}, {initialState:false});
+ var state = await app.getAXState({includeScreenshot:false});`);
+ assert.equal(result.isError, false, text(result));
+ assert.equal(images(result).length, 0);
+ result = await session.run("await app.getScreenshot({includeAX:false});");
+ assert.equal(result.isError, false, text(result));
+ assert.equal(images(result).length, 1);
+ result = await session.run(`var fresh = await app.getAXState({includeScreenshot:false,emit:false});
+ var index = fresh.match(/^\\s*(\\d+)\\s+[^\\n]*Read-back input(?:[\\s,]|$)/m)[1];
+ await app.setValue(index, "worker-live-中文🙂");`);
+ assert.equal(result.isError, false, text(result));
+ await waitFor(() => JSON.parse(readFileSync(stateFile, "utf8")).text === "worker-live-中文🙂");
+ console.log("macOS real AX/native MCP/CLI/Worker JS: collection gates, PNG, snapshot/query invalidation, coordinate refusal and element recovery passed.");
+} finally {
+ await session?.close();
+ native?.close();
+ if (fixture) {
+ fixture.kill("SIGTERM");
+ if (fixture.exitCode === null && fixture.signalCode === null) await new Promise(resolve => {
+ const timer = setTimeout(() => { fixture.kill("SIGKILL"); resolve(); }, 3000);
+ fixture.once("exit", () => { clearTimeout(timer); resolve(); });
+ });
+ }
+ rmSync(directory, { recursive: true, force: true });
+}
diff --git a/scripts/tests/windows-observation-js-smoke.mjs b/scripts/tests/windows-observation-js-smoke.mjs
new file mode 100644
index 0000000..38bb8d7
--- /dev/null
+++ b/scripts/tests/windows-observation-js-smoke.mjs
@@ -0,0 +1,46 @@
+#!/usr/bin/env node
+// The PowerShell desktop smoke owns this fixture and its lifecycle.
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { JsonLinePeer, WorkerJavaScriptSession } from "../node-repl/open-computer-use-repl.mjs";
+
+const [nativePath, app, stateFile] = process.argv.slice(2);
+const native = new JsonLinePeer({ command: nativePath, args: ["mcp"], env: process.env });
+let session;
+async function run(code, expectedError) {
+ const result = await session.run(code, 20000);
+ const text = result.content.filter(item => item.type === "text").map(item => item.text).join("\n");
+ assert.equal(!!result.isError, !!expectedError, text);
+ if (expectedError) assert.match(text, expectedError);
+ return result;
+}
+try {
+ await native.initialize();
+ session = new WorkerJavaScriptSession({ native });
+ const ax = await run(`var app = await cua.getApp(${JSON.stringify(app)}, {initialState:false});
+ var state = await app.getAXState({includeScreenshot:false});
+ function inputIndex(state) {
+ var line = state.split("\\n").find(line => /^\\s*\\d+\\s+edit Observation input(?:\\s|$)/.test(line));
+ if(!line) throw Error("Fixture input not found in AX");
+ return line.match(/^\\s*(\\d+)/)[1];
+ }
+ var oldIndex = inputIndex(state);`);
+ assert.equal(ax.content.some(item => item.type === "image"), false);
+ const screenshot = await run("await app.getScreenshot({includeAX:false});");
+ assert.equal(screenshot.content.filter(item => item.type === "image").length, 1);
+ await run('await app.setValue(oldIndex, "must-not-write");', /unknown element_index/);
+ await run(`var state = await app.getAXState({includeScreenshot:false,emit:false});
+ await app.setValue(inputIndex(state), "worker-live-中文");`);
+ const deadline = Date.now() + 8000;
+ while (JSON.parse(readFileSync(stateFile, "utf8")).text !== "worker-live-中文") {
+ assert.ok(Date.now() < deadline, "Real Windows text did not update");
+ await new Promise(resolve => setTimeout(resolve, 50));
+ }
+ const full = await run("await app.getAXStateAndScreenshot();");
+ assert.equal(full.content.filter(item => item.type === "image").length, 1);
+ assert.ok(full.content.some(item => item.type === "text" && item.text.includes("worker-live-中文")));
+ console.log("Windows Worker JS → native MCP → WPF: channels, stale-index errors, Unicode update and default full observation passed.");
+} finally {
+ await session?.close();
+ native.close();
+}
diff --git a/scripts/tests/windows-observation-smoke.ps1 b/scripts/tests/windows-observation-smoke.ps1
new file mode 100644
index 0000000..7568382
--- /dev/null
+++ b/scripts/tests/windows-observation-smoke.ps1
@@ -0,0 +1,117 @@
+param([Parameter(Mandatory = $true)][string]$NativePath)
+$ErrorActionPreference = "Stop"
+if ($env:OS -ne "Windows_NT") { throw "Windows desktop required" }
+$NativePath = (Resolve-Path -LiteralPath $NativePath).Path
+$directory = Join-Path ([System.IO.Path]::GetTempPath()) ("ocu-observation-" + [Guid]::NewGuid())
+[void](New-Item -ItemType Directory -Path $directory)
+$stateFile = Join-Path $directory "state.json"
+$fixture = $null
+$native = $null
+$requestId = 0
+$sequence = 0
+
+function Assert($condition, $message) { if (-not $condition) { throw $message } }
+function Wait-For($test, $description) {
+ $deadline = [DateTime]::UtcNow.AddSeconds(15)
+ do {
+ try { if (& $test) { return } } catch { }
+ Start-Sleep -Milliseconds 50
+ } while ([DateTime]::UtcNow -lt $deadline)
+ throw "Timeout: $description"
+}
+function Read-State { [System.IO.File]::ReadAllText($stateFile) | ConvertFrom-Json }
+function Send-Request($method, $parameters) {
+ $script:requestId += 1
+ $message = @{ jsonrpc = "2.0"; id = $script:requestId; method = $method; params = $parameters }
+ $native.StandardInput.WriteLine(($message | ConvertTo-Json -Depth 20 -Compress))
+ $native.StandardInput.Flush()
+ $read = $native.StandardOutput.ReadLineAsync()
+ Assert ($read.Wait(40000)) "MCP timed out"
+ Assert (-not [string]::IsNullOrWhiteSpace($read.Result)) "MCP exited"
+ $response = $read.Result | ConvertFrom-Json
+ Assert ($response.id -eq $script:requestId -and $null -eq $response.error) "Unexpected RPC response"
+ return $response.result
+}
+function Call-Tool($tool, $arguments) {
+ if (-not $arguments.ContainsKey("app")) { $arguments.app = [string]$fixture.Id }
+ Send-Request "tools/call" @{ name = $tool; arguments = $arguments }
+}
+function Text($result) { ($result.content | Where-Object type -eq "text" | ForEach-Object text) -join "`n" }
+function Images($result) { @($result.content | Where-Object type -eq "image") }
+function Input-Index($result) {
+ $matches = [regex]::Matches((Text $result), '(?m)^\s*(\d+)\s+edit Observation input(?:\s|$)')
+ Assert ($matches.Count -eq 1) (Text $result)
+ $matches[0].Groups[1].Value
+}
+function Change-Window($kind) {
+ $script:sequence += 1
+ $message = @{ sequence = $script:sequence; kind = $kind } | ConvertTo-Json -Compress
+ [System.IO.File]::WriteAllText(($stateFile + ".command"), $message, [System.Text.Encoding]::UTF8)
+ Wait-For { (Read-State).sequence -eq $script:sequence } "fixture window command"
+ Start-Sleep -Milliseconds 200
+}
+try {
+ $fixtureSource = Join-Path $PSScriptRoot "fixtures/WindowsObservationFixture.ps1"
+ $fixture = Start-Process powershell.exe -ArgumentList @("-NoProfile", "-STA", "-File", "`"$fixtureSource`"", "-StatePath", "`"$stateFile`"") -PassThru -RedirectStandardOutput (Join-Path $directory "fixture.out") -RedirectStandardError (Join-Path $directory "fixture.err")
+ Wait-For { Test-Path -LiteralPath $stateFile } "WPF startup"
+ Wait-For { (Get-Process -Id $fixture.Id).MainWindowHandle -ne [IntPtr]::Zero } "visible WPF main window"
+ $start = New-Object System.Diagnostics.ProcessStartInfo
+ $start.FileName = $NativePath
+ $start.Arguments = "mcp"
+ $start.UseShellExecute = $false
+ $start.RedirectStandardInput = $true
+ $start.RedirectStandardOutput = $true
+ $start.RedirectStandardError = $true
+ $start.StandardOutputEncoding = [System.Text.Encoding]::UTF8
+ $native = New-Object System.Diagnostics.Process
+ $native.StartInfo = $start
+ [void]$native.Start()
+ $null = Send-Request "initialize" @{ protocolVersion = "2025-03-26"; capabilities = @{}; clientInfo = @{ name = "windows-observation-smoke"; version = "1" } }
+ $full = Call-Tool "get_app_state" @{}
+ Assert (-not $full.isError -and (Images $full).Count -eq 1) (Text $full)
+ Write-Output "Normal WPF window returned a validated PNG."
+ $oldIndex = Input-Index $full
+ $image = Call-Tool "get_app_state" @{ include_ax = $false }
+ Assert (-not $image.isError -and (Images $image).Count -eq 1) (Text $image)
+ Assert ((Text $image) -match "indexes invalidated" -and (Text $image) -notmatch "Observation input|initial-value") "AX not omitted"
+ $stale = Call-Tool "set_value" @{ element_index = $oldIndex; value = "must-not-write" }
+ Assert $stale.isError "Stale index accepted"
+ $ax = Call-Tool "get_app_state" @{ include_screenshot = $false }
+ Assert (-not $ax.isError -and (Images $ax).Count -eq 0) (Text $ax)
+ $coordinates = Call-Tool "click" @{ x = 10; y = 10 }
+ Assert $coordinates.isError "AX-only coordinate action accepted"
+ $value = "live-" + [char]0x4E2D + [char]0x6587
+ $write = Call-Tool "set_value" @{ element_index = (Input-Index $ax); value = $value }
+ Assert (-not $write.isError) (Text $write)
+ Wait-For { (Read-State).text -eq $value } "real UIA value update"
+ Write-Output "Selective channels, index refusal/recovery and real Unicode text update passed."
+ foreach ($kind in @("tiny", "minimize", "cloak")) {
+ Change-Window $kind
+ if ($kind -eq "tiny") {
+ $state = Read-State
+ Assert ($state.width -eq 1 -and $state.height -eq 1) "Window did not become 1x1: $($state.width)x$($state.height)"
+ }
+ if ($kind -eq "minimize") { Assert (Read-State).minimized "Window did not minimize" }
+ if ($kind -eq "cloak") { Assert ((Read-State).cloakHResult -eq 0 -and (Read-State).cloaked -ne 0) "Window did not cloak" }
+ $unusable = Call-Tool "get_app_state" @{}
+ Assert ((Images $unusable).Count -eq 0) "$kind window returned an image"
+ Change-Window "restore"
+ $restored = Call-Tool "get_app_state" @{}
+ Assert (-not $restored.isError -and (Images $restored).Count -eq 1) (Text $restored)
+ Write-Output "$kind window omitted its image; restored window returned a PNG."
+ }
+ $cli = & $NativePath snapshot --include-screenshot false ([string]$fixture.Id)
+ Assert ($LASTEXITCODE -eq 0 -and ($cli -join "`n") -match "include_screenshot=false") "CLI flag not forwarded"
+ & node (Join-Path $PSScriptRoot "windows-observation-js-smoke.mjs") $NativePath ([string]$fixture.Id) $stateFile
+ Assert ($LASTEXITCODE -eq 0) "Worker JS desktop smoke failed"
+ Write-Output "Windows real WPF/native MCP/CLI: channels, stale indexes, AX recovery, coordinate refusal, Unicode text update, 1x1/minimized/cloaked screenshot filtering passed."
+} finally {
+ if ($null -ne $native -and -not $native.HasExited) {
+ $native.StandardInput.Close()
+ if (-not $native.WaitForExit(3000)) { $native.Kill(); $native.WaitForExit() }
+ }
+ if ($null -ne $fixture -and -not $fixture.HasExited) { $fixture.Kill(); $fixture.WaitForExit() }
+ $fixtureError = Join-Path $directory "fixture.err"
+ if (Test-Path -LiteralPath $fixtureError) { Get-Content -LiteralPath $fixtureError | Write-Output }
+ Remove-Item -LiteralPath $directory -Recurse -Force
+}
diff --git a/scripts/tests/windows-observation.test.ps1 b/scripts/tests/windows-observation.test.ps1
new file mode 100644
index 0000000..b9e8e3e
--- /dev/null
+++ b/scripts/tests/windows-observation.test.ps1
@@ -0,0 +1,67 @@
+param([Parameter(Mandatory = $true)][string]$RuntimePath)
+$ErrorActionPreference = "Stop"
+
+# Execute the production observation functions with instrumented UIA/capture
+# dependencies. This can run under pwsh on Linux; it is not a Windows UI test.
+$tokens = $null
+$errors = $null
+$ast = [System.Management.Automation.Language.Parser]::ParseFile($RuntimePath, [ref]$tokens, [ref]$errors)
+if ($errors.Count -gt 0) { throw ($errors | Out-String) }
+$functions = $ast.FindAll({ param($node)
+ $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and
+ $node.Name -in @("Resolve-ObservationOptions", "Build-Snapshot")
+}, $true)
+if ($functions.Count -ne 2) { throw "Production observation functions missing" }
+foreach ($function in $functions) { Invoke-Expression $function.Extent.Text }
+
+$DefaultTextLimit = 500
+$AccessibilityTreeMaxNodeCount = 1200
+$AccessibilityTreeMaxDepth = 64
+function Resolve-App($query) { [pscustomobject]@{ ProcessName = $query; Id = 42; MainWindowTitle = "Fixture" } }
+function Get-MainElement($process) { "window" }
+function Get-WindowBounds($process, $element) { [pscustomobject]@{ x = 0; y = 0; width = 300; height = 200 } }
+function Limit-Text($text, $limit) { $text }
+function Render-Tree($element, $bounds, $limit, $nodes, $depth) {
+ $script:calls.tree += 1
+ [pscustomobject]@{ lines = @("Input"); records = @([pscustomobject]@{ index = 7 }) }
+}
+function Get-FocusedSummary($processId, $limit) { $script:calls.focus += 1; "focus" }
+function Get-SelectedText($processId, $limit) { $script:calls.selection += 1; "selection" }
+function Capture-WindowPngBase64($bounds) { $script:calls.capture += 1; "png" }
+function Assert($condition, $message) { if (-not $condition) { throw $message } }
+
+foreach ($mode in @(
+ @{ ax = $true; image = $true },
+ @{ ax = $true; image = $false },
+ @{ ax = $false; image = $true }
+)) {
+ $script:calls = @{ tree = 0; focus = 0; selection = 0; capture = 0 }
+ $snapshot = Build-Snapshot "Fixture" -IncludeAX $mode.ax -IncludeScreenshot $mode.image
+ Assert ($snapshot.app.pid -eq 42 -and $snapshot.windowBounds.width -eq 300) "Window metadata missing"
+ Assert ($script:calls.tree -eq [int]$mode.ax -and $script:calls.focus -eq [int]$mode.ax -and
+ $script:calls.selection -eq [int]$mode.ax -and $script:calls.capture -eq [int]$mode.image) "Disabled collection still executed"
+ if ($mode.ax) {
+ Assert ($snapshot.elements.Count -eq 1 -and $snapshot.selectedText -eq "selection") "AX missing"
+ } else {
+ Assert ($snapshot.elements.Count -eq 0 -and $snapshot.treeLines.Count -eq 0 -and
+ $null -eq $snapshot.focusedSummary -and $null -eq $snapshot.selectedText) "AX was not omitted"
+ }
+ Assert (($snapshot.screenshotPngBase64 -eq "png") -eq $mode.image) "Screenshot mode mismatch"
+}
+
+$defaults = Resolve-ObservationOptions ('{}' | ConvertFrom-Json)
+Assert ($defaults.include_ax -and $defaults.include_screenshot) "Defaults changed"
+foreach ($payload in @('{"include_ax":false}', '{"include_screenshot":false}')) {
+ $flags = Resolve-ObservationOptions ($payload | ConvertFrom-Json)
+ Assert (-not ($flags.include_ax -and $flags.include_screenshot)) "False flag lost"
+}
+foreach ($payload in @('{"include_ax":"false"}', '{"include_ax":0}', '{"include_screenshot":null}',
+ '{"include_ax":false,"include_screenshot":false}')) {
+ $failed = $false
+ try { $null = Resolve-ObservationOptions ($payload | ConvertFrom-Json) } catch { $failed = $true }
+ Assert $failed "Invalid observation accepted"
+}
+$failed = $false
+try { $null = Build-Snapshot "Fixture" -IncludeAX $false -IncludeScreenshot $false } catch { $failed = $true }
+Assert $failed "Empty snapshot accepted"
+Write-Output "PowerShell production functions: parse, defaults, strict booleans and all collection modes passed (instrumented dependencies)."
diff --git a/skills/open-computer-use/references/usage.md b/skills/open-computer-use/references/usage.md
index 84048c7..51d2899 100644
--- a/skills/open-computer-use/references/usage.md
+++ b/skills/open-computer-use/references/usage.md
@@ -52,6 +52,33 @@ await app.typeText("Hello");
await app.getAXState();
```
+### Selective observation (opt-in)
+
+The native `get_app_state` tool on macOS, Windows and Linux accepts `include_ax` and
+`include_screenshot`. Both default to `true`; at least one must remain true.
+The REPL forwards the corresponding optional `includeAX` / `includeScreenshot`
+booleans:
+
+```js
+var editor = await cua.getApp("gedit", { initialState: false });
+await editor.getScreenshot({ includeAX: false }); // no AX, focus or selection walk
+await editor.click([100, 100]); // window-relative coordinates
+await editor.getAXState({ includeScreenshot: false }); // no capture or PNG encoding
+// Use newly observed element indexes only after inspecting the AX result.
+```
+
+Screenshot-only observation invalidates prior element indexes and cached query
+aliases for that app, even if the window title is unchanged. Its result explicitly
+reports this. Coordinate actions use the newly observed window bounds; element
+actions require a fresh AX observation. Existing calls without these options keep
+their full native observation behavior. These flags are implemented on all three native platforms; action read-back is unchanged.
+
+These options select collection, not image deduplication. An unchanged screenshot
+is still returned when requested. macOS/Linux JS actions already disable automatic
+read-back, and getAXState does not emit images; includeScreenshot:false additionally skips
+native capture on all three platforms. See `docs/selective-observation.md` for real GTK
+and Worker JS validation.
+
The code-first surface also forwards macOS background-operation options:
```js