👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
feat: add selective AX and screenshot collection across platforms (#101)
draplater31a7d7da5f28
diff --git a/.github/workflows/windows-observation.yml b/.github/workflows/windows-observation.ymlnew file mode 100644index 0000000..ef48f36--- /dev/null+++ b/.github/workflows/windows-observation.yml@@ -0,0 +1,47 @@+name: Windows observation++on:+ pull_request:+ paths:+ - 'apps/OpenComputerUseWindows/**'+ - 'scripts/tests/windows-observation*'+ - 'scripts/tests/fixtures/WindowsObservationFixture.ps1'+ - '.github/workflows/windows-observation.yml'+ workflow_dispatch:++permissions:+ contents: read++concurrency:+ group: windows-observation-${{ github.ref }}+ cancel-in-progress: true++jobs:+ windows-observation:+ runs-on: windows-2025+ timeout-minutes: 15+ defaults:+ run:+ shell: pwsh+ steps:+ - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd+ - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e+ with:+ node-version: '24'+ - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c+ with:+ go-version: '1.26.1'+ cache: false+ - name: Windows Go tests and build+ working-directory: apps/OpenComputerUseWindows+ run: |+ go test ./...+ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }+ go build -o "$env:RUNNER_TEMP/ocu-observation.exe" .+ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }+ - name: PowerShell collection branches+ run: |+ powershell.exe -NoProfile -NonInteractive -File ./scripts/tests/windows-observation.test.ps1 -RuntimePath apps/OpenComputerUseWindows/runtime.ps1+ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }+ - name: Real WPF and native MCP+ run: ./scripts/tests/windows-observation-smoke.ps1 -NativePath "$env:RUNNER_TEMP/ocu-observation.exe"diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swiftindex 9c7deb7..94b8c5b 100644--- a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift@@ -523,9 +523,9 @@ private final class AppAgentConnection: @unchecked Sendable {let service = ComputerUseService()return CLIProxyResponse(stdout: (service.listApps().primaryText ?? "") + "\n", stderr: "", exitCode: EXIT_SUCCESS)- case let .snapshot(app, textLimit, treeLimits):+ case let .snapshot(app, textLimit, treeLimits, observation):let service = ComputerUseService()- let text = try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits).primaryText ?? ""+ let text = try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits, observation: observation).textContentreturn CLIProxyResponse(stdout: text + "\n", stderr: "", exitCode: EXIT_SUCCESS)case let .call(invocation):diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swiftindex fef1c19..7c4ce10 100644--- a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift@@ -65,9 +65,9 @@ enum OpenComputerUseMain {case .listApps:let service = ComputerUseService()print(service.listApps().primaryText ?? "")- case let .snapshot(app, textLimit, treeLimits):+ case let .snapshot(app, textLimit, treeLimits, observation):let service = ComputerUseService()- print(try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits).primaryText ?? "")+ print(try service.getAppState(app: app, textLimit: textLimit, treeLimits: treeLimits, observation: observation).textContent)case let .call(invocation):if VisualCursorSupport.isEnabled {_ = NSApplication.shared.setActivationPolicy(.accessory)diff --git a/apps/OpenComputerUseLinux/main.go b/apps/OpenComputerUseLinux/main.goindex 28b68d8..30e98d2 100644--- a/apps/OpenComputerUseLinux/main.go+++ b/apps/OpenComputerUseLinux/main.go@@ -88,6 +88,7 @@ type elementRecord struct {}type appSnapshot struct {+ ScreenshotOmitted bool `json:"-"`App appDescriptor `json:"app"`WindowTitle string `json:"windowTitle,omitempty"`WindowBounds *frame `json:"windowBounds,omitempty"`@@ -116,6 +117,9 @@ func (s *appSnapshot) renderedText() string {fmt.Sprintf("Window: %q, App: %s.", title, s.App.Name),}lines = append(lines, s.TreeLines...)+ if s.ScreenshotOmitted {+ lines = append(lines, "Screenshot omitted by include_screenshot=false")+ }if strings.TrimSpace(s.SelectedText) != "" {lines = append(lines, "", fmt.Sprintf("Selected text: [%s]", s.SelectedText))} else if strings.TrimSpace(s.FocusedSummary) != "" {@@ -139,6 +143,9 @@ func (s *appSnapshot) result() toolCallResult {}type linuxRequest struct {+ IncludeAX *bool `json:"include_ax,omitempty"`+ IncludeScreenshot *bool `json:"include_screenshot,omitempty"`+Tool string `json:"tool"`App string `json:"app,omitempty"`Element *elementRecord `json:"element,omitempty"`@@ -215,7 +222,18 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {if windowPlacement != "keep" {return textResult("window_placement '"+windowPlacement+"' is not supported on Linux", true)}- return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth)+ includeAX, err := optionalBool(args, "include_ax")+ if err != nil {+ return textResult(err.Error(), true)+ }+ includeScreenshot, err := optionalBool(args, "include_screenshot")+ if err != nil {+ return textResult(err.Error(), true)+ }+ if includeAX != nil && !*includeAX && includeScreenshot != nil && !*includeScreenshot {+ return textResult("At least one of include_ax or include_screenshot must be true", true)+ }+ return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth, includeAX, includeScreenshot)case "click":clickMethod, err := parseClickMethod(optionalString(args, "click_method"))if err != nil {@@ -284,11 +302,11 @@ func (s *service) listApps() toolCallResult {return textResult(response.Text, false)}-func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int) toolCallResult {+func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int, includeAX, includeScreenshot *bool) toolCallResult {if app == "" {return textResult("Missing required argument: app", true)}- request := linuxRequest{Tool: "get_app_state", App: app}+ request := linuxRequest{Tool: "get_app_state", App: app, IncludeAX: includeAX, IncludeScreenshot: includeScreenshot}if textLimit != nil {request.TextLimit = textLimit.runtimeValue()}@@ -302,7 +320,11 @@ func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maif result.IsError {return result}- return snapshot.result()+ result = snapshot.result()+ if includeAX != nil && !*includeAX {+ result.Content[0].Text += "\nAccessibility omitted; previous element indexes invalidated. Read AX before element-targeted actions."+ }+ return result}func (s *service) click(app, elementIndex string, x, y *float64, clickCount int, mouseButton, clickMethod string) toolCallResult {@@ -328,6 +350,9 @@ func (s *service) click(app, elementIndex string, x, y *float64, clickCount int,if snapshot == nil {return textResult("No app state is available for "+app+". Run get_app_state before action tools.", true)}+ if elementIndex == "" && snapshot.ScreenshotOmitted {+ return textResult("Screenshot omitted by include_screenshot=false; request a screenshot before screenshot-coordinate actions.", true)+ }request := linuxRequest{Tool: "click",App: app,@@ -414,6 +439,9 @@ func (s *service) drag(app string, fromX, fromY, toX, toY *float64) toolCallResuif snapshot == nil {return textResult("No app state is available for "+app+". Run get_app_state before action tools.", true)}+ if snapshot.ScreenshotOmitted {+ return textResult("Screenshot omitted by include_screenshot=false; request a screenshot before screenshot-coordinate actions.", true)+ }return s.actionResult(app, linuxRequest{Tool: "drag", App: app, FromX: fromX, FromY: fromY, ToX: toX, ToY: toY, WindowBounds: snapshot.WindowBounds})}@@ -512,6 +540,25 @@ func (s *service) refreshSnapshot(app string, request linuxRequest) (*appSnapshoif response.Snapshot == nil {return nil, textResult("Linux runtime did not return an app snapshot.", true)}+ if request.IncludeScreenshot != nil && !*request.IncludeScreenshot {+ response.Snapshot.ScreenshotPNGBase64 = ""+ response.Snapshot.ScreenshotOmitted = true+ }+ if request.IncludeAX != nil && !*request.IncludeAX {+ // A screenshot cannot validate AX indexes, even if the window title matches.+ // Invalidate every alias of the prior observation, including PID queries.+ previous := s.currentSnapshot(app)+ for alias, cached := range s.snapshots {+ if cached == previous || cached.App.PID == response.Snapshot.App.PID ||+ (previous != nil && cached.App.PID == previous.App.PID) {+ delete(s.snapshots, alias)+ }+ }+ response.Snapshot.Elements = nil+ response.Snapshot.TreeLines = nil+ response.Snapshot.FocusedSummary = ""+ response.Snapshot.SelectedText = ""+ }s.rememberSnapshot(app, response.Snapshot)return response.Snapshot, toolCallResult{}}@@ -1011,6 +1058,18 @@ func optionalString(args map[string]any, key string) string {return value}+func optionalBool(args map[string]any, key string) (*bool, error) {+ value, exists := args[key]+ if !exists {+ return nil, nil+ }+ boolean, ok := value.(bool)+ if !ok {+ return nil, fmt.Errorf("%s must be a boolean", key)+ }+ return &boolean, nil+}+func requiredElementIndex(args map[string]any) string {return strings.TrimSpace(optionalElementIndex(args))}@@ -1240,7 +1299,7 @@ func toolDefinitions() []toolDefinition {},{Name: "get_app_state",- Description: "Get the state of an already running app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.",+ Description: "Get the state of an already running app's key window. By default returns a screenshot and accessibility tree; include_ax/include_screenshot can opt out of either collection. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.",Annotations: readOnlyAnnotations(),InputSchema: objectSchema(map[string]any{"app": stringProperty("App name or bundle identifier"),@@ -1248,6 +1307,9 @@ func toolDefinitions() []toolDefinition {"max_tree_nodes": positiveIntegerProperty("Maximum accessibility tree nodes to render. Defaults to 1200."),"max_tree_depth": positiveIntegerProperty("Maximum accessibility tree depth to render. Defaults to 64."),"window_placement": enumStringProperty("keep (default) leaves the window where it is. agent_display and restore are macOS-only and not supported on Linux.", windowPlacementValues),++ "include_ax": map[string]any{"type": "boolean", "default": true, "description": "Include accessibility tree, focus and selection. False skips AX collection and invalidates prior element indexes; coordinate actions remain available."},+ "include_screenshot": map[string]any{"type": "boolean", "default": true, "description": "Capture and return a screenshot. False skips capture and PNG encoding."},}, []string{"app"}),},{@@ -1402,13 +1464,16 @@ func runCLI(args []string, stdout io.Writer) error {fmt.Fprintln(stdout, result.Content[0].Text)return nilcase "snapshot":- app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(args[1:])+ snapshotArgs, observation, err := parseSnapshotObservationArgs(args[1:])if err != nil {return err}- toolArgs := map[string]any{- "app": app,+ app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(snapshotArgs)+ if err != nil {+ return err}+ toolArgs := observation+ toolArgs["app"] = appif textLimit != nil {toolArgs["text_limit"] = textLimit.runtimeValue()}@@ -1443,6 +1508,37 @@ func runCLI(args []string, stdout io.Writer) error {}}+func parseSnapshotObservationArgs(args []string) ([]string, map[string]any, error) {+ remaining := []string{}+ options := map[string]any{}+ for index := 0; index < len(args); index++ {+ option := args[index]+ if option != "--include-ax" && option != "--include-screenshot" {+ remaining = append(remaining, option)+ continue+ }+ index+++ if index >= len(args) {+ return nil, nil, fmt.Errorf("%s requires true or false", option)+ }+ var value bool+ switch strings.ToLower(args[index]) {+ case "true":+ value = true+ case "false":+ value = false+ default:+ return nil, nil, fmt.Errorf("%s requires true or false", option)+ }+ key := strings.ReplaceAll(strings.TrimPrefix(option, "--"), "-", "_")+ options[key] = value+ }+ if options["include_ax"] == false && options["include_screenshot"] == false {+ return nil, nil, errors.New("At least one of include_ax or include_screenshot must be true")+ }+ return remaining, options, nil+}+func parseSnapshotArgs(args []string) (string, *textLimit, *int, *int, error) {var app stringvar textLimit *textLimit@@ -1746,7 +1842,7 @@ func helpText(command string) string {case "call":return "Usage:\n open-computer-use call <tool> [--args '<json-object>']\n open-computer-use call --calls '<json-array>'\n\nThe JSON array form keeps all calls in one process so element_index state can be reused.\n"case "snapshot":- return "Usage:\n open-computer-use snapshot [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>\n\nPrint the current Linux AT-SPI snapshot for the target app.\n"+ return "Usage:\n open-computer-use snapshot [--include-ax true|false] [--include-screenshot true|false] [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>\n\nPrint the current Linux AT-SPI snapshot for the target app.\n"default:return `Open Computer Use for Linuxdiff --git a/apps/OpenComputerUseLinux/main_test.go b/apps/OpenComputerUseLinux/main_test.goindex 13c4f7c..17fd6ae 100644--- a/apps/OpenComputerUseLinux/main_test.go+++ b/apps/OpenComputerUseLinux/main_test.go@@ -3,9 +3,11 @@ package mainimport ("bytes""encoding/json"+ "fmt""net""os""path/filepath"+ "strconv""strings""testing")@@ -118,6 +120,143 @@ func TestActionWithoutReadBackPropagatesBridgeErrors(t *testing.T) {}}+func observationPythonResponse(t *testing.T, response string) string {+ t.Helper()+ dir := t.TempDir()+ requestPath := filepath.Join(dir, "request.json")+ script := "#!/bin/sh\ncat \"$2\" > \"$OCU_TEST_OBSERVATION_REQUEST\"\nprintf '%s\\n' \"$OCU_TEST_OBSERVATION_RESPONSE\"\n"+ if err := os.WriteFile(filepath.Join(dir, "python3"), []byte(script), 0o700); err != nil {+ t.Fatal(err)+ }+ t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))+ t.Setenv("OCU_TEST_OBSERVATION_REQUEST", requestPath)+ t.Setenv("OCU_TEST_OBSERVATION_RESPONSE", response)+ return requestPath+}++func TestSelectiveObservationRequestAndDefaultCompatibility(t *testing.T) {+ for _, flags := range []map[string]any{{}, {"include_ax": false}, {"include_screenshot": false}, {"include_ax": true, "include_screenshot": true}} {+ t.Run(fmt.Sprint(flags), func(t *testing.T) {+ path := observationPythonResponse(t, `{"ok":true,"snapshot":{"app":{"name":"Editor","pid":42}}}`)+ args := map[string]any{"app": "Editor"}+ for key, value := range flags {+ args[key] = value+ }+ if result := newService().callTool("get_app_state", args); result.IsError {+ t.Fatalf("observation failed: %#v", result)+ }+ data, err := os.ReadFile(path)+ if err != nil {+ t.Fatal(err)+ }+ var request map[string]any+ if err := json.Unmarshal(data, &request); err != nil {+ t.Fatal(err)+ }+ for _, key := range []string{"include_ax", "include_screenshot"} {+ want, present := flags[key]+ got, sent := request[key]+ if present != sent || got != want {+ t.Fatalf("%s serialized as %#v (%v), want %#v (%v)", key, got, sent, want, present)+ }+ }+ })+ }+}++func TestSelectiveObservationRejectsInvalidOptions(t *testing.T) {+ for _, args := range []map[string]any{+ {"app": "Editor", "include_ax": "false"},+ {"app": "Editor", "include_screenshot": nil},+ {"app": "Editor", "include_ax": false, "include_screenshot": false},+ } {+ if result := newService().callTool("get_app_state", args); !result.IsError {+ t.Fatalf("invalid options accepted: %#v", args)+ }+ }+}++func TestSelectiveObservationSchemaKeepsFullDefaults(t *testing.T) {+ tool := findToolDefinition(t, "get_app_state")+ properties := tool.InputSchema["properties"].(map[string]any)+ for _, name := range []string{"include_ax", "include_screenshot"} {+ property := properties[name].(map[string]any)+ if property["type"] != "boolean" || property["default"] != true {+ t.Fatalf("%s schema = %#v", name, property)+ }+ }+ if required := tool.InputSchema["required"].([]string); len(required) != 1 || required[0] != "app" {+ t.Fatalf("observation options should be optional: %#v", required)+ }+}++func TestScreenshotOnlyInvalidatesIndexesAcrossWindowAndProcessChanges(t *testing.T) {+ for _, pid := range []int{42, 43} {+ t.Run(strconv.Itoa(pid), func(t *testing.T) {+ path := observationPythonResponse(t, fmt.Sprintf(`{"ok":true,"snapshot":{"app":{"name":"Editor","pid":%d},"windowTitle":"New Window","windowBounds":{"width":300,"height":200},"screenshotPngBase64":"cG5n"}}`, pid))+ s := newService()+ old := &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, WindowTitle: "Old Window", Elements: []elementRecord{{Index: 7}}}+ s.rememberSnapshot("old-window-query", old)+ s.snapshots["editor"] = old+ other := &appSnapshot{App: appDescriptor{Name: "Other", PID: 99}}+ s.rememberSnapshot("Other", other)+ result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_ax": false})+ if result.IsError || len(result.Content) != 2 || !strings.Contains(result.Content[0].Text, "indexes invalidated") {+ t.Fatalf("screenshot-only result = %#v", result)+ }+ if s.currentSnapshot("old-window-query") != nil || s.currentSnapshot("Other") != other {+ t.Fatal("old alias retained or unrelated app modified")+ }+ if pid != 42 && s.currentSnapshot("42") != nil {+ t.Fatal("old PID alias retained across process change")+ }+ if _, err := lookupElement(s.currentSnapshot("Editor"), "7"); err == nil {+ t.Fatal("stale index remains usable")+ }+ if result := s.setValue("Editor", "7", "bad"); !result.IsError {+ t.Fatal("stale element action accepted")+ }+ x, y := 10.0, 20.0+ if result := s.click("Editor", "", &x, &y, 1, "left", "auto"); result.IsError {+ t.Fatalf("coordinate action unavailable: %#v", result)+ }+ data, err := os.ReadFile(path)+ if err != nil || !strings.Contains(string(data), `"tool":"click"`) {+ t.Fatalf("coordinate click not sent: %s, %v", data, err)+ }+ t.Setenv("OCU_TEST_OBSERVATION_RESPONSE", fmt.Sprintf(`{"ok":true,"snapshot":{"app":{"name":"Editor","pid":%d},"elements":[{"index":8}]}}`, pid))+ if result := s.callTool("get_app_state", map[string]any{"app": "Editor"}); result.IsError {+ t.Fatalf("AX refresh failed: %#v", result)+ }+ if _, err := lookupElement(s.currentSnapshot("Editor"), "8"); err != nil {+ t.Fatal("explicit AX did not restore indexes")+ }+ })+ }+}++func TestScreenshotOnlyInvalidatesOlderAliasesWhenProcessChanges(t *testing.T) {+ observationPythonResponse(t, `{"ok":true,"snapshot":{"app":{"name":"Editor","pid":43},"windowBounds":{"width":300,"height":200},"screenshotPngBase64":"cG5n"}}`)+ s := newService()+ older := &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 6}}}+ s.rememberSnapshot("older-query", older)+ s.rememberSnapshot("Editor", &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 7}}})+ unrelated := &appSnapshot{App: appDescriptor{Name: "Other", PID: 99}}+ s.rememberSnapshot("Other", unrelated)+ result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_ax": false})+ if result.IsError {+ t.Fatalf("screenshot-only observation failed: %#v", result)+ }+ for _, alias := range []string{"older-query", "42"} {+ if s.currentSnapshot(alias) != nil {+ t.Fatalf("stale AX alias %q survived process replacement", alias)+ }+ }+ if s.currentSnapshot("Other") != unrelated {+ t.Fatal("unrelated app observation invalidated")+ }+}+func TestClickMethodSchemaAndParser(t *testing.T) {tool := findToolDefinition(t, "click")properties := tool.InputSchema["properties"].(map[string]any)diff --git a/apps/OpenComputerUseLinux/observation_cli_test.go b/apps/OpenComputerUseLinux/observation_cli_test.gonew file mode 100644index 0000000..546f6b7--- /dev/null+++ b/apps/OpenComputerUseLinux/observation_cli_test.go@@ -0,0 +1,29 @@+package main++import (+ "strings"+ "testing"+)++func TestSnapshotObservationFlagsPreserveBudgetsAndBooleanTypes(t *testing.T) {+ args, flags, err := parseSnapshotObservationArgs([]string{"--include-screenshot", "false", "--max-tree-nodes", "2000", "Editor"})+ if err != nil {+ t.Fatal(err)+ }+ app, _, nodes, _, err := parseSnapshotArgs(args)+ if err != nil || app != "Editor" || nodes == nil || *nodes != 2000 || flags["include_screenshot"] != false {+ t.Fatalf("%v %v %v", args, flags, err)+ }+ for _, bad := range [][]string{{"--include-ax"}, {"--include-screenshot", "0"}, {"--include-ax", "no"}, {"--include-ax", "false", "--include-screenshot", "false"}} {+ if _, _, err := parseSnapshotObservationArgs(bad); err == nil {+ t.Fatalf("invalid flags accepted: %v", bad)+ }+ }+ _, defaults, err := parseSnapshotObservationArgs([]string{"Editor"})+ if err != nil || len(defaults) != 0 {+ t.Fatalf("legacy call changed: %v %v", defaults, err)+ }+ if !strings.Contains(helpText("snapshot"), "--include-screenshot") {+ t.Fatal("snapshot help omits observation flags")+ }+}diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.pyindex 3fc4c41..d5f72e0 100644--- a/apps/OpenComputerUseLinux/runtime.py+++ b/apps/OpenComputerUseLinux/runtime.py@@ -442,21 +442,24 @@ def focus_details(window, text_limit=DEFAULT_TEXT_LIMIT, max_nodes=MAX_ELEMENTS,return summary, selection, False-def build_snapshot(query, text_limit=DEFAULT_TEXT_LIMIT, max_tree_nodes=MAX_ELEMENTS, max_tree_depth=MAX_DEPTH):+def build_snapshot(query, text_limit=DEFAULT_TEXT_LIMIT, max_tree_nodes=MAX_ELEMENTS, max_tree_depth=MAX_DEPTH, include_ax=True, include_screenshot=True):app = resolve_app(query)window_index, window = main_window(app)bounds = extents(window)- records, lines = render_tree(- window,- bounds,- [window_index],- text_limit=text_limit,- max_tree_nodes=max_tree_nodes,- max_tree_depth=max_tree_depth,- )- focus, selection, focus_truncated = focus_details(window, text_limit, max_tree_nodes, max_tree_depth)- if focus_truncated:- lines.append("[Focus search incomplete: node/depth budget reached.]")+ records, lines = [], []+ focus, selection = None, None+ if include_ax:+ records, lines = render_tree(+ window,+ bounds,+ [window_index],+ text_limit=text_limit,+ max_tree_nodes=max_tree_nodes,+ max_tree_depth=max_tree_depth,+ )+ focus, selection, focus_truncated = focus_details(window, text_limit, max_tree_nodes, max_tree_depth)+ if focus_truncated:+ lines.append("[Focus search incomplete: node/depth budget reached.]")pid = node_pid(app)return {"app": {@@ -466,7 +469,7 @@ def build_snapshot(query, text_limit=DEFAULT_TEXT_LIMIT, max_tree_nodes=MAX_ELEM},"windowTitle": limit_text(node_name(window), text_limit=text_limit),"windowBounds": bounds,- "screenshotPngBase64": capture_window_png(bounds),+ "screenshotPngBase64": capture_window_png(bounds) if include_screenshot else None,"treeLines": lines,"focusedSummary": focus,"selectedText": selection,@@ -803,6 +806,11 @@ def perform_operation(operation):if tool == "list_apps":return {"ok": True, "text": list_apps_text()}if tool == "get_app_state":+ for key in ("include_ax", "include_screenshot"):+ if key in operation and not isinstance(operation[key], bool):+ raise RuntimeError(key + " must be a boolean")+ if not operation.get("include_ax", True) and not operation.get("include_screenshot", True):+ raise RuntimeError("At least one of include_ax or include_screenshot must be true")return {"ok": True,"snapshot": build_snapshot(@@ -810,6 +818,8 @@ def perform_operation(operation):text_limit=parse_text_limit(operation.get("text_limit"), DEFAULT_TEXT_LIMIT),max_tree_nodes=positive_int(operation.get("max_tree_nodes"), MAX_ELEMENTS),max_tree_depth=positive_int(operation.get("max_tree_depth"), MAX_DEPTH),+ include_ax=operation.get("include_ax", True),+ include_screenshot=operation.get("include_screenshot", True),),}diff --git a/apps/OpenComputerUseLinux/runtime_test.py b/apps/OpenComputerUseLinux/runtime_test.pyindex 05280b0..25cc814 100644--- a/apps/OpenComputerUseLinux/runtime_test.py+++ b/apps/OpenComputerUseLinux/runtime_test.py@@ -172,6 +172,71 @@ class RuntimeKeyTests(unittest.TestCase):self.assertEqual(self.events, [])+class RuntimeSelectiveObservationTests(unittest.TestCase):+ def setUp(self):+ self.runtime = load_runtime()+ self.app = object()+ self.window = object()+ self.patches = []+ for name, value in {+ "resolve_app": self.app,+ "main_window": (0, self.window),+ "extents": {"x": 1, "y": 2, "width": 300, "height": 200},+ "node_pid": 42,+ "node_name": "Editor",+ "render_tree": ([{"index": 7}], ["tree"]),+ "focus_details": ("focus", "selection", False),+ "capture_window_png": "png",+ }.items():+ patch = mock.patch.object(self.runtime, name, return_value=value)+ self.patches.append(patch)+ patch.start()+ self.addCleanup(patch.stop)++ def test_defaults_still_collect_full_state(self):+ state = self.runtime.build_snapshot("Editor")+ self.assertEqual(state["elements"], [{"index": 7}])+ self.assertEqual(state["screenshotPngBase64"], "png")+ self.runtime.focus_details.assert_called_once()++ def test_screenshot_only_skips_tree_focus_and_selection(self):+ state = self.runtime.build_snapshot("Editor", include_ax=False)+ self.assertEqual(state["screenshotPngBase64"], "png")+ self.assertEqual(state["elements"], [])+ self.assertEqual(state["treeLines"], [])+ self.assertIsNone(state["focusedSummary"])+ self.assertIsNone(state["selectedText"])+ for name in ("render_tree", "focus_details"):+ getattr(self.runtime, name).assert_not_called()++ def test_ax_only_skips_capture(self):+ state = self.runtime.build_snapshot("Editor", include_screenshot=False)+ self.assertEqual(state["treeLines"], ["tree"])+ self.assertIsNone(state["screenshotPngBase64"])+ self.runtime.capture_window_png.assert_not_called()++ def test_ax_only_preserves_bounded_focus_reporting(self):+ self.runtime.focus_details.return_value = (None, None, True)+ state = self.runtime.build_snapshot("Editor", text_limit=20, max_tree_nodes=2,+ max_tree_depth=1, include_screenshot=False)+ self.runtime.focus_details.assert_called_once_with(self.window, 20, 2, 1)+ self.assertIn("[Focus search incomplete: node/depth budget reached.]", state["treeLines"])+ self.runtime.capture_window_png.assert_not_called()++ def test_tool_forwards_explicit_observation_flags(self):+ state = self.runtime.perform_operation({"tool": "get_app_state", "app": "Editor", "include_ax": False})+ self.assertTrue(state["ok"])+ self.runtime.render_tree.assert_not_called()++ def test_invalid_or_empty_observations_are_rejected(self):+ for flags in ({"include_ax": "false"}, {"include_screenshot": None},+ {"include_ax": 0}, {"include_ax": False, "include_screenshot": False}):+ with self.subTest(flags=flags):+ with self.assertRaises(RuntimeError):+ self.runtime.perform_operation({"tool": "get_app_state", "app": "Editor", **flags})+ self.runtime.resolve_app.assert_not_called()++class HiddenDetailsTests(unittest.TestCase):def setUp(self):self.runtime = load_runtime()diff --git a/apps/OpenComputerUseWindows/main.go b/apps/OpenComputerUseWindows/main.goindex 47bef73..8b99ffb 100644--- a/apps/OpenComputerUseWindows/main.go+++ b/apps/OpenComputerUseWindows/main.go@@ -93,6 +93,7 @@ type elementRecord struct {}type appSnapshot struct {+ ScreenshotOmitted bool `json:"-"`App appDescriptor `json:"app"`WindowTitle string `json:"windowTitle,omitempty"`WindowBounds *frame `json:"windowBounds,omitempty"`@@ -121,7 +122,9 @@ func (s *appSnapshot) renderedText() string {fmt.Sprintf("Window: %q, App: %s.", title, s.App.Name),}lines = append(lines, s.TreeLines...)- if !s.hasUsableScreenshot() {+ if s.ScreenshotOmitted {+ lines = append(lines, "", "Screenshot omitted by include_screenshot=false")+ } else if !s.hasUsableScreenshot() {lines = append(lines, "", "Screenshot unavailable. Restore a visible, non-minimized, on-screen window before using screenshot-coordinate actions; element_index actions may still work.")}if strings.TrimSpace(s.SelectedText) != "" {@@ -198,28 +201,30 @@ func validateScreenshotPNGBase64(encoded string) string {}type psRequest struct {- Tool string `json:"tool"`- App string `json:"app,omitempty"`- Element *elementRecord `json:"element,omitempty"`- X *float64 `json:"x,omitempty"`- Y *float64 `json:"y,omitempty"`- FromX *float64 `json:"from_x,omitempty"`- FromY *float64 `json:"from_y,omitempty"`- ToX *float64 `json:"to_x,omitempty"`- ToY *float64 `json:"to_y,omitempty"`- ClickCount int `json:"click_count,omitempty"`- MouseButton string `json:"mouse_button,omitempty"`- ClickMethod string `json:"click_method,omitempty"`- Action string `json:"action,omitempty"`- Direction string `json:"direction,omitempty"`- Pages float64 `json:"pages,omitempty"`- Text string `json:"text,omitempty"`- Key string `json:"key,omitempty"`- Value string `json:"value,omitempty"`- WindowBounds *frame `json:"windowBounds,omitempty"`- TextLimit any `json:"text_limit,omitempty"`- MaxTreeNodes int `json:"max_tree_nodes,omitempty"`- MaxTreeDepth int `json:"max_tree_depth,omitempty"`+ IncludeAX *bool `json:"include_ax,omitempty"`+ IncludeScreenshot *bool `json:"include_screenshot,omitempty"`+ Tool string `json:"tool"`+ App string `json:"app,omitempty"`+ Element *elementRecord `json:"element,omitempty"`+ X *float64 `json:"x,omitempty"`+ Y *float64 `json:"y,omitempty"`+ FromX *float64 `json:"from_x,omitempty"`+ FromY *float64 `json:"from_y,omitempty"`+ ToX *float64 `json:"to_x,omitempty"`+ ToY *float64 `json:"to_y,omitempty"`+ ClickCount int `json:"click_count,omitempty"`+ MouseButton string `json:"mouse_button,omitempty"`+ ClickMethod string `json:"click_method,omitempty"`+ Action string `json:"action,omitempty"`+ Direction string `json:"direction,omitempty"`+ Pages float64 `json:"pages,omitempty"`+ Text string `json:"text,omitempty"`+ Key string `json:"key,omitempty"`+ Value string `json:"value,omitempty"`+ WindowBounds *frame `json:"windowBounds,omitempty"`+ TextLimit any `json:"text_limit,omitempty"`+ MaxTreeNodes int `json:"max_tree_nodes,omitempty"`+ MaxTreeDepth int `json:"max_tree_depth,omitempty"`}type textLimit struct {@@ -243,10 +248,11 @@ type psResponse struct {type service struct {snapshots map[string]*appSnapshot+ runBridge func(psRequest) (*psResponse, error)}func newService() *service {- return &service{snapshots: map[string]*appSnapshot{}}+ return &service{snapshots: map[string]*appSnapshot{}, runBridge: runPowerShell}}func (s *service) callTool(name string, args map[string]any) toolCallResult {@@ -273,7 +279,18 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {if windowPlacement != "keep" {return textResult("window_placement '"+windowPlacement+"' is not supported on Windows", true)}- return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth)+ includeAX, err := optionalBool(args, "include_ax")+ if err != nil {+ return textResult(err.Error(), true)+ }+ includeScreenshot, err := optionalBool(args, "include_screenshot")+ if err != nil {+ return textResult(err.Error(), true)+ }+ if includeAX != nil && !*includeAX && includeScreenshot != nil && !*includeScreenshot {+ return textResult("At least one of include_ax or include_screenshot must be true", true)+ }+ return s.getAppState(requiredString(args, "app"), textLimit, maxTreeNodes, maxTreeDepth, includeAX, includeScreenshot)case "click":clickMethod, err := parseClickMethod(optionalString(args, "click_method"))if err != nil {@@ -329,7 +346,7 @@ func (s *service) callTool(name string, args map[string]any) toolCallResult {}func (s *service) listApps() toolCallResult {- response, err := runPowerShell(psRequest{Tool: "list_apps"})+ response, err := s.runBridge(psRequest{Tool: "list_apps"})if err != nil {return textResult(err.Error(), true)}@@ -342,11 +359,11 @@ func (s *service) listApps() toolCallResult {return textResult(response.Text, false)}-func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int) toolCallResult {+func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maxTreeDepth *int, includeAX, includeScreenshot *bool) toolCallResult {if app == "" {return textResult("Missing required argument: app", true)}- request := psRequest{Tool: "get_app_state", App: app}+ request := psRequest{Tool: "get_app_state", App: app, IncludeAX: includeAX, IncludeScreenshot: includeScreenshot}if textLimit != nil {request.TextLimit = textLimit.runtimeValue()}@@ -360,7 +377,11 @@ func (s *service) getAppState(app string, textLimit *textLimit, maxTreeNodes, maif result.IsError {return result}- return snapshot.result()+ result = snapshot.result()+ if includeAX != nil && !*includeAX {+ result.Content[0].Text += "\nAccessibility omitted; previous element indexes invalidated. Read AX before element-targeted actions."+ }+ return result}func (s *service) click(app, elementIndex string, x, y *float64, clickCount int, mouseButton, clickMethod string) toolCallResult {@@ -479,7 +500,7 @@ func (s *service) drag(app string, fromX, fromY, toX, toY *float64) toolCallResu}func screenshotCoordinatesUnavailableResult(app string) toolCallResult {- return textResult("No usable screenshot is available for "+app+". Restore a visible, non-minimized, on-screen window and run get_app_state again before using screenshot coordinates.", true)+ return textResult("No usable screenshot is available for "+app+". Run get_app_state with include_screenshot=true on a visible, non-minimized, on-screen window before using screenshot coordinates.", true)}func (s *service) typeText(app, text, keyMethod string) toolCallResult {@@ -545,7 +566,7 @@ func (s *service) currentSnapshot(app string) *appSnapshot {}func (s *service) refreshSnapshot(app string, request psRequest) (*appSnapshot, toolCallResult) {- response, err := runPowerShell(request)+ response, err := s.runBridge(request)if err != nil {return nil, textResult(err.Error(), true)}@@ -555,6 +576,25 @@ func (s *service) refreshSnapshot(app string, request psRequest) (*appSnapshot,if response.Snapshot == nil {return nil, textResult("Windows runtime did not return an app snapshot.", true)}+ if request.IncludeScreenshot != nil && !*request.IncludeScreenshot {+ response.Snapshot.ScreenshotPNGBase64 = ""+ response.Snapshot.ScreenshotOmitted = true+ }+ if request.IncludeAX != nil && !*request.IncludeAX {+ // A screenshot cannot validate AX indexes, even if the window title matches.+ // Invalidate every alias of the prior observation, including PID queries.+ previous := s.currentSnapshot(app)+ for alias, cached := range s.snapshots {+ if cached == previous || cached.App.PID == response.Snapshot.App.PID ||+ (previous != nil && cached.App.PID == previous.App.PID) {+ delete(s.snapshots, alias)+ }+ }+ response.Snapshot.Elements = nil+ response.Snapshot.TreeLines = nil+ response.Snapshot.FocusedSummary = ""+ response.Snapshot.SelectedText = ""+ }response.Snapshot.sanitizeScreenshot()s.rememberSnapshot(app, response.Snapshot)return response.Snapshot, toolCallResult{}@@ -641,6 +681,18 @@ func optionalString(args map[string]any, key string) string {return value}+func optionalBool(args map[string]any, key string) (*bool, error) {+ value, exists := args[key]+ if !exists {+ return nil, nil+ }+ boolean, ok := value.(bool)+ if !ok {+ return nil, fmt.Errorf("%s must be a boolean", key)+ }+ return &boolean, nil+}+func requiredElementIndex(args map[string]any) string {return strings.TrimSpace(optionalElementIndex(args))}@@ -861,14 +913,16 @@ func toolDefinitions() []toolDefinition {},{Name: "get_app_state",- Description: "Get the state of an already running app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.",+ Description: "Get the state of an already running app's key window. By default returns a screenshot and accessibility tree; include_ax/include_screenshot can opt out of either collection. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.",Annotations: readOnlyAnnotations(),InputSchema: objectSchema(map[string]any{- "app": stringProperty("App name or bundle identifier"),- "text_limit": textLimitProperty("Maximum text characters to return. Use \"max\" for full text. Defaults to 500."),- "max_tree_nodes": positiveIntegerProperty("Maximum accessibility tree nodes to render. Defaults to 1200."),- "max_tree_depth": positiveIntegerProperty("Maximum accessibility tree depth to render. Defaults to 64."),- "window_placement": enumStringProperty("keep (default) leaves the window where it is. agent_display and restore are macOS-only and not supported on Windows.", windowPlacementValues),+ "include_ax": map[string]any{"type": "boolean", "default": true, "description": "Include accessibility tree, focus and selection. False skips AX collection and invalidates prior element indexes; coordinate actions remain available."},+ "include_screenshot": map[string]any{"type": "boolean", "default": true, "description": "Capture and return a screenshot. False skips capture and PNG encoding."},+ "app": stringProperty("App name or bundle identifier"),+ "text_limit": textLimitProperty("Maximum text characters to return. Use \"max\" for full text. Defaults to 500."),+ "max_tree_nodes": positiveIntegerProperty("Maximum accessibility tree nodes to render. Defaults to 1200."),+ "max_tree_depth": positiveIntegerProperty("Maximum accessibility tree depth to render. Defaults to 64."),+ "window_placement": enumStringProperty("keep (default) leaves the window where it is. agent_display and restore are macOS-only and not supported on Windows.", windowPlacementValues),}, []string{"app"}),},{@@ -1023,13 +1077,16 @@ func runCLI(args []string, stdout io.Writer) error {fmt.Fprintln(stdout, result.Content[0].Text)return nilcase "snapshot":- app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(args[1:])+ snapshotArgs, observation, err := parseSnapshotObservationArgs(args[1:])if err != nil {return err}- toolArgs := map[string]any{- "app": app,+ app, textLimit, maxTreeNodes, maxTreeDepth, err := parseSnapshotArgs(snapshotArgs)+ if err != nil {+ return err}+ toolArgs := observation+ toolArgs["app"] = appif textLimit != nil {toolArgs["text_limit"] = textLimit.runtimeValue()}@@ -1064,6 +1121,37 @@ func runCLI(args []string, stdout io.Writer) error {}}+func parseSnapshotObservationArgs(args []string) ([]string, map[string]any, error) {+ remaining := []string{}+ options := map[string]any{}+ for index := 0; index < len(args); index++ {+ option := args[index]+ if option != "--include-ax" && option != "--include-screenshot" {+ remaining = append(remaining, option)+ continue+ }+ index+++ if index >= len(args) {+ return nil, nil, fmt.Errorf("%s requires true or false", option)+ }+ var value bool+ switch strings.ToLower(args[index]) {+ case "true":+ value = true+ case "false":+ value = false+ default:+ return nil, nil, fmt.Errorf("%s requires true or false", option)+ }+ key := strings.ReplaceAll(strings.TrimPrefix(option, "--"), "-", "_")+ options[key] = value+ }+ if options["include_ax"] == false && options["include_screenshot"] == false {+ return nil, nil, errors.New("At least one of include_ax or include_screenshot must be true")+ }+ return remaining, options, nil+}+func parseSnapshotArgs(args []string) (string, *textLimit, *int, *int, error) {var app stringvar textLimit *textLimit@@ -1367,7 +1455,7 @@ func helpText(command string) string {case "call":return "Usage:\n open-computer-use.exe call <tool> [--args '<json-object>']\n open-computer-use.exe call --calls '<json-array>'\n\nThe JSON array form keeps all calls in one process so element_index state can be reused.\n"case "snapshot":- return "Usage:\n open-computer-use.exe snapshot [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>\n\nPrint the current Windows UI Automation snapshot for the target app.\n"+ return "Usage:\n open-computer-use.exe snapshot [--include-ax true|false] [--include-screenshot true|false] [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>\n\nPrint the current Windows UI Automation snapshot for the target app.\n"default:return `Open Computer Use for Windowsdiff --git a/apps/OpenComputerUseWindows/observation_cli_test.go b/apps/OpenComputerUseWindows/observation_cli_test.gonew file mode 100644index 0000000..546f6b7--- /dev/null+++ b/apps/OpenComputerUseWindows/observation_cli_test.go@@ -0,0 +1,29 @@+package main++import (+ "strings"+ "testing"+)++func TestSnapshotObservationFlagsPreserveBudgetsAndBooleanTypes(t *testing.T) {+ args, flags, err := parseSnapshotObservationArgs([]string{"--include-screenshot", "false", "--max-tree-nodes", "2000", "Editor"})+ if err != nil {+ t.Fatal(err)+ }+ app, _, nodes, _, err := parseSnapshotArgs(args)+ if err != nil || app != "Editor" || nodes == nil || *nodes != 2000 || flags["include_screenshot"] != false {+ t.Fatalf("%v %v %v", args, flags, err)+ }+ for _, bad := range [][]string{{"--include-ax"}, {"--include-screenshot", "0"}, {"--include-ax", "no"}, {"--include-ax", "false", "--include-screenshot", "false"}} {+ if _, _, err := parseSnapshotObservationArgs(bad); err == nil {+ t.Fatalf("invalid flags accepted: %v", bad)+ }+ }+ _, defaults, err := parseSnapshotObservationArgs([]string{"Editor"})+ if err != nil || len(defaults) != 0 {+ t.Fatalf("legacy call changed: %v %v", defaults, err)+ }+ if !strings.Contains(helpText("snapshot"), "--include-screenshot") {+ t.Fatal("snapshot help omits observation flags")+ }+}diff --git a/apps/OpenComputerUseWindows/observation_test.go b/apps/OpenComputerUseWindows/observation_test.gonew file mode 100644index 0000000..e3cb946--- /dev/null+++ b/apps/OpenComputerUseWindows/observation_test.go@@ -0,0 +1,115 @@+package main++import (+ "encoding/json"+ "strings"+ "testing"+)++func TestObservationRequestSerializationAndChannels(t *testing.T) {+ for _, flags := range []map[string]any{{}, {"include_ax": false}, {"include_screenshot": false}, {"include_ax": true, "include_screenshot": true}} {+ s := newService()+ s.runBridge = func(request psRequest) (*psResponse, error) {+ data, err := json.Marshal(request)+ if err != nil {+ t.Fatal(err)+ }+ var sent map[string]any+ if err := json.Unmarshal(data, &sent); err != nil {+ t.Fatal(err)+ }+ for _, key := range []string{"include_ax", "include_screenshot"} {+ want, present := flags[key]+ got, exists := sent[key]+ if present != exists || want != got {+ t.Fatalf("%s serialized as %#v, want %#v", key, sent, flags)+ }+ }+ return &psResponse{OK: true, Snapshot: &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42},+ WindowBounds: &frame{Width: 300, Height: 200}, ScreenshotPNGBase64: pngBase64(t, 300, 200),+ TreeLines: []string{"entry"}, FocusedSummary: "focus", SelectedText: "中文🙂", Elements: []elementRecord{{Index: 7}}}}, nil+ }+ args := map[string]any{"app": "Editor"}+ for key, value := range flags {+ args[key] = value+ }+ result := s.callTool("get_app_state", args)+ if result.IsError {+ t.Fatal(result)+ }+ if flags["include_screenshot"] == false {+ if len(result.Content) != 1 || !strings.Contains(result.Content[0].Text, "include_screenshot=false") {+ t.Fatal(result)+ }+ } else if len(result.Content) != 2 {+ t.Fatal(result)+ }+ if flags["include_ax"] == false {+ cached := s.currentSnapshot("Editor")+ if len(cached.Elements) != 0 || len(cached.TreeLines) != 0 || cached.FocusedSummary != "" || cached.SelectedText != "" {+ t.Fatal(cached)+ }+ if !strings.Contains(result.Content[0].Text, "indexes invalidated") {+ t.Fatal(result)+ }+ }+ }+}++func TestObservationRejectsInvalidOptionsBeforeBridge(t *testing.T) {+ s := newService()+ s.runBridge = func(psRequest) (*psResponse, error) { t.Fatal("invalid observation reached bridge"); return nil, nil }+ for _, args := range []map[string]any{{"include_ax": "false"}, {"include_screenshot": nil}, {"include_ax": 0}, {"include_ax": false, "include_screenshot": false}} {+ args["app"] = "Editor"+ if result := s.callTool("get_app_state", args); !result.IsError {+ t.Fatal(args)+ }+ }+}++func TestObservationInvalidatesOlderAliasesAcrossProcessReplacement(t *testing.T) {+ s := newService()+ s.rememberSnapshot("older-query", &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 6}}})+ s.rememberSnapshot("Editor", &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 7}}})+ other := &appSnapshot{App: appDescriptor{Name: "Other", PID: 99}}+ s.rememberSnapshot("Other", other)+ s.runBridge = func(request psRequest) (*psResponse, error) {+ return &psResponse{OK: true, Snapshot: &appSnapshot{App: appDescriptor{Name: "Editor", PID: 43},+ WindowBounds: &frame{Width: 300, Height: 200}, ScreenshotPNGBase64: pngBase64(t, 300, 200), Elements: []elementRecord{{Index: 8}}}}, nil+ }+ if result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_ax": false}); result.IsError {+ t.Fatal(result)+ }+ if s.currentSnapshot("older-query") != nil || s.currentSnapshot("42") != nil || s.currentSnapshot("Other") != other {+ t.Fatal("stale aliases retained or unrelated app invalidated")+ }+ if result := s.setValue("Editor", "7", "bad"); !result.IsError {+ t.Fatal("stale element action accepted")+ }+ x, y := 10.0, 20.0+ if result := s.click("Editor", "", &x, &y, 1, "left", "auto"); result.IsError {+ t.Fatal("screenshot-only coordinate action failed", result)+ }+ if result := s.callTool("get_app_state", map[string]any{"app": "Editor", "include_screenshot": false}); result.IsError {+ t.Fatal(result)+ }+ if _, err := lookupElement(s.currentSnapshot("Editor"), "8"); err != nil {+ t.Fatal("AX refresh did not restore indexes", err)+ }+ if result := s.click("Editor", "", &x, &y, 1, "left", "auto"); !result.IsError {+ t.Fatal("AX-only observation accepted screenshot coordinates")+ }+ if result := s.setValue("Editor", "8", "works"); result.IsError {+ t.Fatal("AX-only element action failed", result)+ }+}++func TestObservationSchemaDefaults(t *testing.T) {+ props := findToolDefinition(t, "get_app_state").InputSchema["properties"].(map[string]any)+ for _, key := range []string{"include_ax", "include_screenshot"} {+ value := props[key].(map[string]any)+ if value["type"] != "boolean" || value["default"] != true {+ t.Fatal(value)+ }+ }+}diff --git a/apps/OpenComputerUseWindows/runtime.ps1 b/apps/OpenComputerUseWindows/runtime.ps1index 3cf4df0..6cbe7ad 100644--- a/apps/OpenComputerUseWindows/runtime.ps1+++ b/apps/OpenComputerUseWindows/runtime.ps1@@ -692,11 +692,38 @@ function Get-SelectedText($processId, $TextLimit = $script:DefaultTextLimit) {return $null}-function Build-Snapshot([string]$query, $TextLimit = $script:DefaultTextLimit, [int]$MaxTreeNodes = $script:AccessibilityTreeMaxNodeCount, [int]$MaxTreeDepth = $script:AccessibilityTreeMaxDepth) {+function Resolve-ObservationOptions($operation) {+ $values = @{ include_ax = $true; include_screenshot = $true }+ foreach ($key in @("include_ax", "include_screenshot")) {+ $property = $operation.PSObject.Properties[$key]+ if ($null -ne $property) {+ if ($property.Value -isnot [bool]) { throw "$key must be a boolean" }+ $values[$key] = $property.Value+ }+ }+ if (-not $values.include_ax -and -not $values.include_screenshot) {+ throw "At least one of include_ax or include_screenshot must be true"+ }+ return [pscustomobject]$values+}++function Build-Snapshot([string]$query, $TextLimit = $script:DefaultTextLimit, [int]$MaxTreeNodes = $script:AccessibilityTreeMaxNodeCount, [int]$MaxTreeDepth = $script:AccessibilityTreeMaxDepth, [bool]$IncludeAX = $true, [bool]$IncludeScreenshot = $true) {+ if (-not $IncludeAX -and -not $IncludeScreenshot) {+ throw "At least one of include_ax or include_screenshot must be true"+ }$process = Resolve-App $query$element = Get-MainElement $process$bounds = Get-WindowBounds $process $element- $rendered = Render-Tree $element $bounds $TextLimit $MaxTreeNodes $MaxTreeDepth+ $rendered = @{ lines = @(); records = @() }+ $focus = $null+ $selection = $null+ $image = $null+ if ($IncludeAX) {+ $rendered = Render-Tree $element $bounds $TextLimit $MaxTreeNodes $MaxTreeDepth+ $focus = Get-FocusedSummary $process.Id $TextLimit+ $selection = Get-SelectedText $process.Id $TextLimit+ }+ if ($IncludeScreenshot) { $image = Capture-WindowPngBase64 $bounds }[pscustomobject]@{app = [pscustomobject]@{name = $process.ProcessName@@ -705,10 +732,10 @@ function Build-Snapshot([string]$query, $TextLimit = $script:DefaultTextLimit, [}windowTitle = Limit-Text $process.MainWindowTitle $TextLimitwindowBounds = $bounds- screenshotPngBase64 = Capture-WindowPngBase64 $bounds+ screenshotPngBase64 = $imagetreeLines = @($rendered.lines)- focusedSummary = Get-FocusedSummary $process.Id $TextLimit- selectedText = Get-SelectedText $process.Id $TextLimit+ focusedSummary = $focus+ selectedText = $selectionelements = @($rendered.records)}}@@ -982,7 +1009,8 @@ try {if ($operation.tool -eq "list_apps") {$response = [pscustomobject]@{ ok = $true; text = (List-Apps) }} elseif ($operation.tool -eq "get_app_state") {- $response = [pscustomobject]@{ ok = $true; snapshot = (Build-Snapshot $operation.app (Resolve-TextLimit $operation.text_limit) ([int]$operation.max_tree_nodes) ([int]$operation.max_tree_depth)) }+ $observation = Resolve-ObservationOptions $operation+ $response = [pscustomobject]@{ ok = $true; snapshot = (Build-Snapshot $operation.app (Resolve-TextLimit $operation.text_limit) ([int]$operation.max_tree_nodes) ([int]$operation.max_tree_depth) $observation.include_ax $observation.include_screenshot) }} else {$process = Resolve-App $operation.app$hwnd = [IntPtr]$process.MainWindowHandlediff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex 8fc5d79..012785f 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -74,6 +74,7 @@- `open-computer-use call <tool> --args '{...}'` 会直接输出 MCP-style JSON result;`open-computer-use call --calls '[...]'` / `--calls-file <path>` 会在同一进程里顺序执行 JSON 数组里的 tool calls,并复用同一个 `ComputerUseService` 内存态,因此 `get_app_state` 之后的 action tool 可以继续使用同一轮 snapshot 的 `element_index`。序列执行默认会在成功的相邻操作之间 sleep 1 秒,也可以用 `--sleep <seconds>` 覆盖;遇到 `isError=true` 的 tool result 后停止。- 对真实 app 的 `get_app_state` / action tool 入口,当前只保留一层密码管理器 bundle denylist:bundle-id 直传时直接返回 safety denial;名称匹配时默认不解析到这些 app。终端、Chrome / Atlas 和系统组件不再属于内置阻止目标。- 普通 app 的 element frame 当前按“窗口左上角为原点”的 window-relative 坐标输出,便于后续把 `element_index` 和截图坐标统一到同一套参考系。+- 三个平台的显式 `get_app_state` 均支持 `include_ax` / `include_screenshot`,默认 true,至少保留一项。JS 对应 `includeAX` / `includeScreenshot`,CLI snapshot 提供同名连字符参数。关闭 AX 跳过树、焦点和选区,并使旧快照 / 查询索引及别名失效;关闭截图跳过捕获与编码,元素操作仍可用,截图坐标操作须重新获取图片。参数经 macOS app agent 转发;不改变动作回读,也不做自动图片去重。见 [按需观察](selective-observation.md)。- macOS 额外提供 `query` 和 `select_text`,当前 native surface 为 11 个工具;Linux / Windows 仍为 9 个核心工具。JS `app.selectText(elementIndex, text, {prefix?, suffix?, selection?})` 复用快照或 query 索引,支持选中文字及前后光标。按字面文本和 UTF-16 定位、拒绝歧义;写入前及返回前检查会话,验证文本未变且实际选区等于请求,即使关闭完整回读也不能省略验证。失败通过 native `isError` / JS 异常传出;结果刷新使用 read-only recovery,不通过激活、键盘或真实指针补救。- `click` / `set_value` 在执行真实动作前后,会额外驱动一层透明 `SoftwareCursorOverlay` window:两者的移动阶段现在共用一条 heading-driven 的官方风格 motion 内核,显式把“当前 cursor 朝向”和“最终 resting pose”一起喂给选路器,优先生成需要时先掉头、再沿车头方向推进的 C 形/单侧大弧轨迹;首次显示时按官方 binary 的 fresh state 从 AppKit 全局 `(0,0)` window origin 生成起点,后续动作继续复用上一帧 visible tip。真正显示出来的 cursor 不再直接等于 path sample,而是经过一层独立的 visual dynamics 状态,把 visible tip、velocity、angle 和 fog/offset 持续推进。`click` 结尾会衔接 click pulse 和更明显但仍然很小的 rotate wobble,`set_value` 则只做 settle / idle,不给 pulse;两者收尾后会在目标点继续保持 idle 状态,等待下一次动作时 tip 保持 anchored、只保留可感知的小角度摆动;只有连续 30 秒没有新动作时才做 cleanup,这样连续 tool call 不会反复从 fresh `(0,0)` 起步;如果宿主在任务 / turn 结束时发出 `turn-ended`,cursor 会立即消失并清掉本轮位置状态。- overlay 用目标点相对窗口的锚点跟随窗口移动;行进、settle、点击反馈和 idle 都读取最新缓存帧,发现位置变化时直接重定位可见 tip 并清掉旧速度。窗口帧在后台串行读取,最多 30 Hz、同时最多一个请求,避免 AX 阻塞渲染线程;优先读取快照持有的 AX 窗口,仅在没有 AX 元素时使用窗口列表,读取失败保留最后有效帧。只保留当前窗口的追踪状态,reset / idle 超时后释放;这层跟随只调整显示位置,输入投递仍走既有路径。@@ -127,7 +128,7 @@- 这 9 个 tool 的协议面与 macOS 主线保持一致:`list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value`。其中 element-targeted action 会优先复用上一轮 `get_app_state` 的 runtime id / automation metadata,coordinate action 使用 screenshot/window-relative 坐标。- Windows `click_method=accessibility` 映射到 UI Automation pattern,`app_post` 映射到 HWND `PostMessage`;macOS-only 的 `sky_click` 和没有实现的 `global` 都会在 snapshot lookup 前明确返回 unsupported。`type_text` / `press_key` 的 `key_method=sky_key` 同样在 snapshot lookup 前返回 unsupported。`auto` 仍保持 UIA 优先、window message fallback 的现有行为。- Windows UI Automation 需要运行在已登录用户的桌面 session 里。通过 SSH 作为脱离桌面的后台进程运行时,PowerShell 可以启动并返回 JSON,但系统可能不给它暴露顶层窗口;这种情况下 `list_apps` 会是空,`get_app_state` 可能返回 `appNotFound(...)`。-- 当前 Windows 侧仍是功能性第一版:没有 visual cursor overlay、没有 installer/onboarding、没有 code signing,也没有独立的 Windows smoke fixture。后续 TODO 记录在 `docs/exec-plans/active/20260422-windows-computer-use-runtime.md`。+- Windows 已有独立 WPF 观察 fixture 和 Windows PR smoke,覆盖观察通道及截图边界。其余工具 / toolkit 覆盖,以及 visual cursor overlay、installer/onboarding、code signing 仍待补齐。后续 TODO 记录在 `docs/exec-plans/active/20260422-windows-computer-use-runtime.md`。### 7. Linux Runtimediff --git a/docs/CICD.md b/docs/CICD.mdindex 589239c..3638317 100644--- a/docs/CICD.md+++ b/docs/CICD.md@@ -19,6 +19,10 @@所有 GitHub Actions 都已经 pin 到 commit SHA。后续升级 action 时,也要继续保持这个约束。+## Windows 观察回归++`.github/workflows/windows-observation.yml` 在相关 PR 上运行 Windows Go 测试、PowerShell 生产采集函数测试,以及独立 WPF / native MCP smoke,也支持手动触发。工作流仅需 `contents: read`,沿用固定 SHA 的 checkout / setup-node / setup-go,不发布制品。桌面 smoke 使用自己的窗口,验证采集开关、索引与 UIA 设值,以及正常 / 1x1 / 最小化 / cloaked 窗口的截图边界。复现命令见 [按需观察](selective-observation.md)。+## 推荐接入顺序1. 保留 `ci.yml`,作为仓库的基础门禁。diff --git a/docs/QUALITY_SCORE.md b/docs/QUALITY_SCORE.mdindex 407622c..7002660 100644--- a/docs/QUALITY_SCORE.md+++ b/docs/QUALITY_SCORE.md@@ -12,7 +12,7 @@| 区域 | 评分 | 原因 | 下一步 || --- | --- | --- | --- || 产品面 | B | 已经有 Swift 本地 `computer-use` MCP server、默认 app 模式权限引导,以及共享的 9 个核心 tools;macOS 另有 query、select_text,JS 可调用。 | 继续收敛复杂 AX 场景下的 state rendering 细节、权限 UI 和更清晰的用户错误提示。 |-| Windows runtime | C | 已新增独立 Go `.exe`,通过 Windows UI Automation + Win32 window message 暴露同样 9 个 tools、MCP server 和 `call --calls`;默认不再自动启动 app、执行 `SetFocus`,或让 `type_text` 走可能抢前台的 UIA text fallback;截图会过滤隐藏、最小化、cloaked、离屏和退化窗口,并在 Go 层复核 PNG,但整体仍是功能性第一版。 | 把当前实机 screenshot 边界用例收敛成仓库内 Windows fixture / 交互式 smoke,再补 installer/signing,以及更原生的 Go UIA 实现或更稳定的 bridge。 |+| Windows runtime | C | 已新增独立 Go `.exe`,通过 Windows UI Automation + Win32 window message 暴露同样 9 个 tools、MCP server 和 `call --calls`;默认不再自动启动 app、执行 `SetFocus`,或让 `type_text` 走可能抢前台的 UIA text fallback;截图会过滤隐藏、最小化、cloaked、离屏和退化窗口,并在 Go 层复核 PNG,但整体仍是功能性第一版。 | 维护 WPF 观察 / 截图边界 fixture 和 Windows PR smoke,扩展更多工具与 toolkit;补 installer/signing,以及更原生的 Go UIA 实现或更稳定的 bridge。 || Linux runtime | C | 已新增独立 Go binary,通过 Python GI / AT-SPI2 暴露同样 9 个 tools、MCP server 和 `call --calls`;Ubuntu GNOME VM 已跑通 `list_apps`、MCP tools list 和 Text Editor 8-tool sequence,并已接入 npm bundled artifact 分发,但截图在 GNOME Wayland 下仍只能 best-effort,coordinate input 也不是通用后台模型。 | 扩展已有的隔离 GTK 动作回读 smoke,补完整工具覆盖、portal/compositor screenshot 路径,以及更原生的 Go D-Bus/libatspi bridge。 || macOS Locked Use | D | 已有签名 Broker / 插件、双进程保护、原生租约与恢复入口,离线和遮罩 rehearsal 有回归;安装恢复和无许可双保护释放已实测,独立无遮罩 Return 解锁 / 重锁已实测,受保护自动解锁 / AX / SCK / 重锁及隔离 legacy Keychain 已有一轮实测通过,完整会话认证状态 / 故障链路尚未通过实测;正常启用改为管理员 enable 与签名校验,普通 enable / 正常 MCP 的完整 GUI 闭环已实测。 | 物理覆盖已确认,继续完成 Data Protection / 会话认证状态、Secure Input / 崩溃 / 显示器故障回归及正式发布制品检查。 || 架构文档 | B | 顶层结构、fixture bridge、app 模式和验证路径已经落文档。 | 后续补 release artifact、code signing / notarization 和 host 集成方式。 |diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.mdindex 3bf1844..06cc2ad 100644--- a/docs/RELIABILITY.md+++ b/docs/RELIABILITY.md@@ -19,7 +19,9 @@- macOS 其他 Desktop 实机回归(需要第二个 Desktop,测试会短暂切换 Space 启动目标):`OPEN_COMPUTER_USE_RUN_CROSS_SPACE_LIVE_TEST=1 swift test --filter CrossSpaceLiveTests`- Linux runtime:`(cd apps/OpenComputerUseLinux && go test ./...)`、`./scripts/build-open-computer-use-linux.sh --arch arm64`- Linux 隐藏控件实测:`bash scripts/run-linux-read-back-smoke.sh /path/to/linux/open-computer-use --hidden-checks`,在私有 D-Bus / Xvfb 中验证真实 GTK 隐藏状态、属性读取、索引及页面切换;依赖与范围见 [测试说明](linux-hidden-details.md)。+- Linux 按需观察实测:`bash scripts/run-linux-read-back-smoke.sh /path/to/linux/open-computer-use --observation-checks`,验证真实 GTK 采集通道、native MCP / Worker JS 输出、旧索引拒绝与恢复;另外需要 Node.js,见 [测试说明](selective-observation.md)。- Windows runtime:`(cd apps/OpenComputerUseWindows && go test ./...)`、`./scripts/build-open-computer-use-windows.sh --arch amd64`;截图相关变更还要在 Windows 交互桌面验证正常窗口有 image、1x1/minimized/cloaked 窗口无 image。+- Windows 观察实测:`pwsh -File scripts/tests/windows-observation-smoke.ps1 -NativePath /path/to/open-computer-use.exe`,使用独立 WPF 窗口走真实 UIA / native MCP,验证采集开关、索引与截图边界;专用 PR 工作流见 [CI 说明](CICD.md)。- 本地诊断:- `open-computer-use doctor`- `open-computer-use snapshot <app>`diff --git a/docs/exec-plans/completed/20261017-pr101-selective-observation.md b/docs/exec-plans/completed/20261017-pr101-selective-observation.mdnew file mode 100644index 0000000..cc440bb--- /dev/null+++ b/docs/exec-plans/completed/20261017-pr101-selective-observation.md@@ -0,0 +1,42 @@+# PR #101: cross-platform selective observation++## Goal++Integrate current main, review and fix selective AX/screenshot collection, extend it to macOS and Windows at the user's request, verify through CLI/native MCP/current JS, and push follow-up commits to the original PR. Do not merge.++## Scope and decisions++- Both channels default to true; at least one is required. Existing calls and action read-back behavior stay compatible.+- AX-only skips capture/encoding; screenshot-only skips tree/focus/selection and invalidates old snapshot aliases, including PID replacement and macOS query indexes. Fresh AX restores element actions.+- AX-only cannot validate screenshot coordinates: require a fresh image observation before coordinate click/drag.+- Use disposable AppKit, GTK and WPF fixtures. macOS follows normal signing/session/permission admission; Linux uses private Xvfb/D-Bus on the existing container engine; Windows uses the hosted Actions desktop. No personal apps or local desktop VM.+- Automatic unchanged-image suppression remains separate. It needs exact image comparison, first/reset/forced output and session/app/window scope; unchanged AX does not establish unchanged pixels.++## Progress++- [x] Merge main and retain existing bounded-focus, hidden-node and read-back behavior.+- [x] Add macOS collection gates, schema/dispatcher/app-agent/CLI support and scoped cache/query invalidation.+- [x] Add Windows Go/PowerShell collection gates, strict options, cache behavior and protocol tests.+- [x] Expose matching snapshot flags on all platforms and reject AX-only coordinate input.+- [x] Verify normal signed macOS release and real GTK through CLI/native MCP/Worker JS; run relevant suites/builds.+- [x] Complete hosted real Windows WPF/UIA/MCP/CLI/Worker JS and screenshot-boundary regression.+- [x] Synchronize final docs/history/PR, remotes and original PR branch; leave PR open.++## Validation and findings++- Main through `7cb8bb7` integrated, preserving the original contribution.+- Older aliases after PID replacement initially retained stale indexes; tests now cover invalidation for both previous/new PIDs and preserve unrelated applications.+- macOS CLI originally printed only its first text block; it now retains omission notes too.+- Swift: 382 tests, 8 skipped, zero failures; selective-observation 8/8. Existing native tool and cursor smoke passed.+- Normally signed macOS release + real AppKit AX/native MCP/CLI/Worker JS passed valid PNG, trace-confirmed collection gates, snapshot/query invalidation, coordinate refusal, real value update and recovery. Fixture cannot become key/main.+- Linux: Go 31, Python 35 and real GTK/AT-SPI/Xvfb passed collection helper gates, PNG, Unicode selection, old name/PID index refusal, coordinate actions, AX recovery and CLI/Worker JS. Existing focus and hidden-detail checks passed.+- Node: 44 passed. Windows Go: 29 passed. Production PowerShell function tests passed under PowerShell 7 and actual Windows PowerShell 5.1 with instrumented dependencies.+- Linux/Windows ARM64/AMD64 builds and targeted vet passed; syntax/docs/whitespace/action-pinning checks passed.+- WinForms fixture did not expose a settable ValuePattern; the positive UIA-write fixture was changed to WPF instead of broadening production input fallbacks. The real WPF/UIA/CLI/native MCP/Worker JS desktop smoke passed on Windows 2025.+- Temporary Linux containers and build artifacts removed. No Wayland/other-toolkit performance claim or version bump.++## Outcome++All three platforms now provide the same explicit observation choices through CLI/native MCP/JS. Real desktop tests passed for AppKit, GTK and WPF; the Windows job additionally verified exact 1x1 Win32, minimized and DWM-cloaked screenshot suppression and image recovery. WPF DPI layout is avoided only for the exact one-pixel fixture; production screenshot thresholds and input fallbacks were unchanged.++Windows desktop evidence: [Windows observation run](https://github.com/iFurySt/open-computer-use/actions/runs/39018965035) at `ae8ec2f`. Final follow-up commits are pushed to #101; the PR remains open for the maintainer to merge. Automatic image deduplication remains in the debt tracker.diff --git a/docs/exec-plans/tech-debt-tracker.md b/docs/exec-plans/tech-debt-tracker.mdindex a1055be..c3e424c 100644--- a/docs/exec-plans/tech-debt-tracker.md+++ b/docs/exec-plans/tech-debt-tracker.md@@ -4,4 +4,5 @@| 日期 | 区域 | 债务描述 | 为什么会存在 | 计划中的后续动作 || --- | --- | --- | --- | --- |+| 2026-10-17 | 重复截图 | 请求截图时仍会返回未变化的画面。 | #101 已统一三个平台的显式采集开关;相同图像省略是独立能力。 | 评估按会话 / 应用 / 窗口隔离的精确图像去重;首帧、重置与强制返回必须明确,不能用 AX 未变推断画面未变,也不把去重当成跳过截图采集。 || 2026-04-17 | 普通 app AX snapshot | Finder 路径已经能拿到前台窗口子树并输出 window-relative frame,但当前还缺更多真实 app 回归样本,无法证明这套 rooting / traversal 对复杂 app 都稳定。 | 这一轮先把 Finder 这类真实 app 的坐标换算和窗口子树收敛好,再把 deterministic 回归继续留给 fixture。 | 增加 Safari / System Settings / Activity Monitor 等真实 app 样本验证,并继续收敛 `kAXMainWindowAttribute`、focused element parent chain 和多窗口回退策略。 |diff --git a/docs/histories/2026-10/20261015-2023-linux-selective-observation.md b/docs/histories/2026-10/20261015-2023-linux-selective-observation.mdnew file mode 100644index 0000000..967ef8e--- /dev/null+++ b/docs/histories/2026-10/20261015-2023-linux-selective-observation.md@@ -0,0 +1,66 @@+## [2026-10-15 20:23] | Task: Add opt-in selective Linux observation++### Execution Context+- Agent: Codex+- Runtime: Linux development environment++### User Request+Offer cheaper explicit observation as a compatibility-preserving option, following the existing lightweight app binding approach.++### Changes+- Add optional Linux `get_app_state` booleans `include_ax` and `include_screenshot`, both defaulting to true.+- Skip AX tree, focus and selection collection for screenshot-only reads; skip capture/PNG encoding for AX-only reads.+- Forward optional REPL `includeAX` and `includeScreenshot` without changing legacy calls.+- Explicitly invalidate prior AX indexes and their aliases after screenshot-only reads, retain current window geometry for coordinate actions, and report the invalidation.+- Add Python collection-path tests, Go protocol/cache/schema tests, Node compatibility tests, usage documentation and release notes.++### Design Intent+Changing `getScreenshot()` defaults would also change its implicit index refresh. Keep existing behavior and make selective observation opt-in. A screenshot cannot validate old indexes, including after a window/process switch, so clear them rather than guessing from matching titles. A fresh AX observation restores indexes.++### Validation+- Linux Go suite and Python bridge unit tests.+- Node REPL regression tests, including unchanged legacy request arguments.+- REPL tests: 23 passed. Full Node suite: 36/37 passed; the staged npm packaging test requires macOS `lipo` and fails identically on the unmodified base checkout on Linux.+- Documentation and whitespace checks.+- No live desktop timing, macOS runtime or Windows runtime validation is claimed. This change is independent of the action read-back fix and does not change input delivery or bridge process lifetime.++### Files Modified+- `apps/OpenComputerUseLinux/main.go`+- `apps/OpenComputerUseLinux/main_test.go`+- `apps/OpenComputerUseLinux/runtime.py`+- `apps/OpenComputerUseLinux/runtime_test.py`+- `scripts/node-repl/open-computer-use-repl.mjs`+- `scripts/node-repl/open-computer-use-repl.test.mjs`+- `skills/open-computer-use/references/usage.md`+- `docs/ARCHITECTURE.md`+- `docs/releases/feature-release-notes.md`++## 2026-10-17:同步主线、修复别名遗漏并补实测++用户要求处理 #101,基于最新 main 解决冲突、检查测试并推送,补真实 GTK;同时评估截图可选与未变化图片重复返回的关系。++- 合入主线 `7cb8bb7`(含 #103),保留原贡献提交及两边的 Go / Python 测试、架构和发布记录。将条件 AX 采集接入现有一次有界 focus_details,保留焦点预算提示及隐藏节点选项。+- 发现只取截图时,应用 PID 改变后较早观察的查询别名可能残留。新增独立测试先复现失败,再清理新旧 PID 对应的全部观察别名,同时保留其他应用缓存。+- JS 在两种观察通道同时关闭时提前拒绝,不产生 native 请求;Persistent / Worker 两种会话均测试参数、错误及旧调用兼容性。+- 扩展 `--observation-checks`:在真实 GTK 上包装生产采集函数,确认 AX-only 没有截图、image-only 没有树或焦点读取,完整模式同时采集;检查真实 PNG、中文 / emoji 选区、名称 / PID 别名下旧索引拒绝、坐标点击及 AX 刷新后的设值。独立 GTK 状态确认结果。+- 新增 Worker JS → native MCP → GTK smoke,验证选择通道、图片输出、旧索引异常、AX 恢复及默认完整观察。环境为现有引擎上的临时 Debian 12 ARM64 容器、私有 Xvfb / D-Bus、GTK 3 / AT-SPI 2 和 Node 24,不启动桌面 VM。++验证:Python 35 项、Linux Go 30 项、Node 44 项通过;Linux ARM64 / AMD64 构建、targeted vet、shell / Python / Node 语法检查通过;按需观察真实链路、原有默认动作回读、焦点 / Unicode 选区和隐藏详情 smoke 均通过。文档与 whitespace 检查通过。未运行 Wayland 或其他 toolkit,不宣称全任务提速,无版本 bump。++需求评估:#101 提供 Linux 显式通道选择,不做截图去重,也未实现其他平台的通道开关。现有 macOS / Linux JS 动作已禁用自动回读,getAXState 不输出图片,但仍可能浪费 native 采集。未变化图片省略通常仍需截图后比较,主要节省传输和模型输入;后续的跨平台支持、会话 / 应用 / 窗口隔离、首帧 / 重置 / 强制返回和精确图像比较已记录到技术债追踪,不以 AX 未变作为画面未变的证据。++验证结束后已删除临时容器和独立构建制品,未影响其他容器服务。++## 2026-10-17:按用户要求统一三个平台++用户明确要求扩展 macOS / Windows,而不只实现 Linux。本阶段取代前文的 Linux-only 范围;自动图片去重仍单独跟踪。++- macOS 新增观察选项、严格布尔校验、schema / dispatcher / app-agent CLI 转发,截图关闭时跳过捕获和编码,AX 关闭时跳过树、选区、菜单及懒加载重试。fixture 同步行为。纯截图清理旧进程快照别名和 query 索引,旧别名不能触发隐式刷新继续使用旧编号。+- Windows Go / PowerShell 支持同样的参数、默认和空观察拒绝;按通道跳过生产采集函数,防御性清理结果中的禁用通道及旧缓存。新增纯传输测试接口,不改变生产 Windows / PowerShell 宿主限制。+- 三个平台的 snapshot CLI 新增 --include-ax / --include-screenshot true|false。只取 AX 后拒绝截图坐标输入,保留元素操作。macOS CLI 改为打印所有文本说明但不输出图片字节,修复实测发现的省略原因丢失。+- macOS 正常签名 release + 自有 AppKit 窗口实测通过:CLI / MCP / Worker JS、PNG、timing 分支证据、普通 / query 索引失效、坐标拒绝与 AX 恢复;窗口不能成为 key/main,未修改系统准入或权限。+- 本地检查:Swift 382 项(8 跳过、0 失败),观察专项 8 项;Linux Go 31 项、Python 35 项、Node 44 项;Windows Go 29 项与生产 PowerShell 函数分支测试通过;Linux / Windows 双架构构建及 targeted vet 通过。真实 GTK 观察、焦点、隐藏详情及 macOS 原生工具 / cursor smoke 通过。+- 增加专门的 Windows Actions 检查与临时 WPF 夹具,真实 Windows 2025 桌面上的 Go / Windows PowerShell 5.1 / native MCP / CLI / Worker JS 全部通过。覆盖三个通道模式、旧索引拒绝、AX 恢复、真实 Unicode 设值和坐标拒绝;正常窗口返回 PNG,物理 1×1 Win32 小窗口、最小化及 DWM-cloaked 窗口不返回图片,恢复后重新返回 PNG。工作流使用仓库既有固定 SHA actions 和 contents:read,无发布步骤。+- 初版 WinForms 控件不提供可写 ValuePattern,正向设值测试改用标准 WPF TextBox,没有扩大生产输入 fallback。夹具状态文件用无 BOM UTF-8 原子写入,避免 Windows PowerShell 空字符串与 null 的文件替换差异。精确 1×1 用同进程 Win32 窗口,避开 WPF 高 DPI 最小尺寸取整。+- Windows [完整实测记录](https://github.com/iFurySt/open-computer-use/actions/runs/39018965035) 通过于 `ae8ec2f`。覆盖 AppKit / GTK / WPF,不宣称所有 toolkit、Wayland 或完整任务性能均已验证。+- 文档同步为三平台使用方式、可复现测试命令、Windows CI 和当前覆盖范围;原 execution plan 完成归档,PR 保持开放,未执行合并。diff --git a/docs/linux-selective-observation.md b/docs/linux-selective-observation.mdnew file mode 100644index 0000000..931b6be--- /dev/null+++ b/docs/linux-selective-observation.md@@ -0,0 +1,3 @@+# Linux 按需观察++现已扩展到 macOS、Windows 和 Linux,完整参数、CLI / MCP 用法及测试见[按需观察](selective-observation.md)。diff --git a/docs/references/js-repl.md b/docs/references/js-repl.mdindex a4259c9..46c3aed 100644--- a/docs/references/js-repl.md+++ b/docs/references/js-repl.md@@ -54,6 +54,15 @@ typeText(text, options?)performSecondaryAction(elementIndex, action)```+On macOS, Windows and Linux, observation methods also accept optional `includeAX` and+`includeScreenshot` booleans, both defaulting to true. Use+`app.getAXState({includeScreenshot:false})` to skip native capture, or+`app.getScreenshot({includeAX:false})` to skip the AX/focus walk. The latter+invalidates previous element indexes; read AX again before element actions.+Disabling both channels is rejected before a native call. These options do not+deduplicate unchanged images. See [selective observation](../selective-observation.md)+for collection semantics and real GTK/Worker validation.+On macOS, `selectText` selects exact text inside a text element. `prefix` and `suffix`match the immediately adjacent text to disambiguate repeated occurrences. `selection`is `text` (default), `cursor_before` or `cursor_after`. Snapshot and `query` indexesdiff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.mdindex c63ff06..cfa1d00 100644--- a/docs/releases/feature-release-notes.md+++ b/docs/releases/feature-release-notes.md@@ -13,6 +13,7 @@| 2026-10-15 | Linux 动作回读 | REPL 批量操作可省去每个动作后被丢弃的完整界面观察。 | Linux 开始支持现有的 `OPEN_COMPUTER_USE_ACTION_READ_BACK=0`,成功动作返回 `ok`,保留最后一次显式观察的索引缓存;默认原生 MCP 回读与显式 `get_app_state` 保持不变。 || 2026-10-15 | Linux 焦点与选区 | 大型界面减少重复查找,预算不足时明确提示,选中文字能正确返回。 | 复用快照已选窗口,一次查找生成焦点摘要与选区;按节点和深度预算停止读取,使用标准 AT-SPI 选区接口,并补隔离 GTK 验证。 || 2026-10-15 | Linux 隐藏控件详情 | 隐藏节点较多时,可减少不需要的属性读取。 | 新增默认关闭的 `OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN`;保留节点、索引和可见子节点,状态未知时完整读取,补充真实 GTK 标签页及 native MCP 回归。详见 [使用与验证](../linux-hidden-details.md)。 |+| 2026-10-15 | 按需观察 | 只需控件信息时可省去截图,只需画面时可省去 AX 遍历。 | 统一 macOS / Windows / Linux 的 include_ax / include_screenshot、JS 和 CLI snapshot 参数,默认完整采集;纯截图清理旧索引,只取 AX 拒绝截图坐标输入,补真实 AppKit / GTK / WPF、Windows 截图边界与 PowerShell 分支验证。详见 [使用与验证](../selective-observation.md)。 || 2026-10-14 | 1.1.0 正式发布 | 用户可直接安装 Locked Use,在锁屏遮罩期间继续 GUI 操作,也可用定向查询和持久截图配置减少工具开销。 | 统一版本为 1.1.0;签名 universal macOS App 内嵌服务、Guardian、安装器和授权插件,普通 enable 请求权限,补齐冷启动握手隔离和发布制品检查;新增 app.query()、doctor --json 与 ocu config。 || 2026-10-10 | macOS Locked Use 验证版 | 可在已登录 Mac 锁屏时,通过全屏遮罩继续 GUI 自动化,并在结束或本地接管后安全重锁。 | 正式组件采用 OCU Guardian / OCULockService / OCULockInstaller / OCULockAuth 短名称,开发 App 保留 Dev 区分;新增签名 Broker / 授权插件、双进程显示器保护、一次性许可、租约排空 / 恢复及构建和实机验证脚本;管理员显式启用 validation profile,生产 backend 在完整验证前保持关闭。 || 2026-10-10 | macOS 定向 AX 查询 | 知道控件文案时,可以直接查询并操作,不必先获取截图和完整 AX tree。 | 新增 native `query` 和 JS `app.query()`,按文字、角色和窗口查找元素,返回完整性元数据;查询索引仅在同一会话内有效,操作前重新验证目标和位置。 |diff --git a/docs/selective-observation.md b/docs/selective-observation.mdnew file mode 100644index 0000000..4b834a6--- /dev/null+++ b/docs/selective-observation.md@@ -0,0 +1,82 @@+# 按需观察++macOS、Windows、Linux 的 `get_app_state` 都可以选择读取 AX 或截图。原生参数 `include_ax` / `include_screenshot` 对应 JS 的 `includeAX` / `includeScreenshot`,均默认 `true`,不能同时设为 `false`。++| 需要什么 | JS 调用 | 实际采集 |+| --- | --- | --- |+| 只读控件和选区 | `await app.getAXState({includeScreenshot: false})` | AX、焦点、选区;不截图、不编码 |+| 只看画面 | `await app.getScreenshot({includeAX: false})` | 截图;不遍历 AX、不查焦点和选区 |+| 两者都要 | `await app.getAXStateAndScreenshot()` | 完整 AX 和截图 |++```js+var app = await cua.getApp("gedit", {initialState: false});+await app.getAXState({includeScreenshot: false});+```++只取截图后,旧元素索引及查询别名会失效,包括应用重启前的索引。坐标操作仍可使用本次观察的窗口位置;元素操作要先重新读取 AX。窗口标题相同不代表旧索引仍然有效。++不传参数的调用保持原行为。只取 AX 后可继续操作元素,但截图坐标点击 / 拖拽要先重新获取截图;只取截图后则要重新读取 AX 才能使用元素索引。macOS 的 query 索引也会随纯截图观察失效。++## CLI 和 MCP++JS 的 `ocu js` / `ocu repl` 和 MCP `js` 使用上面的调用。直接调用原生 CLI / MCP 时使用 snake_case:++```sh+ocu call get_app_state --args '{"app":"TextEdit","include_screenshot":false}'+ocu snapshot --include-screenshot false TextEdit+ocu snapshot --include-ax false TextEdit+```++原生 MCP `get_app_state` 接收同样的 `include_ax` / `include_screenshot` 布尔值。`snapshot` 两个参数都接受 `true` / `false`,不能同时关闭。`snapshot` 一直只打印文本;需要图片内容时使用 JS 或 `call` / MCP。++这两个选项只控制本次显式观察,不改变动作回读。++## 和重复截图的关系++macOS / Linux 下的 JS 动作已默认关闭自动回读;`getAXState()` 本身也不向模型输出图片,但默认仍会在 native 层采集图片后丢弃。`includeScreenshot: false` 可以在三个平台连这部分采集和编码一起省掉。直接使用 native MCP 时,它也会省去返回的 image block。++这里不做截图比较:请求了截图,即使画面相同也照常返回。“画面未变则省略图片”需要单独实现,通常仍要先截图再比较,主要减少传输和模型输入。后续可按会话、应用和窗口保存精确图像指纹,首次观察及状态重置时返回图片,并提供强制返回方式;不能仅凭 AX 没变化就认定画面没变化。++## 测试++```sh+swift test+python3 apps/OpenComputerUseLinux/runtime_test.py+(cd apps/OpenComputerUseLinux && go test ./...)+(cd apps/OpenComputerUseWindows && go test ./...)+node --test scripts/node-repl/*.test.mjs+```++真实 GTK 测试复用 [私有 D-Bus / Xvfb 环境](linux-action-read-back.md),另外需要 Node.js:++```sh+(cd apps/OpenComputerUseLinux && go build -o /tmp/ocu-observation-test .)+bash scripts/run-linux-read-back-smoke.sh /tmp/ocu-observation-test --observation-checks+```++脚本包装真实生产采集函数,确认关闭的通道没有被调用;再通过 native MCP 和 Worker JS 检查 PNG、选区、旧索引拒绝、坐标操作及 AX 恢复。GTK 独立导出状态确认动作结果。这是 X11 / GTK / AT-SPI 测试,不代表 Wayland 或完整任务性能测试。++macOS 使用正常签名且已授权的 OCU 可执行文件验证真实 AppKit 窗口,无需修改准入配置:++```sh+node scripts/tests/macos-observation-smoke.mjs "/path/to/Open Computer Use.app/Contents/MacOS/OpenComputerUse"+```++它通过真实 AX / native MCP / CLI / Worker JS 验证三个采集模式、PNG、快照和 query 索引失效、坐标拒绝和元素恢复,并用现有 timing 日志确认跳过的采集分支未执行。窗口禁止成为 key/main window;脚本不向个人应用输入。++Windows 的 PowerShell 采集分支可用以下命令单独验证:++```powershell+pwsh -NoProfile -File scripts/tests/windows-observation.test.ps1 -RuntimePath apps/OpenComputerUseWindows/runtime.ps1+```++这个测试解析生产脚本并执行其中的观察函数,使用带计数的 UIA / 截图依赖验证分支。它不等于 Windows 桌面实测。++Windows 桌面测试需要已登录的交互会话、Windows PowerShell 和 Node.js:++```powershell+go -C apps/OpenComputerUseWindows build -o "$env:TEMP/ocu-observation.exe" .+pwsh -File scripts/tests/windows-observation-smoke.ps1 -NativePath "$env:TEMP/ocu-observation.exe"+```++脚本创建自己的 WPF 窗口,通过真实 UIA / native MCP / CLI / Worker JS 检查通道选择、旧索引拒绝、AX 恢复、Unicode 设值和截图坐标拒绝;再用同进程的 Win32 小窗口验证物理 1×1 边界,并检查 WPF 最小化、cloaked 时不返回图片,恢复后重新返回 PNG。`.github/workflows/windows-observation.yml` 在 Windows runner 执行这组检查。WPF 的通过结果不代表所有 WinForms / WinUI / Electron 控件都支持相同 UIA pattern。diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swiftindex 3a3db1e..2fbfc30 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift@@ -148,6 +148,7 @@ public struct AppSnapshot {let elements: [Int: ElementRecord]var screenshotNote: String? = nil+ var observation: SnapshotObservationOptions = .defaultspublic var renderedText: String {renderedText(style: .fullState)@@ -161,6 +162,9 @@ public struct AppSnapshot {lines.append("App=\(appReference) (pid \(app.pid))")lines.append("Window: \(quoted(displayTitle)), App: \(app.name).")lines.append(contentsOf: treeLines)+ if !observation.includeAX {+ lines.append(SnapshotObservationOptions.accessibilityOmittedNote)+ }if let selectedText, !selectedText.isEmpty {lines.append("")@@ -184,10 +188,12 @@ enum SnapshotBuilder {for app: RunningAppDescriptor,textLimit: SnapshotTextLimit = .defaults,treeLimits: AccessibilityTreeLimits = .defaults,- recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation+ recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation,+ observation: SnapshotObservationOptions = .defaults) throws -> AppSnapshot {+ try observation.validate()if app.name == FixtureBridge.appName, let fixtureState = try FixtureBridge.readState() {- return buildFixtureSnapshot(app: app, state: fixtureState)+ return buildFixtureSnapshot(app: app, state: fixtureState, observation: observation)}try requireUsableComputerUseSession()@@ -198,8 +204,8 @@ enum SnapshotBuilder {}let appElement = AXUIElementCreateApplication(app.pid)- let lazyWebAccessibility = enableBestEffortAccessibilityModes(appElement)- || appHasLazyWebAccessibility(bundleURL: app.runningApplication.bundleURL)+ let lazyWebAccessibility = observation.includeAX && (enableBestEffortAccessibilityModes(appElement)+ || appHasLazyWebAccessibility(bundleURL: app.runningApplication.bundleURL))let systemWide = AXUIElementCreateSystemWide()var focusedApplication = copyElement(systemWide, attribute: kAXFocusedApplicationAttribute)var focusedWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)@@ -221,8 +227,8 @@ enum SnapshotBuilder {// the tree, the capture and every element frame refer to the same window;// the title/area heuristics remain the fallback.var windowCapture = TimingLog.measure("snapshot.window_capture") {- WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: rootWindow))- ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle)+ WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: rootWindow), capture: observation.includeScreenshot)+ ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle, capture: observation.includeScreenshot)}if windowCapture == nil,recoveryPolicy == .allowActivation,@@ -231,8 +237,8 @@ enum SnapshotBuilder {if let recoveredWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) {rootWindow = recoveredWindowwindowTitle = stringValue(of: recoveredWindow, attribute: kAXTitleAttribute)- windowCapture = WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: recoveredWindow))- ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle)+ windowCapture = WindowCapture.resolve(for: app.pid, exactWindowID: SkyLightSPI.shared.windowID(for: recoveredWindow), capture: observation.includeScreenshot)+ ?? WindowCapture.resolve(for: app.pid, titleHint: windowTitle, capture: observation.includeScreenshot)}}@@ -250,7 +256,8 @@ enum SnapshotBuilder {systemWide: systemWide,textLimit: textLimit,treeLimits: treeLimits,- lazyWebAccessibility: lazyWebAccessibility+ lazyWebAccessibility: lazyWebAccessibility,+ observation: observation)}@@ -264,11 +271,21 @@ enum SnapshotBuilder {systemWide: AXUIElement,textLimit: SnapshotTextLimit,treeLimits: AccessibilityTreeLimits,- lazyWebAccessibility: Bool+ lazyWebAccessibility: Bool,+ observation: SnapshotObservationOptions) -> AppSnapshot {let windowBounds = windowCapture.bounds- let screenshot = windowCapture.screenshotResult()+ let screenshot = observation.includeScreenshot ? windowCapture.screenshotResult()+ : ScreenshotEncodingResult(data: nil, note: SnapshotObservationOptions.screenshotOmittedNote)let screenshotData = screenshot.data+ guard observation.includeAX else {+ return AppSnapshot(app: app, windowTitle: windowTitle, windowBounds: windowBounds,+ targetWindowID: windowCapture.windowID, targetWindowLayer: windowCapture.layer,+ screenshotData: screenshotData, mode: .accessibility, treeLines: [],+ focusedSummary: nil, focusedElement: nil, selectedText: nil,+ windowElement: rootElement, elements: [:], screenshotNote: screenshot.note,+ observation: observation)+ }let focusedElement = preferredFocusedElement(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)let selectedText = focusedElement.flatMap { copySelectedText($0, textLimit: textLimit) }let context = RenderContext(@@ -330,7 +347,8 @@ enum SnapshotBuilder {selectedText: selectedText,windowElement: rootElement,elements: renderer.records,- screenshotNote: screenshot.note+ screenshotNote: screenshot.note,+ observation: observation)}@@ -444,14 +462,15 @@ enum SnapshotBuilder {return copyElement(appElement, attribute: kAXFocusedUIElementAttribute)}- private static func buildFixtureSnapshot(app: RunningAppDescriptor, state: FixtureAppState) -> AppSnapshot {+ static func buildFixtureSnapshot(app: RunningAppDescriptor, state: FixtureAppState,+ observation: SnapshotObservationOptions = .defaults) -> AppSnapshot {var lines: [String] = []var records: [Int: ElementRecord] = [:]let focusedIdentifier = state.focusedIdentifiervar focusedSummary: String?- for element in state.elements.sorted(by: { $0.index < $1.index }) {+ for element in (observation.includeAX ? state.elements.sorted(by: { $0.index < $1.index }) : []) {let titleSegment = element.title.map { " \($0)" } ?? ""let valueSegment = element.value.map { " Value: \($0)" } ?? ""let actionsSegment = element.actions.isEmpty ? "" : " Secondary Actions: \(element.actions.joined(separator: ", "))"@@ -485,9 +504,11 @@ enum SnapshotBuilder {treeLines: lines,focusedSummary: focusedSummary,focusedElement: nil,- selectedText: state.selectedText,+ selectedText: observation.includeAX ? state.selectedText : nil,windowElement: nil,- elements: records+ elements: records,+ screenshotNote: observation.includeScreenshot ? nil : SnapshotObservationOptions.screenshotOmittedNote,+ observation: observation)}}diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swiftindex 76485aa..a74e823 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift@@ -541,6 +541,7 @@ func shouldPreferContainingWebRowAXClickCandidate(public final class ComputerUseService {private var snapshotsByApp: [String: AppSnapshot] = [:]+ private var invalidatedSnapshotAliases: Set<String> = []// Read per call: the app agent applies the caller's OPEN_COMPUTER_USE_* variables// for the duration of each request.private var actionReadBack: Bool { actionReadBackEnabled(environment: ProcessInfo.processInfo.environment) }@@ -593,14 +594,16 @@ public final class ComputerUseService {app query: String,textLimit: SnapshotTextLimit = .defaults,treeLimits: AccessibilityTreeLimits = .defaults,- windowPlacement: WindowPlacement = .keep+ windowPlacement: WindowPlacement = .keep,+ observation: SnapshotObservationOptions = .defaults) throws -> ToolCallResult {+ try observation.validate()guard windowPlacement != .keep else {- return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits), style: .fullState)+ return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, observation: observation), style: .fullState)}// Explicit placements never activate or raise; they only move the frame.- let snapshot = try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly)+ let snapshot = try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly, observation: observation)guard snapshot.mode != .fixture else {throw ComputerUseError.message("window_placement '\(windowPlacement.rawValue)' is not supported for fixture apps")}@@ -616,7 +619,7 @@ public final class ComputerUseService {break}- return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly), style: .fullState)+ return snapshotResult(for: try refreshSnapshot(for: query, textLimit: textLimit, treeLimits: treeLimits, recoveryPolicy: .readOnly, observation: observation), style: .fullState)}public func click(@@ -641,6 +644,9 @@ public final class ComputerUseService {let snapshot = try snapshotForAction(app: query, elementIndex: elementIndex)let button = MouseButtonKind(rawValue: mouseButton.lowercased()) ?? .left+ if elementIndex == nil, !snapshot.observation.includeScreenshot {+ throw ComputerUseError.stateUnavailable(SnapshotObservationOptions.screenshotOmittedNote + "; request a screenshot before screenshot-coordinate actions")+ }if snapshot.mode == .fixture {guard clickMethod == .auto else {throw ComputerUseError.message(@@ -878,6 +884,9 @@ public final class ComputerUseService {public func drag(app query: String, fromX: Double, fromY: Double, toX: Double, toY: Double) throws -> ToolCallResult {let snapshot = try currentSnapshot(for: query)+ guard snapshot.observation.includeScreenshot else {+ throw ComputerUseError.stateUnavailable(SnapshotObservationOptions.screenshotOmittedNote + "; request a screenshot before screenshot-coordinate actions")+ }if snapshot.mode == .fixture {try FixtureBridge.post(FixtureCommand(kind: "drag", identifier: "fixture-drag-pad", x: fromX, y: fromY, toX: toX, toY: toY))Thread.sleep(forTimeInterval: 0.15)@@ -1185,7 +1194,7 @@ public final class ComputerUseService {return dict}- private func currentSnapshot(for query: String) throws -> AppSnapshot {+ func currentSnapshot(for query: String) throws -> AppSnapshot {if let snapshot = snapshotsByApp[query.lowercased()] {if snapshot.mode != .fixture {do {@@ -1198,6 +1207,9 @@ public final class ComputerUseService {return snapshot}+ if invalidatedSnapshotAliases.contains(query.lowercased()) {+ throw ComputerUseError.stateUnavailable(SnapshotObservationOptions.accessibilityOmittedNote)+ }return try refreshSnapshot(for: query)}@@ -1206,27 +1218,47 @@ public final class ComputerUseService {for query: String,textLimit: SnapshotTextLimit = .defaults,treeLimits: AccessibilityTreeLimits = .defaults,- recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation+ recoveryPolicy: SnapshotRecoveryPolicy = .allowActivation,+ observation: SnapshotObservationOptions = .defaults) throws -> AppSnapshot {let app = try AppDiscovery.resolve(query)let snapshot = try SnapshotBuilder.build(for: app,textLimit: textLimit,treeLimits: treeLimits,- recoveryPolicy: recoveryPolicy+ recoveryPolicy: recoveryPolicy,+ observation: observation)+ rememberSnapshot(snapshot, for: query)+ return snapshot+ }++ func rememberSnapshot(_ snapshot: AppSnapshot, for query: String) {+ let app = snapshot.app+ if !snapshot.observation.includeAX {+ var pids: Set<Int32> = [app.pid]+ if let previous = snapshotsByApp[query.lowercased()] { pids.insert(previous.app.pid) }+ let aliases = snapshotsByApp.compactMap { pids.contains($0.value.app.pid) ? $0.key : nil }+ for alias in aliases {+ snapshotsByApp.removeValue(forKey: alias)+ invalidatedSnapshotAliases.insert(alias)+ }+ targetedElements.removeAll(forPIDs: pids)+ }+let keys = Set([query.lowercased(),+ String(app.pid),app.name.lowercased(),(app.bundleIdentifier ?? "").lowercased(),].filter { !$0.isEmpty })for key in keys {snapshotsByApp[key] = snapshot+ invalidatedSnapshotAliases.remove(key)}- return snapshot}private func lookupElement(snapshot: AppSnapshot, index: String) throws -> ElementRecord {diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swiftindex e104de9..cfa984c 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift@@ -67,7 +67,11 @@ public final class ComputerUseToolDispatcher {maxNodeCount: try optionalPositiveInt("max_tree_nodes", in: arguments),maxDepth: try optionalPositiveInt("max_tree_depth", in: arguments)),- windowPlacement: try parseWindowPlacement(optionalString("window_placement", in: arguments))+ windowPlacement: try parseWindowPlacement(optionalString("window_placement", in: arguments)),+ observation: SnapshotObservationOptions(+ includeAX: try strictObservationBool("include_ax", in: arguments),+ includeScreenshot: try strictObservationBool("include_screenshot", in: arguments)+ ))case "click":return try service.click(@@ -154,6 +158,14 @@ public final class ComputerUseToolDispatcher {return value}+ private func strictObservationBool(_ key: String, in arguments: [String: Any]) throws -> Bool {+ guard let value = arguments[key] else { return true }+ guard let number = value as? NSNumber, CFGetTypeID(number) == CFBooleanGetTypeID() else {+ throw ComputerUseError.invalidArguments("\(key) must be a boolean")+ }+ return number.boolValue+ }+private func optionalBool(_ key: String, in arguments: [String: Any]) -> Bool? {switch arguments[key] {case let value as Bool: return valuediff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swiftindex 0eeb6fa..60dd85b 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift@@ -7,7 +7,7 @@ public enum OpenComputerUseCLICommand: Equatable {case lockedUseStatus(json: Bool)case lockedUseManagement(action: String, validation: Bool)case listApps- case snapshot(app: String, textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults)+ case snapshot(app: String, textLimit: SnapshotTextLimit = .defaults, treeLimits: AccessibilityTreeLimits = .defaults, observation: SnapshotObservationOptions = .defaults)case call(OpenComputerUseCallInvocation)case turnEnded(payload: String?)case help(command: String?)@@ -175,12 +175,14 @@ public func openComputerUseHelpText(command: String? = nil) -> String {case "snapshot":return """Usage:- open-computer-use snapshot [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>+ open-computer-use snapshot [--include-ax true|false] [--include-screenshot true|false] [--text-limit <positive-int|max>] [--max-tree-nodes <positive-int>] [--max-tree-depth <positive-int>] <app>Arguments:<app> App name or bundle identifier to inspect.Options:+ --include-ax Include AX/focus/selection (true by default). False invalidates prior indexes.+ --include-screenshot Include screenshot capture (true by default). At least one channel must be true.--text-limit Override the default 500 character text limit. Use `max` for full text.--max-tree-nodes Override the default 1200 node accessibility tree budget.--max-tree-depth Override the default 64 level accessibility tree depth.@@ -301,6 +303,7 @@ private func parseSnapshot(arguments: [String]) throws -> OpenComputerUseCLICommreturn .help(command: "snapshot")}+ var observation = SnapshotObservationOptions.defaultsvar app: String?var textLimit = SnapshotTextLimit.defaultsvar maxTreeNodes: Int?@@ -310,6 +313,13 @@ private func parseSnapshot(arguments: [String]) throws -> OpenComputerUseCLICommwhile index < arguments.count {let argument = arguments[index]switch argument {+ case "--include-ax", "--include-screenshot":+ index += 1+ guard index < arguments.count, let value = Bool(arguments[index].lowercased()) else {+ throw OpenComputerUseCLIError(message: "\(argument) requires true or false", helpCommand: "snapshot")+ }+ if argument == "--include-ax" { observation.includeAX = value }+ else { observation.includeScreenshot = value }case "--text-limit":index += 1guard index < arguments.count else {@@ -348,13 +358,17 @@ private func parseSnapshot(arguments: [String]) throws -> OpenComputerUseCLICommthrow OpenComputerUseCLIError(message: "snapshot requires an app name or bundle identifier", helpCommand: "snapshot")}+ guard observation.includeAX || observation.includeScreenshot else {+ throw OpenComputerUseCLIError(message: "At least one of include_ax or include_screenshot must be true", helpCommand: "snapshot")+ }return .snapshot(app: app,textLimit: textLimit,treeLimits: AccessibilityTreeLimits.defaults.replacing(maxNodeCount: maxTreeNodes,maxDepth: maxTreeDepth- )+ ),+ observation: observation)}diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SnapshotObservationOptions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SnapshotObservationOptions.swiftnew file mode 100644index 0000000..942d826--- /dev/null+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SnapshotObservationOptions.swift@@ -0,0 +1,21 @@+import Foundation++public struct SnapshotObservationOptions: Equatable, Sendable {+ public var includeAX: Bool+ public var includeScreenshot: Bool++ public static let defaults = SnapshotObservationOptions()+ public static let accessibilityOmittedNote = "Accessibility omitted; previous element indexes invalidated. Read AX before element-targeted actions."+ public static let screenshotOmittedNote = "Screenshot omitted by include_screenshot=false"++ public init(includeAX: Bool = true, includeScreenshot: Bool = true) {+ self.includeAX = includeAX+ self.includeScreenshot = includeScreenshot+ }++ public func validate() throws {+ guard includeAX || includeScreenshot else {+ throw ComputerUseError.invalidArguments("At least one of include_ax or include_screenshot must be true")+ }+ }+}diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swiftindex b58381d..c8a6cff 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TargetedQuery.swift@@ -60,6 +60,10 @@ final class TargetedElementRegistry<Value> {}func remove(_ index: Int) { entries.removeValue(forKey: index) }+ func removeAll(forPIDs pids: Set<Int32>) {+ entries = entries.filter { !pids.contains($0.value.app.pid) }+ order.removeAll { entries[$0] == nil }+ }func clear() { entries.removeAll(); order.removeAll() }}diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swiftindex e53da12..d6c0fd3 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift@@ -94,6 +94,8 @@ public enum ToolDefinitions {annotations: defaultAnnotations(),inputSchema: objectSchema(properties: [+ "include_ax": ["type": "boolean", "default": true, "description": "Include accessibility tree, focus and selection. False skips collection and invalidates previous element indexes; read AX again before element actions."],+ "include_screenshot": ["type": "boolean", "default": true, "description": "Capture and return a screenshot. False skips capture and encoding; screenshot-coordinate actions require a fresh screenshot."],"app": stringProperty(description: "App name or bundle identifier"),"text_limit": textLimitProperty(description: "Maximum text characters to return. Use \"max\" for full text. Defaults to 500."),"max_tree_nodes": positiveIntegerProperty(description: "Maximum accessibility tree nodes to render. Defaults to 1200."),diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swiftindex d1f4737..37276e6 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift@@ -43,6 +43,13 @@ public struct ToolCallResult: @unchecked Sendable {content.first(where: { $0.dictionary["type"] as? String == "text" })?.dictionary["text"] as? String}+ /// Plain-text CLI output includes observation notes but never image bytes.+ public var textContent: String {+ content.compactMap { item in+ item.dictionary["type"] as? String == "text" ? item.dictionary["text"] as? String : nil+ }.joined(separator: "\n")+ }+public var asDictionary: [String: Any] {["content": content.map(\.dictionary),diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/SelectiveObservationTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/SelectiveObservationTests.swiftnew file mode 100644index 0000000..e196be7--- /dev/null+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/SelectiveObservationTests.swift@@ -0,0 +1,114 @@+import AppKit+import XCTest+@testable import OpenComputerUseKit++final class SelectiveObservationTests: XCTestCase {+ private func app(pid: Int32 = 42, name: String = "ObservationFixture") -> RunningAppDescriptor {+ .init(name: name, bundleIdentifier: "test.\(name)", pid: pid, runningApplication: .current)+ }++ private func snapshot(pid: Int32 = 42, name: String = "ObservationFixture",+ options: SnapshotObservationOptions = .defaults) -> AppSnapshot {+ let state = FixtureAppState(windowTitle: "test", windowBounds: .init(rect: CGRect(x: 0, y: 0, width: 300, height: 200)),+ focusedIdentifier: "input", elements: [+ .init(identifier: "input", index: 7, role: "text", title: "Input", value: "中文🙂", actions: [],+ frame: .init(rect: CGRect(x: 10, y: 10, width: 100, height: 30)))+ ], selectedText: "中文🙂")+ return SnapshotBuilder.buildFixtureSnapshot(app: app(pid: pid, name: name), state: state, observation: options)+ }++ func testEmptyObservationFailsBeforeResolvingApp() {+ XCTAssertThrowsError(try ComputerUseService().getAppState(app: "missing-test-app",+ observation: .init(includeAX: false, includeScreenshot: false))) {+ XCTAssertTrue(String(describing: $0).contains("At least one"))+ }+ }++ func testDispatcherRequiresActualJSONBooleans() {+ let dispatcher = ComputerUseToolDispatcher()+ for key in ["include_ax", "include_screenshot"] {+ for value: Any in ["false", 0, 1, NSNull(), [], [:]] {+ XCTAssertThrowsError(try dispatcher.callTool(name: "get_app_state", arguments: ["app": "missing-test-app", key: value])) {+ XCTAssertTrue(String(describing: $0).contains("\(key) must be a boolean"))+ }+ }+ }+ }++ func testSchemaHasMatchingFullDefaults() throws {+ let tool = try XCTUnwrap(ToolDefinitions.all.first { $0.name == "get_app_state" })+ let properties = try XCTUnwrap(tool.inputSchema["properties"] as? [String: Any])+ for key in ["include_ax", "include_screenshot"] {+ let value = try XCTUnwrap(properties[key] as? [String: Any])+ XCTAssertEqual(value["type"] as? String, "boolean")+ XCTAssertEqual(value["default"] as? Bool, true)+ }+ }++ func testCLIObservationOptionsAndDefaultCompatibility() throws {+ XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["snapshot", "TextEdit"]), .snapshot(app: "TextEdit"))+ XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["snapshot", "--include-screenshot", "false", "TextEdit"]),+ .snapshot(app: "TextEdit", observation: .init(includeScreenshot: false)))+ XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["snapshot", "TextEdit", "--include-ax", "FALSE"]),+ .snapshot(app: "TextEdit", observation: .init(includeAX: false)))+ for args in [["--include-ax"], ["--include-screenshot", "0"], ["--include-ax", "no"],+ ["--include-ax", "false", "--include-screenshot", "false"]] {+ XCTAssertThrowsError(try parseOpenComputerUseCLI(arguments: ["snapshot", "TextEdit"] + args))+ }+ }++ func testFixtureChannelsPreserveOrOmitAXAndCoordinateGuidance() {+ let full = snapshot()+ XCTAssertEqual(full.selectedText, "中文🙂")+ XCTAssertEqual(full.elements.count, 1)+ let image = snapshot(options: .init(includeAX: false))+ XCTAssertTrue(image.elements.isEmpty)+ XCTAssertTrue(image.treeLines.isEmpty)+ XCTAssertNil(image.focusedSummary)+ XCTAssertNil(image.selectedText)+ XCTAssertTrue(image.renderedText.contains("indexes invalidated"))+ let ax = snapshot(options: .init(includeScreenshot: false))+ XCTAssertEqual(ax.elements.count, 1)+ XCTAssertEqual(ax.selectedText, "中文🙂")+ XCTAssertNil(ax.screenshotData)+ XCTAssertEqual(ax.screenshotNote, SnapshotObservationOptions.screenshotOmittedNote)+ }++ func testScreenshotOnlyInvalidatesOlderAliasesAcrossPIDChangeAndAXRestoresIndexes() throws {+ let service = ComputerUseService()+ service.rememberSnapshot(snapshot(), for: "older-query")+ service.rememberSnapshot(snapshot(), for: "ObservationFixture")+ service.rememberSnapshot(snapshot(pid: 99, name: "Other"), for: "Other")+ service.rememberSnapshot(snapshot(pid: 43, options: .init(includeAX: false)), for: "ObservationFixture")+ XCTAssertThrowsError(try service.currentSnapshot(for: "older-query"))+ XCTAssertThrowsError(try service.currentSnapshot(for: "42"))+ XCTAssertTrue(try service.currentSnapshot(for: "ObservationFixture").elements.isEmpty)+ XCTAssertEqual(try service.currentSnapshot(for: "Other").elements.count, 1)+ service.rememberSnapshot(snapshot(pid: 43, options: .init(includeScreenshot: false)), for: "ObservationFixture")+ XCTAssertEqual(try service.currentSnapshot(for: "ObservationFixture").elements.count, 1)+ }++ func testQueryIndexInvalidationIsScopedAndNeverReissuesIDs() throws {+ let registry = TargetedElementRegistry<String>()+ let first = TargetedAppIdentity(pid: 42, bundleIdentifier: "a", launchedAt: nil)+ let other = TargetedAppIdentity(pid: 99, bundleIdentifier: "b", launchedAt: nil)+ let a = registry.insert(app: first) { _ in "a" }+ let b = registry.insert(app: other) { _ in "b" }+ registry.removeAll(forPIDs: [42])+ XCTAssertThrowsError(try registry.resolve(a, app: first))+ XCTAssertEqual(try registry.resolve(b, app: other), "b")+ XCTAssertGreaterThan(registry.insert(app: first) { _ in "new" }, b)+ }++ func testAXOnlyRejectsCoordinateInputBeforeAnyFixtureCommand() {+ let service = ComputerUseService()+ service.rememberSnapshot(snapshot(options: .init(includeScreenshot: false)), for: "fixture")+ XCTAssertThrowsError(try service.click(app: "fixture", elementIndex: nil, x: 10, y: 10,+ clickCount: 1, mouseButton: "left")) {+ XCTAssertTrue(String(describing: $0).contains("include_screenshot=false"))+ }+ XCTAssertThrowsError(try service.drag(app: "fixture", fromX: 1, fromY: 1, toX: 2, toY: 2)) {+ XCTAssertTrue(String(describing: $0).contains("include_screenshot=false"))+ }+ }+}diff --git a/scripts/node-repl/open-computer-use-repl.mjs b/scripts/node-repl/open-computer-use-repl.mjsindex 89b798c..5e5c780 100755--- a/scripts/node-repl/open-computer-use-repl.mjs+++ b/scripts/node-repl/open-computer-use-repl.mjs@@ -41,6 +41,7 @@ The runtime exposes an asynchronous app-bound API:- \`await app.getAXState({ emit?, textLimit?, maxTreeNodes?, maxTreeDepth?, windowPlacement? })\`- \`await app.getScreenshot({ emit? })\`- \`await app.getAXStateAndScreenshot(options?)\`+- Optional observation on macOS, Windows and Linux: \`app.getScreenshot({ includeAX: false })\` skips AX/focus/selection and invalidates prior element indexes; refresh AX before element actions. \`app.getAXState({ includeScreenshot: false })\` skips capture; screenshot-coordinate actions require a fresh screenshot. Omitted options retain full native observation.- \`await app.click(elementIndexOrPoint, { mouseButton?, clickCount?, clickMethod? })\`- \`await app.scroll(elementIndex, direction, pages?)\`- \`await app.drag([fromX, fromY], [toX, toY])\`@@ -258,6 +259,15 @@ function optionsToSnapshotArgs(options = {}) {if (options.maxTreeNodes !== undefined) args.max_tree_nodes = options.maxTreeNodes;if (options.maxTreeDepth !== undefined) args.max_tree_depth = options.maxTreeDepth;if (options.windowPlacement !== undefined) args.window_placement = options.windowPlacement;+ for (const [option, argument] of [["includeAX", "include_ax"], ["includeScreenshot", "include_screenshot"]]) {+ if (options[option] !== undefined) {+ if (typeof options[option] !== "boolean") throw new Error(`${option} must be a boolean`);+ args[argument] = options[option];+ }+ }+ if (args.include_ax === false && args.include_screenshot === false) {+ throw new Error("At least one of includeAX or includeScreenshot must be true");+ }return args;}diff --git a/scripts/node-repl/open-computer-use-repl.test.mjs b/scripts/node-repl/open-computer-use-repl.test.mjsindex f343bdc..5c7aaad 100644--- a/scripts/node-repl/open-computer-use-repl.test.mjs+++ b/scripts/node-repl/open-computer-use-repl.test.mjs@@ -102,6 +102,42 @@ test("app-bound API composes actions and state in one js call", async () => {assert.ok(result.content.some(item => item.type === "text" && item.text === "state:Text"));});+for (const Session of [PersistentJavaScriptSession, WorkerJavaScriptSession]) {+ test(`${Session.name}: selective observation is explicit and legacy calls keep their arguments`, async t => {+ const native = mockNative();+ const session = new Session({ native });+ t.after(() => session.close?.());+ const result = await session.run(`+ var selectiveApp = await cua.getApp("Text", { initialState: false });+ await selectiveApp.getScreenshot({ includeAX: false, emit: false });+ await selectiveApp.getAXState({ includeScreenshot: false, emit: false });+ await selectiveApp.getScreenshot({ emit: false });+ await selectiveApp.getAXState({ emit: false });+ `);+ assert.equal(result.isError, false);+ assert.deepEqual(native.calls.map(call => call.arguments), [+ { app: "Text", include_ax: false, text_limit: 1 },+ { app: "Text", include_screenshot: false },+ { app: "Text", text_limit: 1 },+ { app: "Text" },+ ]);+ const count = native.calls.length;+ for (const code of [+ 'await selectiveApp.getScreenshot({ includeAX: "false" })',+ 'await selectiveApp.getAXState({ includeScreenshot: null })',+ ]) {+ const invalid = await session.run(code);+ assert.equal(invalid.isError, true);+ assert.match(invalid.content.at(-1).text, /must be a boolean/);+ }+ assert.equal(native.calls.length, count);+ const empty = await session.run('await selectiveApp.getAXStateAndScreenshot({ includeAX: false, includeScreenshot: false })');+ assert.equal(empty.isError, true);+ assert.match(empty.content.at(-1).text, /At least one/);+ assert.equal(native.calls.length, count);+ });+}+test("app-bound API forwards background-operation options", async () => {const native = mockNative();const session = new PersistentJavaScriptSession({ native });diff --git a/scripts/run-linux-read-back-smoke.sh b/scripts/run-linux-read-back-smoke.shindex 8a9507f..608c637 100755--- a/scripts/run-linux-read-back-smoke.sh+++ b/scripts/run-linux-read-back-smoke.sh@@ -3,7 +3,7 @@ set -euo pipefailrepo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"if [[ "$(uname -s)" != Linux || $# -lt 1 ]]; then- echo "Usage (Linux): bash scripts/run-linux-read-back-smoke.sh /path/to/open-computer-use [--focus-checks | --hidden-checks]" >&2+ echo "Usage (Linux): bash scripts/run-linux-read-back-smoke.sh /path/to/open-computer-use [--focus-checks | --hidden-checks | --observation-checks]" >&2exit 2fidiff --git a/scripts/tests/fixtures/MacOSObservationFixture.swift b/scripts/tests/fixtures/MacOSObservationFixture.swiftnew file mode 100644index 0000000..d5b5d2c--- /dev/null+++ b/scripts/tests/fixtures/MacOSObservationFixture.swift@@ -0,0 +1,25 @@+import AppKit++final class ObservationWindow: NSWindow {+ override var canBecomeKey: Bool { false }+ override var canBecomeMain: Bool { false }+}++let app = NSApplication.shared+app.setActivationPolicy(.accessory)+let window = ObservationWindow(contentRect: NSRect(x: 200, y: 200, width: 420, height: 180),+ styleMask: [.titled, .closable], backing: .buffered, defer: false)+window.title = "OCU Observation Fixture"+window.isReleasedWhenClosed = false+let input = NSTextField(frame: NSRect(x: 20, y: 75, width: 380, height: 30))+input.stringValue = "initial-中文🙂"+input.setAccessibilityLabel("Read-back input")+window.contentView?.addSubview(input)+window.orderFront(nil)+let statePath = CommandLine.arguments[1]+let timer = Timer.scheduledTimer(withTimeInterval: 0.03, repeats: true) { _ in+ let data = try! JSONSerialization.data(withJSONObject: ["text": input.stringValue])+ try? data.write(to: URL(fileURLWithPath: statePath), options: .atomic)+}+Timer.scheduledTimer(withTimeInterval: 120, repeats: false) { _ in app.terminate(nil) }+app.run()diff --git a/scripts/tests/fixtures/WindowsObservationFixture.ps1 b/scripts/tests/fixtures/WindowsObservationFixture.ps1new file mode 100644index 0000000..9d30f81--- /dev/null+++ b/scripts/tests/fixtures/WindowsObservationFixture.ps1@@ -0,0 +1,105 @@+param([Parameter(Mandatory = $true)][string]$StatePath)+$ErrorActionPreference = "Stop"+Add-Type -AssemblyName PresentationFramework+Add-Type -TypeDefinition @"+using System;+using System.Runtime.InteropServices;+public static class ObservationDwm {+ [StructLayout(LayoutKind.Sequential)]+ public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }+ [DllImport("user32.dll")]+ public static extern bool GetWindowRect(IntPtr hwnd, out RECT rect);+ [DllImport("user32.dll", CharSet = CharSet.Unicode)]+ private static extern IntPtr CreateWindowEx(uint extendedStyle, string windowClass, string title, uint style,+ int x, int y, int width, int height, IntPtr parent, IntPtr menu, IntPtr instance, IntPtr parameter);+ [DllImport("user32.dll")]+ public static extern bool DestroyWindow(IntPtr hwnd);+ public static IntPtr CreateTinyWindow() {+ return CreateWindowEx(0, "STATIC", "OCU Windows Observation Fixture", 0x90000000u,+ 80, 80, 1, 1, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero);+ }+ [DllImport("dwmapi.dll")]+ public static extern int DwmSetWindowAttribute(IntPtr hwnd, int attr, ref int value, int size);+ [DllImport("dwmapi.dll")]+ public static extern int DwmGetWindowAttribute(IntPtr hwnd, int attr, out int value, int size);+}+"@+$window = New-Object System.Windows.Window+$window.Title = "OCU Windows Observation Fixture"+$window.ShowInTaskbar = $true+$window.Left = 80+$window.Top = 80+$window.Width = 600+$window.Height = 280+$inputBox = New-Object System.Windows.Controls.TextBox+[System.Windows.Automation.AutomationProperties]::SetName($inputBox, "Observation input")+$inputBox.Margin = New-Object System.Windows.Thickness(20)+$inputBox.Height = 30+$inputBox.VerticalAlignment = "Center"+$inputBox.Text = "initial-value"+$window.Content = $inputBox+$state = @{ sequence = 0; text = ""; width = 0; height = 0; minimized = $false; cloakHResult = 0; cloaked = 0 }+$stateEncoding = New-Object System.Text.UTF8Encoding($false)+$tinyWindow = @{ handle = [IntPtr]::Zero }+$commandPath = $StatePath + ".command"+$timer = New-Object System.Windows.Threading.DispatcherTimer+$timer.Interval = [TimeSpan]::FromMilliseconds(50)+$timer.Add_Tick({+ $hwnd = (New-Object System.Windows.Interop.WindowInteropHelper($window)).Handle+ if (Test-Path -LiteralPath $commandPath) {+ try {+ $command = [System.IO.File]::ReadAllText($commandPath) | ConvertFrom-Json+ if ($command.sequence -gt $state.sequence) {+ switch ($command.kind) {+ "tiny" {+ # WPF rounds a one-DIP window to multiple physical+ # pixels on high-DPI desktops. Use an owned Win32+ # fixture window for the exact physical 1x1 case.+ $window.Hide()+ $tinyWindow.handle = [ObservationDwm]::CreateTinyWindow()+ }+ "minimize" { $window.WindowState = "Minimized" }+ "cloak" {+ $value = 1+ $state.cloakHResult = [ObservationDwm]::DwmSetWindowAttribute($hwnd, 13, [ref]$value, 4)+ }+ "restore" {+ $value = 0+ [void][ObservationDwm]::DwmSetWindowAttribute($hwnd, 13, [ref]$value, 4)+ if ($tinyWindow.handle -ne [IntPtr]::Zero) {+ [void][ObservationDwm]::DestroyWindow($tinyWindow.handle)+ $tinyWindow.handle = [IntPtr]::Zero+ }+ $window.WindowState = "Normal"+ $window.Width = 600+ $window.Height = 280+ $window.Show()+ }+ }+ $state.sequence = $command.sequence+ }+ } catch { }+ }+ $hwnd = if ($tinyWindow.handle -ne [IntPtr]::Zero) { $tinyWindow.handle }+ else { (New-Object System.Windows.Interop.WindowInteropHelper($window)).Handle }+ $cloaked = 0+ [void][ObservationDwm]::DwmGetWindowAttribute($hwnd, 14, [ref]$cloaked, 4)+ $state.cloaked = $cloaked+ $state.text = $inputBox.Text+ $rect = New-Object ObservationDwm+RECT+ [void][ObservationDwm]::GetWindowRect($hwnd, [ref]$rect)+ $state.width = $rect.Right - $rect.Left+ $state.height = $rect.Bottom - $rect.Top+ $state.minimized = $window.WindowState -eq "Minimized"+ $temporary = $StatePath + ".tmp"+ [System.IO.File]::WriteAllText($temporary, ($state | ConvertTo-Json -Compress), $stateEncoding)+ try {+ if (Test-Path -LiteralPath $StatePath) { [System.IO.File]::Replace($temporary, $StatePath, ($StatePath + ".bak")) }+ else { [System.IO.File]::Move($temporary, $StatePath) }+ } catch [System.IO.IOException] { }+})+$timer.Start()+$window.Add_Closed({ $timer.Stop() })+$application = New-Object System.Windows.Application+$application.ShutdownMode = "OnMainWindowClose"+[void]$application.Run($window)diff --git a/scripts/tests/linux-observation-js-smoke.mjs b/scripts/tests/linux-observation-js-smoke.mjsnew file mode 100644index 0000000..d106c33--- /dev/null+++ b/scripts/tests/linux-observation-js-smoke.mjs@@ -0,0 +1,56 @@+#!/usr/bin/env node+// Runs inside the private GTK smoke session; Python owns fixture lifecycle.+import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import { JsonLinePeer, WorkerJavaScriptSession } from "../node-repl/open-computer-use-repl.mjs";++const [nativePath, appName, stateFile] = process.argv.slice(2);+assert.ok(nativePath && appName && stateFile, "Run through --observation-checks");+const native = new JsonLinePeer({ command: nativePath, args: ["mcp"],+ env: { ...process.env, OPEN_COMPUTER_USE_ACTION_READ_BACK: "0" } });+let session;+async function run(code, expectedError) {+ const result = await session.run(code);+ const text = result.content.filter(item => item.type === "text").map(item => item.text).join("\n");+ if (expectedError) {+ assert.equal(result.isError, true, text);+ assert.match(text, expectedError);+ } else assert.equal(result.isError, false, text);+ return result;+}+try {+ await native.initialize();+ const request = native.request.bind(native);+ native.request = async (method, params, timeout) => {+ const result = await request(method, params, timeout);+ if (method === "tools/call" && params.name === "get_app_state" && !result.isError) {+ const images = result.content.filter(item => item.type === "image");+ assert.equal(images.length, params.arguments.include_screenshot === false ? 0 : 1);+ }+ return result;+ };+ session = new WorkerJavaScriptSession({ native });+ const ax = await run(`var app = await cua.getApp(${JSON.stringify(appName)}, {initialState:false});+ var ax = await app.getAXState({includeScreenshot:false});+ var oldIndex = ax.match(/^\\s*(\\d+)\\s+[^\\n]*Read-back input(?:\\s|$)/m)[1];`);+ assert.equal(ax.content.some(item => item.type === "image"), false);+ const shot = await run(`var screenshot = await app.getScreenshot({includeAX:false});+ if(screenshot.subarray(0,8).toString('hex') !== '89504e470d0a1a0a') throw Error('Invalid PNG');`);+ assert.equal(shot.content.filter(item => item.type === "image").length, 1);+ await run(`await app.setValue(oldIndex, "must-not-write");`, /unknown element_index/);+ await run(`var fresh = await app.getAXState({includeScreenshot:false,emit:false});+ var index = fresh.match(/^\\s*(\\d+)\\s+[^\\n]*Read-back input(?:\\s|$)/m)[1];+ await app.setValue(index, "worker-live-中文🙂");`);+ const deadline = Date.now() + 5000;+ while (JSON.parse(readFileSync(stateFile, "utf8")).text !== "worker-live-中文🙂") {+ assert.ok(Date.now() < deadline, "GTK value did not update");+ await new Promise(resolve => setTimeout(resolve, 30));+ }+ const full = await run(`await app.getAXStateAndScreenshot();`);+ assert.equal(full.content.filter(item => item.type === "image").length, 1);+ assert.ok(full.content.some(item => item.type === "text" && item.text.includes("worker-live-中文🙂")));+ console.log("Worker JS → native MCP → GTK: selective collection, PNG output, stale-index error and AX recovery passed.");+} finally {+ await session?.close();+ native.close();+}diff --git a/scripts/tests/linux-read-back-smoke.py b/scripts/tests/linux-read-back-smoke.pyindex cebad77..b6181e8 100644--- a/scripts/tests/linux-read-back-smoke.py+++ b/scripts/tests/linux-read-back-smoke.py@@ -1,7 +1,8 @@#!/usr/bin/env python3-"""Exercise native Linux MCP actions against an isolated real GTK fixture."""+"""Exercise native Linux MCP observation and actions against an isolated GTK fixture."""import argparse+import base64from contextlib import ExitStackimport importlib.utilimport json@@ -10,6 +11,7 @@ from pathlib import Pathimport reimport selectimport subprocess+import structimport sysimport tempfileimport time@@ -285,14 +287,98 @@ def check_hidden(client, native, name, state_file, stderr):print("Native MCP: default/opt-in output, stable indexes, page changes and full detail restoration passed.")-def smoke(native, focus_checks=False, hidden_checks=False):+def check_observation(client, native, name, state_file, stderr):+ wait_for(lambda: json.loads(state_file.read_text())["selected"] == "中文🙂", "fixture selection")+ runtime_path = Path(__file__).resolve().parents[2] / "apps/OpenComputerUseLinux/runtime.py"+ spec = importlib.util.spec_from_file_location("observation_smoke_runtime", runtime_path)+ runtime = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(runtime)++ def png(data):+ raw = base64.b64decode(data, validate=True)+ assert raw[:8] == b"\x89PNG\r\n\x1a\n" and raw[12:16] == b"IHDR", raw[:24]+ width, height = struct.unpack(">II", raw[16:24])+ assert width > 0 and height > 0, (width, height)++ def snapshot(**flags):+ with ExitStack() as stack:+ calls = {key: stack.enter_context(mock.patch.object(runtime, key, wraps=getattr(runtime, key)))+ for key in ("render_tree", "focus_details", "capture_window_png")}+ result = runtime.build_snapshot(name, **flags)+ return result, {key: query.call_count for key, query in calls.items()}++ full, counts = snapshot()+ assert counts == {"render_tree": 1, "focus_details": 1, "capture_window_png": 1}, counts+ png(full["screenshotPngBase64"])+ assert full["selectedText"] == "中文🙂", full+ ax, counts = snapshot(include_screenshot=False)+ assert counts == {"render_tree": 1, "focus_details": 1, "capture_window_png": 0}, counts+ assert ax["screenshotPngBase64"] is None and ax["elements"] == full["elements"], ax+ assert ax["selectedText"] == "中文🙂", ax+ image, counts = snapshot(include_ax=False)+ assert counts == {"render_tree": 0, "focus_details": 0, "capture_window_png": 1}, counts+ png(image["screenshotPngBase64"])+ assert image["elements"] == [] and image["treeLines"] == [], image+ assert image["focusedSummary"] is None and image["selectedText"] is None, image+ assert image["windowBounds"] == full["windowBounds"], image+ print("Real GTK collection: full, AX-only (no capture), image-only (no tree/focus) passed.")++ ax = client.call("get_app_state", name, include_screenshot=False)+ expect_snapshot(ax)+ assert all(item["type"] != "image" for item in ax["content"]), ax+ assert "Selected text: [中文🙂]" in text(ax), text(ax)+ refused = client.call("click", name, x=10, y=10)+ assert refused.get("isError") and "include_screenshot=false" in text(refused), refused+ # The standalone snapshot and JSON call entry points must preserve flags.+ cli = subprocess.run([native, "snapshot", "--include-screenshot", "false", name],+ check=True, capture_output=True, text=True, timeout=40)+ assert "Read-back input" in cli.stdout and "include_screenshot=false" in cli.stdout, cli.stdout+ cli = subprocess.run([native, "call", "get_app_state", "--args",+ json.dumps({"app": name, "include_screenshot": False})],+ check=True, capture_output=True, text=True, timeout=40)+ assert all(item["type"] != "image" for item in json.loads(cli.stdout)["content"]), cli.stdout+ old_index = element_index(ax, "Read-back input")+ invalid = client.call("get_app_state", name, include_ax=False, include_screenshot=False)+ assert invalid.get("isError"), invalid+ image = client.call("get_app_state", name, include_ax=False)+ assert not image.get("isError"), image+ images = [item for item in image["content"] if item["type"] == "image"]+ assert len(images) == 1, image+ png(images[0]["data"])+ assert "indexes invalidated" in text(image), text(image)+ for omitted in ("Read-back input", "The focused UI element", "Selected text:"):+ assert omitted not in text(image), text(image)+ for alias in (name, str(full["app"]["pid"])):+ result = client.call("set_value", alias, element_index=old_index, value="must-not-write")+ assert result.get("isError"), result+ assert json.loads(state_file.read_text())["text"] == "prefix-中文🙂-suffix"++ button = next(row for row in full["elements"] if row["name"] == "Increment")+ bounds = button["frame"]+ result = client.call("click", name, x=bounds["x"] + bounds["width"] / 2,+ y=bounds["y"] + bounds["height"] / 2)+ assert not result.get("isError"), result+ wait_for(lambda: json.loads(state_file.read_text())["count"] == 1, "coordinate click after image-only read")+ refreshed = client.call("get_app_state", name, include_screenshot=False)+ expect_snapshot(refreshed)+ result = client.call("set_value", name, element_index=element_index(refreshed, "Read-back input"),+ value="selective-中文🙂")+ assert not result.get("isError"), result+ wait_for(lambda: json.loads(state_file.read_text())["text"] == "selective-中文🙂", "AX index recovery")+ print("Native MCP: valid PNG, omitted channels, stale name/PID indexes refused, coordinate action and AX recovery passed.")++ subprocess.run(["node", str(Path(__file__).with_name("linux-observation-js-smoke.mjs")),+ native, name, str(state_file)], check=True, timeout=30)+++def smoke(native, focus_checks=False, hidden_checks=False, observation_checks=False):with tempfile.TemporaryDirectory(prefix="ocu-read-back-smoke-") as directory:directory = Path(directory)state_file = directory / "state.json"name = "OCUReadBackFixture" + str(os.getpid())with (directory / "stderr.log").open("w+") as stderr:arguments = [sys.executable, __file__, "--fixture", name, "--state-file", str(state_file)]- if focus_checks:+ if focus_checks or observation_checks:arguments.append("--focus-checks")if hidden_checks:arguments.append("--hidden-checks")@@ -308,6 +394,9 @@ def smoke(native, focus_checks=False, hidden_checks=False):if focus_checks:check_focus(client, name, state_file)return+ if observation_checks:+ check_observation(client, native, name, state_file, stderr)+ returnif hidden_checks:check_hidden(client, native, name, state_file, stderr)observed = client.call("get_app_state", name)@@ -381,6 +470,7 @@ if __name__ == "__main__":checks = parser.add_mutually_exclusive_group()checks.add_argument("--focus-checks", action="store_true")checks.add_argument("--hidden-checks", action="store_true")+ checks.add_argument("--observation-checks", action="store_true")parser.add_argument("--state-file", type=Path)args = parser.parse_args()if args.fixture:@@ -388,4 +478,4 @@ if __name__ == "__main__":else:if not args.native:parser.error("--native is required")- smoke(str(Path(args.native).resolve()), args.focus_checks, args.hidden_checks)+ smoke(str(Path(args.native).resolve()), args.focus_checks, args.hidden_checks, args.observation_checks)diff --git a/scripts/tests/macos-observation-smoke.mjs b/scripts/tests/macos-observation-smoke.mjsnew file mode 100644index 0000000..73df8fe--- /dev/null+++ b/scripts/tests/macos-observation-smoke.mjs@@ -0,0 +1,111 @@+#!/usr/bin/env node+import assert from "node:assert/strict";+import { spawn, spawnSync } from "node:child_process";+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import path from "node:path";+import { fileURLToPath } from "node:url";+import { JsonLinePeer, WorkerJavaScriptSession } from "../node-repl/open-computer-use-repl.mjs";++assert.equal(process.platform, "darwin");+assert.ok(process.argv[2], "Pass the normally signed OCU app executable with existing permissions/admission");+const nativePath = path.resolve(process.argv[2]);+const directory = mkdtempSync(path.join(tmpdir(), "ocu-observation-"));+const fixturePath = path.join(directory, "OCUObservationFixture");+const stateFile = path.join(directory, "state.json");+const appName = "OCUObservationFixture";+const env = { ...process.env, OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY: "1",+ OPEN_COMPUTER_USE_VISUAL_CURSOR: "0", OPEN_COMPUTER_USE_ACTION_READ_BACK: "0",+ OPEN_COMPUTER_USE_DEBUG_TIMING: "1", OPEN_COMPUTER_USE_CONFIG_FILE: path.join(directory, "config.json") };+for (const key of Object.keys(env)) if (key.startsWith("OPEN_COMPUTER_USE_IMAGE_")) delete env[key];+writeFileSync(env.OPEN_COMPUTER_USE_CONFIG_FILE, "{}");+let fixture, native, session;+const text = result => result.content.filter(item => item.type === "text").map(item => item.text).join("\n");+const images = result => result.content.filter(item => item.type === "image");+function indexOf(state) {+ const match = state.match(/^\s*(\d+)\s+[^\n]*Read-back input(?:[\s,]|$)/m);+ assert.ok(match, state);+ return match[1];+}+async function waitFor(test) {+ const deadline = Date.now() + 8000;+ while (!test()) { assert.ok(Date.now() < deadline, "Fixture timeout"); await new Promise(r => setTimeout(r, 30)); }+}+async function call(name, args, expectedError) {+ const result = await native.request("tools/call", { name, arguments: { app: appName, ...args } });+ assert.equal(!!result.isError, !!expectedError, text(result));+ if (expectedError) assert.match(text(result), expectedError);+ return result;+}+async function observe(args) {+ native.stderr = "";+ const result = await call("get_app_state", args);+ await new Promise(r => setTimeout(r, 50));+ if (args.include_screenshot === false) {+ assert.equal(images(result).length, 0);+ assert.doesNotMatch(native.stderr, /timing snapshot\.capture_(?:hw|sck) /);+ assert.match(native.stderr, /timing snapshot\.tree_walk /);+ } else {+ assert.equal(images(result).length, 1, text(result));+ assert.equal(Buffer.from(images(result)[0].data, "base64").subarray(0, 8).toString("hex"), "89504e470d0a1a0a");+ assert.match(native.stderr, /timing snapshot\.capture_(?:hw|sck) /);+ }+ if (args.include_ax === false) {+ assert.doesNotMatch(native.stderr, /timing snapshot\.tree_walk /);+ assert.ok(!text(result).includes("Read-back input"));+ assert.match(text(result), /indexes invalidated/);+ }+ return result;+}+try {+ const source = fileURLToPath(new URL("./fixtures/MacOSObservationFixture.swift", import.meta.url));+ const build = spawnSync("swiftc", [source, "-o", fixturePath], { encoding: "utf8" });+ assert.equal(build.status, 0, build.stderr);+ fixture = spawn(fixturePath, [stateFile], { stdio: ["ignore", "ignore", "ignore"] });+ await waitFor(() => existsSync(stateFile));+ native = new JsonLinePeer({ command: nativePath, args: ["mcp"], env });+ await native.initialize();+ const full = await observe({});+ const oldIndex = indexOf(text(full));+ const queried = JSON.parse(text(await call("query", { role: "AXTextField" })));+ assert.equal(queried.matches.length, 1);+ await observe({ include_ax: false });+ await call("set_value", { element_index: oldIndex, value: "bad" }, /unknown element_index/);+ await call("set_value", { element_index: queried.matches[0].index, value: "bad" }, /expired|no longer available/);+ const ax = await observe({ include_screenshot: false });+ await call("click", { x: 10, y: 10 }, /include_screenshot=false/);+ await call("set_value", { element_index: indexOf(text(ax)), value: "native-live-中文🙂" });+ await waitFor(() => JSON.parse(readFileSync(stateFile, "utf8")).text === "native-live-中文🙂");+ for (const command of [["snapshot", "--include-screenshot", "false", appName],+ ["call", "get_app_state", "--args", JSON.stringify({ app: appName, include_screenshot: false })]]) {+ const cli = spawnSync(nativePath, command, { env, encoding: "utf8", timeout: 15000 });+ assert.equal(cli.status, 0, cli.stderr);+ assert.match(cli.stdout, /include_screenshot=false/);+ if (command[0] === "call") assert.equal(images(JSON.parse(cli.stdout)).length, 0);+ }+ session = new WorkerJavaScriptSession({ native });+ let result = await session.run(`var app = await cua.getApp(${JSON.stringify(appName)}, {initialState:false});+ var state = await app.getAXState({includeScreenshot:false});`);+ assert.equal(result.isError, false, text(result));+ assert.equal(images(result).length, 0);+ result = await session.run("await app.getScreenshot({includeAX:false});");+ assert.equal(result.isError, false, text(result));+ assert.equal(images(result).length, 1);+ result = await session.run(`var fresh = await app.getAXState({includeScreenshot:false,emit:false});+ var index = fresh.match(/^\\s*(\\d+)\\s+[^\\n]*Read-back input(?:[\\s,]|$)/m)[1];+ await app.setValue(index, "worker-live-中文🙂");`);+ assert.equal(result.isError, false, text(result));+ await waitFor(() => JSON.parse(readFileSync(stateFile, "utf8")).text === "worker-live-中文🙂");+ console.log("macOS real AX/native MCP/CLI/Worker JS: collection gates, PNG, snapshot/query invalidation, coordinate refusal and element recovery passed.");+} finally {+ await session?.close();+ native?.close();+ if (fixture) {+ fixture.kill("SIGTERM");+ if (fixture.exitCode === null && fixture.signalCode === null) await new Promise(resolve => {+ const timer = setTimeout(() => { fixture.kill("SIGKILL"); resolve(); }, 3000);+ fixture.once("exit", () => { clearTimeout(timer); resolve(); });+ });+ }+ rmSync(directory, { recursive: true, force: true });+}diff --git a/scripts/tests/windows-observation-js-smoke.mjs b/scripts/tests/windows-observation-js-smoke.mjsnew file mode 100644index 0000000..38bb8d7--- /dev/null+++ b/scripts/tests/windows-observation-js-smoke.mjs@@ -0,0 +1,46 @@+#!/usr/bin/env node+// The PowerShell desktop smoke owns this fixture and its lifecycle.+import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import { JsonLinePeer, WorkerJavaScriptSession } from "../node-repl/open-computer-use-repl.mjs";++const [nativePath, app, stateFile] = process.argv.slice(2);+const native = new JsonLinePeer({ command: nativePath, args: ["mcp"], env: process.env });+let session;+async function run(code, expectedError) {+ const result = await session.run(code, 20000);+ const text = result.content.filter(item => item.type === "text").map(item => item.text).join("\n");+ assert.equal(!!result.isError, !!expectedError, text);+ if (expectedError) assert.match(text, expectedError);+ return result;+}+try {+ await native.initialize();+ session = new WorkerJavaScriptSession({ native });+ const ax = await run(`var app = await cua.getApp(${JSON.stringify(app)}, {initialState:false});+ var state = await app.getAXState({includeScreenshot:false});+ function inputIndex(state) {+ var line = state.split("\\n").find(line => /^\\s*\\d+\\s+edit Observation input(?:\\s|$)/.test(line));+ if(!line) throw Error("Fixture input not found in AX");+ return line.match(/^\\s*(\\d+)/)[1];+ }+ var oldIndex = inputIndex(state);`);+ assert.equal(ax.content.some(item => item.type === "image"), false);+ const screenshot = await run("await app.getScreenshot({includeAX:false});");+ assert.equal(screenshot.content.filter(item => item.type === "image").length, 1);+ await run('await app.setValue(oldIndex, "must-not-write");', /unknown element_index/);+ await run(`var state = await app.getAXState({includeScreenshot:false,emit:false});+ await app.setValue(inputIndex(state), "worker-live-中文");`);+ const deadline = Date.now() + 8000;+ while (JSON.parse(readFileSync(stateFile, "utf8")).text !== "worker-live-中文") {+ assert.ok(Date.now() < deadline, "Real Windows text did not update");+ await new Promise(resolve => setTimeout(resolve, 50));+ }+ const full = await run("await app.getAXStateAndScreenshot();");+ assert.equal(full.content.filter(item => item.type === "image").length, 1);+ assert.ok(full.content.some(item => item.type === "text" && item.text.includes("worker-live-中文")));+ console.log("Windows Worker JS → native MCP → WPF: channels, stale-index errors, Unicode update and default full observation passed.");+} finally {+ await session?.close();+ native.close();+}diff --git a/scripts/tests/windows-observation-smoke.ps1 b/scripts/tests/windows-observation-smoke.ps1new file mode 100644index 0000000..7568382--- /dev/null+++ b/scripts/tests/windows-observation-smoke.ps1@@ -0,0 +1,117 @@+param([Parameter(Mandatory = $true)][string]$NativePath)+$ErrorActionPreference = "Stop"+if ($env:OS -ne "Windows_NT") { throw "Windows desktop required" }+$NativePath = (Resolve-Path -LiteralPath $NativePath).Path+$directory = Join-Path ([System.IO.Path]::GetTempPath()) ("ocu-observation-" + [Guid]::NewGuid())+[void](New-Item -ItemType Directory -Path $directory)+$stateFile = Join-Path $directory "state.json"+$fixture = $null+$native = $null+$requestId = 0+$sequence = 0++function Assert($condition, $message) { if (-not $condition) { throw $message } }+function Wait-For($test, $description) {+ $deadline = [DateTime]::UtcNow.AddSeconds(15)+ do {+ try { if (& $test) { return } } catch { }+ Start-Sleep -Milliseconds 50+ } while ([DateTime]::UtcNow -lt $deadline)+ throw "Timeout: $description"+}+function Read-State { [System.IO.File]::ReadAllText($stateFile) | ConvertFrom-Json }+function Send-Request($method, $parameters) {+ $script:requestId += 1+ $message = @{ jsonrpc = "2.0"; id = $script:requestId; method = $method; params = $parameters }+ $native.StandardInput.WriteLine(($message | ConvertTo-Json -Depth 20 -Compress))+ $native.StandardInput.Flush()+ $read = $native.StandardOutput.ReadLineAsync()+ Assert ($read.Wait(40000)) "MCP timed out"+ Assert (-not [string]::IsNullOrWhiteSpace($read.Result)) "MCP exited"+ $response = $read.Result | ConvertFrom-Json+ Assert ($response.id -eq $script:requestId -and $null -eq $response.error) "Unexpected RPC response"+ return $response.result+}+function Call-Tool($tool, $arguments) {+ if (-not $arguments.ContainsKey("app")) { $arguments.app = [string]$fixture.Id }+ Send-Request "tools/call" @{ name = $tool; arguments = $arguments }+}+function Text($result) { ($result.content | Where-Object type -eq "text" | ForEach-Object text) -join "`n" }+function Images($result) { @($result.content | Where-Object type -eq "image") }+function Input-Index($result) {+ $matches = [regex]::Matches((Text $result), '(?m)^\s*(\d+)\s+edit Observation input(?:\s|$)')+ Assert ($matches.Count -eq 1) (Text $result)+ $matches[0].Groups[1].Value+}+function Change-Window($kind) {+ $script:sequence += 1+ $message = @{ sequence = $script:sequence; kind = $kind } | ConvertTo-Json -Compress+ [System.IO.File]::WriteAllText(($stateFile + ".command"), $message, [System.Text.Encoding]::UTF8)+ Wait-For { (Read-State).sequence -eq $script:sequence } "fixture window command"+ Start-Sleep -Milliseconds 200+}+try {+ $fixtureSource = Join-Path $PSScriptRoot "fixtures/WindowsObservationFixture.ps1"+ $fixture = Start-Process powershell.exe -ArgumentList @("-NoProfile", "-STA", "-File", "`"$fixtureSource`"", "-StatePath", "`"$stateFile`"") -PassThru -RedirectStandardOutput (Join-Path $directory "fixture.out") -RedirectStandardError (Join-Path $directory "fixture.err")+ Wait-For { Test-Path -LiteralPath $stateFile } "WPF startup"+ Wait-For { (Get-Process -Id $fixture.Id).MainWindowHandle -ne [IntPtr]::Zero } "visible WPF main window"+ $start = New-Object System.Diagnostics.ProcessStartInfo+ $start.FileName = $NativePath+ $start.Arguments = "mcp"+ $start.UseShellExecute = $false+ $start.RedirectStandardInput = $true+ $start.RedirectStandardOutput = $true+ $start.RedirectStandardError = $true+ $start.StandardOutputEncoding = [System.Text.Encoding]::UTF8+ $native = New-Object System.Diagnostics.Process+ $native.StartInfo = $start+ [void]$native.Start()+ $null = Send-Request "initialize" @{ protocolVersion = "2025-03-26"; capabilities = @{}; clientInfo = @{ name = "windows-observation-smoke"; version = "1" } }+ $full = Call-Tool "get_app_state" @{}+ Assert (-not $full.isError -and (Images $full).Count -eq 1) (Text $full)+ Write-Output "Normal WPF window returned a validated PNG."+ $oldIndex = Input-Index $full+ $image = Call-Tool "get_app_state" @{ include_ax = $false }+ Assert (-not $image.isError -and (Images $image).Count -eq 1) (Text $image)+ Assert ((Text $image) -match "indexes invalidated" -and (Text $image) -notmatch "Observation input|initial-value") "AX not omitted"+ $stale = Call-Tool "set_value" @{ element_index = $oldIndex; value = "must-not-write" }+ Assert $stale.isError "Stale index accepted"+ $ax = Call-Tool "get_app_state" @{ include_screenshot = $false }+ Assert (-not $ax.isError -and (Images $ax).Count -eq 0) (Text $ax)+ $coordinates = Call-Tool "click" @{ x = 10; y = 10 }+ Assert $coordinates.isError "AX-only coordinate action accepted"+ $value = "live-" + [char]0x4E2D + [char]0x6587+ $write = Call-Tool "set_value" @{ element_index = (Input-Index $ax); value = $value }+ Assert (-not $write.isError) (Text $write)+ Wait-For { (Read-State).text -eq $value } "real UIA value update"+ Write-Output "Selective channels, index refusal/recovery and real Unicode text update passed."+ foreach ($kind in @("tiny", "minimize", "cloak")) {+ Change-Window $kind+ if ($kind -eq "tiny") {+ $state = Read-State+ Assert ($state.width -eq 1 -and $state.height -eq 1) "Window did not become 1x1: $($state.width)x$($state.height)"+ }+ if ($kind -eq "minimize") { Assert (Read-State).minimized "Window did not minimize" }+ if ($kind -eq "cloak") { Assert ((Read-State).cloakHResult -eq 0 -and (Read-State).cloaked -ne 0) "Window did not cloak" }+ $unusable = Call-Tool "get_app_state" @{}+ Assert ((Images $unusable).Count -eq 0) "$kind window returned an image"+ Change-Window "restore"+ $restored = Call-Tool "get_app_state" @{}+ Assert (-not $restored.isError -and (Images $restored).Count -eq 1) (Text $restored)+ Write-Output "$kind window omitted its image; restored window returned a PNG."+ }+ $cli = & $NativePath snapshot --include-screenshot false ([string]$fixture.Id)+ Assert ($LASTEXITCODE -eq 0 -and ($cli -join "`n") -match "include_screenshot=false") "CLI flag not forwarded"+ & node (Join-Path $PSScriptRoot "windows-observation-js-smoke.mjs") $NativePath ([string]$fixture.Id) $stateFile+ Assert ($LASTEXITCODE -eq 0) "Worker JS desktop smoke failed"+ Write-Output "Windows real WPF/native MCP/CLI: channels, stale indexes, AX recovery, coordinate refusal, Unicode text update, 1x1/minimized/cloaked screenshot filtering passed."+} finally {+ if ($null -ne $native -and -not $native.HasExited) {+ $native.StandardInput.Close()+ if (-not $native.WaitForExit(3000)) { $native.Kill(); $native.WaitForExit() }+ }+ if ($null -ne $fixture -and -not $fixture.HasExited) { $fixture.Kill(); $fixture.WaitForExit() }+ $fixtureError = Join-Path $directory "fixture.err"+ if (Test-Path -LiteralPath $fixtureError) { Get-Content -LiteralPath $fixtureError | Write-Output }+ Remove-Item -LiteralPath $directory -Recurse -Force+}diff --git a/scripts/tests/windows-observation.test.ps1 b/scripts/tests/windows-observation.test.ps1new file mode 100644index 0000000..b9e8e3e--- /dev/null+++ b/scripts/tests/windows-observation.test.ps1@@ -0,0 +1,67 @@+param([Parameter(Mandatory = $true)][string]$RuntimePath)+$ErrorActionPreference = "Stop"++# Execute the production observation functions with instrumented UIA/capture+# dependencies. This can run under pwsh on Linux; it is not a Windows UI test.+$tokens = $null+$errors = $null+$ast = [System.Management.Automation.Language.Parser]::ParseFile($RuntimePath, [ref]$tokens, [ref]$errors)+if ($errors.Count -gt 0) { throw ($errors | Out-String) }+$functions = $ast.FindAll({ param($node)+ $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and+ $node.Name -in @("Resolve-ObservationOptions", "Build-Snapshot")+}, $true)+if ($functions.Count -ne 2) { throw "Production observation functions missing" }+foreach ($function in $functions) { Invoke-Expression $function.Extent.Text }++$DefaultTextLimit = 500+$AccessibilityTreeMaxNodeCount = 1200+$AccessibilityTreeMaxDepth = 64+function Resolve-App($query) { [pscustomobject]@{ ProcessName = $query; Id = 42; MainWindowTitle = "Fixture" } }+function Get-MainElement($process) { "window" }+function Get-WindowBounds($process, $element) { [pscustomobject]@{ x = 0; y = 0; width = 300; height = 200 } }+function Limit-Text($text, $limit) { $text }+function Render-Tree($element, $bounds, $limit, $nodes, $depth) {+ $script:calls.tree += 1+ [pscustomobject]@{ lines = @("Input"); records = @([pscustomobject]@{ index = 7 }) }+}+function Get-FocusedSummary($processId, $limit) { $script:calls.focus += 1; "focus" }+function Get-SelectedText($processId, $limit) { $script:calls.selection += 1; "selection" }+function Capture-WindowPngBase64($bounds) { $script:calls.capture += 1; "png" }+function Assert($condition, $message) { if (-not $condition) { throw $message } }++foreach ($mode in @(+ @{ ax = $true; image = $true },+ @{ ax = $true; image = $false },+ @{ ax = $false; image = $true }+)) {+ $script:calls = @{ tree = 0; focus = 0; selection = 0; capture = 0 }+ $snapshot = Build-Snapshot "Fixture" -IncludeAX $mode.ax -IncludeScreenshot $mode.image+ Assert ($snapshot.app.pid -eq 42 -and $snapshot.windowBounds.width -eq 300) "Window metadata missing"+ Assert ($script:calls.tree -eq [int]$mode.ax -and $script:calls.focus -eq [int]$mode.ax -and+ $script:calls.selection -eq [int]$mode.ax -and $script:calls.capture -eq [int]$mode.image) "Disabled collection still executed"+ if ($mode.ax) {+ Assert ($snapshot.elements.Count -eq 1 -and $snapshot.selectedText -eq "selection") "AX missing"+ } else {+ Assert ($snapshot.elements.Count -eq 0 -and $snapshot.treeLines.Count -eq 0 -and+ $null -eq $snapshot.focusedSummary -and $null -eq $snapshot.selectedText) "AX was not omitted"+ }+ Assert (($snapshot.screenshotPngBase64 -eq "png") -eq $mode.image) "Screenshot mode mismatch"+}++$defaults = Resolve-ObservationOptions ('{}' | ConvertFrom-Json)+Assert ($defaults.include_ax -and $defaults.include_screenshot) "Defaults changed"+foreach ($payload in @('{"include_ax":false}', '{"include_screenshot":false}')) {+ $flags = Resolve-ObservationOptions ($payload | ConvertFrom-Json)+ Assert (-not ($flags.include_ax -and $flags.include_screenshot)) "False flag lost"+}+foreach ($payload in @('{"include_ax":"false"}', '{"include_ax":0}', '{"include_screenshot":null}',+ '{"include_ax":false,"include_screenshot":false}')) {+ $failed = $false+ try { $null = Resolve-ObservationOptions ($payload | ConvertFrom-Json) } catch { $failed = $true }+ Assert $failed "Invalid observation accepted"+}+$failed = $false+try { $null = Build-Snapshot "Fixture" -IncludeAX $false -IncludeScreenshot $false } catch { $failed = $true }+Assert $failed "Empty snapshot accepted"+Write-Output "PowerShell production functions: parse, defaults, strict booleans and all collection modes passed (instrumented dependencies)."diff --git a/skills/open-computer-use/references/usage.md b/skills/open-computer-use/references/usage.mdindex 84048c7..51d2899 100644--- a/skills/open-computer-use/references/usage.md+++ b/skills/open-computer-use/references/usage.md@@ -52,6 +52,33 @@ await app.typeText("Hello");await app.getAXState();```+### Selective observation (opt-in)++The native `get_app_state` tool on macOS, Windows and Linux accepts `include_ax` and+`include_screenshot`. Both default to `true`; at least one must remain true.+The REPL forwards the corresponding optional `includeAX` / `includeScreenshot`+booleans:++```js+var editor = await cua.getApp("gedit", { initialState: false });+await editor.getScreenshot({ includeAX: false }); // no AX, focus or selection walk+await editor.click([100, 100]); // window-relative coordinates+await editor.getAXState({ includeScreenshot: false }); // no capture or PNG encoding+// Use newly observed element indexes only after inspecting the AX result.+```++Screenshot-only observation invalidates prior element indexes and cached query+aliases for that app, even if the window title is unchanged. Its result explicitly+reports this. Coordinate actions use the newly observed window bounds; element+actions require a fresh AX observation. Existing calls without these options keep+their full native observation behavior. These flags are implemented on all three native platforms; action read-back is unchanged.++These options select collection, not image deduplication. An unchanged screenshot+is still returned when requested. macOS/Linux JS actions already disable automatic+read-back, and getAXState does not emit images; includeScreenshot:false additionally skips+native capture on all three platforms. See `docs/selective-observation.md` for real GTK+and Worker JS validation.+The code-first surface also forwards macOS background-operation options:```js