👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
feat(config): add persistent screenshot settings and output formats
ifuryst2cf3eae57666
diff --git a/Package.resolved b/Package.resolvednew file mode 100644index 0000000..2c66bfe--- /dev/null+++ b/Package.resolved@@ -0,0 +1,15 @@+{+ "originHash" : "ddf5857bfef42315d0aa9bdd7ff832a14fc1f7dff5df23672cd6202a96da51a8",+ "pins" : [+ {+ "identity" : "libwebp-xcode",+ "kind" : "remoteSourceControl",+ "location" : "https://github.com/SDWebImage/libwebp-Xcode.git",+ "state" : {+ "revision" : "2b5256c29ff4e20f2a0d5ee863b62b1a22144434",+ "version" : "1.6.0"+ }+ }+ ],+ "version" : 3+}diff --git a/Package.swift b/Package.swiftindex e43a5a6..a70eca5 100644--- a/Package.swift+++ b/Package.swift@@ -33,6 +33,9 @@ let package = Package(targets: ["StandaloneCursor"]),],+ dependencies: [+ .package(url: "https://github.com/SDWebImage/libwebp-Xcode.git", exact: "1.6.0"),+ ],targets: [.target(name: "OpenComputerUseVirtualDisplayShim",@@ -44,7 +47,7 @@ let package = Package(),.target(name: "OpenComputerUseKit",- dependencies: ["OpenComputerUseVirtualDisplayShim"],+ dependencies: ["OpenComputerUseVirtualDisplayShim", .product(name: "libwebp", package: "libwebp-Xcode")],path: "packages/OpenComputerUseKit/Sources/OpenComputerUseKit"),.executableTarget(diff --git a/README.md b/README.mdindex 876c2a3..db69823 100644--- a/README.md+++ b/README.md@@ -204,3 +204,7 @@ Cursor Motion is an open-source cursor motion system for macOS, based on public## License[MIT](./LICENSE).++### Persistent configuration++Use `ocu config` to inspect settings and `ocu config set image.format jpg` to persist a macOS screenshot preference. Environment overrides remain supported. See [configuration](docs/configuration.md) ([简体中文](docs/configuration.zh-CN.md)) for settings and CLI commands.diff --git a/README.zh-CN.md b/README.zh-CN.mdindex d0cebc5..0508228 100644--- a/README.zh-CN.md+++ b/README.zh-CN.md@@ -159,3 +159,7 @@ Cursor Motion 是一个面向 macOS 的开源光标运动系统,基于 Softwar## License[MIT](./LICENSE)。++### 持久化配置++使用 `ocu config` 查看生效配置,用 `ocu config set image.format jpg` 保存 macOS 截图偏好。默认文件是 `~/.config/ocu/config.json`,ENV 优先于文件。详见[中文配置说明](docs/configuration.zh-CN.md)([English](docs/configuration.md))。diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.mdindex 38cd295..f930ba8 100644--- a/THIRD_PARTY_NOTICES.md+++ b/THIRD_PARTY_NOTICES.md@@ -59,3 +59,37 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHERLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THESOFTWARE.++## libwebp++WebP screenshots statically link libwebp 1.6.0 via SDWebImage/libwebp-Xcode, pinned to revision `2b5256c29ff4e20f2a0d5ee863b62b1a22144434` in Package.resolved. License: BSD-3-Clause.++Copyright (c) 2010, Google Inc. All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are+met:++ * Redistributions of source code must retain the above copyright+ notice, this list of conditions and the following disclaimer.++ * Redistributions in binary form must reproduce the above copyright+ notice, this list of conditions and the following disclaimer in+ the documentation and/or other materials provided with the+ distribution.++ * Neither the name of Google nor the names of its contributors may+ be used to endorse or promote products derived from this software+ without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swiftindex e4e764b..b79b96b 100644--- a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift@@ -144,9 +144,13 @@ enum MacOSAppAgentProxy {}private static func proxiedEnvironment() -> [String: String] {- ProcessInfo.processInfo.environment.filter { key, _ in+ var environment = ProcessInfo.processInfo.environment.filter { key, _ inkey.hasPrefix("OPEN_COMPUTER_USE_")}+ if let url = try? OCUConfiguration.fileURL(environment: OCUConfiguration.environment) {+ environment["OPEN_COMPUTER_USE_CONFIG_FILE"] = url.path+ }+ return environment}}@@ -425,18 +429,14 @@ private enum AppAgentEnvironment {private static let lock = NSLock()static func withOverrides<T>(_ overrides: [String: String], _ body: () throws -> T) rethrows -> T {- guard !overrides.isEmpty else {- return try body()- }-lock.lock()defer { lock.unlock() }- let previousValues = Dictionary(- uniqueKeysWithValues: overrides.keys.map { key in- (key, ProcessInfo.processInfo.environment[key])- }- )+ let keys = Set(overrides.keys).union(OCUConfiguration.environmentKeys)+ let previousValues = Dictionary(uniqueKeysWithValues: keys.map { key in+ (key, getenv(key).map { String(cString: $0) })+ })+ for key in OCUConfiguration.environmentKeys where overrides[key] == nil { unsetenv(key) }for (key, value) in overrides {setenv(key, value, 1)}diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex c3f6541..f3a7df0 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -144,7 +144,7 @@- 开源版当前不复刻官方闭源实现里的 caller signing、私有 IPC、完整 overlay choreography 和 plugin 自安装逻辑。- 因为官方 `SkyComputerUseClient` 带有宿主侧 launch constraints,普通 stdio MCP client 在本机上可能被系统直接杀掉;如果要探测官方 bundled `computer-use`,`scripts/computer-use-cli` 的 app-server 模式现在只适合做工具清单和协议面观察。官方 `1.0.755` 的真实 tool call 还会经过 service-side sender authorization / active IPC client 追踪,外部 raw helper 即使走已签名 Codex binary,也可能返回 `Sender process is not authenticated`;需要真实使用官方工具时应走正常 Codex agent/tool 调用链,开源版则继续提供可直连的 `open-computer-use` MCP server。- 当前权限引导已经具备可运行 app、深链、拖拽辅助,以及一版更接近官方的 accessory panel 入场动画和返回 affordance;点击链路也已经补上独立 visual cursor、官方 asset fallback 和相对目标 window 的排序逻辑,并且在 overlay 可见期间会持续重申“排在目标 window 之上”,避免用户手动激活目标 app 后 cursor 被目标窗口重新盖住;但整体还没有完全复刻官方那套嵌入式 choreography / host 集成 / session approval 体验。-- screenshot 当前通过 `ScreenCaptureKit` 捕获目标窗口,并以 MCP `image` content block 的 base64 PNG 返回,不再把普通 app 截图落盘到仓库或临时目录;编码前会按最大尺寸和目标字节数自适应缩小,避免复杂页面的大 PNG 触发 host 侧 MCP result 降级,同时 coordinate tools 继续按实际返回的 screenshot pixel 尺寸映射坐标;单次 ScreenCaptureKit capture 会设置超时,超时后省略 image block 而不是卡住整个 `get_app_state`。+- screenshot 当前通过 `ScreenCaptureKit` 捕获目标窗口,并以 MCP `image` content block 的 base64 PNG(默认)或配置的 JPG/WebP 返回,不再把普通 app 截图落盘到仓库或临时目录;编码前会按配置的最大长边等比例缩小,或按超限策略与最小总像素数门槛省略截图,同时 coordinate tools 继续按实际返回的 screenshot pixel 尺寸映射坐标;单次 ScreenCaptureKit capture 会设置超时,超时后省略 image block 而不是卡住整个 `get_app_state`。- 会话状态现在是进程内内存态,保存每个 app 最近一次 snapshot 和 element index 映射。## 主要验证路径@@ -169,3 +169,9 @@- `open-computer-use snapshot <app>`- `open-computer-use call list_apps`- `open-computer-use call --calls '[{"tool":"get_app_state","args":{"app":"TextEdit"}}]'`++## 用户配置++macOS screenshot 在每轮捕获读取用户 JSON 配置,优先级为 ENV > 文件 > 默认值。npm launcher 提供 `ocu config` 查看、set、reset 和 path;支持 PNG/JPG/WebP、JPEG 质量、最大长边/超限缩小策略、最小总像素数丢弃与 SCK 超时,保留 hardware capture 优先路径。app-agent 按请求转发解析后的配置路径并清除未提供的 image 环境,防止常驻值污染。见[中文配置说明](configuration.zh-CN.md)与[English](configuration.md)。++截图像素丢弃门槛按 width×height 计算,低于(不含等于)门槛不返回 image,但保留 AX 和原因;配置省略截图后拒绝坐标输入。WebP 无损编码通过固定版本 libwebp 静态链接,随 app/npm 制品带上许可文本。diff --git a/docs/SECURITY.md b/docs/SECURITY.mdindex c485dd6..c461497 100644--- a/docs/SECURITY.md+++ b/docs/SECURITY.md@@ -11,7 +11,7 @@## 数据处理-- 普通 app 的 screenshot 默认只在内存中编码成 PNG,并通过 MCP `image` content block 直接回传;默认不长期持久化。+- 普通 app 的 screenshot 默认只在内存中编码成 PNG(可配置为 JPG 或无损 WebP),并通过 MCP `image` content block 直接回传;默认不长期持久化。- Windows runtime 对隐藏、最小化、DWM cloaked、离屏或退化小窗口 fail closed:保留可读的 UI Automation tree,但不返回 image block,也不允许在缺少有效截图时执行 coordinate click / drag;不会用放大、补边或占位图伪造可用截图。- Linux runtime 的 screenshot 是 best-effort;如果 GNOME Wayland 返回黑图,bridge 会省略 image block,避免把无效截图误当成真实画面。- fixture app 的合成状态只写到本地临时 JSON 文件,目的是支撑 deterministic smoke test;当前写入走原子替换,减少测试期间的读写竞争。diff --git a/docs/SUPPLY_CHAIN_SECURITY.md b/docs/SUPPLY_CHAIN_SECURITY.mdindex 90173b2..34d4d7c 100644--- a/docs/SUPPLY_CHAIN_SECURITY.md+++ b/docs/SUPPLY_CHAIN_SECURITY.md@@ -33,3 +33,7 @@- 让构建过程尽量可重复、可验证。- 如果条件允许,在部署链路里增加对 provenance 的校验。- 把 attestation 校验继续下沉到部署平台或准入层。++## Screenshot WebP codec++macOS SwiftPM pins SDWebImage/libwebp-Xcode to libwebp 1.6.0 with a committed Package.resolved revision. The encoder links into the native executable; no Homebrew codec is required. BSD notices are retained in THIRD_PARTY_NOTICES.md and copied into app Resources and npm packages. Dependency/SBOM review must include this native dependency.diff --git a/docs/configuration.md b/docs/configuration.mdnew file mode 100644index 0000000..072682c--- /dev/null+++ b/docs/configuration.md@@ -0,0 +1,54 @@+# Configuration++[English](configuration.md) | [简体中文](configuration.zh-CN.md)++The config file defaults to `~/.config/ocu/config.json`. You can edit it directly, or create it if it doesn't exist. Here are the defaults:++```json+{+ "image": {+ "format": "png",+ "jpegQuality": 0.8,+ "maxLongEdgePixels": 1280,+ "scaleDownAfterMaxSize": true,+ "discardBelowPixelCount": 64,+ "captureTimeout": 5+ }+}+```++These settings currently apply to macOS screenshots. Changes take effect on the next capture. Environment variables are also supported, with precedence: ENV > config file > defaults.++## Configure from the command line++You can also use `ocu config`. Changes are saved to the file above:++```bash+# View settings+ocu config++# Switch to JPG+ocu config set image.format jpg++# Read one setting+ocu config get image.format++# Restore the default (ENV overrides still take priority)+ocu config reset image.format++# Show the config file path+ocu config path+```++You can use `open-computer-use` in place of `ocu`.++## Settings++| Setting | Default | Description | Example command |+| --- | --- | --- | --- |+| `image.format` | `png` | Output format: `png`, `jpg` or `webp`. WebP is lossless. | `ocu config set image.format webp` |+| `image.jpegQuality` | `0.8` | JPG quality, from `0` to `1`. Only affects JPG. | `ocu config set image.jpegQuality 0.9` |+| `image.maxLongEdgePixels` | `1280` | Maximum long edge in pixels. Integer `1–16384`; `null` disables the limit. | `ocu config set image.maxLongEdgePixels 1920` |+| `image.scaleDownAfterMaxSize` | `true` | If the long edge exceeds the limit, `true` resizes proportionally and `false` omits the screenshot. | `ocu config set image.scaleDownAfterMaxSize false` |+| `image.discardBelowPixelCount` | `64` | Omit when width × height is below this value; equality is retained. Integer `0–268435456`; `0` disables filtering. For example, `4096` is the area of `64×64`. Resized images are checked too. | `ocu config set image.discardBelowPixelCount 4096` |+| `image.captureTimeout` | `5` | ScreenCaptureKit fallback capture timeout, in seconds. Range `0.01–300`. | `ocu config set image.captureTimeout 10` |diff --git a/docs/configuration.zh-CN.md b/docs/configuration.zh-CN.mdnew file mode 100644index 0000000..b66d366--- /dev/null+++ b/docs/configuration.zh-CN.md@@ -0,0 +1,54 @@+# 配置++[English](configuration.md) | [简体中文](configuration.zh-CN.md)++配置文件默认在 `~/.config/ocu/config.json`,可以直接编辑。如果文件还不存在,可以创建一个。下面是默认配置:++```json+{+ "image": {+ "format": "png",+ "jpegQuality": 0.8,+ "maxLongEdgePixels": 1280,+ "scaleDownAfterMaxSize": true,+ "discardBelowPixelCount": 64,+ "captureTimeout": 5+ }+}+```++目前这些配置用于 macOS 截图,修改后在下一次截图时生效。也支持环境变量,优先级是 ENV > 配置文件 > 默认值。++## 命令行配置++也可以通过 `ocu config` 来配置,修改会保存到上面的文件里:++```bash+# 查看配置+ocu config++# 改成 JPG+ocu config set image.format jpg++# 查看某一项+ocu config get image.format++# 恢复默认(如果有 ENV,仍优先使用 ENV)+ocu config reset image.format++# 查看配置文件路径+ocu config path+```++`ocu` 也可以换成 `open-computer-use`。++## 配置项++| 配置项 | 默认值 | 说明 | 命令示例 |+| --- | --- | --- | --- |+| `image.format` | `png` | 输出格式,可选 `png`、`jpg`、`webp`。WebP 为无损编码。 | `ocu config set image.format webp` |+| `image.jpegQuality` | `0.8` | JPG 质量,范围 `0–1`,只影响 JPG。 | `ocu config set image.jpegQuality 0.9` |+| `image.maxLongEdgePixels` | `1280` | 图片长边上限,单位像素,范围 `1–16384`,整数。设为 `null` 关闭限制。 | `ocu config set image.maxLongEdgePixels 1920` |+| `image.scaleDownAfterMaxSize` | `true` | 超过长边上限时,`true` 等比例缩小,`false` 丢弃截图。 | `ocu config set image.scaleDownAfterMaxSize false` |+| `image.discardBelowPixelCount` | `64` | 宽 × 高小于此值时丢弃,等于时保留。范围 `0–268435456`,整数;`0` 关闭过滤。比如 `4096` 就是 `64×64` 的面积。缩小后的图片也会检查。 | `ocu config set image.discardBelowPixelCount 4096` |+| `image.captureTimeout` | `5` | ScreenCaptureKit 后备截图的超时,单位秒,范围 `0.01–300`。 | `ocu config set image.captureTimeout 10` |diff --git a/docs/exec-plans/completed/20261010-persistent-config.md b/docs/exec-plans/completed/20261010-persistent-config.mdnew file mode 100644index 0000000..593c087--- /dev/null+++ b/docs/exec-plans/completed/20261010-persistent-config.md@@ -0,0 +1,21 @@+# Persistent OCU configuration++## Goal and scope+Provide `ocu config` and a user JSON settings file, with environment overrides, for macOS screenshot encoding and bounds. Keep PNG defaults and the #65 hardware capture path. Windows/Linux screenshot policy is unchanged. Work from remote main independently of local experimental branches.++## Decisions and risks+- Use `$XDG_CONFIG_HOME/ocu/config.json`, otherwise `~/.config/ocu/config.json`; absolute `OPEN_COMPUTER_USE_CONFIG_FILE` overrides the file.+- Precedence: valid environment override, valid persisted value, built-in default. Reload each capture. Invalid runtime settings warn and fall back; CLI rejects invalid writes and malformed files.+- Screenshot settings are separate from safety gates. Atomic private writes; no implicit configuration writes during reads.+- Preserve screenshot coordinate mapping and bound PNG, JPG and lossless WebP. Keep existing public PNG helpers for compatibility.++- Final decision: remove byte budgets, their scale floor and ENV overrides; encode once after the long-edge cap. Stale budget settings are ignored and no longer listed or configurable.+- Follow-up: explicit resize/discard fields; minimum image area is width × height (default 64 pixels), checked before/after resizing. Preserve AX and explain omission, reject screenshot coordinates when omitted. Nullable maximum long edge; lossless WebP uses pinned static libwebp with bundled license.++## Validation and progress+- [x] Implement CLI, native loader and proxy isolation.+- [x] Test persistence, precedence, malformed files, encoding, bounds and packaging.+- [x] Synchronize docs/history and prepare a PR consolidating #72/#43 with both contributors credited.++## Results+Swift 192 tests (7 skipped) and Node 30 tests passed, including isolated npm staging. Docs/whitespace passed. Full smoke failed at fixture type_text; clean origin/main independently reproduced it. No signed App/release or real-desktop capture validation. Implementation is prepared on current main for the replacement PR, with Co-authored-by credit and existing PR heads untouched.diff --git a/docs/histories/2026-10/20261010-1429-persistent-config.md b/docs/histories/2026-10/20261010-1429-persistent-config.mdnew file mode 100644index 0000000..2a99832--- /dev/null+++ b/docs/histories/2026-10/20261010-1429-persistent-config.md@@ -0,0 +1,30 @@+# Persistent screenshot configuration++## User request+Provide a user config file, environment overrides and `ocu config` commands so ordinary users can persist screenshot settings without shell environment setup, while reviewing PRs #72 and #43.++## Changes+- Added npm launcher config list/get/set/reset/path/help; atomic 0600 JSON writes, unknown-field preservation and validation. Default path uses XDG_CONFIG_HOME or the user's `.config/ocu/config.json`; an absolute explicit file override is supported.+- Native macOS snapshots reload configuration per capture. Valid ENV > file > defaults; malformed files warn and use ENV/defaults. App-agent requests forward caller path and isolate image environment settings.+- Added PNG/JPG/lossless WebP encoding with a matching MIME and format-neutral screenshot data; PNG remains the default and the old PNG accessor remains compatible and the internal PNG helper keeps dimension capping. All encodings keep actual returned-image coordinate mapping. Fixed dimension-cap scaling below the previous absolute minScale.+- Preserved #65 hardware capture first and SCK fallback; timeout applies to SCK only. Windows/Linux unchanged. This implementation combines the configurable-bound direction in #43 with the JPEG direction in #72 with the final size-only policy requested by the user.+- Added explicit `maxLongEdgePixels`, `scaleDownAfterMaxSize`, `discardBelowPixelCount` settings. Minimum size means width × height, as requested: 63 pixels are omitted and 64 retained by default; zero disables this filter. Filtering also applies after resizing. Oversize images scale down by default, or are omitted when scaling is disabled. Nullable size bounds disable that limit. Legacy config names remain readable.+- Omitted screenshots carry a reason while preserving AX output; coordinate clicks/drags reject missing screenshot coordinates.+- Pinned libwebp-Xcode 1.6.0, linked libwebp statically and included its BSD license in npm/App packaging. JPG quality does not affect lossless WebP.+- Final user decision: removed maxBytes and byteBudgetMinScale together with the retry loop, old internal PNG byte limits and ENV overrides. Encoding runs once after the long-edge cap; stale file/ENV budget values have no effect, and CLI rejects removed settings. Regression tests verify exact retained dimensions for all formats.+- Added paired English/Chinese configuration guides linked from their respective READMEs. Following the user’s writing preference, simplified both guides to the file location and default JSON, basic CLI usage, then one complete settings/defaults/commands table. Removed detailed runtime, ENV, compatibility and integration sections; retained ENV precedence and the key pixel-area semantics. Corrected the architecture screenshot paragraph to remove the superseded byte-budget behavior.+- Updated READMEs, architecture/security, usage and feature notes; added configuration documentation.++## Verification+- Swift: 192 tests, 7 opt-in live tests skipped, zero failures.+- Node: 30 tests passed, including actual staging/launcher testing with inert native packaging fixtures; no release artifact validation claimed.+- Native tests cover file reload, malformed-file fallback, precedence, all three encodings, bounds and coordinate mapping. Node covers persistence, invalid writes, file privacy and packaging.+- Documentation check and git diff whitespace check passed.+- Full smoke failed at the fixture type_text append assertion. An independent unmodified origin/main snapshot reproduced the same failure. Cursor-idle phase was not reached; real desktop readability/capture is unverified.+- System Node could not load its simdjson library; used the bundled Codex Node runtime without changing system installation.++## Attribution and delivery+- Integrates the screenshot-format/API cleanup and byte-cap removal from [#72](https://github.com/iFurySt/open-codex-computer-use/pull/72) by @hyprcat (ATG), and the configurable capture/size controls, scaling fix and request environment isolation from [#43](https://github.com/iFurySt/open-codex-computer-use/pull/43) by @aikins01 (Aikins Laryea). The final implementation adapts both contributions to persistent configuration and the agreed size-only policy.+- Consolidated the local development commits with Co-authored-by trailers using the original contributors’ commit identities. The replacement PR credits both authors and closes #72/#43 when merged; their original branches remain untouched.+- Synced with current main before delivery, retaining the v1.0.0 release notes alongside the new feature entry. Post-sync Swift and Node tests passed.+- PR delivery is authorized; no merge, release, signed App replacement or live desktop automation. The pre-existing smoke failure remains a separate follow-up.diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.mdindex 2886e31..5dd76d5 100644--- a/docs/releases/feature-release-notes.md+++ b/docs/releases/feature-release-notes.md@@ -4,6 +4,7 @@| 日期 | 功能域 | 用户价值 | 变更摘要 || --- | --- | --- | --- |+| 2026-10-10 | macOS 截图配置 | 普通用户可用 `ocu config` 持久化截图偏好,无需维护 shell ENV。 | 新增用户 JSON、ENV 覆盖、PNG/JPG/WebP 格式、明确的超大缩小/丢弃开关、最小总像素数过滤、质量/SCK 超时;取消字节预算缩小,支持关闭长边上限,省略截图时说明原因,保留 #65 捕获路径与坐标映射。 || 2026-10-10 | 1.0.0 正式发布 | 用户可通过 npm 或直接下载使用跨平台桌面自动化,并以持久 JavaScript 会话编排操作。 | 基于当前 main 发布 `1.0.0`:统一所有版本源,交付 macOS 后台键盘、遮挡/其他 Space 窗口读取及显式 agent display,包含 Linux 快捷键与 Windows 无效截图修复;Linux / Windows runtime 继续标为实验性。 || 2026-10-09 | Linux 按键 | Linux 上 `press_key` 的 Enter、Tab、Escape、方向键和 `ctrl+a` 等快捷键会送达正确的按键,Agent 可以提交表单、切换焦点和使用快捷键。 | Linux bridge 改用 `KeySynthType.SYM` 发送具名键 keysym,并以 `LOCKMODIFIERS` / `UNLOCKMODIFIERS` 保持修饰键;此前 keysym 被当作 keycode,Enter 会输入 `4`,修饰键不生效。 || 2026-10-09 | Windows 截图可靠性 | 极小、最小化、隐藏、cloaked 或离屏窗口不会再向模型返回不可用的 PNG;极端情况下仍可读取 accessibility tree,并明确限制依赖截图坐标的操作。 | Windows bridge 在截图前校验窗口状态、virtual screen 交集和最小尺寸,Go 层再次解码 PNG header 复核;无有效截图时省略 image block,coordinate click / drag fail closed,不做放大、补边或占位图。 |diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swiftindex 83cd477..6c236d7 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift@@ -91,10 +91,7 @@ public struct SnapshotTextLimit: Equatable, Sendable {let accessibilityTreeMaxNodeCount = AccessibilityTreeLimits.defaultMaxNodeCountlet accessibilityTreeMaxDepth = AccessibilityTreeLimits.defaultMaxDepth-let screenshotCaptureTimeout: TimeInterval = 5-let screenshotResultMaxPNGBytes = 900_000let screenshotResultMaxDimension: CGFloat = 1280-let screenshotResultMinScale: CGFloat = 0.25private let windowVisibilityRecoveryDelay: TimeInterval = 0.7private let axWebAreaRole = "AXWebArea"// Chrome needs up to ~2s after AXManualAccessibility before the web tree exists.@@ -129,7 +126,12 @@ public struct AppSnapshot {public let windowBounds: CGRect?let targetWindowID: CGWindowID?let targetWindowLayer: Int?- public let screenshotPNGData: Data?+ public let screenshotData: Data?+ @available(*, deprecated, message: "Use screenshotData; screenshots may be JPEG")+ public var screenshotPNGData: Data? {+ guard let data = screenshotData, data.starts(with: [0x89, 0x50, 0x4e, 0x47]) else { return nil }+ return data+ }let mode: SnapshotModelet treeLines: [String]let focusedSummary: String?@@ -139,6 +141,7 @@ public struct AppSnapshot {let windowElement: AXUIElement?let elements: [Int: ElementRecord]+ var screenshotNote: String? = nilpublic var renderedText: String {renderedText(style: .fullState)@@ -256,7 +259,8 @@ enum SnapshotBuilder {lazyWebAccessibility: Bool) -> AppSnapshot {let windowBounds = windowCapture.bounds- let screenshotPNGData = windowCapture.pngDataIfAvailable()+ let screenshot = windowCapture.screenshotResult()+ let screenshotData = screenshot.datalet focusedElement = preferredFocusedElement(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)let selectedText = focusedElement.flatMap { copySelectedText($0, textLimit: textLimit) }let context = RenderContext(@@ -310,14 +314,15 @@ enum SnapshotBuilder {windowBounds: windowBounds,targetWindowID: windowCapture.windowID,targetWindowLayer: windowCapture.layer,- screenshotPNGData: screenshotPNGData,+ screenshotData: screenshotData,mode: .accessibility,treeLines: treeLines,focusedSummary: renderer.focusedSummary,focusedElement: focusedElement,selectedText: selectedText,windowElement: rootElement,- elements: renderer.records+ elements: renderer.records,+ screenshotNote: screenshot.note)}@@ -467,7 +472,7 @@ enum SnapshotBuilder {windowBounds: state.windowBounds.cgRect,targetWindowID: nil,targetWindowLayer: nil,- screenshotPNGData: nil,+ screenshotData: nil,mode: .fixture,treeLines: lines,focusedSummary: focusedSummary,@@ -498,6 +503,7 @@ private struct WindowCapture {let layer: Intlet bounds: CGRectlet image: CGImage?+ var imageConfig = ImageCaptureConfig.defaults/// Exact binding: the window the AX tree was walked from.static func resolve(for pid: pid_t, exactWindowID: CGWindowID?) -> WindowCapture? {@@ -516,7 +522,8 @@ private struct WindowCapture {else {continue}- return WindowCapture(windowID: exactWindowID, layer: layer, bounds: bounds, image: captureImage(windowID: exactWindowID, bounds: bounds))+ let config = ImageCaptureConfig.current+ return WindowCapture(windowID: exactWindowID, layer: layer, bounds: bounds, image: captureImage(windowID: exactWindowID, bounds: bounds, config: config), imageConfig: config)}return nil}@@ -562,22 +569,23 @@ private struct WindowCapture {return nil}- let image = captureImage(windowID: best.windowID, bounds: best.bounds)+ let config = ImageCaptureConfig.current+ let image = captureImage(windowID: best.windowID, bounds: best.bounds, config: config)- return WindowCapture(windowID: best.windowID, layer: best.layer, bounds: best.bounds, image: image)+ return WindowCapture(windowID: best.windowID, layer: best.layer, bounds: best.bounds, image: image, imageConfig: config)}/// WindowServer's hardware window capture first (any Space, covered or not,/// ~10-40ms), ScreenCaptureKit when it is unavailable.- private static func captureImage(windowID: CGWindowID, bounds: CGRect) -> CGImage? {+ private static func captureImage(windowID: CGWindowID, bounds: CGRect, config: ImageCaptureConfig) -> CGImage? {if let image = TimingLog.measure("snapshot.capture_hw") { SkyLightSPI.shared.hardwareCaptureWindow(windowID) } {return image}- return TimingLog.measure("snapshot.capture_sck") { captureImageWithScreenCaptureKit(windowID: windowID, bounds: bounds) }+ return TimingLog.measure("snapshot.capture_sck") { captureImageWithScreenCaptureKit(windowID: windowID, bounds: bounds, config: config) }}- private static func captureImageWithScreenCaptureKit(windowID: CGWindowID, bounds: CGRect) -> CGImage? {- try? BlockingAsyncBridge.run(timeout: screenshotCaptureTimeout) {+ private static func captureImageWithScreenCaptureKit(windowID: CGWindowID, bounds: CGRect, config: ImageCaptureConfig) -> CGImage? {+ try? BlockingAsyncBridge.run(timeout: config.captureTimeout) {let shareableContent = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)guard let window = shareableContent.windows.first(where: { $0.windowID == windowID }) else {return nil@@ -603,13 +611,11 @@ private struct WindowCapture {?? 1}- func pngDataIfAvailable() -> Data? {- guard let image else {- return nil- }-- return boundedScreenshotPNGData(for: image)+ func screenshotResult() -> ScreenshotEncodingResult {+ guard let image else { return ScreenshotEncodingResult(data: nil, note: nil) }+ return encodeScreenshot(for: image, config: imageConfig)}+}struct WindowCaptureCandidate {@@ -655,39 +661,55 @@ func preferredWindowCaptureCandidate(_ candidates: [WindowCaptureCandidate], titfunc boundedScreenshotPNGData(for image: CGImage,- maxBytes: Int = screenshotResultMaxPNGBytes,- maxDimension: CGFloat = screenshotResultMaxDimension,- minScale: CGFloat = screenshotResultMinScale+ maxDimension: CGFloat = screenshotResultMaxDimension) -> Data? {- guard image.width > 0, image.height > 0, maxBytes > 0 else {- return nil- }-- let original = pngData(for: image)- let largestDimension = CGFloat(max(image.width, image.height))- var scale = min(1, maxDimension / largestDimension)+ boundedScreenshotData(for: image, config: ImageCaptureConfig(+ discardBelowPixelCount: 0, maxDimension: maxDimension+ ))+}- if scale >= 1, let original, original.count <= maxBytes {- return original- }+struct ScreenshotEncodingResult {+ let data: Data?+ let note: String?+}- var best = original- while scale >= minScale {- guard let resized = resizedCGImage(image, scale: scale),- let data = pngData(for: resized)- else {- break- }+func boundedScreenshotData(for image: CGImage, config: ImageCaptureConfig) -> Data? {+ encodeScreenshot(for: image, config: config).data+}- best = data- if data.count <= maxBytes {- return data+func encodeScreenshot(for image: CGImage, config: ImageCaptureConfig) -> ScreenshotEncodingResult {+ func omitted(_ reason: String) -> ScreenshotEncodingResult { .init(data: nil, note: "Screenshot omitted: " + reason) }+ guard image.width > 0, image.height > 0 else { return omitted("invalid dimensions") }+ guard config.maxDimension.map({ $0.isFinite && $0 >= 1 && $0.rounded(.down) == $0 }) ?? true else {+ return omitted("invalid image configuration")+ }+ func isTiny(_ width: Int, _ height: Int) -> Bool {+ Double(width) * Double(height) < Double(config.discardBelowPixelCount)+ }+ if isTiny(image.width, image.height) {+ return omitted("\(image.width)×\(image.height) has fewer than \(config.discardBelowPixelCount) pixels")+ }+ let longEdge = CGFloat(max(image.width, image.height))+ if let maximum = config.maxDimension, longEdge > maximum, !config.scaleDownAfterMaxSize {+ return omitted("long edge \(Int(longEdge)) exceeds maxLongEdgePixels \(Int(maximum)); scaleDownAfterMaxSize is false")+ }+ func encode(_ image: CGImage) -> Data? {+ switch config.format {+ case "png": return pngData(for: image)+ case "webp": return losslessWebPData(for: image)+ case "jpg", "jpeg": return NSBitmapImageRep(cgImage: image).representation(using: .jpeg, properties: [.compressionFactor: config.jpegQuality])+ default: return nil}-- scale *= 0.85}-- return best+ let scale = min(1, (config.maxDimension ?? longEdge) / longEdge)+ let width = max(1, Int((CGFloat(image.width) * scale).rounded(.down)))+ let height = max(1, Int((CGFloat(image.height) * scale).rounded(.down)))+ if isTiny(width, height) {+ return omitted("resizing would fall below discardBelowPixelCount \(config.discardBelowPixelCount)")+ }+ guard let resized = scale == 1 ? image : resizedCGImage(image, scale: scale),+ let data = encode(resized) else { return omitted("\(config.format) encoding failed") }+ return .init(data: data, note: nil)}private func pngData(for image: CGImage) -> Data? {@@ -696,8 +718,8 @@ private func pngData(for image: CGImage) -> Data? {}private func resizedCGImage(_ image: CGImage, scale: CGFloat) -> CGImage? {- let width = max(1, Int((CGFloat(image.width) * scale).rounded()))- let height = max(1, Int((CGFloat(image.height) * scale).rounded()))+ let width = max(1, Int((CGFloat(image.width) * scale).rounded(.down)))+ let height = max(1, Int((CGFloat(image.height) * scale).rounded(.down)))let colorSpace = CGColorSpaceCreateDeviceRGB()let bitmapInfo = CGImageAlphaInfo.premultipliedLast.rawValuediff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swiftindex 4349590..b0779cd 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift@@ -667,7 +667,7 @@ public final class ComputerUseService {pulseVisualCursor(at: cursorTarget, clickCount: clickCount, mouseButton: button)} else if let x, let y {let screenshotPoint = CGPoint(x: x, y: y)- let point = screenshotPixelToWindowPointInSnapshot(snapshot: snapshot, point: screenshotPoint)+ let point = try screenshotPixelToWindowPointInSnapshot(snapshot: snapshot, point: screenshotPoint)let targetPoint = try windowPointToGlobalPoint(snapshot: snapshot, point: point)let cursorTarget = makeVisualCursorTarget(at: targetPoint,@@ -1757,15 +1757,18 @@ public final class ComputerUseService {private func screenshotToGlobalPoint(snapshot: AppSnapshot, x: Double, y: Double) throws -> CGPoint {try windowPointToGlobalPoint(snapshot: snapshot,- point: screenshotPixelToWindowPointInSnapshot(+ point: try screenshotPixelToWindowPointInSnapshot(snapshot: snapshot,point: CGPoint(x: x, y: y)))}- private func screenshotPixelToWindowPointInSnapshot(snapshot: AppSnapshot, point: CGPoint) -> CGPoint {- screenshotPixelToWindowPoint(+ private func screenshotPixelToWindowPointInSnapshot(snapshot: AppSnapshot, point: CGPoint) throws -> CGPoint {+ if snapshot.screenshotData == nil, let note = snapshot.screenshotNote {+ throw ComputerUseError.stateUnavailable(note + "; coordinate input requires a delivered screenshot")+ }+ return screenshotPixelToWindowPoint(point,screenshotPixelSize: screenshotPixelSize(snapshot: snapshot),windowBounds: snapshot.windowBounds@@ -1774,8 +1777,8 @@ public final class ComputerUseService {private func screenshotPixelSize(snapshot: AppSnapshot) -> CGSize? {guard- let screenshotPNGData = snapshot.screenshotPNGData,- let imageSource = CGImageSourceCreateWithData(screenshotPNGData as CFData, nil),+ let screenshotData = snapshot.screenshotData,+ let imageSource = CGImageSourceCreateWithData(screenshotData as CFData, nil),let properties = CGImageSourceCopyPropertiesAtIndex(imageSource, 0, nil) as? [CFString: Any],let pixelWidth = properties[kCGImagePropertyPixelWidth] as? CGFloat,let pixelHeight = properties[kCGImagePropertyPixelHeight] as? CGFloat,@@ -2036,8 +2039,9 @@ public final class ComputerUseService {private func snapshotResult(for snapshot: AppSnapshot, style: SnapshotTextStyle) -> ToolCallResult {var content = [ToolResultContentItem.text(snapshot.renderedText(style: style))]- if let screenshotPNGData = snapshot.screenshotPNGData {- content.append(.pngImage(screenshotPNGData))+ if let note = snapshot.screenshotNote { content.append(.text(note)) }+ if let screenshotData = snapshot.screenshotData {+ content.append(.screenshotImage(screenshotData))}return ToolCallResult(content: content)}diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ImageCaptureConfig.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ImageCaptureConfig.swiftnew file mode 100644index 0000000..63a5d21--- /dev/null+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ImageCaptureConfig.swift@@ -0,0 +1,122 @@+import Foundation+import CoreGraphics+import Darwin++/// Shared path resolution for native calls and app-agent request forwarding.+public enum OCUConfiguration {+ public static let environmentKeys: Set<String> = [+ "OPEN_COMPUTER_USE_CONFIG_FILE", "OPEN_COMPUTER_USE_IMAGE_FORMAT",+ "OPEN_COMPUTER_USE_IMAGE_JPEG_QUALITY", "OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT",+ "OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION",+ "OPEN_COMPUTER_USE_IMAGE_SCALE_DOWN_AFTER_MAX_SIZE",+ "OPEN_COMPUTER_USE_IMAGE_DISCARD_BELOW_PIXEL_COUNT",+ ]+ public static var environment: [String: String] {+ let keys = environmentKeys.union(["HOME", "XDG_CONFIG_HOME"])+ return Dictionary(uniqueKeysWithValues: keys.compactMap { key in+ getenv(key).map { (key, String(cString: $0)) }+ })+ }+ public static func fileURL(environment: [String: String]) throws -> URL {+ if let override = environment["OPEN_COMPUTER_USE_CONFIG_FILE"], !override.isEmpty {+ guard override.hasPrefix("/") else { throw ConfigError.invalidPath }+ return URL(fileURLWithPath: override)+ }+ let base = environment["XDG_CONFIG_HOME"].flatMap { $0.isEmpty ? nil : $0 }+ ?? (environment["HOME"] ?? FileManager.default.homeDirectoryForCurrentUser.path) + "/.config"+ guard base.hasPrefix("/") else { throw ConfigError.invalidPath }+ return URL(fileURLWithPath: base).appendingPathComponent("ocu/config.json")+ }+ enum ConfigError: Error { case invalidPath }+}++struct ImageCaptureConfig: Equatable {+ var format = "png"+ var jpegQuality: CGFloat = 0.8+ var captureTimeout: TimeInterval = 5+ var scaleDownAfterMaxSize = true+ var discardBelowPixelCount = 64+ var maxDimension: CGFloat? = 1280+ static let defaults = ImageCaptureConfig()++ static var current: ImageCaptureConfig {+ let env = OCUConfiguration.environment+ var document: [String: Any] = [:]+ do {+ let url = try OCUConfiguration.fileURL(environment: env)+ if FileManager.default.fileExists(atPath: url.path) {+ guard let root = try JSONSerialization.jsonObject(with: Data(contentsOf: url)) as? [String: Any],+ root["image"] == nil || root["image"] is [String: Any] else {+ throw CocoaError(.fileReadCorruptFile)+ }+ document = root["image"] as? [String: Any] ?? [:]+ }+ } catch {+ warn("Cannot read config: \(error). Using environment/defaults.")+ }+ return resolve(image: document, environment: env, warning: warn)+ }++ static func resolve(image: [String: Any], environment: [String: String], warning: (String) -> Void = { _ in }) -> ImageCaptureConfig {+ var image = image+ // Compatibility for settings persisted by the first configuration implementation.+ if image["maxLongEdgePixels"] == nil { image["maxLongEdgePixels"] = image["maxDimension"] }+ var result = defaults+ func number(_ name: String, _ envKey: String, _ fallback: Double, min: Double, max: Double, integer: Bool = false) -> Double {+ func valid(_ value: Any, text: Bool) -> Double? {+ let n: Double?+ if text, let raw = value as? String { n = Double(raw.trimmingCharacters(in: .whitespacesAndNewlines)) }+ else if let raw = value as? NSNumber, CFGetTypeID(raw) != CFBooleanGetTypeID() { n = raw.doubleValue }+ else { n = nil }+ guard let n, n.isFinite, n >= min, n <= max, !integer || n.rounded(.down) == n else { return nil }+ return n+ }+ var value = fallback+ if let raw = image[name], !(raw is NSNull) {+ if let n = valid(raw, text: false) { value = n } else { warning("Invalid image.\(name); using default") }+ }+ if let raw = environment[envKey], raw != "null" {+ if let n = valid(raw, text: true) { value = n } else { warning("Invalid \(envKey); using file/default") }+ }+ return value+ }+ if let value = image["format"] {+ if let value = value as? String, ["png", "jpg", "jpeg", "webp"].contains(value) { result.format = value == "jpeg" ? "jpg" : value }+ else { warning("Invalid image.format; using png") }+ }+ if let value = environment["OPEN_COMPUTER_USE_IMAGE_FORMAT"] {+ if ["png", "jpg", "jpeg", "webp"].contains(value) { result.format = value == "jpeg" ? "jpg" : value }+ else { warning("Invalid OPEN_COMPUTER_USE_IMAGE_FORMAT; using file/default") }+ }+ result.jpegQuality = number("jpegQuality", "OPEN_COMPUTER_USE_IMAGE_JPEG_QUALITY", 0.8, min: 0, max: 1)+ result.captureTimeout = number("captureTimeout", "OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT", 5, min: 0.01, max: 300)+ result.maxDimension = number("maxLongEdgePixels", "OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION", 1280, min: 1, max: 16384, integer: true)+ func boolean(_ name: String, _ key: String, _ fallback: Bool) -> Bool {+ var value = fallback+ if let raw = image[name] {+ if let raw = raw as? NSNumber, CFGetTypeID(raw) == CFBooleanGetTypeID() { value = raw.boolValue }+ else { warning("Invalid image.\(name); using default") }+ }+ if let raw = environment[key] {+ if raw == "true" || raw == "false" { value = raw == "true" }+ else { warning("Invalid \(key); using file/default") }+ }+ return value+ }+ func disabled(_ name: String, _ key: String) -> Bool {+ if let raw = environment[key] {+ if raw == "null" { return true }+ let upper = 16384.0+ if let n = Double(raw), n.isFinite, n >= 1, n <= upper, n.rounded(.down) == n { return false }+ }+ return image[name] is NSNull+ }+ result.scaleDownAfterMaxSize = boolean("scaleDownAfterMaxSize", "OPEN_COMPUTER_USE_IMAGE_SCALE_DOWN_AFTER_MAX_SIZE", true)+ result.discardBelowPixelCount = Int(number("discardBelowPixelCount", "OPEN_COMPUTER_USE_IMAGE_DISCARD_BELOW_PIXEL_COUNT", 64, min: 0, max: 268435456, integer: true))+ if disabled("maxLongEdgePixels", "OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION") { result.maxDimension = nil }+ return result+ }+ private static func warn(_ message: String) {+ FileHandle.standardError.write(Data("ocu config: \(message)\n".utf8))+ }+}diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swiftindex 0072cb8..d1f4737 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift@@ -12,6 +12,13 @@ public struct ToolResultContentItem: @unchecked Sendable {)}+ static func screenshotImage(_ data: Data) -> ToolResultContentItem {+ ToolResultContentItem(dictionary: [+ "type": "image", "data": data.base64EncodedString(),+ "mimeType": data.starts(with: [0xff, 0xd8, 0xff]) ? "image/jpeg" : (data.count >= 12 && String(data: data.subdata(in: 8..<12), encoding: .ascii) == "WEBP" ? "image/webp" : "image/png"),+ ])+ }+public static func pngImage(_ data: Data) -> ToolResultContentItem {ToolResultContentItem(dictionary: [diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/WebPEncoding.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/WebPEncoding.swiftnew file mode 100644index 0000000..3fb2a7f--- /dev/null+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/WebPEncoding.swift@@ -0,0 +1,29 @@+import CoreGraphics+import Foundation+import libwebp++/// Encode screenshots losslessly so text and small controls keep their original pixels.+func losslessWebPData(for image: CGImage) -> Data? {+ let stride = image.width * 4+ guard image.width <= 16383, image.height <= 16383 else { return nil }+ var pixels = [UInt8](repeating: 0, count: stride * image.height)+ return pixels.withUnsafeMutableBytes { buffer in+ guard let context = CGContext(data: buffer.baseAddress, width: image.width, height: image.height,+ bitsPerComponent: 8, bytesPerRow: stride, space: CGColorSpaceCreateDeviceRGB(),+ bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue | CGBitmapInfo.byteOrder32Big.rawValue) else { return nil }+ context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))+ let bytes = buffer.bindMemory(to: UInt8.self)+ // CoreGraphics renders premultiplied RGBA; libwebp expects straight RGBA.+ for offset in Swift.stride(from: 0, to: bytes.count, by: 4) {+ let alpha = Int(bytes[offset + 3])+ if alpha > 0 && alpha < 255 {+ for channel in 0..<3 { bytes[offset + channel] = UInt8(min(255, (Int(bytes[offset + channel]) * 255 + alpha / 2) / alpha)) }+ }+ }+ var output: UnsafeMutablePointer<UInt8>?+ let size = WebPEncodeLosslessRGBA(bytes.baseAddress, Int32(image.width), Int32(image.height), Int32(stride), &output)+ guard size > 0, let output else { return nil }+ defer { WebPFree(output) }+ return Data(bytes: output, count: size)+ }+}diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/AgentDisplayLiveTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/AgentDisplayLiveTests.swiftindex 4dc9739..f1949b0 100644--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/AgentDisplayLiveTests.swift+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/AgentDisplayLiveTests.swift@@ -79,7 +79,7 @@ final class AgentDisplayLiveTests: XCTestCase {let parkedSnapshot = try SnapshotBuilder.build(for: descriptor, recoveryPolicy: .readOnly)XCTAssertEqual(parkedSnapshot.targetWindowID, window.id)XCTAssertTrue(parkedSnapshot.treeLines.contains { $0.contains("sky probe paragraph") }, "parked snapshot must include web content")- XCTAssertNotNil(parkedSnapshot.screenshotPNGData)+ XCTAssertNotNil(parkedSnapshot.screenshotData)XCTAssertEqual(NSWorkspace.shared.frontmostApplication?.processIdentifier, frontBefore, "parking must not change the frontmost app")XCTAssertEqual(NSEvent.mouseLocation, mouseBefore, "parking must not move the pointer")diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/AppMatrixLiveTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/AppMatrixLiveTests.swiftindex 179aa14..40745d8 100644--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/AppMatrixLiveTests.swift+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/AppMatrixLiveTests.swift@@ -81,7 +81,7 @@ final class AppMatrixLiveTests: XCTestCase {let nodes = snapshot.elements.countlet hasWeb = snapshot.treeLines.contains { $0.contains("HTML content") }let covered = snapshot.treeLines.contains { $0.hasPrefix("Note: this window is covered") }- let shot = snapshot.screenshotPNGData?.count ?? 0+ let shot = snapshot.screenshotData?.count ?? 0guard let windowID = snapshot.targetWindowID, let bounds = snapshot.windowBounds else {rows.append(row(name, engine, "snapshot ok nodes=\(nodes) web=\(hasWeb) shot=\(shot)B \(snapMs)ms; no window id"))continue@@ -104,7 +104,7 @@ final class AppMatrixLiveTests: XCTestCase {let parkedMs = Int((TimingLog.now() - s0) * 1000)let parkedWeb = parkedSnapshot.treeLines.contains { $0.contains("HTML content") }let onDisplay = AgentDisplay.shared.displayBounds.map { $0.contains(CGPoint(x: (parkedSnapshot.windowBounds?.minX ?? -1) + 1, y: (parkedSnapshot.windowBounds?.minY ?? -1) + 1)) } ?? false- var parkDetail = "PARK \(parkMs)ms onDisplay=\(onDisplay) snapshot \(parkedMs)ms nodes=\(parkedSnapshot.elements.count) web=\(parkedWeb) shot=\(parkedSnapshot.screenshotPNGData?.count ?? 0)B; " + inputRound(parkedSnapshot, descriptor, app.processIdentifier)+ var parkDetail = "PARK \(parkMs)ms onDisplay=\(onDisplay) snapshot \(parkedMs)ms nodes=\(parkedSnapshot.elements.count) web=\(parkedWeb) shot=\(parkedSnapshot.screenshotData?.count ?? 0)B; " + inputRound(parkedSnapshot, descriptor, app.processIdentifier)let r0 = TimingLog.now()try AgentDisplay.shared.restore(windowID: windowID)let restoredBounds = (try? SnapshotBuilder.build(for: descriptor, recoveryPolicy: .readOnly))?.windowBoundsdiff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/CrossSpaceLiveTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/CrossSpaceLiveTests.swiftindex 646799f..9ea33d8 100644--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/CrossSpaceLiveTests.swift+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/CrossSpaceLiveTests.swift@@ -123,7 +123,7 @@ final class CrossSpaceLiveTests: XCTestCase {let parkedSnapshot = try SnapshotBuilder.build(for: descriptor, recoveryPolicy: .readOnly)XCTAssertEqual(parkedSnapshot.targetWindowID, window.id, "snapshot must resolve the off-screen window")XCTAssertTrue(parkedSnapshot.treeLines.contains { $0.contains("sky probe") }, "parked snapshot must include web content")- XCTAssertNotNil(parkedSnapshot.screenshotPNGData, "parked window must still be captured")+ XCTAssertNotNil(parkedSnapshot.screenshotData, "parked window must still be captured")XCTAssertEqual(activeSpace(cid), homeSpace, "snapshot must not switch Spaces")XCTAssertNotEqual(NSWorkspace.shared.frontmostApplication?.processIdentifier, window.pid, "snapshot must not activate Chrome")diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/ImageCaptureConfigTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/ImageCaptureConfigTests.swiftnew file mode 100644index 0000000..c78973d--- /dev/null+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/ImageCaptureConfigTests.swift@@ -0,0 +1,95 @@+import AppKit+import ImageIO+import Darwin+import XCTest+@testable import OpenComputerUseKit++final class ImageCaptureConfigTests: XCTestCase {+ func testPrecedenceAndInvalidOverrides() {+ let file: [String: Any] = ["format": "jpeg", "maxDimension": 640]+ let env = ["OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION": "800"]+ let config = ImageCaptureConfig.resolve(image: file, environment: env)+ XCTAssertEqual(config.format, "jpg")+ XCTAssertEqual(config.maxDimension, 800)+ XCTAssertEqual(config.captureTimeout, 5)+ XCTAssertEqual(ImageCaptureConfig.resolve(image: ["maxDimension": true, "format": "gif"], environment: [:]), .defaults)+ }+ func testPathResolution() throws {+ XCTAssertEqual(try OCUConfiguration.fileURL(environment: ["HOME": "/users/test"]).path, "/users/test/.config/ocu/config.json")+ XCTAssertEqual(try OCUConfiguration.fileURL(environment: ["XDG_CONFIG_HOME": "/settings"]).path, "/settings/ocu/config.json")+ XCTAssertEqual(try OCUConfiguration.fileURL(environment: ["OPEN_COMPUTER_USE_CONFIG_FILE": "/settings/custom.json"]).path, "/settings/custom.json")+ XCTAssertThrowsError(try OCUConfiguration.fileURL(environment: ["XDG_CONFIG_HOME": "relative"]))+ }+ func testNativeReloadsPersistedConfigurationAndFallsBackOnMalformedFile() throws {+ let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)+ try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)+ let file = directory.appendingPathComponent("config.json")+ let keys = OCUConfiguration.environmentKeys+ let previous = Dictionary(uniqueKeysWithValues: keys.map { ($0, getenv($0).map { String(cString: $0) }) })+ for key in keys { unsetenv(key) }+ defer {+ for (key, value) in previous {+ if let value { setenv(key, value, 1) } else { unsetenv(key) }+ }+ try? FileManager.default.removeItem(at: directory)+ }+ setenv("OPEN_COMPUTER_USE_CONFIG_FILE", file.path, 1)+ try Data(#"{"image":{"format":"jpeg","maxDimension":640}}"#.utf8).write(to: file, options: .atomic)+ XCTAssertEqual(ImageCaptureConfig.current.format, "jpg")+ XCTAssertEqual(ImageCaptureConfig.current.maxDimension, 640)+ try Data(#"{"image":{"format":"png","maxDimension":800}}"#.utf8).write(to: file, options: .atomic)+ XCTAssertEqual(ImageCaptureConfig.current.maxDimension, 800)+ setenv("OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION", "1024", 1)+ XCTAssertEqual(ImageCaptureConfig.current.maxDimension, 1024)+ try Data("broken".utf8).write(to: file, options: .atomic)+ XCTAssertEqual(ImageCaptureConfig.current.maxDimension, 1024)+ XCTAssertEqual(ImageCaptureConfig.current.format, "png")+ }+ func testExplicitPoliciesAndNullableLimits() {+ let config = ImageCaptureConfig.resolve(image: ["maxLongEdgePixels": NSNull(), "scaleDownAfterMaxSize": false, "discardBelowPixelCount": 4096], environment: [:])+ XCTAssertNil(config.maxDimension)+ XCTAssertFalse(config.scaleDownAfterMaxSize)+ XCTAssertEqual(config.discardBelowPixelCount, 4096)+ let overridden = ImageCaptureConfig.resolve(image: ["maxLongEdgePixels": NSNull()], environment: ["OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION": "1000"])+ XCTAssertEqual(overridden.maxDimension, 1000)+ }+ func testPixelAreaDiscardBoundaryAndOversizePolicy() throws {+ func image(_ width: Int, _ height: Int) throws -> CGImage {+ let context = try XCTUnwrap(CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue))+ return try XCTUnwrap(context.makeImage())+ }+ let below = encodeScreenshot(for: try image(7, 9), config: .defaults)+ XCTAssertNil(below.data)+ XCTAssertTrue(below.note?.contains("64 pixels") == true)+ XCTAssertNotNil(encodeScreenshot(for: try image(8, 8), config: .defaults).data)+ XCTAssertNotNil(encodeScreenshot(for: try image(1, 64), config: .defaults).data)+ let oversized = try image(800, 600)+ let omitted = encodeScreenshot(for: oversized, config: ImageCaptureConfig(scaleDownAfterMaxSize: false, maxDimension: 80))+ XCTAssertNil(omitted.data)+ XCTAssertTrue(omitted.note?.contains("scaleDownAfterMaxSize is false") == true)+ let resizedTooSmall = encodeScreenshot(for: oversized, config: ImageCaptureConfig(discardBelowPixelCount: 10000, maxDimension: 80))+ XCTAssertNil(resizedTooSmall.data)+ XCTAssertTrue(resizedTooSmall.note?.contains("resizing would fall below") == true)+ let unbounded = encodeScreenshot(for: oversized, config: ImageCaptureConfig(maxDimension: nil))+ XCTAssertNotNil(unbounded.data)+ XCTAssertNil(unbounded.note)+ }+ func testEncodingBoundsAndCoordinateMapping() throws {+ let context = try XCTUnwrap(CGContext(data: nil, width: 800, height: 600, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue))+ context.setFillColor(NSColor.blue.cgColor)+ context.fill(CGRect(x: 0, y: 0, width: 800, height: 600))+ let image = try XCTUnwrap(context.makeImage())+ for format in ["png", "jpg", "webp"] {+ let config = ImageCaptureConfig.resolve(image: ["format": format, "maxLongEdgePixels": 80, "maxBytes": 1, "byteBudgetMinScale": 0.01, "minScale": 0.01], environment: ["OPEN_COMPUTER_USE_IMAGE_MAX_BYTES": "1", "OPEN_COMPUTER_USE_IMAGE_MIN_SCALE": "0.01"])+ let data = try XCTUnwrap(boundedScreenshotData(for: image, config: config))+ let source = try XCTUnwrap(CGImageSourceCreateWithData(data as CFData, nil))+ let decoded = try XCTUnwrap(CGImageSourceCreateImageAtIndex(source, 0, nil))+ XCTAssertEqual(decoded.width, 80)+ XCTAssertEqual(decoded.height, 60)+ XCTAssertEqual(ToolResultContentItem.screenshotImage(data).dictionary["mimeType"] as? String, format == "jpg" ? "image/jpeg" : "image/\(format)")+ let mapped = screenshotPixelToWindowPoint(CGPoint(x: 40, y: 30), screenshotPixelSize: CGSize(width: 80, height: 60), windowBounds: CGRect(x: 0, y: 0, width: 800, height: 600))+ XCTAssertEqual(mapped, CGPoint(x: 400, y: 300))++ }+ }+}diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OcclusionKeepAliveLiveTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OcclusionKeepAliveLiveTests.swiftindex 0278104..9edc9cb 100644--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OcclusionKeepAliveLiveTests.swift+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OcclusionKeepAliveLiveTests.swift@@ -91,7 +91,7 @@ final class OcclusionKeepAliveLiveTests: XCTestCase {XCTAssertTrue(coveredSnapshot.treeLines.contains { $0.contains("sky probe") }, "covered snapshot must still include the web content")XCTAssertFalse(coveredSnapshot.treeLines.contains { $0.hasPrefix("Note: this window is covered") })- XCTAssertNotNil(coveredSnapshot.screenshotPNGData)+ XCTAssertNotNil(coveredSnapshot.screenshotData)XCTAssertNotEqual(NSWorkspace.shared.frontmostApplication?.processIdentifier, window.pid, "snapshots must not activate Chrome")WindowOcclusionKeepAlive.shared.releaseAll()diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swiftindex 89a5ac2..952acfa 100644--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift@@ -233,19 +233,18 @@ final class OpenComputerUseKitTests: XCTestCase {let image = try makeNoisyTestImage(width: 800, height: 600)let data = try XCTUnwrap(boundedScreenshotPNGData(for: image,- maxBytes: 50_000,- maxDimension: 320,- minScale: 0.05+ maxDimension: 320))let size = try imageSize(in: data)- XCTAssertLessThanOrEqual(data.count, 50_000)+ XCTAssertEqual(size.width, 320)+ XCTAssertEqual(size.height, 240)XCTAssertLessThanOrEqual(max(size.width, size.height), 320)}func testBoundedScreenshotPNGDataKeepsSmallScreenshotsAtOriginalSize() throws {let image = try makeSolidTestImage(width: 32, height: 24)- let data = try XCTUnwrap(boundedScreenshotPNGData(for: image, maxBytes: 1_000_000, maxDimension: 320))+ let data = try XCTUnwrap(boundedScreenshotPNGData(for: image, maxDimension: 320))let size = try imageSize(in: data)XCTAssertEqual(size.width, 32)@@ -2502,7 +2501,7 @@ final class OpenComputerUseKitTests: XCTestCase {windowBounds: nil,targetWindowID: nil,targetWindowLayer: nil,- screenshotPNGData: nil,+ screenshotData: nil,mode: .accessibility,treeLines: treeLines,focusedSummary: focusedSummary,diff --git a/scripts/build-open-computer-use-app.sh b/scripts/build-open-computer-use-app.shindex dd5ef9d..d596b51 100755--- a/scripts/build-open-computer-use-app.sh+++ b/scripts/build-open-computer-use-app.sh@@ -317,6 +317,7 @@ mkdir -p "${iconset_dir}""${iconset_build_script}" "${icon_master_png}" "${iconset_dir}"iconutil -c icns "${iconset_dir}" -o "${resources_dir}/${bundle_icon_name}"cp "${cursor_reference_source}" "${resources_dir}/official-software-cursor-window-252.png"+cp "${repo_root}/THIRD_PARTY_NOTICES.md" "${resources_dir}/THIRD_PARTY_NOTICES.md"cat > "${contents_dir}/Info.plist" <<PLIST<?xml version="1.0" encoding="UTF-8"?>diff --git a/scripts/node-repl/ocu-config.mjs b/scripts/node-repl/ocu-config.mjsnew file mode 100644index 0000000..db8decc--- /dev/null+++ b/scripts/node-repl/ocu-config.mjs@@ -0,0 +1,128 @@+import { existsSync, readFileSync, mkdirSync, writeFileSync, renameSync, rmSync } from "node:fs";+import { homedir } from "node:os";+import path from "node:path";+import { randomUUID } from "node:crypto";++export const settings = {+ "image.format": { default: "png", env: "OPEN_COMPUTER_USE_IMAGE_FORMAT", choices: ["png", "jpg", "webp"] },+ "image.jpegQuality": { default: 0.8, env: "OPEN_COMPUTER_USE_IMAGE_JPEG_QUALITY", min: 0, max: 1 },+ "image.maxLongEdgePixels": { default: 1280, env: "OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION", min: 1, max: 16384, integer: true, nullable: true },+ "image.scaleDownAfterMaxSize": { default: true, env: "OPEN_COMPUTER_USE_IMAGE_SCALE_DOWN_AFTER_MAX_SIZE", boolean: true },+ "image.discardBelowPixelCount": { default: 64, env: "OPEN_COMPUTER_USE_IMAGE_DISCARD_BELOW_PIXEL_COUNT", min: 0, max: 268435456, integer: true },+ "image.captureTimeout": { default: 5, env: "OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT", min: 0.01, max: 300 },+};++export function configPath(env = process.env) {+ if (env.OPEN_COMPUTER_USE_CONFIG_FILE) {+ if (!path.isAbsolute(env.OPEN_COMPUTER_USE_CONFIG_FILE)) throw new Error("OPEN_COMPUTER_USE_CONFIG_FILE must be absolute");+ return env.OPEN_COMPUTER_USE_CONFIG_FILE;+ }+ const base = env.XDG_CONFIG_HOME || path.join(env.HOME || homedir(), ".config");+ if (!path.isAbsolute(base)) throw new Error("XDG_CONFIG_HOME must be absolute");+ return path.join(base, "ocu", "config.json");+}++export function validateSetting(key, raw, { fromText = false } = {}) {+ const spec = settings[key];+ if (!spec) throw new Error(`Unknown setting: ${key}`);+ if (spec.nullable && (raw === null || (fromText && raw === "null"))) return null;+ if (spec.boolean) {+ if (typeof raw === "boolean") return raw;+ if (fromText && ["true", "false"].includes(raw)) return raw === "true";+ throw new Error(`${key} must be true or false`);+ }+ if (key === "image.format" && raw === "jpeg") raw = "jpg";+ const value = fromText && !spec.choices ? (String(raw).trim() ? Number(raw) : NaN) : raw;+ if (spec.choices) {+ if (!spec.choices.includes(value)) throw new Error(`${key} must be ${spec.choices.join(" or ")}`);+ } else if (typeof value !== "number" || !Number.isFinite(value) || value < spec.min || value > spec.max || (spec.integer && !Number.isSafeInteger(value))) {+ throw new Error(`${key} must be ${spec.integer ? "an integer" : "a number"} between ${spec.min} and ${spec.max}`);+ }+ return value;+}++export function readConfig(file) {+ if (!existsSync(file)) return {};+ let value;+ try { value = JSON.parse(readFileSync(file, "utf8")); }+ catch (error) { throw new Error(`Cannot read config ${file}: ${error.message}`); }+ if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Config must be a JSON object");+ if (value.image !== undefined && (!value.image || typeof value.image !== "object" || Array.isArray(value.image))) throw new Error("Config image must be an object");+ return value;+}++export function inspectConfig(env = process.env) {+ const file = configPath(env);+ const persisted = readConfig(file);+ const values = {}, sources = {}, warnings = [];+ for (const [key, spec] of Object.entries(settings)) {+ const name = key.split(".")[1];+ const legacyName = { maxLongEdgePixels: "maxDimension" }[name];+ if (persisted.image?.[name] === undefined && legacyName && persisted.image?.[legacyName] !== undefined) persisted.image[name] = persisted.image[legacyName];+ let value = spec.default, source = "default";+ if (persisted.image?.[name] !== undefined) {+ try { value = validateSetting(key, persisted.image[name]); source = "file"; }+ catch (error) { warnings.push(error.message); }+ }+ if (env[spec.env] !== undefined) {+ try { value = validateSetting(key, env[spec.env], { fromText: true }); source = "env"; }+ catch (error) { warnings.push(`${spec.env}: ${error.message}`); }+ }+ values[key] = value; sources[key] = source;+ }+ return { path: file, values, sources, warnings };+}++function save(file, config) {+ mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });+ const temporary = `${file}.${randomUUID()}.tmp`;+ try {+ writeFileSync(temporary, JSON.stringify(config, null, 2) + "\n", { mode: 0o600, flag: "wx" });+ renameSync(temporary, file);+ } finally { rmSync(temporary, { force: true }); }+}++export function configCommand(argv, env = process.env) {+ if (argv.length === 1 && ["--help", "-h"].includes(argv[0])) return configHelp;+ const action = argv[0] || "list";+ if (action === "path" && argv.length === 1) return configPath(env);+ if (action === "list" && (argv.length <= 1 || (argv.length === 2 && argv[1] === "--json"))) {+ const report = inspectConfig(env);+ if (argv[1] === "--json") return JSON.stringify(report, null, 2);+ return [`Config: ${report.path}`, "Screenshot settings apply to macOS.", ...Object.entries(report.values).map(([key, value]) => `${key} = ${value} (${report.sources[key]})`), ...report.warnings.map(w => `Warning: ${w}`)].join("\n");+ }+ if (action === "get" && argv.length === 2) {+ validateKey(argv[1]); return String(inspectConfig(env).values[argv[1]]);+ }+ if (action === "set" && argv.length === 3) {+ const [key, raw] = argv.slice(1), value = validateSetting(key, raw, { fromText: true });+ const file = configPath(env), config = readConfig(file);+ config.image ??= {}; config.image[key.split(".")[1]] = value;+ save(file, config);+ return `${key} = ${value} saved to ${file}${env[settings[key].env] !== undefined ? `\nWarning: ${settings[key].env} overrides this saved setting.` : ""}`;+ }+ if (action === "reset" && argv.length === 2) {+ const key = argv[1]; validateKey(key);+ const file = configPath(env), config = readConfig(file);+ if (config.image) {+ const name = key.split(".")[1]; delete config.image[name];+ const legacy = { maxLongEdgePixels: "maxDimension" }[name];+ if (legacy) delete config.image[legacy];+ }+ if (existsSync(file)) save(file, config);+ return `${key} reset (environment overrides still apply)`;+ }+ throw new Error("Usage: ocu config [list [--json] | path | get KEY | set KEY VALUE | reset KEY]");+}+function validateKey(key) { if (!settings[key]) throw new Error(`Unknown setting: ${key}`); }+export const configHelp = `Usage: ocu config [list [--json] | path | get KEY | set KEY VALUE | reset KEY]+Settings (macOS screenshots): ${Object.keys(settings).join(", ")}+image.maxLongEdgePixels: maximum long edge in pixels (null disables).+image.scaleDownAfterMaxSize: true resizes oversized images; false omits them.+image.discardBelowPixelCount: omit when width * height is below this total pixel count (0 disables).+Formats: png (default), jpg, webp (lossless); jpeg is a jpg alias.+Precedence: environment > config file > defaults. Changes apply on the next capture.+Examples:+ ocu config set image.format jpg+ ocu config set image.maxLongEdgePixels 1024+ ocu config reset image.format`;diff --git a/scripts/node-repl/ocu-config.test.mjs b/scripts/node-repl/ocu-config.test.mjsnew file mode 100644index 0000000..34437e9--- /dev/null+++ b/scripts/node-repl/ocu-config.test.mjs@@ -0,0 +1,107 @@+import { spawnSync } from "node:child_process";+import { fileURLToPath } from "node:url";+import test from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync, rmSync, readFileSync, writeFileSync, statSync, cpSync, mkdirSync } from "node:fs";+import { tmpdir } from "node:os";+import path from "node:path";+import { configCommand, configPath, inspectConfig } from "./ocu-config.mjs";+import { main } from "./open-computer-use-cli.mjs";+function fixture(t) {+ const directory = mkdtempSync(path.join(tmpdir(), "ocu-config-test-"));+ t.after(() => rmSync(directory, { recursive: true, force: true }));+ return { HOME: directory, OPEN_COMPUTER_USE_CONFIG_FILE: path.join(directory, "config.json") };+}+test("config paths and read-only defaults", t => {+ const env = fixture(t);+ assert.equal(configPath({ HOME: "/users/test" }), "/users/test/.config/ocu/config.json");+ assert.equal(configPath({ XDG_CONFIG_HOME: "/settings" }), "/settings/ocu/config.json");+ assert.throws(() => configPath({ XDG_CONFIG_HOME: "relative" }), /absolute/);+ assert.equal(inspectConfig(env).values["image.format"], "png");+ assert.throws(() => statSync(env.OPEN_COMPUTER_USE_CONFIG_FILE), /ENOENT/);+});+test("persistent settings, environment precedence and reset", t => {+ const env = fixture(t);+ configCommand(["set", "image.format", "jpg"], env);+ configCommand(["set", "image.maxLongEdgePixels", "640"], env);+ assert.equal(inspectConfig(env).values["image.format"], "jpg");+ const report = inspectConfig({ ...env, OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION: "800" });+ assert.equal(report.values["image.maxLongEdgePixels"], 800);+ assert.equal(report.sources["image.maxLongEdgePixels"], "env");+ assert.equal(inspectConfig({ ...env, OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION: "NaN" }).values["image.maxLongEdgePixels"], 640);+ configCommand(["reset", "image.format"], env);+ assert.equal(inspectConfig(env).values["image.format"], "png");+ assert.equal(statSync(env.OPEN_COMPUTER_USE_CONFIG_FILE).mode & 0o777, 0o600);+});+test("invalid settings and malformed files never overwrite existing content", t => {+ const env = fixture(t);+ configCommand(["set", "image.format", "jpg"], env);+ const before = readFileSync(env.OPEN_COMPUTER_USE_CONFIG_FILE, "utf8");+ for (const [key, value] of [["image.maxLongEdgePixels", "0"], ["image.maxBytes", "Infinity"], ["image.byteBudgetMinScale", "1.2"], ["image.format", "gif"], ["safety.global", "1"]]) {+ assert.throws(() => configCommand(["set", key, value], env));+ }+ assert.equal(readFileSync(env.OPEN_COMPUTER_USE_CONFIG_FILE, "utf8"), before);+ writeFileSync(env.OPEN_COMPUTER_USE_CONFIG_FILE, "broken");+ assert.throws(() => configCommand(["set", "image.format", "png"], env), /Cannot read config/);+ assert.equal(readFileSync(env.OPEN_COMPUTER_USE_CONFIG_FILE, "utf8"), "broken");+});+test("unknown config fields are preserved on writes", t => {+ const env = fixture(t);+ writeFileSync(env.OPEN_COMPUTER_USE_CONFIG_FILE, JSON.stringify({ future: { enabled: true }, image: { future: "keep" } }));+ configCommand(["set", "image.format", "jpg"], env);+ const saved = JSON.parse(readFileSync(env.OPEN_COMPUTER_USE_CONFIG_FILE));+ assert.equal(saved.future.enabled, true);+ assert.equal(saved.image.future, "keep");+});+test("launcher config works without native artifacts or spawning", async t => {+ const env = fixture(t), output = [];+ const exit = await main({ packageRoot: env.HOME, platformPackages: {}, env, argv: ["config", "list", "--json"], output: { write: s => output.push(s) }, spawnFn: () => { throw new Error("unexpected spawn"); } });+ assert.equal(exit, 0);+ assert.equal(JSON.parse(output.join("")).values["image.format"], "png");+});++test("staged npm launcher includes config module and persists without a native launch", t => {+ const env = fixture(t);+ const root = path.join(env.HOME, "repo");+ const source = fileURLToPath(new URL("../../", import.meta.url));+ mkdirSync(root);+ for (const entry of ["scripts", "skills", "plugins", ".agents", "LICENSE", "THIRD_PARTY_NOTICES.md"]) cpSync(path.join(source, entry), path.join(root, entry), { recursive: true });+ // Packaging fixtures only: config must never execute these native artifacts.+ for (const relative of ["Open Computer Use.app/Contents/MacOS/OpenComputerUse", "linux/arm64/open-computer-use", "linux/amd64/open-computer-use", "windows/arm64/open-computer-use.exe", "windows/amd64/open-computer-use.exe"]) {+ const file = path.join(root, "dist", relative);+ mkdirSync(path.dirname(file), { recursive: true });+ writeFileSync(file, "fixture", { mode: 0o755 });+ }+ const build = spawnSync(process.execPath, [path.join(root, "scripts/npm/build-packages.mjs"), "--skip-build"], { encoding: "utf8" });+ assert.equal(build.status, 0, build.stderr);+ const launcher = path.join(root, "dist/npm/open-computer-use/bin/open-computer-use");+ const run = spawnSync(process.execPath, [launcher, "config", "set", "image.format", "jpg"], { env: { ...process.env, ...env }, encoding: "utf8" });+ assert.equal(run.status, 0, run.stderr);+ assert.equal(inspectConfig(env).values["image.format"], "jpg");+});++test("explicit resize/drop settings, pixel-count threshold and nullable limits", t => {+ const env = fixture(t);+ configCommand(["set", "image.scaleDownAfterMaxSize", "false"], env);+ configCommand(["set", "image.discardBelowPixelCount", "4096"], env);+ configCommand(["set", "image.maxLongEdgePixels", "null"], env);+ configCommand(["set", "image.format", "webp"], env);+ const values = inspectConfig(env).values;+ assert.equal(values["image.scaleDownAfterMaxSize"], false);+ assert.equal(values["image.discardBelowPixelCount"], 4096);+ assert.equal(values["image.maxLongEdgePixels"], null);+ assert.equal(values["image.format"], "webp");+ assert.throws(() => configCommand(["set", "image.scaleDownAfterMaxSize", "yes"], env), /true or false/);+ assert.throws(() => configCommand(["set", "image.discardBelowPixelCount", "-1"], env));+});++test("removed byte budgets are rejected and stale values are ignored", t => {+ const env = fixture(t);+ writeFileSync(env.OPEN_COMPUTER_USE_CONFIG_FILE, JSON.stringify({ image: { maxLongEdgePixels: 640, maxBytes: 1, byteBudgetMinScale: 0.01, minScale: 0.01 } }));+ const report = inspectConfig({ ...env, OPEN_COMPUTER_USE_IMAGE_MAX_BYTES: "1", OPEN_COMPUTER_USE_IMAGE_MIN_SCALE: "0.01" });+ assert.equal(report.values["image.maxLongEdgePixels"], 640);+ for (const key of ["image.maxBytes", "image.byteBudgetMinScale"]) {+ assert.equal(key in report.values, false);+ assert.throws(() => configCommand(["set", key, "1"], env), /Unknown setting/);+ }+});diff --git a/scripts/node-repl/open-computer-use-cli.mjs b/scripts/node-repl/open-computer-use-cli.mjsindex f1f1055..0b88af6 100644--- a/scripts/node-repl/open-computer-use-cli.mjs+++ b/scripts/node-repl/open-computer-use-cli.mjs@@ -1,3 +1,4 @@+import { configCommand, configHelp } from "./ocu-config.mjs";import { spawn } from "node:child_process";import { constants as fsConstants, accessSync, existsSync, readFileSync } from "node:fs";import path from "node:path";@@ -122,6 +123,7 @@ Commands:mcp Start the native 9-tool stdio MCP server.js <code> Run JavaScript once with the asynchronous cua API.repl Start a persistent interactive JavaScript session.+ config View or persist macOS screenshot settings.capabilities Report Node, adapter, native, js, and repl availability.doctor Print permission status and launch onboarding if needed on macOS.list-apps Print running or recently used apps.@@ -480,6 +482,14 @@ export async function main({writeLine(output, installHelp(installCommands.get(argv[1]), installerUsage(argv[1])));return 0;}+ if (command === "config") {+ writeLine(output, configCommand(argv.slice(1), env));+ return 0;+ }+ if (command === "help" && argv[1] === "config") {+ writeLine(output, configHelp);+ return 0;+ }if (command === "capabilities") {const extra = argv.slice(1);if (extra.includes("-h") || extra.includes("--help")) writeLine(output, capabilitiesHelp());diff --git a/scripts/npm/build-packages.mjs b/scripts/npm/build-packages.mjsindex 81d47e4..0a5465f 100644--- a/scripts/npm/build-packages.mjs+++ b/scripts/npm/build-packages.mjs@@ -410,6 +410,7 @@ function renderMetaPackageJson(packageName, version) {"skills/open-computer-use/","README.md","LICENSE",+ "THIRD_PARTY_NOTICES.md",],};}@@ -426,6 +427,7 @@ function copyInstallerScripts(packageRoot) {mkdirSync(path.join(packageRoot, "scripts", "node-repl"), { recursive: true });for (const scriptName of ["open-computer-use-cli.mjs",+ "ocu-config.mjs","open-computer-use-repl.mjs","open-computer-use-kernel.mjs",]) {@@ -501,6 +503,7 @@ function stageMetaPackage(packageName, version, outDir) {recursive: true,});cpSync(path.join(repoRoot, "LICENSE"), path.join(packageRoot, "LICENSE"));+ cpSync(path.join(repoRoot, "THIRD_PARTY_NOTICES.md"), path.join(packageRoot, "THIRD_PARTY_NOTICES.md"));copyBundledRuntimes(packageRoot, packageName);copyInstallerScripts(packageRoot);diff --git a/skills/open-computer-use/references/usage.md b/skills/open-computer-use/references/usage.mdindex d21ea36..2815f59 100644--- a/skills/open-computer-use/references/usage.md+++ b/skills/open-computer-use/references/usage.md@@ -236,3 +236,7 @@ The Linux runtime uses AT-SPI2 through the desktop session bus. It must run in a## SafetyPause and ask the user before actions that affect external systems or sensitive local state, including sending messages, submitting forms, deleting files, approving prompts, uploading files, or interacting with password managers.++## Persisted screenshot settings++Use `ocu config` to inspect effective values and their sources. `ocu config set image.format jpg` or `ocu config set image.maxLongEdgePixels 1024` persists macOS screenshot preferences in the user config file. ENV overrides take precedence; `ocu config reset KEY` removes a saved preference. Windows/Linux screenshot behavior is unchanged.