Craft Anything: understand how any software works, and craft your own.
Rust
Bound combined macOS UI observation output (R5.6) (#1087)
Tomáš Doubekc7bb13bd1de3
diff --git a/crates/engine/src/tools/apple_runtime.rs b/crates/engine/src/tools/apple_runtime.rsindex ad4341d1..94f3cecb 100644--- a/crates/engine/src/tools/apple_runtime.rs+++ b/crates/engine/src/tools/apple_runtime.rs@@ -471,6 +471,7 @@ mod macos {} else {format!("Skipped the accessibility tree for pid {pid}, as requested.")};+ let truncated = result.get("truncated").and_then(Value::as_bool).unwrap_or(false);let screenshot_limitation = if p.screenshot && result.get("screenshot").is_none() {let error = result.get("screenshot_error");Some(Limitation::new(@@ -490,6 +491,11 @@ mod macos {.summary(summary).confidence(Confidence::Observed).authority(Authority::OurRun);+ if truncated {+ b = b+ .limitation(Limitation::new("tree_truncated", "The accessibility tree reached the node or output limit; only a prefix is included."))+ .coverage(Coverage { status: craft_anything_evidence::CoverageStatus::Partial, ..Coverage::default() });+ }if let Some(limitation) = screenshot_limitation {b = b.limitation(limitation).coverage(Coverage { status: craft_anything_evidence::CoverageStatus::Partial, ..Coverage::default() });}@@ -652,6 +658,30 @@ mod tests {}}+ /// Exercise the real serializer with synthetic AX values and image data; no UI permissions.+ #[cfg(target_os = "macos")]+ #[test]+ fn macos_budgets_combined_tree_and_screenshot_output() {+ use craft_anything_platform::process::Process;+ use std::time::Duration;+ let root = tempfile::tempdir().unwrap();+ let source = include_str!("apple_runtime/native_ui_helper.swift").split("let input = FileHandle.standardInput").next().unwrap();+ let fixture = include_str!("apple_runtime/output_budget_test.swift");+ let src = root.path().join("budget.swift");+ let bin = root.path().join("budget");+ std::fs::write(&src, format!("{source}\n{fixture}")).unwrap();+ Process::new("xcrun")+ .args(["swiftc", "-Onone", "-module-cache-path"])+ .arg(root.path().join("module-cache"))+ .arg(&src)+ .arg("-o")+ .arg(&bin)+ .timeout(Duration::from_secs(60))+ .run_checked()+ .unwrap();+ Process::new(&bin).timeout(Duration::from_secs(30)).run_checked().unwrap();+ }+/// Run the real Swift helper without reading app UI or requiring Accessibility permission./// Both zero and positive node limits must return an empty, non-truncated tree and evidence/// that records the skipped read.diff --git a/crates/engine/src/tools/apple_runtime/native_ui_helper.swift b/crates/engine/src/tools/apple_runtime/native_ui_helper.swiftindex 74c66f47..dde5554f 100644--- a/crates/engine/src/tools/apple_runtime/native_ui_helper.swift+++ b/crates/engine/src/tools/apple_runtime/native_ui_helper.swift@@ -139,6 +139,33 @@ func findElement(_ root: AXUIElement, _ needle: String, _ budget: inout Int) ->return nil}+// Match the runner's stdout cap, including the newline added by emit. Measure JSON bytes:+// base64 slashes and AX strings can expand when escaped by JSONSerialization.+func boundedObservation(_ observation: [String: Any]) -> [String: Any] {+ let stdoutCap = 16 * 1024 * 1024+ var result = observation+ var nodes = result["nodes"] as? [[String: Any]] ?? []+ while true {+ if let data = try? JSONSerialization.data(withJSONObject: result, options: []),+ data.count < stdoutCap {+ return result+ }+ if !nodes.isEmpty {+ // Keep a prefix of the tree and the image. Halving bounds serialization attempts+ // even when one AX value alone is larger than the entire output budget.+ nodes = Array(nodes.prefix(nodes.count / 2))+ result["nodes"] = nodes+ result["truncated"] = true+ } else if result.removeValue(forKey: "screenshot") != nil {+ result.merge(screenshotFailure("screenshot_output_limit",+ "The screenshot exceeded the helper output budget.")) { _, new in new }+ } else {+ // Serialization failure must also produce a small, valid response.+ return ["error": "Could not serialize the observation within the output budget."]+ }+ }+}+func emit(_ obj: [String: Any]) {if let data = try? JSONSerialization.data(withJSONObject: obj, options: []),let text = String(data: data, encoding: .utf8) {@@ -168,7 +195,7 @@ case "observe":if req.screenshot ?? false {result.merge(await captureScreenshot(pid: pid, windowID: req.window_id)) { _, new in new }}- emit(result)+ emit(boundedObservation(result))case "walkthrough":let app = AXUIElementCreateApplication(pid)var steps: [[String: Any]] = []diff --git a/crates/engine/src/tools/apple_runtime/output_budget_test.swift b/crates/engine/src/tools/apple_runtime/output_budget_test.swiftnew file mode 100644index 00000000..a4ae5211--- /dev/null+++ b/crates/engine/src/tools/apple_runtime/output_budget_test.swift@@ -0,0 +1,36 @@+// Synthetic regression fixture compiled with the production helper functions.+func serializedSize(_ value: Any) -> Int {+ return try! JSONSerialization.data(withJSONObject: value).count+}++let cap = 16 * 1024 * 1024+let image: [String: Any] = ["base64": String(repeating: "A", count: 8 * 1024 * 1024), "mime_type": "image/png"]+// Each component fits separately, but their combined serialized response does not.+let nodes: [[String: Any]] = (0..<16).map { ["depth": $0, "value": String(repeating: "界\\\"\n", count: 100_000)] }+let combined: [String: Any] = ["nodes": nodes, "screenshot": image, "truncated": false]+assert(serializedSize(nodes) < cap)+assert(serializedSize(image) < cap)+assert(serializedSize(combined) >= cap)+let bounded = boundedObservation(combined)+let encoded = try JSONSerialization.data(withJSONObject: bounded)+assert(encoded.count + 1 <= cap)+let decoded = try JSONSerialization.jsonObject(with: encoded) as! [String: Any]+let kept = decoded["nodes"] as! [[String: Any]]+assert(!kept.isEmpty && kept.count < nodes.count)+assert(decoded["truncated"] as? Bool == true)+assert((decoded["screenshot"] as! [String: Any])["base64"] as? String == image["base64"] as? String)+assert(kept.first?["value"] as? String == nodes.first?["value"] as? String)++let small: [String: Any] = ["nodes": [["title": "small"]], "truncated": false]+assert(NSDictionary(dictionary: boundedObservation(small)).isEqual(to: small))+let skipped: [String: Any] = ["nodes": [], "screenshot": image, "truncated": false]+assert(NSDictionary(dictionary: boundedObservation(skipped)).isEqual(to: skipped))+let hugeNode: [String: Any] = ["nodes": [["value": String(repeating: "x", count: cap)]], "truncated": false]+let trimmed = boundedObservation(hugeNode)+assert((trimmed["nodes"] as! [[String: Any]]).isEmpty)+assert(trimmed["truncated"] as? Bool == true)+let hugeImage: [String: Any] = ["nodes": [], "screenshot": ["base64": String(repeating: "/", count: cap)], "truncated": false]+let refused = boundedObservation(hugeImage)+assert(refused["screenshot"] == nil)+assert((refused["screenshot_error"] as! [String: Any])["code"] as? String == "screenshot_output_limit")+assert(serializedSize(refused) + 1 <= cap)diff --git a/docs/ROADMAP.md b/docs/ROADMAP.mdindex 87c08109..4a15ac43 100644--- a/docs/ROADMAP.md+++ b/docs/ROADMAP.md@@ -487,7 +487,9 @@ Golden rule 3: the result is evidence. Each fix needs a Mac to test on.— claimed: mavericzz, branch bugfix/r5.6-accessibility-denied; PR #887- [x] macOS: `runtime_ui_read` captures a requested screenshot (#248): one on-screen windowbelonging to the pid, restricted by `window_id` when supplied; PNG/base64 up to 2048 px- per side. Missing Screen Recording permission, a missing window or a capture failure produces+ per side. Combined JSON (including escaping and the newline) stays within 16 MiB by+ trimming the tree with explicit truncation and partial coverage, preserving the image.+ Missing Screen Recording permission, a missing window or a capture failure producesan explicit screenshot limitation and partial evidence. Denied Accessibility permission withthe tree enabled (the default) returns a tool error before capture; `accessibility: false`skips the tree. Requires macOS 14+ and existing Screen Recording permission; opens no dialogdiff --git a/docs/STATUS.md b/docs/STATUS.mdindex 64ddba7b..76132697 100644--- a/docs/STATUS.md+++ b/docs/STATUS.md@@ -12,7 +12,7 @@ workspace clippy, layers, fmt and the secret scan of all history were clean. Chi| Disabled chat backends | Naming a backend that is off refuses chat and either side of compare with a settings error and guidance, before API or coding-agent setup | Isolated hub regressions for disabled API and agent backends, both compare positions, and enabled/automatic controls. Invalid tunnel settings detect API setup before the guard. No live provider or agent was used || Runner-owned app windows | Runtime capture tries the launched process and its descendants; the captured window owner's PID is also used for the UI tree. Unrelated processes with matching names are excluded. Process discovery has a deadline, output cap and candidate limit | Hermetic regressions cover runner/child/grandchild ownership, unrelated windows, direct launches, cycles, malformed rows, caps and errors. A macOS process snapshot finds an owned child without opening a window. Windows and real Go GUI capture have not been run || New-project toolchains | New Python, Go, .NET, Maven, Gradle and Make projects use the same build catalogue as existing projects (`engine/src/toolchains.toml`). Shared launch recipes let screenshots, self-tests and Play start Python `main.py`, `app.py` or `__main__.py`, Go with a root `main.go`, and one root `.csproj`. Python uses a project-local virtual environment when present, resolves its executable to an absolute path even for relative project folders, and forces fresh syntax compilation; .NET passes app options after `--`. Manifest markers take priority over standalone Python files. Runtime window-capture requests normalize relative project and runner paths to absolute paths. Existing packaged programs, Rust, Electron and Godot keep launch priority. Multiple `.csproj` matches or a capped scan do not choose a project. Other layouts still need a packaged program or another launch recipe | Catalogue and launch regressions cover argv boundaries, ambiguous entrypoints, scan limits and Windows/local interpreter selection, including relative project folders and unchanged argument boundaries. Capture-request tests cover relative placements and PATH results, local interpreters and optional UI-tree output without opening a window. An owned interpreter stand-in runs the full new-project build/screenshot path, rejects failed builds, and verifies a failed screenshot cannot reuse an old PNG. An opt-in real Python 3.15.2 fixture on macOS passes build/screenshot checks, ignores invalid virtualenv code, and rejects a syntax error even when its mtime matches the cached version. Without `compileall -f` that same syntax-error probe incorrectly exits zero. Real Go/.NET/Maven/Gradle runs, Windows and GUI apps have not been verified for this change |-| macOS optional UI tree and screenshots | `runtime_ui_read` with `accessibility: false` skips the tree and its permission check, returns empty nodes without truncation, and records the skipped read (#307). Walkthroughs still require Accessibility. With `screenshot: true`, macOS 14+ returns one owned-by-pid window as PNG/base64, restricted by `window_id` when supplied; without it, the first matching on-screen window. Images are limited to 2048 px per side and 6 MiB PNG. Missing Screen Recording permission, a missing window or a capture failure is an explicit screenshot limitation with partial coverage (#248). Denied Accessibility permission returns a tool error before capture when the tree is enabled (the default); set `accessibility: false` to skip the tree. Permission dialogs are never opened | Engine regressions compile/run the real Swift helper: skipped trees at node limits 0/1, screenshot-off control, and impossible-window screenshot errors propagated to evidence. Live helper probe on macOS 27.2 captured only an owned AppKit fixture: valid 320 × 192 PNG (14,400 bytes), matching window id; screenshot-off returned no image/error. No granted-permission tree read, walkthrough, macOS 14 run or MCP transport run |+| macOS optional UI tree and screenshots | `runtime_ui_read` with `accessibility: false` skips the tree and its permission check, returns empty nodes without truncation, and records the skipped read (#307). Walkthroughs still require Accessibility. With `screenshot: true`, macOS 14+ returns one owned-by-pid window as PNG/base64, restricted by `window_id` when supplied; without it, the first matching on-screen window. Images are limited to 2048 px per side and 6 MiB PNG. Combined JSON stays within the 16 MiB helper output cap; oversized trees retain a prefix with explicit truncation and partial coverage while preserving the image. Missing Screen Recording permission, a missing window or a capture failure is an explicit screenshot limitation with partial coverage (#248). Denied Accessibility permission returns a tool error before capture when the tree is enabled (the default); set `accessibility: false` to skip the tree. Permission dialogs are never opened | Synthetic Swift serializer regression covers combined tree/image overflow, escaped Unicode values, a single oversized node, an oversized image, and unchanged small/skipped trees. Engine regressions compile/run the real Swift helper: skipped trees at node limits 0/1, screenshot-off control, and impossible-window screenshot errors propagated to evidence. Live helper probe on macOS 27.2 captured only an owned AppKit fixture: valid 320 × 192 PNG (14,400 bytes), matching window id; screenshot-off returned no image/error. No granted-permission tree read, walkthrough, macOS 14 run or MCP transport run || DNS question boundaries | A question is retained only when its name, type and class are complete. A missing class byte marks the DNS message incomplete; preceding complete questions remain available, and incomplete questions do not enter the query inventory | Four packet regressions cover both truncation lengths, multiple questions, complete class values and the capture summary. No live capture was used || Reject impossible IPv4 header and total lengths | IPv4 decoding rejects impossible header lengths and reports incomplete base headers or options. | Synthetic byte fixtures in `pcap_ipv4_lengths` integration regressions. Cover 10–19 captured header bytes, total lengths below IHL, missing options, complete options, truncated transport headers with readable ports, and a cut UDP payload with both ports and its captured length preserved. || Validate the version in IPv6 frames | IPv6 decoding checks the packet version before reporting IPv6 facts, including captures declaring raw IPv6; explicit raw IPv4 likewise uses IPv4 validation. | Synthetic byte fixtures in `pcap_ipv6_version` integration regressions. Cover incorrect versions behind the IPv6 EtherType, mismatched explicit raw IPv4 and IPv6 link types, matching explicit and generic raw packets, and valid IPv6 traffic-class bits. |