Skip to content

Craft Anything: understand how any software works, and craft your own.

Rust

Bound combined macOS UI observation output (R5.6) (#1087)

Tomáš Doubekc7bb13bd1de3
diff --git a/crates/engine/src/tools/apple_runtime.rs b/crates/engine/src/tools/apple_runtime.rs
index ad4341d1..94f3cecb 100644
--- a/crates/engine/src/tools/apple_runtime.rs
+++ b/crates/engine/src/tools/apple_runtime.rs
@@ -471,6 +471,7 @@ mod macos {
} else {
format!("Skipped the accessibility tree for pid {pid}, as requested.")
};
+ let truncated = result.get("truncated").and_then(Value::as_bool).unwrap_or(false);
let screenshot_limitation = if p.screenshot && result.get("screenshot").is_none() {
let error = result.get("screenshot_error");
Some(Limitation::new(
@@ -490,6 +491,11 @@ mod macos {
.summary(summary)
.confidence(Confidence::Observed)
.authority(Authority::OurRun);
+ if truncated {
+ b = b
+ .limitation(Limitation::new("tree_truncated", "The accessibility tree reached the node or output limit; only a prefix is included."))
+ .coverage(Coverage { status: craft_anything_evidence::CoverageStatus::Partial, ..Coverage::default() });
+ }
if let Some(limitation) = screenshot_limitation {
b = b.limitation(limitation).coverage(Coverage { status: craft_anything_evidence::CoverageStatus::Partial, ..Coverage::default() });
}
@@ -652,6 +658,30 @@ mod tests {
}
}
+ /// Exercise the real serializer with synthetic AX values and image data; no UI permissions.
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn macos_budgets_combined_tree_and_screenshot_output() {
+ use craft_anything_platform::process::Process;
+ use std::time::Duration;
+ let root = tempfile::tempdir().unwrap();
+ let source = include_str!("apple_runtime/native_ui_helper.swift").split("let input = FileHandle.standardInput").next().unwrap();
+ let fixture = include_str!("apple_runtime/output_budget_test.swift");
+ let src = root.path().join("budget.swift");
+ let bin = root.path().join("budget");
+ std::fs::write(&src, format!("{source}\n{fixture}")).unwrap();
+ Process::new("xcrun")
+ .args(["swiftc", "-Onone", "-module-cache-path"])
+ .arg(root.path().join("module-cache"))
+ .arg(&src)
+ .arg("-o")
+ .arg(&bin)
+ .timeout(Duration::from_secs(60))
+ .run_checked()
+ .unwrap();
+ Process::new(&bin).timeout(Duration::from_secs(30)).run_checked().unwrap();
+ }
+
/// Run the real Swift helper without reading app UI or requiring Accessibility permission.
/// Both zero and positive node limits must return an empty, non-truncated tree and evidence
/// that records the skipped read.
diff --git a/crates/engine/src/tools/apple_runtime/native_ui_helper.swift b/crates/engine/src/tools/apple_runtime/native_ui_helper.swift
index 74c66f47..dde5554f 100644
--- a/crates/engine/src/tools/apple_runtime/native_ui_helper.swift
+++ b/crates/engine/src/tools/apple_runtime/native_ui_helper.swift
@@ -139,6 +139,33 @@ func findElement(_ root: AXUIElement, _ needle: String, _ budget: inout Int) ->
return nil
}
+// Match the runner's stdout cap, including the newline added by emit. Measure JSON bytes:
+// base64 slashes and AX strings can expand when escaped by JSONSerialization.
+func boundedObservation(_ observation: [String: Any]) -> [String: Any] {
+ let stdoutCap = 16 * 1024 * 1024
+ var result = observation
+ var nodes = result["nodes"] as? [[String: Any]] ?? []
+ while true {
+ if let data = try? JSONSerialization.data(withJSONObject: result, options: []),
+ data.count < stdoutCap {
+ return result
+ }
+ if !nodes.isEmpty {
+ // Keep a prefix of the tree and the image. Halving bounds serialization attempts
+ // even when one AX value alone is larger than the entire output budget.
+ nodes = Array(nodes.prefix(nodes.count / 2))
+ result["nodes"] = nodes
+ result["truncated"] = true
+ } else if result.removeValue(forKey: "screenshot") != nil {
+ result.merge(screenshotFailure("screenshot_output_limit",
+ "The screenshot exceeded the helper output budget.")) { _, new in new }
+ } else {
+ // Serialization failure must also produce a small, valid response.
+ return ["error": "Could not serialize the observation within the output budget."]
+ }
+ }
+}
+
func emit(_ obj: [String: Any]) {
if let data = try? JSONSerialization.data(withJSONObject: obj, options: []),
let text = String(data: data, encoding: .utf8) {
@@ -168,7 +195,7 @@ case "observe":
if req.screenshot ?? false {
result.merge(await captureScreenshot(pid: pid, windowID: req.window_id)) { _, new in new }
}
- emit(result)
+ emit(boundedObservation(result))
case "walkthrough":
let app = AXUIElementCreateApplication(pid)
var steps: [[String: Any]] = []
diff --git a/crates/engine/src/tools/apple_runtime/output_budget_test.swift b/crates/engine/src/tools/apple_runtime/output_budget_test.swift
new file mode 100644
index 00000000..a4ae5211
--- /dev/null
+++ b/crates/engine/src/tools/apple_runtime/output_budget_test.swift
@@ -0,0 +1,36 @@
+// Synthetic regression fixture compiled with the production helper functions.
+func serializedSize(_ value: Any) -> Int {
+ return try! JSONSerialization.data(withJSONObject: value).count
+}
+
+let cap = 16 * 1024 * 1024
+let image: [String: Any] = ["base64": String(repeating: "A", count: 8 * 1024 * 1024), "mime_type": "image/png"]
+// Each component fits separately, but their combined serialized response does not.
+let nodes: [[String: Any]] = (0..<16).map { ["depth": $0, "value": String(repeating: "界\\\"\n", count: 100_000)] }
+let combined: [String: Any] = ["nodes": nodes, "screenshot": image, "truncated": false]
+assert(serializedSize(nodes) < cap)
+assert(serializedSize(image) < cap)
+assert(serializedSize(combined) >= cap)
+let bounded = boundedObservation(combined)
+let encoded = try JSONSerialization.data(withJSONObject: bounded)
+assert(encoded.count + 1 <= cap)
+let decoded = try JSONSerialization.jsonObject(with: encoded) as! [String: Any]
+let kept = decoded["nodes"] as! [[String: Any]]
+assert(!kept.isEmpty && kept.count < nodes.count)
+assert(decoded["truncated"] as? Bool == true)
+assert((decoded["screenshot"] as! [String: Any])["base64"] as? String == image["base64"] as? String)
+assert(kept.first?["value"] as? String == nodes.first?["value"] as? String)
+
+let small: [String: Any] = ["nodes": [["title": "small"]], "truncated": false]
+assert(NSDictionary(dictionary: boundedObservation(small)).isEqual(to: small))
+let skipped: [String: Any] = ["nodes": [], "screenshot": image, "truncated": false]
+assert(NSDictionary(dictionary: boundedObservation(skipped)).isEqual(to: skipped))
+let hugeNode: [String: Any] = ["nodes": [["value": String(repeating: "x", count: cap)]], "truncated": false]
+let trimmed = boundedObservation(hugeNode)
+assert((trimmed["nodes"] as! [[String: Any]]).isEmpty)
+assert(trimmed["truncated"] as? Bool == true)
+let hugeImage: [String: Any] = ["nodes": [], "screenshot": ["base64": String(repeating: "/", count: cap)], "truncated": false]
+let refused = boundedObservation(hugeImage)
+assert(refused["screenshot"] == nil)
+assert((refused["screenshot_error"] as! [String: Any])["code"] as? String == "screenshot_output_limit")
+assert(serializedSize(refused) + 1 <= cap)
diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md
index 87c08109..4a15ac43 100644
--- a/docs/ROADMAP.md
+++ b/docs/ROADMAP.md
@@ -487,7 +487,9 @@ Golden rule 3: the result is evidence. Each fix needs a Mac to test on.
— claimed: mavericzz, branch bugfix/r5.6-accessibility-denied; PR #887
- [x] macOS: `runtime_ui_read` captures a requested screenshot (#248): one on-screen window
belonging to the pid, restricted by `window_id` when supplied; PNG/base64 up to 2048 px
- per side. Missing Screen Recording permission, a missing window or a capture failure produces
+ per side. Combined JSON (including escaping and the newline) stays within 16 MiB by
+ trimming the tree with explicit truncation and partial coverage, preserving the image.
+ Missing Screen Recording permission, a missing window or a capture failure produces
an explicit screenshot limitation and partial evidence. Denied Accessibility permission with
the tree enabled (the default) returns a tool error before capture; `accessibility: false`
skips the tree. Requires macOS 14+ and existing Screen Recording permission; opens no dialog
diff --git a/docs/STATUS.md b/docs/STATUS.md
index 64ddba7b..76132697 100644
--- a/docs/STATUS.md
+++ b/docs/STATUS.md
@@ -12,7 +12,7 @@ workspace clippy, layers, fmt and the secret scan of all history were clean. Chi
| Disabled chat backends | Naming a backend that is off refuses chat and either side of compare with a settings error and guidance, before API or coding-agent setup | Isolated hub regressions for disabled API and agent backends, both compare positions, and enabled/automatic controls. Invalid tunnel settings detect API setup before the guard. No live provider or agent was used |
| Runner-owned app windows | Runtime capture tries the launched process and its descendants; the captured window owner's PID is also used for the UI tree. Unrelated processes with matching names are excluded. Process discovery has a deadline, output cap and candidate limit | Hermetic regressions cover runner/child/grandchild ownership, unrelated windows, direct launches, cycles, malformed rows, caps and errors. A macOS process snapshot finds an owned child without opening a window. Windows and real Go GUI capture have not been run |
| New-project toolchains | New Python, Go, .NET, Maven, Gradle and Make projects use the same build catalogue as existing projects (`engine/src/toolchains.toml`). Shared launch recipes let screenshots, self-tests and Play start Python `main.py`, `app.py` or `__main__.py`, Go with a root `main.go`, and one root `.csproj`. Python uses a project-local virtual environment when present, resolves its executable to an absolute path even for relative project folders, and forces fresh syntax compilation; .NET passes app options after `--`. Manifest markers take priority over standalone Python files. Runtime window-capture requests normalize relative project and runner paths to absolute paths. Existing packaged programs, Rust, Electron and Godot keep launch priority. Multiple `.csproj` matches or a capped scan do not choose a project. Other layouts still need a packaged program or another launch recipe | Catalogue and launch regressions cover argv boundaries, ambiguous entrypoints, scan limits and Windows/local interpreter selection, including relative project folders and unchanged argument boundaries. Capture-request tests cover relative placements and PATH results, local interpreters and optional UI-tree output without opening a window. An owned interpreter stand-in runs the full new-project build/screenshot path, rejects failed builds, and verifies a failed screenshot cannot reuse an old PNG. An opt-in real Python 3.15.2 fixture on macOS passes build/screenshot checks, ignores invalid virtualenv code, and rejects a syntax error even when its mtime matches the cached version. Without `compileall -f` that same syntax-error probe incorrectly exits zero. Real Go/.NET/Maven/Gradle runs, Windows and GUI apps have not been verified for this change |
-| macOS optional UI tree and screenshots | `runtime_ui_read` with `accessibility: false` skips the tree and its permission check, returns empty nodes without truncation, and records the skipped read (#307). Walkthroughs still require Accessibility. With `screenshot: true`, macOS 14+ returns one owned-by-pid window as PNG/base64, restricted by `window_id` when supplied; without it, the first matching on-screen window. Images are limited to 2048 px per side and 6 MiB PNG. Missing Screen Recording permission, a missing window or a capture failure is an explicit screenshot limitation with partial coverage (#248). Denied Accessibility permission returns a tool error before capture when the tree is enabled (the default); set `accessibility: false` to skip the tree. Permission dialogs are never opened | Engine regressions compile/run the real Swift helper: skipped trees at node limits 0/1, screenshot-off control, and impossible-window screenshot errors propagated to evidence. Live helper probe on macOS 27.2 captured only an owned AppKit fixture: valid 320 × 192 PNG (14,400 bytes), matching window id; screenshot-off returned no image/error. No granted-permission tree read, walkthrough, macOS 14 run or MCP transport run |
+| macOS optional UI tree and screenshots | `runtime_ui_read` with `accessibility: false` skips the tree and its permission check, returns empty nodes without truncation, and records the skipped read (#307). Walkthroughs still require Accessibility. With `screenshot: true`, macOS 14+ returns one owned-by-pid window as PNG/base64, restricted by `window_id` when supplied; without it, the first matching on-screen window. Images are limited to 2048 px per side and 6 MiB PNG. Combined JSON stays within the 16 MiB helper output cap; oversized trees retain a prefix with explicit truncation and partial coverage while preserving the image. Missing Screen Recording permission, a missing window or a capture failure is an explicit screenshot limitation with partial coverage (#248). Denied Accessibility permission returns a tool error before capture when the tree is enabled (the default); set `accessibility: false` to skip the tree. Permission dialogs are never opened | Synthetic Swift serializer regression covers combined tree/image overflow, escaped Unicode values, a single oversized node, an oversized image, and unchanged small/skipped trees. Engine regressions compile/run the real Swift helper: skipped trees at node limits 0/1, screenshot-off control, and impossible-window screenshot errors propagated to evidence. Live helper probe on macOS 27.2 captured only an owned AppKit fixture: valid 320 × 192 PNG (14,400 bytes), matching window id; screenshot-off returned no image/error. No granted-permission tree read, walkthrough, macOS 14 run or MCP transport run |
| DNS question boundaries | A question is retained only when its name, type and class are complete. A missing class byte marks the DNS message incomplete; preceding complete questions remain available, and incomplete questions do not enter the query inventory | Four packet regressions cover both truncation lengths, multiple questions, complete class values and the capture summary. No live capture was used |
| Reject impossible IPv4 header and total lengths | IPv4 decoding rejects impossible header lengths and reports incomplete base headers or options. | Synthetic byte fixtures in `pcap_ipv4_lengths` integration regressions. Cover 10–19 captured header bytes, total lengths below IHL, missing options, complete options, truncated transport headers with readable ports, and a cut UDP payload with both ports and its captured length preserved. |
| Validate the version in IPv6 frames | IPv6 decoding checks the packet version before reporting IPv6 facts, including captures declaring raw IPv6; explicit raw IPv4 likewise uses IPv4 validation. | Synthetic byte fixtures in `pcap_ipv6_version` integration regressions. Cover incorrect versions behind the IPv6 EtherType, mismatched explicit raw IPv4 and IPv6 link types, matching explicit and generic raw packets, and valid IPv6 traffic-class bits. |