Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

nanocua: reach graff's model, and say why a window can't be seen

Rach Pradhane709ba1d93b5
diff --git a/apps/nanocua/README.md b/apps/nanocua/README.md
index 854f2328..8d4b8c16 100644
--- a/apps/nanocua/README.md
+++ b/apps/nanocua/README.md
@@ -48,7 +48,8 @@ named. `click` with a ref presses the element through accessibility when it can;
otherwise, and for keys, text and scrolling, events are posted to that app alone,
so your pointer stays where it is and your keyboard stays yours. Screenshots
capture that app's window, even behind others, with the mapping from image pixels
-back to screen points. This isn't a sandbox: it's your real Mac, your apps and
+back to screen points. A window on another Space or minimized is out of reach:
+macOS shows neither its picture nor its elements, and the reply says so. This isn't a sandbox: it's your real Mac, your apps and
your logins. And some apps ignore events they get in the background (Chromium's
web content is the likely one); refs still work there.
diff --git a/apps/nanocua/src/mac/backend.zig b/apps/nanocua/src/mac/backend.zig
index c18092d3..15045b01 100644
--- a/apps/nanocua/src/mac/backend.zig
+++ b/apps/nanocua/src/mac/backend.zig
@@ -24,6 +24,8 @@ extern "c" fn dispatch_async_f(queue: objc.dispatch_queue_t, ctx: ?*anyopaque, w
extern "c" fn getpid() c_int;
extern "c" fn usleep(us: c_uint) c_int;
+const off_this_space = "That app's window isn't on the user's current Space, or is minimized, so macOS shows nanocua neither its picture nor its elements. Tell the user once; don't retry.";
+
/// What one agent is working on. Agents connect one socket each: refs from
/// one's tree mean nothing to another, and neither moves the other's target.
pub const Session = struct {
@@ -274,6 +276,7 @@ const Job = struct {
if (pid == m.self_pid) return j.fail("That is nanocua itself, which agents don't see. Pass pid or app (see list_apps).", .{});
const max: usize = @intCast(std.math.clamp(j.max_nodes orelse 400, 1, 2000));
try ax.writeTree(&j.session.refs, w, pid, .{ .max_nodes = max });
+ if (windows.main(pid)) |win| if (!win.onscreen) try w.print("\n{s}\n", .{off_this_space});
j.session.last = pid;
},
.screenshot => try j.screenshot(),
@@ -547,6 +550,8 @@ const Job = struct {
const w = windows.main(pid) orelse return j.fail("App {d} has no window to capture.", .{pid});
rect = w.frame;
j.session.last = pid;
+ // macOS refuses these (SCStream error -3811), with no better reason to pass on.
+ if (!w.onscreen) return j.fail("{s}", .{off_this_space});
break :blk capture.window(m.gpa, w.id, w.frame, max);
},
};
diff --git a/apps/nanocua/src/mac/windows.zig b/apps/nanocua/src/mac/windows.zig
index a7dedd92..2a02e8e8 100644
--- a/apps/nanocua/src/mac/windows.zig
+++ b/apps/nanocua/src/mac/windows.zig
@@ -29,6 +29,13 @@ pub fn better(a: Window, b: Window) bool {
return a.frame.size.width * a.frame.size.height > b.frame.size.width * b.frame.size.height;
}
+/// Whether it's big enough to be a window someone works in. An app can have
+/// layer 0 strips and icons and nothing else: Finder with its windows closed has a
+/// menu-bar-sized strip, whose picture is blank.
+pub fn sizedLikeAWindow(w: Window) bool {
+ return w.frame.size.width >= 100 and w.frame.size.height >= 40;
+}
+
fn int(v: ?*anyopaque) ?i64 {
const x = v orelse return null;
if (cf.CFGetTypeID(x) != cf.CFNumberGetTypeID()) return null;
@@ -56,6 +63,7 @@ pub fn main(pid: i32) ?Window {
for (0..n) |i| {
const d: cf.CFDictionaryRef = @ptrCast(cf.CFArrayGetValueAtIndex(list, @intCast(i)) orelse continue);
const w = read(d, pid) orelse continue;
+ if (!sizedLikeAWindow(w)) continue;
if (best == null or better(w, best.?)) best = w;
}
return best;
@@ -99,6 +107,18 @@ test "the main window: on screen first, then the largest" {
try std.testing.expect(better(of(10, 10, true), of(1352, 849, false)));
}
+test "strips and icons aren't an app's main window" {
+ const of = struct {
+ fn f(w: f64, h: f64) Window {
+ return .{ .id = 1, .frame = .{ .size = .{ .width = w, .height = h } }, .onscreen = false };
+ }
+ }.f;
+ try std.testing.expect(!sizedLikeAWindow(of(1352, 29)));
+ try std.testing.expect(!sizedLikeAWindow(of(64, 64)));
+ try std.testing.expect(sizedLikeAWindow(of(1352, 47)));
+ try std.testing.expect(sizedLikeAWindow(of(260, 400)));
+}
+
test "a window contains the points in its frame" {
const w: Window = .{ .id = 1, .frame = .{ .origin = .{ .x = 100, .y = 50 }, .size = .{ .width = 200, .height = 100 } } };
try std.testing.expect(contains(w, 100, 50) and contains(w, 299, 149));
diff --git a/apps/nanocua/src/tools.zig b/apps/nanocua/src/tools.zig
index e85d2fcd..39c08f5d 100644
--- a/apps/nanocua/src/tools.zig
+++ b/apps/nanocua/src/tools.zig
@@ -34,17 +34,25 @@ pub const Tool = enum {
return std.meta.stringToEnum(Tool, name);
}
+ /// When to use nanocua and when to stop. Here as well as in the handshake's
+ /// `instructions`, which clients may ignore: graff does.
+ pub const guidance =
+ \\ Use nanocua for another app's real UI; to show or test your own work, prefer the project's own commands.
+ \\If a call says nanocua is stopped, or input or the whole screen is off, tell the user once and carry on
+ \\without it. Don't retry in a loop or set up permissions yourself: they are the user's to give.
+ ;
+
pub fn description(t: Tool) []const u8 {
return switch (t) {
.list_apps =>
\\Lists the running apps with windows: name, bundle id, pid, and which one is frontmost.
- ,
+ ++ guidance,
.get_ui_tree =>
\\The accessibility tree of an app, in the background (pid or app; else the app you last read, else the
\\frontmost): one line per element with a ref like [12], its role, title, value and frame in screen points
\\(top-left origin). The app becomes the one you act in. Use refs with click, type_text, scroll and
\\set_value; they stay valid until the next get_ui_tree. Prefer this over screenshots: it is exact and cheap.
- ,
+ ++ guidance,
.screenshot =>
\\A JPEG of an app's main window, even behind other windows (pid or app; else the app you last read).
\\With whole_screen, if the user allows it: the main display, or a region of it (x, y, width, height).
@@ -199,3 +207,8 @@ test "arguments read leniently" {
try std.testing.expectEqual(@as(?bool, null), a.boolean("text"));
try std.testing.expectEqual(@as(?[]const u8, null), Args.of(null).string("text"));
}
+
+test "the guidance reaches clients that ignore a server's instructions" {
+ for ([_]Tool{ .list_apps, .get_ui_tree }) |t|
+ try std.testing.expect(std.mem.indexOf(u8, t.description(), "permissions yourself") != null);
+}
diff --git a/docs/adr/0314-nanocua-is-grafts-computer-use.md b/docs/adr/0314-nanocua-is-grafts-computer-use.md
index dd41c5ef..bc3be6c6 100644
--- a/docs/adr/0314-nanocua-is-grafts-computer-use.md
+++ b/docs/adr/0314-nanocua-is-grafts-computer-use.md
@@ -33,8 +33,11 @@ app's decision, not graff's or the agent's:
- the real pointer and keyboard and pictures of the whole display need a second
switch; Stop refuses everything; agents cannot reach nanocua's own window.
-Its instructions tell agents to prefer the project's own commands for showing
-their work and not to chase permissions, which is ADR 0209's failure.
+The descriptions of `list_apps` and `get_ui_tree` tell agents to prefer the
+project's own commands for showing their work and not to chase permissions, which
+is ADR 0209's failure. That text is in the tool descriptions because graff doesn't
+read an MCP server's `instructions`, so a handshake's text alone never reaches its
+model. Other clients get it there too.
codegraff holds the original. nanoui's `apps/nanocua` is generated from it by
`scripts/vendor-nanocua.sh` and stays under codegraff's license, not nanoui's