graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Bound directory scans before filtering and mark partial results
Rach Pradhand3b993db7f60
diff --git a/docs/adr/0013-list-dir-lives-in-codedb.md b/docs/adr/0013-list-dir-lives-in-codedb.mdindex 887770e4..0c0cf7e4 100644--- a/docs/adr/0013-list-dir-lives-in-codedb.md+++ b/docs/adr/0013-list-dir-lives-in-codedb.md@@ -30,3 +30,6 @@ extra schema bytes beat this subcommand.- `codedb ls` / `tree` stay index queries.- The model has to read the codedb description (or the prompt sentence) tofind `list_dir`; that is the cost of not growing the catalog.++The in-process scan budget and partial-result contract are refined by+[0184](0184-directory-scan-budgets-count-examined-entries.md).diff --git a/docs/adr/0184-directory-scan-budgets-count-examined-entries.md b/docs/adr/0184-directory-scan-budgets-count-examined-entries.mdnew file mode 100644index 00000000..ee18f8ff--- /dev/null+++ b/docs/adr/0184-directory-scan-budgets-count-examined-entries.md@@ -0,0 +1,28 @@+# 0184. Directory scan budgets count examined entries++Status: accepted++## Context++A limit applied after collecting directory names bounds the rendered tree but+does not bound enumeration or temporary allocation. Counting only visible+entries also lets ignored files and symlinks bypass a scan limit.++## Decision++Directory listings and missing-path suggestions count entries as the iterator+returns them, before filtering or copying names. Stop before requesting another+entry once the scan budget is exhausted. Load nested ignore rules independently+of entry order so truncation cannot accidentally bypass them.++An exhausted budget produces an explicitly partial result. Subtree counts are+lower bounds, and suggestions from an incomplete scan do not claim to be the+closest match across the entire directory. Below the limit, retain ordinary+listing order, ignore rules, and visibility of useful dotfiles.++## Consequences++Directories with many ignored entries can reach the budget earlier. The caller+can narrow the requested path; partial output must never imply a complete tree.+This bounds local navigation work without changing the tool catalog or removing+conversation history. It does not establish model token or cost savings.diff --git a/docs/adr/README.md b/docs/adr/README.mdindex c78c1f35..68a01459 100644--- a/docs/adr/README.md+++ b/docs/adr/README.md@@ -209,6 +209,7 @@ record only when you need the evidence or the edge cases.| [0181](0181-acp-load-replays-saved-conversation.md) | Live ACP loads saved conversations in their selected workspace and replays history without rerunning tools. || [0182](0182-acp-connection-usage-is-an-extension.md) | Connection-scoped ACP usage preserves unknown totals through a custom notification; network retries remain activity indicators. || [0183](0183-request-policy-owns-ambiguous-retries.md) | Ambiguous HTTP/2 failures return to the request retry policy so attempts and unknown usage remain observable. |+| [0184](0184-directory-scan-budgets-count-examined-entries.md) | Directory scan limits count examined entries before filtering; truncated listings and suggestions remain explicitly partial. |## When to write onediff --git a/src/list_dir.zig b/src/list_dir.zigindex 039be87e..f7b569ea 100644--- a/src/list_dir.zig+++ b/src/list_dir.zig@@ -36,7 +36,7 @@ const root_truncation_notice =const walk_truncation_notice =\\- \\Note: there are more than 20000 items in the directory, so not all files may be shown.+ \\Note: the 20000-entry scan limit was reached; the listing and subtree counts may be incomplete.;const Node = struct {@@ -48,6 +48,7 @@ const Node = struct {file_count: usize,ext: std.StringHashMap(usize),expanded: bool,+ partial: bool = false,};fn newNode(arena: Allocator, name: []const u8, is_dir: bool, depth: usize, parent: ?*Node) !*Node {@@ -101,6 +102,7 @@ const Walk = struct {root_abs: []const u8,rules: std.ArrayList(gitignore.Rule),items: usize = 0,+ scanned: usize = 0,truncated: bool = false,};@@ -111,41 +113,43 @@ fn skip(w: *Walk, abs: []const u8, is_dir: bool) !bool {fn fill(w: *Walk, node: *Node, rel: []const u8) !void {const abs = if (rel.len == 0) w.root_abs else try join(w.arena, w.root_abs, rel);- var dir = Io.Dir.cwd().openDir(w.io, abs, .{ .iterate = true, .follow_symlinks = false }) catch return;+ var dir = try Io.Dir.cwd().openDir(w.io, abs, .{ .iterate = true, .follow_symlinks = false });defer dir.close(w.io);- var names: std.ArrayList(struct { name: []const u8, is_dir: bool }) = .empty;- var it = dir.iterate();- var saw_ignore = false;- while (it.next(w.io) catch null) |ent| {- if (ent.kind == .sym_link) continue;- const name = try w.arena.dupe(u8, ent.name);- if (std.mem.eql(u8, name, ".gitignore") and rel.len > 0) saw_ignore = true;- const is_dir = ent.kind == .directory;- try names.append(w.arena, .{ .name = name, .is_dir = is_dir });- }- if (saw_ignore) {- const gi = try join(w.arena, abs, ".gitignore");- const text = Io.Dir.cwd().readFileAlloc(w.io, gi, w.arena, .limited(64 * 1024)) catch null;- if (text) |t| {- const extra = gitignore.parse(w.arena, t, abs) catch &.{};- w.rules.appendSlice(w.arena, extra) catch {};+ // Load rules before the bounded scan, regardless of directory entry order.+ if (rel.len > 0) {+ if (dir.openFile(w.io, ".gitignore", .{ .follow_symlinks = false })) |file| {+ defer file.close(w.io);+ if ((try file.stat(w.io)).kind == .file) {+ var reader = file.reader(w.io, &.{});+ const text = try reader.interface.allocRemaining(w.arena, .limited(64 * 1024));+ const extra = try gitignore.parse(w.arena, text, abs);+ try w.rules.appendSlice(w.arena, extra);+ }+ } else |err| switch (err) {+ error.FileNotFound, error.SymLinkLoop => {},+ else => return err,}}-- for (names.items) |e| {- if (isGitComponent(e.name)) continue;- const child_rel = if (rel.len == 0) e.name else try join(w.arena, rel, e.name);- const child_abs = try join(w.arena, w.root_abs, child_rel);- if (try skip(w, child_abs, e.is_dir)) continue;- const child = try newNode(w.arena, e.name, e.is_dir, node.depth + 1, node);- try node.kids.append(w.arena, child);- if (!e.is_dir) try addFile(w.arena, node, e.name);- w.items += 1;- if (w.items >= max_walk_items) {+ var it = dir.iterate();+ while (true) {+ // Count physical entries, including ignored entries and symlinks. Do not+ // enumerate or allocate an unbounded directory before applying the cap.+ if (w.scanned >= max_walk_items) {w.truncated = true;return;}+ const ent = (try it.next(w.io)) orelse return;+ w.scanned += 1;+ if (ent.kind == .sym_link or isGitComponent(ent.name)) continue;+ const is_dir = ent.kind == .directory;+ const child_abs = try join(w.arena, abs, ent.name);+ if (try skip(w, child_abs, is_dir)) continue;+ const name = try w.arena.dupe(u8, ent.name);+ const child = try newNode(w.arena, name, is_dir, node.depth + 1, node);+ try node.kids.append(w.arena, child);+ if (!is_dir) try addFile(w.arena, node, name);+ w.items += 1;}}@@ -163,9 +167,10 @@ fn nameLess(_: void, a: *Node, b: *Node) bool {return an.len < bn.len;}-fn sortTree(n: *Node) void {+fn sortTree(n: *Node, partial: bool) void {+ n.partial = partial;std.mem.sort(*Node, n.kids.items, {}, nameLess);- for (n.kids.items) |k| sortTree(k);+ for (n.kids.items) |k| sortTree(k, partial);}const ExtPair = struct { k: []const u8, n: usize };@@ -184,7 +189,8 @@ fn summary(arena: Allocator, n: *Node) ![]const u8 {const take = @min(pairs.items.len, top_k_exts);var aw: Io.Writer.Allocating = .init(arena);const word: []const u8 = if (n.file_count == 1) "file" else "files";- try aw.writer.print("[{d} {s} in subtree: ", .{ n.file_count, word });+ if (n.partial) try aw.writer.writeAll("[at least ") else try aw.writer.writeByte('[');+ try aw.writer.print("{d} {s} in subtree: ", .{ n.file_count, word });var shown: usize = 0;for (pairs.items[0..take], 0..) |p, i| {if (i > 0) try aw.writer.writeAll(", ");@@ -307,7 +313,7 @@ fn walkTree(io: Io, arena: Allocator, abs: []const u8) !struct { root: *Node, trif (k.is_dir) try q.append(arena, .{ .n = k, .rel = try join(arena, item.rel, k.name) });}}- sortTree(root);+ sortTree(root, w.truncated);return .{ .root = root, .truncated = w.truncated };}@@ -503,3 +509,77 @@ test "empty directory is a header only" {const out = try listAbs(io, arena_state.allocator(), abs, "empty");try std.testing.expectEqualStrings("- empty/", out);}++test "directory enumeration cannot exceed remaining walk budget" {+ const io = std.testing.io;+ var tmp = std.testing.tmpDir(.{ .iterate = true });+ defer tmp.cleanup();+ for (0..8) |i| {+ var name: [32]u8 = undefined;+ try tmp.dir.writeFile(io, .{ .sub_path = try std.fmt.bufPrint(&name, "entry-{d}.txt", .{i}), .data = "x" });+ }+ var path_buf: [std.fs.max_path_bytes]u8 = undefined;+ const path_len = try tmp.dir.realPath(io, &path_buf);+ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena.deinit();+ var walk: Walk = .{ .io = io, .arena = arena.allocator(), .root_abs = path_buf[0..path_len], .rules = .empty, .items = max_walk_items - 1, .scanned = max_walk_items - 1 };+ const root = try newNode(arena.allocator(), "", true, 0, null);+ try fill(&walk, root, "");+ try std.testing.expectEqual(max_walk_items, walk.scanned);+ try std.testing.expectEqual(@as(usize, 1), root.kids.items.len);+ try std.testing.expect(walk.truncated);+}++test "bounded nested scan loads ignore rules first and labels partial counts" {+ const io = std.testing.io;+ var tmp = std.testing.tmpDir(.{ .iterate = true });+ defer tmp.cleanup();+ try tmp.dir.createDirPath(io, "nested");+ try tmp.dir.writeFile(io, .{ .sub_path = "nested/hidden.log", .data = "x" });+ try tmp.dir.writeFile(io, .{ .sub_path = "nested/.gitignore", .data = "*.log\n" });+ var path_buf: [std.fs.max_path_bytes]u8 = undefined;+ const path_len = try tmp.dir.realPath(io, &path_buf);+ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena.deinit();+ var walk: Walk = .{ .io = io, .arena = arena.allocator(), .root_abs = path_buf[0..path_len], .rules = .empty, .scanned = max_walk_items - 2 };+ const root = try newNode(arena.allocator(), "", true, 0, null);+ try fill(&walk, root, "nested");+ try std.testing.expectEqual(max_walk_items, walk.scanned);+ try std.testing.expect(walk.truncated);+ try std.testing.expectEqual(@as(usize, 1), root.kids.items.len);+ try std.testing.expectEqualStrings(".gitignore", root.kids.items[0].name);+ sortTree(root, walk.truncated);+ try std.testing.expect(std.mem.startsWith(u8, try summary(arena.allocator(), root), "[at least 1 file"));+}++test "nested ignore symlinks stay skipped and unreadable rules fail explicitly" {+ const io = std.testing.io;+ var tmp = std.testing.tmpDir(.{ .iterate = true });+ defer tmp.cleanup();+ try tmp.dir.createDirPath(io, "nested");+ try tmp.dir.writeFile(io, .{ .sub_path = "rules", .data = "*.log\n" });+ try tmp.dir.symLink(io, "../rules", "nested/.gitignore", .{});+ try tmp.dir.writeFile(io, .{ .sub_path = "nested/keep.log", .data = "x" });+ var path_buf: [std.fs.max_path_bytes]u8 = undefined;+ const path_len = try tmp.dir.realPath(io, &path_buf);+ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena.deinit();+ var walk: Walk = .{ .io = io, .arena = arena.allocator(), .root_abs = path_buf[0..path_len], .rules = .empty };+ const root = try newNode(arena.allocator(), "", true, 0, null);+ try fill(&walk, root, "nested");+ try std.testing.expectEqual(@as(usize, 1), root.kids.items.len);+ try std.testing.expectEqualStrings("keep.log", root.kids.items[0].name);+ try tmp.dir.deleteFile(io, "nested/.gitignore");+ const large = try arena.allocator().alloc(u8, 64 * 1024 + 1);+ @memset(large, 'x');+ try tmp.dir.writeFile(io, .{ .sub_path = "nested/.gitignore", .data = large });+ try std.testing.expectError(error.StreamTooLong, fill(&walk, root, "nested"));+ try std.testing.expectError(error.FileNotFound, fill(&walk, root, "missing"));+ try tmp.dir.deleteFile(io, "nested/.gitignore");+ try tmp.dir.createDirPath(io, "nested/.gitignore");+ const directory_root = try newNode(arena.allocator(), "", true, 0, null);+ try fill(&walk, directory_root, "nested");+ sortTree(directory_root, false);+ try std.testing.expectEqualStrings(".gitignore", directory_root.kids.items[0].name);+ try std.testing.expect(directory_root.kids.items[0].is_dir);+}diff --git a/src/list_dir_nearmiss.zig b/src/list_dir_nearmiss.zigindex 6c2cda75..51916bb2 100644--- a/src/list_dir_nearmiss.zig+++ b/src/list_dir_nearmiss.zig@@ -74,6 +74,10 @@ fn byName(_: void, a: Entry, b: Entry) bool {/// read. `resolved` is the path that failed; `display` is what the model/// typed (its parent is echoed back so the next command can be copied).pub fn suggest(io: Io, arena: Allocator, resolved: []const u8, display: []const u8, agent_cwd: ?[]const u8) ?[]const u8 {+ return suggestBudget(io, arena, resolved, display, agent_cwd, max_scanned);+}++fn suggestBudget(io: Io, arena: Allocator, resolved: []const u8, display: []const u8, agent_cwd: ?[]const u8, scan_budget: usize) ?[]const u8 {const target = std.mem.trimEnd(u8, resolved, "/");const leaf = std.fs.path.basename(target);if (leaf.len == 0) return null;@@ -88,16 +92,28 @@ pub fn suggest(io: Io, arena: Allocator, resolved: []const u8, display: []constconst rules = gitignore.loadClimb(io, arena, root_abs) catch return null;var entries: std.ArrayList(Entry) = .empty;var it = dir.iterate();- while (it.next(io) catch null) |entry| {+ var scanned: usize = 0;+ var limited = false;+ while (true) {+ // The budget counts physical entries, including ignored names and+ // symlinks. Never enumerate a huge parent just to find a few matches.+ if (scanned == scan_budget) {+ limited = true;+ break;+ }+ const entry = (it.next(io) catch return null) orelse break;+ scanned += 1;if (entry.kind == .sym_link or std.mem.eql(u8, entry.name, ".git")) continue;- if (entries.items.len == max_scanned) break;const is_dir = entry.kind == .directory;const child_abs = std.fmt.allocPrint(arena, "{s}/{s}", .{ root_abs, entry.name }) catch return null;if (gitignore.ignored(arena, rules, root_abs, child_abs, is_dir) catch return null) continue;const shown = std.fmt.allocPrint(arena, "{s}{s}", .{ entry.name, if (is_dir) "/" else "" }) catch return null;entries.append(arena, .{ .shown = shown, .is_dir = is_dir, .dist = distance(leaf, entry.name) }) catch return null;}- if (entries.items.len == 0) return null;+ if (entries.items.len == 0) {+ if (!limited) return null;+ return std.fmt.allocPrint(arena, " Scan stopped after {d} entries in {s}; no visible entries were found among those scanned. Other matches may exist.", .{ scanned, parent_display }) catch null;+ }var aw: Io.Writer.Allocating = .init(arena);const w = &aw.writer;@@ -105,13 +121,16 @@ pub fn suggest(io: Io, arena: Allocator, resolved: []const u8, display: []constvar close: usize = 0;for (entries.items) |e| {if (e.dist > threshold(leaf) or close == max_close) break;- w.print("{s}{s}", .{ if (close == 0) " Closest match: " else ", ", e.shown }) catch return null;+ w.print("{s}{s}", .{ if (close == 0) (if (limited) " Closest among scanned entries: " else " Closest match: ") else ", ", e.shown }) catch return null;close += 1;}if (close > 0) w.writeAll(".") catch return null;std.mem.sort(Entry, entries.items, {}, byName);- w.print(" Entries in {s} ({d}): ", .{ parent_display, entries.items.len }) catch return null;+ if (limited)+ w.print(" Visible entries among {d} scanned in {s} ({d}): ", .{ scanned, parent_display, entries.items.len }) catch return null+ else+ w.print(" Entries in {s} ({d}): ", .{ parent_display, entries.items.len }) catch return null;var bytes: usize = 0;var listed: usize = 0;for (entries.items) |e| {@@ -122,6 +141,7 @@ pub fn suggest(io: Io, arena: Allocator, resolved: []const u8, display: []const}if (listed < entries.items.len) w.print(", … {d} more", .{entries.items.len - listed}) catch return null;w.writeAll(".") catch return null;+ if (limited) w.print(" Scan stopped after {d} entries; other matches may exist.", .{scanned}) catch return null;return aw.writer.buffered();}@@ -161,3 +181,43 @@ test "not found: names the closest sibling and lists the parent" {try std.testing.expect(std.mem.indexOf(u8, none, "Closest match") == null);try std.testing.expect(std.mem.startsWith(u8, none, " Entries in sub (4): "));}++test "near-miss scan budget counts ignored entries and still reports an empty partial scan" {+ const io = std.testing.io;+ var tmp = std.testing.tmpDir(.{ .iterate = true });+ defer tmp.cleanup();+ try tmp.dir.writeFile(io, .{ .sub_path = ".gitignore", .data = "*\n" });+ for (0..4) |i| {+ var name: [16]u8 = undefined;+ try tmp.dir.writeFile(io, .{ .sub_path = try std.fmt.bufPrint(&name, "ignored-{d}", .{i}), .data = "x" });+ }+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ const arena = arena_state.allocator();+ var path_buf: [std.fs.max_path_bytes]u8 = undefined;+ const n = try tmp.dir.realPath(io, &path_buf);+ const missing = try std.fmt.allocPrint(arena, "{s}/missing", .{path_buf[0..n]});+ const hint = suggestBudget(io, arena, missing, "missing", null, 2) orelse return error.TestUnexpectedResult;+ try std.testing.expect(std.mem.indexOf(u8, hint, "Scan stopped after 2 entries") != null);+ try std.testing.expect(std.mem.indexOf(u8, hint, "no visible entries") != null);+ try std.testing.expect(std.mem.indexOf(u8, hint, "Closest match") == null);+}++test "near-miss partial scan never claims a global closest match" {+ const io = std.testing.io;+ var tmp = std.testing.tmpDir(.{ .iterate = true });+ defer tmp.cleanup();+ try tmp.dir.createDirPath(io, "a");+ try tmp.dir.createDirPath(io, "b");+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ const arena = arena_state.allocator();+ var path_buf: [std.fs.max_path_bytes]u8 = undefined;+ const n = try tmp.dir.realPath(io, &path_buf);+ const missing = try std.fmt.allocPrint(arena, "{s}/ax", .{path_buf[0..n]});+ const hint = suggestBudget(io, arena, missing, "ax", null, 1) orelse return error.TestUnexpectedResult;+ try std.testing.expect(std.mem.indexOf(u8, hint, "Closest among scanned entries:") != null);+ try std.testing.expect(std.mem.indexOf(u8, hint, "Visible entries among 1 scanned") != null);+ try std.testing.expect(std.mem.indexOf(u8, hint, "other matches may exist") != null);+ try std.testing.expect(std.mem.indexOf(u8, hint, "Closest match:") == null);+}