Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Bound directory scans before filtering and mark partial results

Rach Pradhand3b993db7f60
diff --git a/docs/adr/0013-list-dir-lives-in-codedb.md b/docs/adr/0013-list-dir-lives-in-codedb.md
index 887770e4..0c0cf7e4 100644
--- a/docs/adr/0013-list-dir-lives-in-codedb.md
+++ b/docs/adr/0013-list-dir-lives-in-codedb.md
@@ -30,3 +30,6 @@ extra schema bytes beat this subcommand.
- `codedb ls` / `tree` stay index queries.
- The model has to read the codedb description (or the prompt sentence) to
find `list_dir`; that is the cost of not growing the catalog.
+
+The in-process scan budget and partial-result contract are refined by
+[0184](0184-directory-scan-budgets-count-examined-entries.md).
diff --git a/docs/adr/0184-directory-scan-budgets-count-examined-entries.md b/docs/adr/0184-directory-scan-budgets-count-examined-entries.md
new file mode 100644
index 00000000..ee18f8ff
--- /dev/null
+++ b/docs/adr/0184-directory-scan-budgets-count-examined-entries.md
@@ -0,0 +1,28 @@
+# 0184. Directory scan budgets count examined entries
+
+Status: accepted
+
+## Context
+
+A limit applied after collecting directory names bounds the rendered tree but
+does not bound enumeration or temporary allocation. Counting only visible
+entries also lets ignored files and symlinks bypass a scan limit.
+
+## Decision
+
+Directory listings and missing-path suggestions count entries as the iterator
+returns them, before filtering or copying names. Stop before requesting another
+entry once the scan budget is exhausted. Load nested ignore rules independently
+of entry order so truncation cannot accidentally bypass them.
+
+An exhausted budget produces an explicitly partial result. Subtree counts are
+lower bounds, and suggestions from an incomplete scan do not claim to be the
+closest match across the entire directory. Below the limit, retain ordinary
+listing order, ignore rules, and visibility of useful dotfiles.
+
+## Consequences
+
+Directories with many ignored entries can reach the budget earlier. The caller
+can narrow the requested path; partial output must never imply a complete tree.
+This bounds local navigation work without changing the tool catalog or removing
+conversation history. It does not establish model token or cost savings.
diff --git a/docs/adr/README.md b/docs/adr/README.md
index c78c1f35..68a01459 100644
--- a/docs/adr/README.md
+++ b/docs/adr/README.md
@@ -209,6 +209,7 @@ record only when you need the evidence or the edge cases.
| [0181](0181-acp-load-replays-saved-conversation.md) | Live ACP loads saved conversations in their selected workspace and replays history without rerunning tools. |
| [0182](0182-acp-connection-usage-is-an-extension.md) | Connection-scoped ACP usage preserves unknown totals through a custom notification; network retries remain activity indicators. |
| [0183](0183-request-policy-owns-ambiguous-retries.md) | Ambiguous HTTP/2 failures return to the request retry policy so attempts and unknown usage remain observable. |
+| [0184](0184-directory-scan-budgets-count-examined-entries.md) | Directory scan limits count examined entries before filtering; truncated listings and suggestions remain explicitly partial. |
## When to write one
diff --git a/src/list_dir.zig b/src/list_dir.zig
index 039be87e..f7b569ea 100644
--- a/src/list_dir.zig
+++ b/src/list_dir.zig
@@ -36,7 +36,7 @@ const root_truncation_notice =
const walk_truncation_notice =
\\
- \\Note: there are more than 20000 items in the directory, so not all files may be shown.
+ \\Note: the 20000-entry scan limit was reached; the listing and subtree counts may be incomplete.
;
const Node = struct {
@@ -48,6 +48,7 @@ const Node = struct {
file_count: usize,
ext: std.StringHashMap(usize),
expanded: bool,
+ partial: bool = false,
};
fn newNode(arena: Allocator, name: []const u8, is_dir: bool, depth: usize, parent: ?*Node) !*Node {
@@ -101,6 +102,7 @@ const Walk = struct {
root_abs: []const u8,
rules: std.ArrayList(gitignore.Rule),
items: usize = 0,
+ scanned: usize = 0,
truncated: bool = false,
};
@@ -111,41 +113,43 @@ fn skip(w: *Walk, abs: []const u8, is_dir: bool) !bool {
fn fill(w: *Walk, node: *Node, rel: []const u8) !void {
const abs = if (rel.len == 0) w.root_abs else try join(w.arena, w.root_abs, rel);
- var dir = Io.Dir.cwd().openDir(w.io, abs, .{ .iterate = true, .follow_symlinks = false }) catch return;
+ var dir = try Io.Dir.cwd().openDir(w.io, abs, .{ .iterate = true, .follow_symlinks = false });
defer dir.close(w.io);
- var names: std.ArrayList(struct { name: []const u8, is_dir: bool }) = .empty;
- var it = dir.iterate();
- var saw_ignore = false;
- while (it.next(w.io) catch null) |ent| {
- if (ent.kind == .sym_link) continue;
- const name = try w.arena.dupe(u8, ent.name);
- if (std.mem.eql(u8, name, ".gitignore") and rel.len > 0) saw_ignore = true;
- const is_dir = ent.kind == .directory;
- try names.append(w.arena, .{ .name = name, .is_dir = is_dir });
- }
- if (saw_ignore) {
- const gi = try join(w.arena, abs, ".gitignore");
- const text = Io.Dir.cwd().readFileAlloc(w.io, gi, w.arena, .limited(64 * 1024)) catch null;
- if (text) |t| {
- const extra = gitignore.parse(w.arena, t, abs) catch &.{};
- w.rules.appendSlice(w.arena, extra) catch {};
+ // Load rules before the bounded scan, regardless of directory entry order.
+ if (rel.len > 0) {
+ if (dir.openFile(w.io, ".gitignore", .{ .follow_symlinks = false })) |file| {
+ defer file.close(w.io);
+ if ((try file.stat(w.io)).kind == .file) {
+ var reader = file.reader(w.io, &.{});
+ const text = try reader.interface.allocRemaining(w.arena, .limited(64 * 1024));
+ const extra = try gitignore.parse(w.arena, text, abs);
+ try w.rules.appendSlice(w.arena, extra);
+ }
+ } else |err| switch (err) {
+ error.FileNotFound, error.SymLinkLoop => {},
+ else => return err,
}
}
-
- for (names.items) |e| {
- if (isGitComponent(e.name)) continue;
- const child_rel = if (rel.len == 0) e.name else try join(w.arena, rel, e.name);
- const child_abs = try join(w.arena, w.root_abs, child_rel);
- if (try skip(w, child_abs, e.is_dir)) continue;
- const child = try newNode(w.arena, e.name, e.is_dir, node.depth + 1, node);
- try node.kids.append(w.arena, child);
- if (!e.is_dir) try addFile(w.arena, node, e.name);
- w.items += 1;
- if (w.items >= max_walk_items) {
+ var it = dir.iterate();
+ while (true) {
+ // Count physical entries, including ignored entries and symlinks. Do not
+ // enumerate or allocate an unbounded directory before applying the cap.
+ if (w.scanned >= max_walk_items) {
w.truncated = true;
return;
}
+ const ent = (try it.next(w.io)) orelse return;
+ w.scanned += 1;
+ if (ent.kind == .sym_link or isGitComponent(ent.name)) continue;
+ const is_dir = ent.kind == .directory;
+ const child_abs = try join(w.arena, abs, ent.name);
+ if (try skip(w, child_abs, is_dir)) continue;
+ const name = try w.arena.dupe(u8, ent.name);
+ const child = try newNode(w.arena, name, is_dir, node.depth + 1, node);
+ try node.kids.append(w.arena, child);
+ if (!is_dir) try addFile(w.arena, node, name);
+ w.items += 1;
}
}
@@ -163,9 +167,10 @@ fn nameLess(_: void, a: *Node, b: *Node) bool {
return an.len < bn.len;
}
-fn sortTree(n: *Node) void {
+fn sortTree(n: *Node, partial: bool) void {
+ n.partial = partial;
std.mem.sort(*Node, n.kids.items, {}, nameLess);
- for (n.kids.items) |k| sortTree(k);
+ for (n.kids.items) |k| sortTree(k, partial);
}
const ExtPair = struct { k: []const u8, n: usize };
@@ -184,7 +189,8 @@ fn summary(arena: Allocator, n: *Node) ![]const u8 {
const take = @min(pairs.items.len, top_k_exts);
var aw: Io.Writer.Allocating = .init(arena);
const word: []const u8 = if (n.file_count == 1) "file" else "files";
- try aw.writer.print("[{d} {s} in subtree: ", .{ n.file_count, word });
+ if (n.partial) try aw.writer.writeAll("[at least ") else try aw.writer.writeByte('[');
+ try aw.writer.print("{d} {s} in subtree: ", .{ n.file_count, word });
var shown: usize = 0;
for (pairs.items[0..take], 0..) |p, i| {
if (i > 0) try aw.writer.writeAll(", ");
@@ -307,7 +313,7 @@ fn walkTree(io: Io, arena: Allocator, abs: []const u8) !struct { root: *Node, tr
if (k.is_dir) try q.append(arena, .{ .n = k, .rel = try join(arena, item.rel, k.name) });
}
}
- sortTree(root);
+ sortTree(root, w.truncated);
return .{ .root = root, .truncated = w.truncated };
}
@@ -503,3 +509,77 @@ test "empty directory is a header only" {
const out = try listAbs(io, arena_state.allocator(), abs, "empty");
try std.testing.expectEqualStrings("- empty/", out);
}
+
+test "directory enumeration cannot exceed remaining walk budget" {
+ const io = std.testing.io;
+ var tmp = std.testing.tmpDir(.{ .iterate = true });
+ defer tmp.cleanup();
+ for (0..8) |i| {
+ var name: [32]u8 = undefined;
+ try tmp.dir.writeFile(io, .{ .sub_path = try std.fmt.bufPrint(&name, "entry-{d}.txt", .{i}), .data = "x" });
+ }
+ var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const path_len = try tmp.dir.realPath(io, &path_buf);
+ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena.deinit();
+ var walk: Walk = .{ .io = io, .arena = arena.allocator(), .root_abs = path_buf[0..path_len], .rules = .empty, .items = max_walk_items - 1, .scanned = max_walk_items - 1 };
+ const root = try newNode(arena.allocator(), "", true, 0, null);
+ try fill(&walk, root, "");
+ try std.testing.expectEqual(max_walk_items, walk.scanned);
+ try std.testing.expectEqual(@as(usize, 1), root.kids.items.len);
+ try std.testing.expect(walk.truncated);
+}
+
+test "bounded nested scan loads ignore rules first and labels partial counts" {
+ const io = std.testing.io;
+ var tmp = std.testing.tmpDir(.{ .iterate = true });
+ defer tmp.cleanup();
+ try tmp.dir.createDirPath(io, "nested");
+ try tmp.dir.writeFile(io, .{ .sub_path = "nested/hidden.log", .data = "x" });
+ try tmp.dir.writeFile(io, .{ .sub_path = "nested/.gitignore", .data = "*.log\n" });
+ var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const path_len = try tmp.dir.realPath(io, &path_buf);
+ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena.deinit();
+ var walk: Walk = .{ .io = io, .arena = arena.allocator(), .root_abs = path_buf[0..path_len], .rules = .empty, .scanned = max_walk_items - 2 };
+ const root = try newNode(arena.allocator(), "", true, 0, null);
+ try fill(&walk, root, "nested");
+ try std.testing.expectEqual(max_walk_items, walk.scanned);
+ try std.testing.expect(walk.truncated);
+ try std.testing.expectEqual(@as(usize, 1), root.kids.items.len);
+ try std.testing.expectEqualStrings(".gitignore", root.kids.items[0].name);
+ sortTree(root, walk.truncated);
+ try std.testing.expect(std.mem.startsWith(u8, try summary(arena.allocator(), root), "[at least 1 file"));
+}
+
+test "nested ignore symlinks stay skipped and unreadable rules fail explicitly" {
+ const io = std.testing.io;
+ var tmp = std.testing.tmpDir(.{ .iterate = true });
+ defer tmp.cleanup();
+ try tmp.dir.createDirPath(io, "nested");
+ try tmp.dir.writeFile(io, .{ .sub_path = "rules", .data = "*.log\n" });
+ try tmp.dir.symLink(io, "../rules", "nested/.gitignore", .{});
+ try tmp.dir.writeFile(io, .{ .sub_path = "nested/keep.log", .data = "x" });
+ var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const path_len = try tmp.dir.realPath(io, &path_buf);
+ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena.deinit();
+ var walk: Walk = .{ .io = io, .arena = arena.allocator(), .root_abs = path_buf[0..path_len], .rules = .empty };
+ const root = try newNode(arena.allocator(), "", true, 0, null);
+ try fill(&walk, root, "nested");
+ try std.testing.expectEqual(@as(usize, 1), root.kids.items.len);
+ try std.testing.expectEqualStrings("keep.log", root.kids.items[0].name);
+ try tmp.dir.deleteFile(io, "nested/.gitignore");
+ const large = try arena.allocator().alloc(u8, 64 * 1024 + 1);
+ @memset(large, 'x');
+ try tmp.dir.writeFile(io, .{ .sub_path = "nested/.gitignore", .data = large });
+ try std.testing.expectError(error.StreamTooLong, fill(&walk, root, "nested"));
+ try std.testing.expectError(error.FileNotFound, fill(&walk, root, "missing"));
+ try tmp.dir.deleteFile(io, "nested/.gitignore");
+ try tmp.dir.createDirPath(io, "nested/.gitignore");
+ const directory_root = try newNode(arena.allocator(), "", true, 0, null);
+ try fill(&walk, directory_root, "nested");
+ sortTree(directory_root, false);
+ try std.testing.expectEqualStrings(".gitignore", directory_root.kids.items[0].name);
+ try std.testing.expect(directory_root.kids.items[0].is_dir);
+}
diff --git a/src/list_dir_nearmiss.zig b/src/list_dir_nearmiss.zig
index 6c2cda75..51916bb2 100644
--- a/src/list_dir_nearmiss.zig
+++ b/src/list_dir_nearmiss.zig
@@ -74,6 +74,10 @@ fn byName(_: void, a: Entry, b: Entry) bool {
/// read. `resolved` is the path that failed; `display` is what the model
/// typed (its parent is echoed back so the next command can be copied).
pub fn suggest(io: Io, arena: Allocator, resolved: []const u8, display: []const u8, agent_cwd: ?[]const u8) ?[]const u8 {
+ return suggestBudget(io, arena, resolved, display, agent_cwd, max_scanned);
+}
+
+fn suggestBudget(io: Io, arena: Allocator, resolved: []const u8, display: []const u8, agent_cwd: ?[]const u8, scan_budget: usize) ?[]const u8 {
const target = std.mem.trimEnd(u8, resolved, "/");
const leaf = std.fs.path.basename(target);
if (leaf.len == 0) return null;
@@ -88,16 +92,28 @@ pub fn suggest(io: Io, arena: Allocator, resolved: []const u8, display: []const
const rules = gitignore.loadClimb(io, arena, root_abs) catch return null;
var entries: std.ArrayList(Entry) = .empty;
var it = dir.iterate();
- while (it.next(io) catch null) |entry| {
+ var scanned: usize = 0;
+ var limited = false;
+ while (true) {
+ // The budget counts physical entries, including ignored names and
+ // symlinks. Never enumerate a huge parent just to find a few matches.
+ if (scanned == scan_budget) {
+ limited = true;
+ break;
+ }
+ const entry = (it.next(io) catch return null) orelse break;
+ scanned += 1;
if (entry.kind == .sym_link or std.mem.eql(u8, entry.name, ".git")) continue;
- if (entries.items.len == max_scanned) break;
const is_dir = entry.kind == .directory;
const child_abs = std.fmt.allocPrint(arena, "{s}/{s}", .{ root_abs, entry.name }) catch return null;
if (gitignore.ignored(arena, rules, root_abs, child_abs, is_dir) catch return null) continue;
const shown = std.fmt.allocPrint(arena, "{s}{s}", .{ entry.name, if (is_dir) "/" else "" }) catch return null;
entries.append(arena, .{ .shown = shown, .is_dir = is_dir, .dist = distance(leaf, entry.name) }) catch return null;
}
- if (entries.items.len == 0) return null;
+ if (entries.items.len == 0) {
+ if (!limited) return null;
+ return std.fmt.allocPrint(arena, " Scan stopped after {d} entries in {s}; no visible entries were found among those scanned. Other matches may exist.", .{ scanned, parent_display }) catch null;
+ }
var aw: Io.Writer.Allocating = .init(arena);
const w = &aw.writer;
@@ -105,13 +121,16 @@ pub fn suggest(io: Io, arena: Allocator, resolved: []const u8, display: []const
var close: usize = 0;
for (entries.items) |e| {
if (e.dist > threshold(leaf) or close == max_close) break;
- w.print("{s}{s}", .{ if (close == 0) " Closest match: " else ", ", e.shown }) catch return null;
+ w.print("{s}{s}", .{ if (close == 0) (if (limited) " Closest among scanned entries: " else " Closest match: ") else ", ", e.shown }) catch return null;
close += 1;
}
if (close > 0) w.writeAll(".") catch return null;
std.mem.sort(Entry, entries.items, {}, byName);
- w.print(" Entries in {s} ({d}): ", .{ parent_display, entries.items.len }) catch return null;
+ if (limited)
+ w.print(" Visible entries among {d} scanned in {s} ({d}): ", .{ scanned, parent_display, entries.items.len }) catch return null
+ else
+ w.print(" Entries in {s} ({d}): ", .{ parent_display, entries.items.len }) catch return null;
var bytes: usize = 0;
var listed: usize = 0;
for (entries.items) |e| {
@@ -122,6 +141,7 @@ pub fn suggest(io: Io, arena: Allocator, resolved: []const u8, display: []const
}
if (listed < entries.items.len) w.print(", … {d} more", .{entries.items.len - listed}) catch return null;
w.writeAll(".") catch return null;
+ if (limited) w.print(" Scan stopped after {d} entries; other matches may exist.", .{scanned}) catch return null;
return aw.writer.buffered();
}
@@ -161,3 +181,43 @@ test "not found: names the closest sibling and lists the parent" {
try std.testing.expect(std.mem.indexOf(u8, none, "Closest match") == null);
try std.testing.expect(std.mem.startsWith(u8, none, " Entries in sub (4): "));
}
+
+test "near-miss scan budget counts ignored entries and still reports an empty partial scan" {
+ const io = std.testing.io;
+ var tmp = std.testing.tmpDir(.{ .iterate = true });
+ defer tmp.cleanup();
+ try tmp.dir.writeFile(io, .{ .sub_path = ".gitignore", .data = "*\n" });
+ for (0..4) |i| {
+ var name: [16]u8 = undefined;
+ try tmp.dir.writeFile(io, .{ .sub_path = try std.fmt.bufPrint(&name, "ignored-{d}", .{i}), .data = "x" });
+ }
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+ var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const n = try tmp.dir.realPath(io, &path_buf);
+ const missing = try std.fmt.allocPrint(arena, "{s}/missing", .{path_buf[0..n]});
+ const hint = suggestBudget(io, arena, missing, "missing", null, 2) orelse return error.TestUnexpectedResult;
+ try std.testing.expect(std.mem.indexOf(u8, hint, "Scan stopped after 2 entries") != null);
+ try std.testing.expect(std.mem.indexOf(u8, hint, "no visible entries") != null);
+ try std.testing.expect(std.mem.indexOf(u8, hint, "Closest match") == null);
+}
+
+test "near-miss partial scan never claims a global closest match" {
+ const io = std.testing.io;
+ var tmp = std.testing.tmpDir(.{ .iterate = true });
+ defer tmp.cleanup();
+ try tmp.dir.createDirPath(io, "a");
+ try tmp.dir.createDirPath(io, "b");
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+ var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const n = try tmp.dir.realPath(io, &path_buf);
+ const missing = try std.fmt.allocPrint(arena, "{s}/ax", .{path_buf[0..n]});
+ const hint = suggestBudget(io, arena, missing, "ax", null, 1) orelse return error.TestUnexpectedResult;
+ try std.testing.expect(std.mem.indexOf(u8, hint, "Closest among scanned entries:") != null);
+ try std.testing.expect(std.mem.indexOf(u8, hint, "Visible entries among 1 scanned") != null);
+ try std.testing.expect(std.mem.indexOf(u8, hint, "other matches may exist") != null);
+ try std.testing.expect(std.mem.indexOf(u8, hint, "Closest match:") == null);
+}