Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

Respect gitignore during source indexing

anirudhkumar-nanonets740faf8cb10c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 63e05dd8..63813729 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,13 @@
### Fixed
+- **Indexing now respects `.gitignore`.** In Git repositories, graft indexes
+ tracked files plus untracked files that Git does not ignore, so generated
+ output such as `Scripts/bundles/` and `Scripts/transpiled/` is no longer
+ parsed merely because its extension is supported. Nested ignore files,
+ negations, and global Git excludes follow Git's own rules; non-Git directories
+ retain the existing filesystem walk and built-in skip list ([#39]).
+
- **`graft ask` no longer lets normalization undo test-file de-ranking.** Test
files were penalized before lexical scores were normalized, but the strongest
test match was still normalized back to the maximum score. The test prior now
@@ -86,6 +93,7 @@
[#35]: https://github.com/NanoNets/Graft/issues/35
[#36]: https://github.com/NanoNets/Graft/issues/36
[#37]: https://github.com/NanoNets/Graft/issues/37
+[#39]: https://github.com/NanoNets/Graft/issues/39
## 0.8.2
diff --git a/src/graph/build.ts b/src/graph/build.ts
index 79360227..336a9b95 100644
--- a/src/graph/build.ts
+++ b/src/graph/build.ts
@@ -114,9 +114,9 @@ export interface GraphBuildResult {
* directory it lives in (posix, `.` for the root). Found anywhere in the tree, so a
* monorepo whose module is in a subdir (e.g. `backend/go.mod`) resolves too. Lets edge
* resolution map Go import paths to in-repo files. */
-function readGoModules(root: string): GoModule[] {
+function readGoModules(root: string, repoFiles: string[]): GoModule[] {
const mods: GoModule[] = [];
- for (const f of walkDir(root)) {
+ for (const f of repoFiles) {
if (basename(f) !== "go.mod") continue;
try {
const m = readFileSync(f, "utf8").match(/^\s*module\s+(\S+)/m);
@@ -136,8 +136,11 @@ export async function buildGraph(
): Promise<GraphBuildResult> {
const root = resolve(dir);
const outDir = contextDirFor(root, opts.contextDir);
- const files = listSourceStats(root, outDir);
- const discoveredScopes = discoverScopes(root);
+ // Enumerate once: source extraction, scope discovery, and Go module
+ // resolution must agree on the same Git-ignore-aware working-tree view.
+ const repoFiles = walkDir(root);
+ const files = listSourceStats(root, outDir, repoFiles);
+ const discoveredScopes = discoverScopes(root, repoFiles);
const nodes: NodeV1[] = [];
const rawEdges: RawEdge[] = [];
@@ -232,7 +235,7 @@ export async function buildGraph(
files: entries,
});
- const edges = resolveEdges(nodes, rawEdges, { goModules: readGoModules(root) });
+ const edges = resolveEdges(nodes, rawEdges, { goModules: readGoModules(root, repoFiles) });
// graph.json is its own Tier-2 cache: fold in the prior meaning layer so an
// unchanged body is never re-summarized (and a Tier-1-only run never wipes it).
diff --git a/src/graph/fingerprint.ts b/src/graph/fingerprint.ts
index e01c525c..440b0798 100644
--- a/src/graph/fingerprint.ts
+++ b/src/graph/fingerprint.ts
@@ -7,9 +7,9 @@
* with the last build's record get read and hashed, which is what keeps a `touch`
* or a `git checkout` of identical bytes from triggering a pointless rebuild.
*
- * Note graft has no notion of git here (it never shells out to git, never reads
- * `.git`): drift is measured against the bytes in the working tree, so an
- * uncommitted, staged, or committed edit all look the same — which is the point.
+ * Git supplies the visible file set (`tracked + untracked - ignored`) when
+ * available, but drift is still measured against bytes in the working tree:
+ * an uncommitted, staged, or committed edit to an indexed file looks the same.
*
* `<outDir>/.cache/fingerprint.json` is a projection of the extraction cache
* (`extract-cache.ts`) minus the parse results, written by the same build. Keeping
diff --git a/src/graph/scopes.ts b/src/graph/scopes.ts
index becc8432..68dd23d1 100644
--- a/src/graph/scopes.ts
+++ b/src/graph/scopes.ts
@@ -21,9 +21,10 @@
* 7. `label` = `prefix` (both "" for root); ordering is prefix-length desc,
* then lexicographic.
*/
-import { existsSync, readdirSync, readFileSync, statSync, type Dirent } from "node:fs";
+import { existsSync, readdirSync, readFileSync, type Dirent } from "node:fs";
import { join, resolve } from "node:path";
-import { SKIP_DIRS } from "../ingest/fs.js";
+import { SKIP_DIRS, walkDir } from "../ingest/fs.js";
+import { relPosix } from "../util/paths.js";
import type { GraphV1, ScopeV1 } from "./types.js";
/** Project-marker files, checked in this order (also the order `markers` is built in). */
@@ -31,28 +32,17 @@ const MARKERS = ["package.json", "go.mod", "pyproject.toml", "setup.py", "Cargo.
const CANONICAL_ROOT: ScopeV1[] = [{ prefix: "", label: "", markers: [] }];
-/** Recursively collect every directory under `root` (posix rel path, "" = root itself),
- * reusing `walkDir`'s skip rules (dot-dirs, `SKIP_DIRS`) so build-output and dependency
- * trees are never scanned for markers. */
-function collectDirs(root: string): string[] {
- const out: string[] = [""];
- const walk = (absDir: string, relDir: string): void => {
- let entries: Dirent[];
- try {
- entries = readdirSync(absDir, { withFileTypes: true });
- } catch {
- return;
- }
- for (const entry of entries) {
- if (!entry.isDirectory()) continue;
- if (entry.name.startsWith(".") || SKIP_DIRS.has(entry.name)) continue;
- const rel = relDir === "" ? entry.name : `${relDir}/${entry.name}`;
- out.push(rel);
- walk(join(absDir, entry.name), rel);
- }
- };
- walk(root, "");
- return out;
+/** Collect directories represented by the canonical visible file set. Building
+ * this from `walkDir` keeps scope discovery aligned with extraction: an ignored
+ * generated tree cannot become a marker or workspace scope. */
+function collectDirs(root: string, repoFiles: string[]): string[] {
+ const out = new Set<string>([""]);
+ for (const abs of repoFiles) {
+ const parts = relPosix(root, abs).split("/");
+ parts.pop();
+ for (let i = 1; i <= parts.length; i++) out.add(parts.slice(0, i).join("/"));
+ }
+ return [...out];
}
/** Minimal `packages:` list parser for `pnpm-workspace.yaml` — no YAML dependency,
@@ -99,10 +89,9 @@ function readWorkspaceGlobs(root: string): string[] | null {
return null;
}
-/** Resolve one workspace glob to matched dir paths (posix, rel to `root`). Supports
- * only `dir/*` (immediate subdirs) and `dir/**` (any nested depth) forms, implemented
- * with plain `readdir` — no glob dependency, per spec. Unsupported forms resolve empty. */
-function resolveGlob(root: string, pattern: string): string[] {
+/** Resolve one workspace glob against visible directories only. Supports
+ * `dir/*` (immediate subdirs), `dir/**` (any nested depth), and literal dirs. */
+function resolveGlob(dirs: string[], pattern: string): string[] {
const norm = pattern.replace(/\\/g, "/").replace(/\/$/, "");
let base: string;
let recursive: boolean;
@@ -119,50 +108,18 @@ function resolveGlob(root: string, pattern: string): string[] {
base = norm.slice(0, -2);
recursive = false;
} else if (!norm.includes("*")) {
- // Literal entry (no glob chars), e.g. `packages: ['apps/*', 'docs']` — a plain
- // workspace-list entry that names a real directory counts as a workspace match
- // in its own right, same as a resolved glob would. Without this, a literal
- // entry silently resolves to nothing while Rule 2 still strips its
- // `package.json` marker (it's a JS-family dir that never matched the glob
- // set), de-scoping a real workspace package.
- try {
- return existsSync(join(root, norm)) && statSync(join(root, norm)).isDirectory()
- ? [norm]
- : [];
- } catch {
- return [];
- }
+ return dirs.includes(norm) ? [norm] : [];
} else {
return []; // unsupported glob form
}
- const results: string[] = [];
- const listDirs = (absDir: string, relDir: string): string[] => {
- let entries: Dirent[];
- try {
- entries = readdirSync(absDir, { withFileTypes: true });
- } catch {
- return [];
- }
- return entries
- .filter((e) => e.isDirectory() && !e.name.startsWith(".") && !SKIP_DIRS.has(e.name))
- .map((e) => (relDir === "" ? e.name : `${relDir}/${e.name}`));
- };
-
- const baseAbs = base === "" ? root : join(root, base);
- if (!recursive) {
- results.push(...listDirs(baseAbs, base));
- } else {
- const stack = [base];
- while (stack.length) {
- const relDir = stack.pop()!;
- const absDir = relDir === "" ? root : join(root, relDir);
- const children = listDirs(absDir, relDir);
- results.push(...children);
- stack.push(...children);
- }
- }
- return results;
+ return dirs.filter((dir) => {
+ if (dir === "") return false;
+ if (recursive) return base === "" || dir.startsWith(`${base}/`);
+ const slash = dir.lastIndexOf("/");
+ const parent = slash === -1 ? "" : dir.slice(0, slash);
+ return parent === base;
+ });
}
interface Candidate {
@@ -171,9 +128,9 @@ interface Candidate {
}
/** Walk the tree (reusing `walkDir`'s skip rules) and find project-marker dirs. */
-export function discoverScopes(root: string): ScopeV1[] {
+export function discoverScopes(root: string, repoFiles: string[] = walkDir(root)): ScopeV1[] {
const absRoot = resolve(root);
- const dirs = collectDirs(absRoot);
+ const dirs = collectDirs(absRoot, repoFiles);
const markerMap = new Map<string, string[]>();
for (const dir of dirs) {
@@ -185,7 +142,7 @@ export function discoverScopes(root: string): ScopeV1[] {
const workspaceMatches = new Set<string>();
if (workspaceGlobs) {
for (const glob of workspaceGlobs) {
- for (const dir of resolveGlob(absRoot, glob)) workspaceMatches.add(dir);
+ for (const dir of resolveGlob(dirs, glob)) workspaceMatches.add(dir);
}
}
diff --git a/src/graph/source-files.ts b/src/graph/source-files.ts
index 93847e1f..7fd87200 100644
--- a/src/graph/source-files.ts
+++ b/src/graph/source-files.ts
@@ -12,8 +12,8 @@ import { relPosix } from "../util/paths.js";
import { languageOf } from "./extract.js";
/** The source files a graph build parses: supported languages, minus the output dir. */
-export function listSourceFiles(root: string, outDir: string): string[] {
- return walkDir(root).filter((f) => !f.startsWith(outDir) && languageOf(f) !== null);
+export function listSourceFiles(root: string, outDir: string, repoFiles: string[] = walkDir(root)): string[] {
+ return repoFiles.filter((f) => !f.startsWith(outDir) && languageOf(f) !== null);
}
export interface SourceStat {
@@ -32,9 +32,9 @@ export interface SourceStat {
* both the freshness probe and the extraction cache. Files that vanish between
* the walk and the stat are dropped (same fail-soft posture as `walkDir`).
*/
-export function listSourceStats(root: string, outDir: string): SourceStat[] {
+export function listSourceStats(root: string, outDir: string, repoFiles?: string[]): SourceStat[] {
const out: SourceStat[] = [];
- for (const abs of listSourceFiles(root, outDir)) {
+ for (const abs of listSourceFiles(root, outDir, repoFiles)) {
let s: { size: number; mtimeMs: number };
try {
s = statSync(abs);
diff --git a/src/ingest/fs.ts b/src/ingest/fs.ts
index f39682db..466e90fa 100644
--- a/src/ingest/fs.ts
+++ b/src/ingest/fs.ts
@@ -1,8 +1,9 @@
/**
* Filesystem walking used by `init`/`check` to enumerate a repo's source files.
*/
-import { readdirSync, statSync } from "node:fs";
-import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+import { lstatSync, readdirSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
/** Directories that are dependency/build output, never source. */
export const SKIP_DIRS = new Set([
@@ -23,15 +24,62 @@ export const MAX_FILE_BYTES = 1_000_000;
/**
* Recursively list all files under a directory. Skips dot-directories,
* dependency/build directories (node_modules, dist, …), and files over 1 MB.
+ * In a Git worktree, tracked files plus untracked, non-ignored files come from
+ * `git ls-files`; this gives indexing exactly Git's nested `.gitignore`,
+ * negation, and global-exclude semantics. Non-Git directories retain the plain
+ * filesystem walk.
*/
export function walkDir(dir: string): string[] {
+ return gitVisibleFiles(dir) ?? walkFilesystem(dir);
+}
+
+/** Git's canonical working-tree file set, relative to `dir`. Tracked files are
+ * deliberately included even when a later ignore rule matches them; `.gitignore`
+ * only controls untracked files in Git, and graft follows the same contract. */
+function gitVisibleFiles(dir: string): string[] | null {
+ const root = resolve(dir);
+ const result = spawnSync(
+ "git",
+ ["ls-files", "--cached", "--others", "--exclude-standard", "-z", "--"],
+ {
+ cwd: root,
+ encoding: "utf8",
+ stdio: ["ignore", "pipe", "ignore"],
+ maxBuffer: 64 * 1024 * 1024,
+ },
+ );
+ if (result.status !== 0 || result.error || typeof result.stdout !== "string") return null;
+
+ const out: string[] = [];
+ for (const rel of result.stdout.split("\0")) {
+ if (!rel || skippedPath(rel)) continue;
+ const abs = resolve(root, rel);
+ try {
+ const stat = lstatSync(abs);
+ if (!stat.isFile() || stat.size > MAX_FILE_BYTES) continue;
+ } catch {
+ // A tracked file deleted from the working tree is still printed by
+ // `--cached`; absence means it is not part of the current source set.
+ continue;
+ }
+ out.push(abs);
+ }
+ return out;
+}
+
+function skippedPath(path: string): boolean {
+ const segments = path.replace(/\\/g, "/").split("/");
+ return segments.some((segment) => segment.startsWith(".") || SKIP_DIRS.has(segment));
+}
+
+function walkFilesystem(dir: string): string[] {
const out: string[] = [];
for (const entry of readdirSync(dir, { withFileTypes: true })) {
if (entry.name.startsWith(".")) continue;
const full = join(dir, entry.name);
if (entry.isDirectory()) {
if (SKIP_DIRS.has(entry.name)) continue;
- out.push(...walkDir(full));
+ out.push(...walkFilesystem(full));
} else if (entry.isFile()) {
try {
if (statSync(full).size > MAX_FILE_BYTES) continue;
diff --git a/test/graph-incremental.test.ts b/test/graph-incremental.test.ts
index 46dfabff..bb38347f 100644
--- a/test/graph-incremental.test.ts
+++ b/test/graph-incremental.test.ts
@@ -6,6 +6,7 @@
*/
import { test } from "node:test";
import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
import { chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
@@ -159,6 +160,27 @@ test("every file on disk lands in the fingerprint", async () => {
assert.deepEqual(Object.keys(fp!.files).sort(), ["src/app.ts", "src/math.ts"]);
});
+test("gitignored generated files stay out of the graph, fingerprint, and drift probe (#39)", async () => {
+ const d = repo();
+ try {
+ execFileSync("git", ["init", "-q"], { cwd: d });
+ writeFileSync(join(d, ".gitignore"), "Scripts/bundles/\n");
+ mkdirSync(join(d, "Scripts", "bundles"), { recursive: true });
+ const bundle = join(d, "Scripts", "bundles", "app.js");
+ writeFileSync(bundle, "export function generatedBundle(): number { return 1; }\n");
+
+ await buildGraph(d);
+ const graph = readGraph(wiringPath(outOf(d))) as GraphV1;
+ assert.ok(!graph.nodes.some((n) => n.path === "Scripts/bundles/app.js"));
+ assert.deepEqual(Object.keys(readFingerprint(outOf(d))!.files).sort(), ["src/app.ts", "src/math.ts"]);
+
+ writeFileSync(bundle, "export function generatedBundle(): number { return 2; }\n");
+ assert.ok(isClean(probeDrift(d, outOf(d))!), "ignored output changes must not trigger a refresh");
+ } finally {
+ rmSync(d, { recursive: true, force: true });
+ }
+});
+
test("an unreadable file is still recorded, so it can't look new on every probe", async (t) => {
const why = chmodDenialUnavailable();
if (why) return t.skip(why);
diff --git a/test/graph-scopes.test.ts b/test/graph-scopes.test.ts
index 09c725bd..b6b1fdca 100644
--- a/test/graph-scopes.test.ts
+++ b/test/graph-scopes.test.ts
@@ -1,5 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -47,6 +48,23 @@ test("workspace globs are intent: packages/* honored, deeper ignored", () => {
rmSync(d, { recursive: true, force: true });
});
+test("gitignored generated workspace directories cannot become scopes (#39)", () => {
+ const d = fx({
+ ".gitignore": "generated/\n",
+ "package.json": JSON.stringify({ workspaces: ["packages/*", "generated/*"] }),
+ "packages/app/package.json": "{}",
+ "packages/app/src/index.ts": "export const app = 1;",
+ "generated/bundle/package.json": "{}",
+ "generated/bundle/index.ts": "export const generated = 1;",
+ });
+ try {
+ execFileSync("git", ["init", "-q"], { cwd: d });
+ assert.deepEqual(discoverScopes(d).map((s) => s.prefix), ["packages/app"]);
+ } finally {
+ rmSync(d, { recursive: true, force: true });
+ }
+});
+
test("root-only marker -> canonical single scope", () => {
const d = fx({ "package.json": "{}", "src/a.ts": "1" });
const scopes = discoverScopes(d);
diff --git a/test/ingest-fs.test.ts b/test/ingest-fs.test.ts
new file mode 100644
index 00000000..5ae70c5c
--- /dev/null
+++ b/test/ingest-fs.test.ts
@@ -0,0 +1,74 @@
+import { execFileSync } from "node:child_process";
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, relative } from "node:path";
+import { walkDir } from "../src/ingest/fs.js";
+
+function fixture(tag: string): string {
+ const dir = mkdtempSync(join(tmpdir(), `graft-walk-${tag}-`));
+ execFileSync("git", ["init", "-q"], { cwd: dir });
+ return dir;
+}
+
+function write(root: string, path: string, content = "export const value = 1;\n"): void {
+ mkdirSync(join(root, path, ".."), { recursive: true });
+ writeFileSync(join(root, path), content);
+}
+
+function walked(root: string): string[] {
+ return walkDir(root)
+ .map((path) => relative(root, path).replace(/\\/g, "/"))
+ .sort();
+}
+
+test("walkDir respects root and nested .gitignore rules, including negation", () => {
+ const dir = fixture("ignore");
+ try {
+ write(dir, ".gitignore", "Scripts/bundles/\ngenerated/*\n!generated/keep.ts\n");
+ write(dir, "src/app.ts");
+ write(dir, "Scripts/bundles/app.js");
+ write(dir, "generated/drop.ts");
+ write(dir, "generated/keep.ts");
+ write(dir, "packages/tool/.gitignore", "output/\n");
+ write(dir, "packages/tool/index.ts");
+ write(dir, "packages/tool/output/bundle.js");
+
+ assert.deepEqual(walked(dir), [
+ "generated/keep.ts",
+ "packages/tool/index.ts",
+ "src/app.ts",
+ ]);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("walkDir keeps tracked files that match an ignore rule and untracked visible files", () => {
+ const dir = fixture("tracked");
+ try {
+ write(dir, ".gitignore", "*.generated.ts\n");
+ write(dir, "tracked.generated.ts");
+ write(dir, "ignored.generated.ts");
+ write(dir, "visible.ts");
+ execFileSync("git", ["add", "-f", "tracked.generated.ts"], { cwd: dir });
+
+ assert.deepEqual(walked(dir), ["tracked.generated.ts", "visible.ts"]);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("walkDir retains fixed skips and filesystem fallback outside Git", () => {
+ const dir = mkdtempSync(join(tmpdir(), "graft-walk-nongit-"));
+ try {
+ write(dir, "src/app.ts");
+ write(dir, "node_modules/pkg/index.ts");
+ write(dir, ".hidden/secret.ts");
+
+ assert.deepEqual(walked(dir), ["src/app.ts"]);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});