Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
Respect gitignore during source indexing
anirudhkumar-nanonets740faf8cb10c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 63e05dd8..63813729 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,6 +4,13 @@### Fixed+- **Indexing now respects `.gitignore`.** In Git repositories, graft indexes+ tracked files plus untracked files that Git does not ignore, so generated+ output such as `Scripts/bundles/` and `Scripts/transpiled/` is no longer+ parsed merely because its extension is supported. Nested ignore files,+ negations, and global Git excludes follow Git's own rules; non-Git directories+ retain the existing filesystem walk and built-in skip list ([#39]).+- **`graft ask` no longer lets normalization undo test-file de-ranking.** Testfiles were penalized before lexical scores were normalized, but the strongesttest match was still normalized back to the maximum score. The test prior now@@ -86,6 +93,7 @@[#35]: https://github.com/NanoNets/Graft/issues/35[#36]: https://github.com/NanoNets/Graft/issues/36[#37]: https://github.com/NanoNets/Graft/issues/37+[#39]: https://github.com/NanoNets/Graft/issues/39## 0.8.2diff --git a/src/graph/build.ts b/src/graph/build.tsindex 79360227..336a9b95 100644--- a/src/graph/build.ts+++ b/src/graph/build.ts@@ -114,9 +114,9 @@ export interface GraphBuildResult {* directory it lives in (posix, `.` for the root). Found anywhere in the tree, so a* monorepo whose module is in a subdir (e.g. `backend/go.mod`) resolves too. Lets edge* resolution map Go import paths to in-repo files. */-function readGoModules(root: string): GoModule[] {+function readGoModules(root: string, repoFiles: string[]): GoModule[] {const mods: GoModule[] = [];- for (const f of walkDir(root)) {+ for (const f of repoFiles) {if (basename(f) !== "go.mod") continue;try {const m = readFileSync(f, "utf8").match(/^\s*module\s+(\S+)/m);@@ -136,8 +136,11 @@ export async function buildGraph(): Promise<GraphBuildResult> {const root = resolve(dir);const outDir = contextDirFor(root, opts.contextDir);- const files = listSourceStats(root, outDir);- const discoveredScopes = discoverScopes(root);+ // Enumerate once: source extraction, scope discovery, and Go module+ // resolution must agree on the same Git-ignore-aware working-tree view.+ const repoFiles = walkDir(root);+ const files = listSourceStats(root, outDir, repoFiles);+ const discoveredScopes = discoverScopes(root, repoFiles);const nodes: NodeV1[] = [];const rawEdges: RawEdge[] = [];@@ -232,7 +235,7 @@ export async function buildGraph(files: entries,});- const edges = resolveEdges(nodes, rawEdges, { goModules: readGoModules(root) });+ const edges = resolveEdges(nodes, rawEdges, { goModules: readGoModules(root, repoFiles) });// graph.json is its own Tier-2 cache: fold in the prior meaning layer so an// unchanged body is never re-summarized (and a Tier-1-only run never wipes it).diff --git a/src/graph/fingerprint.ts b/src/graph/fingerprint.tsindex e01c525c..440b0798 100644--- a/src/graph/fingerprint.ts+++ b/src/graph/fingerprint.ts@@ -7,9 +7,9 @@* with the last build's record get read and hashed, which is what keeps a `touch`* or a `git checkout` of identical bytes from triggering a pointless rebuild.*- * Note graft has no notion of git here (it never shells out to git, never reads- * `.git`): drift is measured against the bytes in the working tree, so an- * uncommitted, staged, or committed edit all look the same — which is the point.+ * Git supplies the visible file set (`tracked + untracked - ignored`) when+ * available, but drift is still measured against bytes in the working tree:+ * an uncommitted, staged, or committed edit to an indexed file looks the same.** `<outDir>/.cache/fingerprint.json` is a projection of the extraction cache* (`extract-cache.ts`) minus the parse results, written by the same build. Keepingdiff --git a/src/graph/scopes.ts b/src/graph/scopes.tsindex becc8432..68dd23d1 100644--- a/src/graph/scopes.ts+++ b/src/graph/scopes.ts@@ -21,9 +21,10 @@* 7. `label` = `prefix` (both "" for root); ordering is prefix-length desc,* then lexicographic.*/-import { existsSync, readdirSync, readFileSync, statSync, type Dirent } from "node:fs";+import { existsSync, readdirSync, readFileSync, type Dirent } from "node:fs";import { join, resolve } from "node:path";-import { SKIP_DIRS } from "../ingest/fs.js";+import { SKIP_DIRS, walkDir } from "../ingest/fs.js";+import { relPosix } from "../util/paths.js";import type { GraphV1, ScopeV1 } from "./types.js";/** Project-marker files, checked in this order (also the order `markers` is built in). */@@ -31,28 +32,17 @@ const MARKERS = ["package.json", "go.mod", "pyproject.toml", "setup.py", "Cargo.const CANONICAL_ROOT: ScopeV1[] = [{ prefix: "", label: "", markers: [] }];-/** Recursively collect every directory under `root` (posix rel path, "" = root itself),- * reusing `walkDir`'s skip rules (dot-dirs, `SKIP_DIRS`) so build-output and dependency- * trees are never scanned for markers. */-function collectDirs(root: string): string[] {- const out: string[] = [""];- const walk = (absDir: string, relDir: string): void => {- let entries: Dirent[];- try {- entries = readdirSync(absDir, { withFileTypes: true });- } catch {- return;- }- for (const entry of entries) {- if (!entry.isDirectory()) continue;- if (entry.name.startsWith(".") || SKIP_DIRS.has(entry.name)) continue;- const rel = relDir === "" ? entry.name : `${relDir}/${entry.name}`;- out.push(rel);- walk(join(absDir, entry.name), rel);- }- };- walk(root, "");- return out;+/** Collect directories represented by the canonical visible file set. Building+ * this from `walkDir` keeps scope discovery aligned with extraction: an ignored+ * generated tree cannot become a marker or workspace scope. */+function collectDirs(root: string, repoFiles: string[]): string[] {+ const out = new Set<string>([""]);+ for (const abs of repoFiles) {+ const parts = relPosix(root, abs).split("/");+ parts.pop();+ for (let i = 1; i <= parts.length; i++) out.add(parts.slice(0, i).join("/"));+ }+ return [...out];}/** Minimal `packages:` list parser for `pnpm-workspace.yaml` — no YAML dependency,@@ -99,10 +89,9 @@ function readWorkspaceGlobs(root: string): string[] | null {return null;}-/** Resolve one workspace glob to matched dir paths (posix, rel to `root`). Supports- * only `dir/*` (immediate subdirs) and `dir/**` (any nested depth) forms, implemented- * with plain `readdir` — no glob dependency, per spec. Unsupported forms resolve empty. */-function resolveGlob(root: string, pattern: string): string[] {+/** Resolve one workspace glob against visible directories only. Supports+ * `dir/*` (immediate subdirs), `dir/**` (any nested depth), and literal dirs. */+function resolveGlob(dirs: string[], pattern: string): string[] {const norm = pattern.replace(/\\/g, "/").replace(/\/$/, "");let base: string;let recursive: boolean;@@ -119,50 +108,18 @@ function resolveGlob(root: string, pattern: string): string[] {base = norm.slice(0, -2);recursive = false;} else if (!norm.includes("*")) {- // Literal entry (no glob chars), e.g. `packages: ['apps/*', 'docs']` — a plain- // workspace-list entry that names a real directory counts as a workspace match- // in its own right, same as a resolved glob would. Without this, a literal- // entry silently resolves to nothing while Rule 2 still strips its- // `package.json` marker (it's a JS-family dir that never matched the glob- // set), de-scoping a real workspace package.- try {- return existsSync(join(root, norm)) && statSync(join(root, norm)).isDirectory()- ? [norm]- : [];- } catch {- return [];- }+ return dirs.includes(norm) ? [norm] : [];} else {return []; // unsupported glob form}- const results: string[] = [];- const listDirs = (absDir: string, relDir: string): string[] => {- let entries: Dirent[];- try {- entries = readdirSync(absDir, { withFileTypes: true });- } catch {- return [];- }- return entries- .filter((e) => e.isDirectory() && !e.name.startsWith(".") && !SKIP_DIRS.has(e.name))- .map((e) => (relDir === "" ? e.name : `${relDir}/${e.name}`));- };-- const baseAbs = base === "" ? root : join(root, base);- if (!recursive) {- results.push(...listDirs(baseAbs, base));- } else {- const stack = [base];- while (stack.length) {- const relDir = stack.pop()!;- const absDir = relDir === "" ? root : join(root, relDir);- const children = listDirs(absDir, relDir);- results.push(...children);- stack.push(...children);- }- }- return results;+ return dirs.filter((dir) => {+ if (dir === "") return false;+ if (recursive) return base === "" || dir.startsWith(`${base}/`);+ const slash = dir.lastIndexOf("/");+ const parent = slash === -1 ? "" : dir.slice(0, slash);+ return parent === base;+ });}interface Candidate {@@ -171,9 +128,9 @@ interface Candidate {}/** Walk the tree (reusing `walkDir`'s skip rules) and find project-marker dirs. */-export function discoverScopes(root: string): ScopeV1[] {+export function discoverScopes(root: string, repoFiles: string[] = walkDir(root)): ScopeV1[] {const absRoot = resolve(root);- const dirs = collectDirs(absRoot);+ const dirs = collectDirs(absRoot, repoFiles);const markerMap = new Map<string, string[]>();for (const dir of dirs) {@@ -185,7 +142,7 @@ export function discoverScopes(root: string): ScopeV1[] {const workspaceMatches = new Set<string>();if (workspaceGlobs) {for (const glob of workspaceGlobs) {- for (const dir of resolveGlob(absRoot, glob)) workspaceMatches.add(dir);+ for (const dir of resolveGlob(dirs, glob)) workspaceMatches.add(dir);}}diff --git a/src/graph/source-files.ts b/src/graph/source-files.tsindex 93847e1f..7fd87200 100644--- a/src/graph/source-files.ts+++ b/src/graph/source-files.ts@@ -12,8 +12,8 @@ import { relPosix } from "../util/paths.js";import { languageOf } from "./extract.js";/** The source files a graph build parses: supported languages, minus the output dir. */-export function listSourceFiles(root: string, outDir: string): string[] {- return walkDir(root).filter((f) => !f.startsWith(outDir) && languageOf(f) !== null);+export function listSourceFiles(root: string, outDir: string, repoFiles: string[] = walkDir(root)): string[] {+ return repoFiles.filter((f) => !f.startsWith(outDir) && languageOf(f) !== null);}export interface SourceStat {@@ -32,9 +32,9 @@ export interface SourceStat {* both the freshness probe and the extraction cache. Files that vanish between* the walk and the stat are dropped (same fail-soft posture as `walkDir`).*/-export function listSourceStats(root: string, outDir: string): SourceStat[] {+export function listSourceStats(root: string, outDir: string, repoFiles?: string[]): SourceStat[] {const out: SourceStat[] = [];- for (const abs of listSourceFiles(root, outDir)) {+ for (const abs of listSourceFiles(root, outDir, repoFiles)) {let s: { size: number; mtimeMs: number };try {s = statSync(abs);diff --git a/src/ingest/fs.ts b/src/ingest/fs.tsindex f39682db..466e90fa 100644--- a/src/ingest/fs.ts+++ b/src/ingest/fs.ts@@ -1,8 +1,9 @@/*** Filesystem walking used by `init`/`check` to enumerate a repo's source files.*/-import { readdirSync, statSync } from "node:fs";-import { join } from "node:path";+import { spawnSync } from "node:child_process";+import { lstatSync, readdirSync, statSync } from "node:fs";+import { join, resolve } from "node:path";/** Directories that are dependency/build output, never source. */export const SKIP_DIRS = new Set([@@ -23,15 +24,62 @@ export const MAX_FILE_BYTES = 1_000_000;/*** Recursively list all files under a directory. Skips dot-directories,* dependency/build directories (node_modules, dist, …), and files over 1 MB.+ * In a Git worktree, tracked files plus untracked, non-ignored files come from+ * `git ls-files`; this gives indexing exactly Git's nested `.gitignore`,+ * negation, and global-exclude semantics. Non-Git directories retain the plain+ * filesystem walk.*/export function walkDir(dir: string): string[] {+ return gitVisibleFiles(dir) ?? walkFilesystem(dir);+}++/** Git's canonical working-tree file set, relative to `dir`. Tracked files are+ * deliberately included even when a later ignore rule matches them; `.gitignore`+ * only controls untracked files in Git, and graft follows the same contract. */+function gitVisibleFiles(dir: string): string[] | null {+ const root = resolve(dir);+ const result = spawnSync(+ "git",+ ["ls-files", "--cached", "--others", "--exclude-standard", "-z", "--"],+ {+ cwd: root,+ encoding: "utf8",+ stdio: ["ignore", "pipe", "ignore"],+ maxBuffer: 64 * 1024 * 1024,+ },+ );+ if (result.status !== 0 || result.error || typeof result.stdout !== "string") return null;++ const out: string[] = [];+ for (const rel of result.stdout.split("\0")) {+ if (!rel || skippedPath(rel)) continue;+ const abs = resolve(root, rel);+ try {+ const stat = lstatSync(abs);+ if (!stat.isFile() || stat.size > MAX_FILE_BYTES) continue;+ } catch {+ // A tracked file deleted from the working tree is still printed by+ // `--cached`; absence means it is not part of the current source set.+ continue;+ }+ out.push(abs);+ }+ return out;+}++function skippedPath(path: string): boolean {+ const segments = path.replace(/\\/g, "/").split("/");+ return segments.some((segment) => segment.startsWith(".") || SKIP_DIRS.has(segment));+}++function walkFilesystem(dir: string): string[] {const out: string[] = [];for (const entry of readdirSync(dir, { withFileTypes: true })) {if (entry.name.startsWith(".")) continue;const full = join(dir, entry.name);if (entry.isDirectory()) {if (SKIP_DIRS.has(entry.name)) continue;- out.push(...walkDir(full));+ out.push(...walkFilesystem(full));} else if (entry.isFile()) {try {if (statSync(full).size > MAX_FILE_BYTES) continue;diff --git a/test/graph-incremental.test.ts b/test/graph-incremental.test.tsindex 46dfabff..bb38347f 100644--- a/test/graph-incremental.test.ts+++ b/test/graph-incremental.test.ts@@ -6,6 +6,7 @@*/import { test } from "node:test";import assert from "node:assert/strict";+import { execFileSync } from "node:child_process";import { chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { basename, join } from "node:path";@@ -159,6 +160,27 @@ test("every file on disk lands in the fingerprint", async () => {assert.deepEqual(Object.keys(fp!.files).sort(), ["src/app.ts", "src/math.ts"]);});+test("gitignored generated files stay out of the graph, fingerprint, and drift probe (#39)", async () => {+ const d = repo();+ try {+ execFileSync("git", ["init", "-q"], { cwd: d });+ writeFileSync(join(d, ".gitignore"), "Scripts/bundles/\n");+ mkdirSync(join(d, "Scripts", "bundles"), { recursive: true });+ const bundle = join(d, "Scripts", "bundles", "app.js");+ writeFileSync(bundle, "export function generatedBundle(): number { return 1; }\n");++ await buildGraph(d);+ const graph = readGraph(wiringPath(outOf(d))) as GraphV1;+ assert.ok(!graph.nodes.some((n) => n.path === "Scripts/bundles/app.js"));+ assert.deepEqual(Object.keys(readFingerprint(outOf(d))!.files).sort(), ["src/app.ts", "src/math.ts"]);++ writeFileSync(bundle, "export function generatedBundle(): number { return 2; }\n");+ assert.ok(isClean(probeDrift(d, outOf(d))!), "ignored output changes must not trigger a refresh");+ } finally {+ rmSync(d, { recursive: true, force: true });+ }+});+test("an unreadable file is still recorded, so it can't look new on every probe", async (t) => {const why = chmodDenialUnavailable();if (why) return t.skip(why);diff --git a/test/graph-scopes.test.ts b/test/graph-scopes.test.tsindex 09c725bd..b6b1fdca 100644--- a/test/graph-scopes.test.ts+++ b/test/graph-scopes.test.ts@@ -1,5 +1,6 @@import { test } from "node:test";import assert from "node:assert/strict";+import { execFileSync } from "node:child_process";import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";@@ -47,6 +48,23 @@ test("workspace globs are intent: packages/* honored, deeper ignored", () => {rmSync(d, { recursive: true, force: true });});+test("gitignored generated workspace directories cannot become scopes (#39)", () => {+ const d = fx({+ ".gitignore": "generated/\n",+ "package.json": JSON.stringify({ workspaces: ["packages/*", "generated/*"] }),+ "packages/app/package.json": "{}",+ "packages/app/src/index.ts": "export const app = 1;",+ "generated/bundle/package.json": "{}",+ "generated/bundle/index.ts": "export const generated = 1;",+ });+ try {+ execFileSync("git", ["init", "-q"], { cwd: d });+ assert.deepEqual(discoverScopes(d).map((s) => s.prefix), ["packages/app"]);+ } finally {+ rmSync(d, { recursive: true, force: true });+ }+});+test("root-only marker -> canonical single scope", () => {const d = fx({ "package.json": "{}", "src/a.ts": "1" });const scopes = discoverScopes(d);diff --git a/test/ingest-fs.test.ts b/test/ingest-fs.test.tsnew file mode 100644index 00000000..5ae70c5c--- /dev/null+++ b/test/ingest-fs.test.ts@@ -0,0 +1,74 @@+import { execFileSync } from "node:child_process";+import { test } from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join, relative } from "node:path";+import { walkDir } from "../src/ingest/fs.js";++function fixture(tag: string): string {+ const dir = mkdtempSync(join(tmpdir(), `graft-walk-${tag}-`));+ execFileSync("git", ["init", "-q"], { cwd: dir });+ return dir;+}++function write(root: string, path: string, content = "export const value = 1;\n"): void {+ mkdirSync(join(root, path, ".."), { recursive: true });+ writeFileSync(join(root, path), content);+}++function walked(root: string): string[] {+ return walkDir(root)+ .map((path) => relative(root, path).replace(/\\/g, "/"))+ .sort();+}++test("walkDir respects root and nested .gitignore rules, including negation", () => {+ const dir = fixture("ignore");+ try {+ write(dir, ".gitignore", "Scripts/bundles/\ngenerated/*\n!generated/keep.ts\n");+ write(dir, "src/app.ts");+ write(dir, "Scripts/bundles/app.js");+ write(dir, "generated/drop.ts");+ write(dir, "generated/keep.ts");+ write(dir, "packages/tool/.gitignore", "output/\n");+ write(dir, "packages/tool/index.ts");+ write(dir, "packages/tool/output/bundle.js");++ assert.deepEqual(walked(dir), [+ "generated/keep.ts",+ "packages/tool/index.ts",+ "src/app.ts",+ ]);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("walkDir keeps tracked files that match an ignore rule and untracked visible files", () => {+ const dir = fixture("tracked");+ try {+ write(dir, ".gitignore", "*.generated.ts\n");+ write(dir, "tracked.generated.ts");+ write(dir, "ignored.generated.ts");+ write(dir, "visible.ts");+ execFileSync("git", ["add", "-f", "tracked.generated.ts"], { cwd: dir });++ assert.deepEqual(walked(dir), ["tracked.generated.ts", "visible.ts"]);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("walkDir retains fixed skips and filesystem fallback outside Git", () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-walk-nongit-"));+ try {+ write(dir, "src/app.ts");+ write(dir, "node_modules/pkg/index.ts");+ write(dir, ".hidden/secret.ts");++ assert.deepEqual(walked(dir), ["src/app.ts"]);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});