Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
// Copyright 2026 Ix Infrastructure Inc.
import * as fs from "node:fs";
import * as nodePath from "node:path";
import { spawnSync } from "node:child_process";
import { createIgnoreMatcher, parseIgnoreFile, type IgnoreMatcher } from "./ignore-globs.js";
import { SUPPORTED_EXTENSIONS } from "./supported-extensions.js";
/**
* Which files are the repository: one implementation for `ix ingest`/`ix map`,
* `ix watch` and `ix status`.
*
* There were three, and they disagreed. The filesystem walk skipped `tests/`,
* `examples/`, `fixtures/`, Go's `pkg/` and every dot-directory, while
* `git ls-files` -- the path in any git repository -- skipped none of them, so
* the same tree gave a different graph depending on whether it was a git
* checkout. `watch` and `status` had their own lists again. And `git ls-files`
* quoted any non-ASCII name (`"caf\303\251.ts"`), the stat on that string
* failed, and the file was silently dropped.
*
* Now: git lists the files when it can (NUL-separated, unquoted); otherwise the
* walk honours each directory's `.gitignore`, so the two agree on an ordinary
* tree. The walk also skips a short list of directories that are never source
* (build output, dependencies, VCS metadata, caches) by name. In a work tree
* git's own ignore rules decide instead: a tracked file is kept whatever its
* directory is called -- a committed `build/` or `out/` can be source -- and
* only untracked files under those names are skipped. Both count what they
* skipped. Tests and fixtures are in the set; the role classifier is what
* tells them apart.
*
* The walk's `.gitignore` reading is the `.ixignore` subset (see
* `ignore-globs.ts`): no `!` negation, so it can exclude a little more than
* git does.
*/
/**
* Never source: build output, dependencies, VCS metadata, tool caches. Skipped
* by name in a walk, and for untracked files in a work tree; never for a file
* git tracks.
*/
export const SKIPPED_DIRS: ReadonlySet<string> = new Set([
"node_modules", ".git", ".hg", ".svn", "dist", "build", "target", "out", ".next",
"__pycache__", ".tox", ".venv", "venv", ".mypy_cache", ".pytest_cache",
".gradle", ".idea", ".vscode", ".settings", ".vs",
".cache", ".parcel-cache", "coverage", ".nyc_output",
"Pods", ".dart_tool", ".pub-cache", "obj",
".terraform", ".serverless", ".turbo", ".svelte-kit", ".nuxt", ".angular", ".expo",
".ix", ".claude", ".gitnexus",
]);
// Generated file patterns — skipped by filename to avoid ingesting low-value autogenerated code.
// These files inflate parse time and graph size without contributing to architectural signal.
const GENERATED_FILE_SUFFIXES = [
".pb.go", // protobuf generated Go
"_deepcopy.go", // controller-gen deepcopy (Kubernetes ecosystem)
"_mock.go", // gomock generated mocks
".pb.ts", // protobuf generated TypeScript
".pb.js", // protobuf generated JavaScript
"_pb.ts", // alternative protobuf TypeScript naming
"_pb.js", // alternative protobuf JavaScript naming
];
const GENERATED_FILE_PREFIXES = [
"zz_generated", // controller-gen convention (Kubernetes ecosystem)
"mock_", // gomock convention
];
// Generated dependency lockfiles, matched whole. Only the ones whose extension
// SUPPORTED_EXTENSIONS actually discovers are listed: `yarn.lock`,
// `composer.lock`, `Cargo.lock`, `Gemfile.lock`, `poetry.lock` and friends end
// in `.lock`, which is not a discovered extension, so they never reach here.
// A lockfile well under MAX_FILE_BYTES still expands into a graph patch larger
// than a proxy will accept — 646 KB was enough for an HTTP 413 (Ix#523) — and
// it describes a resolved dependency snapshot, not architecture.
const GENERATED_FILE_NAMES = new Set([
"package-lock.json", // npm
"npm-shrinkwrap.json", // npm, published form
"pnpm-lock.yaml", // pnpm
"packages.lock.json", // NuGet
]);
export function isGeneratedFile(basename: string): boolean {
// Lowercased for the name match alone: `isSupportedSourceFile` lowercases
// too, so on a case-insensitive filesystem a `Package-lock.json` is
// discovered and would otherwise slip past a case-sensitive comparison.
if (GENERATED_FILE_NAMES.has(basename.toLowerCase())) return true;
for (const suffix of GENERATED_FILE_SUFFIXES) {
if (basename.endsWith(suffix)) return true;
}
for (const prefix of GENERATED_FILE_PREFIXES) {
if (basename.startsWith(prefix)) return true;
}
return false;
}
export function isSupportedSourceFile(filePath: string): boolean {
const fileName = nodePath.basename(filePath).toLowerCase();
return fileName === "dockerfile"
|| fileName.endsWith(".dockerfile")
|| fileName === "makefile"
|| fileName === "gnumakefile"
|| SUPPORTED_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());
}
/** A file discovery would hand to the parser. */
function isCandidate(filePath: string): boolean {
return isSupportedSourceFile(filePath) && !isGeneratedFile(nodePath.basename(filePath));
}
export type DiscoveryExclude = { matcher: IgnoreMatcher; root: string; onSkip?: () => void };
/** What discovery left out, so it can be reported rather than guessed at. */
export interface DiscoveryCounts {
/** Untracked files under a `SKIPPED_DIRS` directory (git mode) or such directories (walk). */
skippedDirs: number;
/** Listed files that could not be stat'd: permissions, a race with a delete. */
unreadable: number;
}
export function emptyDiscoveryCounts(): DiscoveryCounts {
return { skippedDirs: 0, unreadable: 0 };
}
export function canonicalizeDiscoveredFilePath(filePath: string): string {
try {
return fs.realpathSync.native(filePath);
} catch {
return nodePath.resolve(filePath);
}
}
function inSkippedDir(relativePath: string): boolean {
const parts = relativePath.split(/[/\\]/);
parts.pop();
return parts.some(p => SKIPPED_DIRS.has(p));
}
const GIT_SAFE_CONFIG = [
"-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", "-c", "core.quotePath=false",
];
/**
* The files `git ls-files` reports under `dir`, canonical and de-duplicated, or
* null when `dir` is not in a git work tree.
*
* `-z` and `core.quotePath=false`: without them git quotes a non-ASCII name
* (`"caf\303\251.ts"`), the stat on that string fails, and the file vanished.
*/
export function tryGitLsFiles(
dir: string,
recursive: boolean,
exclude?: DiscoveryExclude,
counts: DiscoveryCounts = emptyDiscoveryCounts(),
): string[] | null {
try {
const result = spawnSync(
"git",
// core.fsmonitor / core.hooksPath: a cloned repository's own .git/config
// can name a command ls-files would run. Never let it.
[...GIT_SAFE_CONFIG, "ls-files", "-z", "--cached", "--others", "--exclude-standard"],
{
cwd: dir,
encoding: "utf-8",
maxBuffer: 64 * 1024 * 1024,
},
);
if (result.status !== 0) return null;
// Asked only when a listed file sits under a skipped name.
let untracked: Set<string> | null = null;
const isUntracked = (entry: string): boolean => {
untracked ??= gitUntracked(dir);
return untracked.has(entry);
};
const files: string[] = [];
const seenCanonical = new Set<string>();
for (const entry of result.stdout.split("\0")) {
if (!entry) continue;
const fullPath = nodePath.resolve(dir, entry);
if (!recursive && nodePath.dirname(fullPath) !== dir) continue;
// Before the extension check, so the count reports what the caller
// excluded rather than what they excluded and Ix would have skipped
// anyway.
if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, fullPath))) {
exclude.onSkip?.();
continue;
}
if (!isCandidate(fullPath)) continue;
// A tracked file stays: git is told what is generated through
// .gitignore, and a repository that commits a `build/` or `out/` tree
// is saying it is source. An untracked one under such a name is what a
// missing .gitignore line would have caught -- a fresh `node_modules`.
if (inSkippedDir(entry) && isUntracked(entry)) { counts.skippedDirs++; continue; }
try {
if (!fs.statSync(fullPath).isFile()) continue;
} catch {
counts.unreadable++;
continue;
}
const canonicalPath = canonicalizeDiscoveredFilePath(fullPath);
if (seenCanonical.has(canonicalPath)) continue;
seenCanonical.add(canonicalPath);
files.push(canonicalPath);
}
return files;
} catch {
return null;
}
}
/** Untracked, not-ignored entries under `dir`, as `git ls-files` spells them. */
function gitUntracked(dir: string): Set<string> {
const result = spawnSync("git", [...GIT_SAFE_CONFIG, "ls-files", "-z", "--others", "--exclude-standard"], {
cwd: dir,
encoding: "utf-8",
maxBuffer: 64 * 1024 * 1024,
});
// Unknown: treat every listed file as tracked, which keeps it.
if (result.status !== 0) return new Set();
return new Set(result.stdout.split("\0").filter(Boolean));
}
/** `.gitignore` matchers in force for a directory: each one's own, with the directory it is relative to. */
type GitignoreStack = Array<{ dir: string; matcher: IgnoreMatcher }>;
function withGitignore(dir: string, stack: GitignoreStack): GitignoreStack {
try {
const patterns = parseIgnoreFile(fs.readFileSync(nodePath.join(dir, ".gitignore"), "utf-8"));
if (patterns.length > 0) return [...stack, { dir, matcher: createIgnoreMatcher(patterns) }];
} catch {
// No .gitignore here, the ordinary case.
}
return stack;
}
function gitignored(stack: GitignoreStack, fullPath: string, isDirectory: boolean): boolean {
return stack.some(({ dir, matcher }) => matcher.matches(nodePath.relative(dir, fullPath), isDirectory));
}
/**
* Walk `dir` for candidate files, honouring every `.gitignore` on the way down
* and skipping `SKIPPED_DIRS`. Yields paths as found; `discoverSourceFiles`
* canonicalizes them.
*/
export function* walkFiles(
dir: string,
recursive: boolean,
exclude?: DiscoveryExclude,
counts: DiscoveryCounts = emptyDiscoveryCounts(),
gitignores: GitignoreStack = [],
): Generator<string> {
let entries: fs.Dirent[];
try { entries = fs.readdirSync(dir, { withFileTypes: true }); }
catch { return; }
const stack = withGitignore(dir, gitignores);
for (const entry of entries) {
const isDirectory = entry.isDirectory();
if (isDirectory && SKIPPED_DIRS.has(entry.name)) { counts.skippedDirs++; continue; }
const full = nodePath.join(dir, entry.name);
// Directories are tested before descending, so an excluded tree costs one
// check rather than one per file inside it.
if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, full), isDirectory)) {
exclude.onSkip?.();
continue;
}
if (gitignored(stack, full, isDirectory)) continue;
if (isDirectory) {
if (recursive) yield* walkFiles(full, true, exclude, counts, stack);
} else if (entry.isFile()) {
if (isCandidate(entry.name)) yield full;
}
}
}
/**
* The source files under `dir`, canonical and de-duplicated: git's list when
* `dir` is in a work tree, else the walk.
*
* `walkLimit` stops the walk after that many files. A git listing is bounded
* by the repository, but a walk is bounded by nothing: a command run from a
* home directory would read the whole disk.
*/
export function discoverSourceFiles(
dir: string,
opts: { recursive?: boolean; exclude?: DiscoveryExclude; counts?: DiscoveryCounts; walkLimit?: number } = {},
): string[] {
const recursive = opts.recursive ?? true;
const counts = opts.counts ?? emptyDiscoveryCounts();
const listed = tryGitLsFiles(dir, recursive, opts.exclude, counts);
if (listed) return listed;
const out: string[] = [];
const seen = new Set<string>();
for (const filePath of walkFiles(dir, recursive, opts.exclude, counts)) {
if (opts.walkLimit !== undefined && out.length >= opts.walkLimit) break;
const canonical = canonicalizeDiscoveredFilePath(filePath);
if (seen.has(canonical)) continue;
seen.add(canonical);
out.push(canonical);
}
return out;
}