Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

12.0 KB
Raw Blame
// Copyright 2026 Ix Infrastructure Inc.

import * as fs from "node:fs";
import * as nodePath from "node:path";
import { spawnSync } from "node:child_process";

import { createIgnoreMatcher, parseIgnoreFile, type IgnoreMatcher } from "./ignore-globs.js";
import { SUPPORTED_EXTENSIONS } from "./supported-extensions.js";

/**
 * Which files are the repository: one implementation for `ix ingest`/`ix map`,
 * `ix watch` and `ix status`.
 *
 * There were three, and they disagreed. The filesystem walk skipped `tests/`,
 * `examples/`, `fixtures/`, Go's `pkg/` and every dot-directory, while
 * `git ls-files` -- the path in any git repository -- skipped none of them, so
 * the same tree gave a different graph depending on whether it was a git
 * checkout. `watch` and `status` had their own lists again. And `git ls-files`
 * quoted any non-ASCII name (`"caf\303\251.ts"`), the stat on that string
 * failed, and the file was silently dropped.
 *
 * Now: git lists the files when it can (NUL-separated, unquoted); otherwise the
 * walk honours each directory's `.gitignore`, so the two agree on an ordinary
 * tree. The walk also skips a short list of directories that are never source
 * (build output, dependencies, VCS metadata, caches) by name. In a work tree
 * git's own ignore rules decide instead: a tracked file is kept whatever its
 * directory is called -- a committed `build/` or `out/` can be source -- and
 * only untracked files under those names are skipped. Both count what they
 * skipped. Tests and fixtures are in the set; the role classifier is what
 * tells them apart.
 *
 * The walk's `.gitignore` reading is the `.ixignore` subset (see
 * `ignore-globs.ts`): no `!` negation, so it can exclude a little more than
 * git does.
 */

/**
 * Never source: build output, dependencies, VCS metadata, tool caches. Skipped
 * by name in a walk, and for untracked files in a work tree; never for a file
 * git tracks.
 */
export const SKIPPED_DIRS: ReadonlySet<string> = new Set([
  "node_modules", ".git", ".hg", ".svn", "dist", "build", "target", "out", ".next",
  "__pycache__", ".tox", ".venv", "venv", ".mypy_cache", ".pytest_cache",
  ".gradle", ".idea", ".vscode", ".settings", ".vs",
  ".cache", ".parcel-cache", "coverage", ".nyc_output",
  "Pods", ".dart_tool", ".pub-cache", "obj",
  ".terraform", ".serverless", ".turbo", ".svelte-kit", ".nuxt", ".angular", ".expo",
  ".ix", ".claude", ".gitnexus",
]);

// Generated file patterns — skipped by filename to avoid ingesting low-value autogenerated code.
// These files inflate parse time and graph size without contributing to architectural signal.
const GENERATED_FILE_SUFFIXES = [
  ".pb.go",        // protobuf generated Go
  "_deepcopy.go",  // controller-gen deepcopy (Kubernetes ecosystem)
  "_mock.go",      // gomock generated mocks
  ".pb.ts",        // protobuf generated TypeScript
  ".pb.js",        // protobuf generated JavaScript
  "_pb.ts",        // alternative protobuf TypeScript naming
  "_pb.js",        // alternative protobuf JavaScript naming
];

const GENERATED_FILE_PREFIXES = [
  "zz_generated",  // controller-gen convention (Kubernetes ecosystem)
  "mock_",         // gomock convention
];

// Generated dependency lockfiles, matched whole. Only the ones whose extension
// SUPPORTED_EXTENSIONS actually discovers are listed: `yarn.lock`,
// `composer.lock`, `Cargo.lock`, `Gemfile.lock`, `poetry.lock` and friends end
// in `.lock`, which is not a discovered extension, so they never reach here.
// A lockfile well under MAX_FILE_BYTES still expands into a graph patch larger
// than a proxy will accept — 646 KB was enough for an HTTP 413 (Ix#523) — and
// it describes a resolved dependency snapshot, not architecture.
const GENERATED_FILE_NAMES = new Set([
  "package-lock.json",    // npm
  "npm-shrinkwrap.json",  // npm, published form
  "pnpm-lock.yaml",       // pnpm
  "packages.lock.json",   // NuGet
]);

export function isGeneratedFile(basename: string): boolean {
  // Lowercased for the name match alone: `isSupportedSourceFile` lowercases
  // too, so on a case-insensitive filesystem a `Package-lock.json` is
  // discovered and would otherwise slip past a case-sensitive comparison.
  if (GENERATED_FILE_NAMES.has(basename.toLowerCase())) return true;
  for (const suffix of GENERATED_FILE_SUFFIXES) {
    if (basename.endsWith(suffix)) return true;
  }
  for (const prefix of GENERATED_FILE_PREFIXES) {
    if (basename.startsWith(prefix)) return true;
  }
  return false;
}

export function isSupportedSourceFile(filePath: string): boolean {
  const fileName = nodePath.basename(filePath).toLowerCase();
  return fileName === "dockerfile"
    || fileName.endsWith(".dockerfile")
    || fileName === "makefile"
    || fileName === "gnumakefile"
    || SUPPORTED_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());
}

/** A file discovery would hand to the parser. */
function isCandidate(filePath: string): boolean {
  return isSupportedSourceFile(filePath) && !isGeneratedFile(nodePath.basename(filePath));
}

export type DiscoveryExclude = { matcher: IgnoreMatcher; root: string; onSkip?: () => void };

/** What discovery left out, so it can be reported rather than guessed at. */
export interface DiscoveryCounts {
  /** Untracked files under a `SKIPPED_DIRS` directory (git mode) or such directories (walk). */
  skippedDirs: number;
  /** Listed files that could not be stat'd: permissions, a race with a delete. */
  unreadable: number;
}

export function emptyDiscoveryCounts(): DiscoveryCounts {
  return { skippedDirs: 0, unreadable: 0 };
}

export function canonicalizeDiscoveredFilePath(filePath: string): string {
  try {
    return fs.realpathSync.native(filePath);
  } catch {
    return nodePath.resolve(filePath);
  }
}

function inSkippedDir(relativePath: string): boolean {
  const parts = relativePath.split(/[/\\]/);
  parts.pop();
  return parts.some(p => SKIPPED_DIRS.has(p));
}

const GIT_SAFE_CONFIG = [
  "-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", "-c", "core.quotePath=false",
];

/**
 * The files `git ls-files` reports under `dir`, canonical and de-duplicated, or
 * null when `dir` is not in a git work tree.
 *
 * `-z` and `core.quotePath=false`: without them git quotes a non-ASCII name
 * (`"caf\303\251.ts"`), the stat on that string fails, and the file vanished.
 */
export function tryGitLsFiles(
  dir: string,
  recursive: boolean,
  exclude?: DiscoveryExclude,
  counts: DiscoveryCounts = emptyDiscoveryCounts(),
): string[] | null {
  try {
    const result = spawnSync(
      "git",
      // core.fsmonitor / core.hooksPath: a cloned repository's own .git/config
      // can name a command ls-files would run. Never let it.
      [...GIT_SAFE_CONFIG, "ls-files", "-z", "--cached", "--others", "--exclude-standard"],
      {
        cwd: dir,
        encoding: "utf-8",
        maxBuffer: 64 * 1024 * 1024,
      },
    );
    if (result.status !== 0) return null;

    // Asked only when a listed file sits under a skipped name.
    let untracked: Set<string> | null = null;
    const isUntracked = (entry: string): boolean => {
      untracked ??= gitUntracked(dir);
      return untracked.has(entry);
    };
    const files: string[] = [];
    const seenCanonical = new Set<string>();
    for (const entry of result.stdout.split("\0")) {
      if (!entry) continue;
      const fullPath = nodePath.resolve(dir, entry);
      if (!recursive && nodePath.dirname(fullPath) !== dir) continue;
      // Before the extension check, so the count reports what the caller
      // excluded rather than what they excluded and Ix would have skipped
      // anyway.
      if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, fullPath))) {
        exclude.onSkip?.();
        continue;
      }
      if (!isCandidate(fullPath)) continue;
      // A tracked file stays: git is told what is generated through
      // .gitignore, and a repository that commits a `build/` or `out/` tree
      // is saying it is source. An untracked one under such a name is what a
      // missing .gitignore line would have caught -- a fresh `node_modules`.
      if (inSkippedDir(entry) && isUntracked(entry)) { counts.skippedDirs++; continue; }
      try {
        if (!fs.statSync(fullPath).isFile()) continue;
      } catch {
        counts.unreadable++;
        continue;
      }
      const canonicalPath = canonicalizeDiscoveredFilePath(fullPath);
      if (seenCanonical.has(canonicalPath)) continue;
      seenCanonical.add(canonicalPath);
      files.push(canonicalPath);
    }
    return files;
  } catch {
    return null;
  }
}

/** Untracked, not-ignored entries under `dir`, as `git ls-files` spells them. */
function gitUntracked(dir: string): Set<string> {
  const result = spawnSync("git", [...GIT_SAFE_CONFIG, "ls-files", "-z", "--others", "--exclude-standard"], {
    cwd: dir,
    encoding: "utf-8",
    maxBuffer: 64 * 1024 * 1024,
  });
  // Unknown: treat every listed file as tracked, which keeps it.
  if (result.status !== 0) return new Set();
  return new Set(result.stdout.split("\0").filter(Boolean));
}

/** `.gitignore` matchers in force for a directory: each one's own, with the directory it is relative to. */
type GitignoreStack = Array<{ dir: string; matcher: IgnoreMatcher }>;

function withGitignore(dir: string, stack: GitignoreStack): GitignoreStack {
  try {
    const patterns = parseIgnoreFile(fs.readFileSync(nodePath.join(dir, ".gitignore"), "utf-8"));
    if (patterns.length > 0) return [...stack, { dir, matcher: createIgnoreMatcher(patterns) }];
  } catch {
    // No .gitignore here, the ordinary case.
  }
  return stack;
}

function gitignored(stack: GitignoreStack, fullPath: string, isDirectory: boolean): boolean {
  return stack.some(({ dir, matcher }) => matcher.matches(nodePath.relative(dir, fullPath), isDirectory));
}

/**
 * Walk `dir` for candidate files, honouring every `.gitignore` on the way down
 * and skipping `SKIPPED_DIRS`. Yields paths as found; `discoverSourceFiles`
 * canonicalizes them.
 */
export function* walkFiles(
  dir: string,
  recursive: boolean,
  exclude?: DiscoveryExclude,
  counts: DiscoveryCounts = emptyDiscoveryCounts(),
  gitignores: GitignoreStack = [],
): Generator<string> {
  let entries: fs.Dirent[];
  try { entries = fs.readdirSync(dir, { withFileTypes: true }); }
  catch { return; }
  const stack = withGitignore(dir, gitignores);

  for (const entry of entries) {
    const isDirectory = entry.isDirectory();
    if (isDirectory && SKIPPED_DIRS.has(entry.name)) { counts.skippedDirs++; continue; }
    const full = nodePath.join(dir, entry.name);
    // Directories are tested before descending, so an excluded tree costs one
    // check rather than one per file inside it.
    if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, full), isDirectory)) {
      exclude.onSkip?.();
      continue;
    }
    if (gitignored(stack, full, isDirectory)) continue;
    if (isDirectory) {
      if (recursive) yield* walkFiles(full, true, exclude, counts, stack);
    } else if (entry.isFile()) {
      if (isCandidate(entry.name)) yield full;
    }
  }
}

/**
 * The source files under `dir`, canonical and de-duplicated: git's list when
 * `dir` is in a work tree, else the walk.
 *
 * `walkLimit` stops the walk after that many files. A git listing is bounded
 * by the repository, but a walk is bounded by nothing: a command run from a
 * home directory would read the whole disk.
 */
export function discoverSourceFiles(
  dir: string,
  opts: { recursive?: boolean; exclude?: DiscoveryExclude; counts?: DiscoveryCounts; walkLimit?: number } = {},
): string[] {
  const recursive = opts.recursive ?? true;
  const counts = opts.counts ?? emptyDiscoveryCounts();
  const listed = tryGitLsFiles(dir, recursive, opts.exclude, counts);
  if (listed) return listed;
  const out: string[] = [];
  const seen = new Set<string>();
  for (const filePath of walkFiles(dir, recursive, opts.exclude, counts)) {
    if (opts.walkLimit !== undefined && out.length >= opts.walkLimit) break;
    const canonical = canonicalizeDiscoveredFilePath(filePath);
    if (seen.has(canonical)) continue;
    seen.add(canonical);
    out.push(canonical);
  }
  return out;
}