Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(ingest): confine discovered files to the directory being ingested
KageBinary6233b498b81c
diff --git a/ix-cli/src/cli/__tests__/ingest-discovery.test.ts b/ix-cli/src/cli/__tests__/ingest-discovery.test.tsindex e6b85b0e..6fb40e26 100644--- a/ix-cli/src/cli/__tests__/ingest-discovery.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-discovery.test.ts@@ -1,6 +1,18 @@-import { describe, expect, it } from 'vitest';+import { execFileSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';-import { dedupeDiscoveredFilePaths, isSupportedSourceFile, needsIndexPrescan } from '../commands/ingest.js';+import { afterEach, describe, expect, it } from 'vitest';++import {+ dedupeDiscoveredFilePaths,+ isSupportedSourceFile,+ discoverIngestFilePaths,+ isWithinDiscoveryRoot,+ needsIndexPrescan,+ tryGitLsFiles,+} from '../commands/ingest.js';describe('dedupeDiscoveredFilePaths', () => {it('collapses alternate discovered paths that point at the same canonical file', () => {@@ -55,6 +67,19 @@ describe('dedupeDiscoveredFilePaths', () => {}});+ it('confines a canonical path to the discovery root', () => {+ const root = join('/repo', 'project');+ expect(isWithinDiscoveryRoot(root, join(root, 'src', 'app.ts'))).toBe(true);+ expect(isWithinDiscoveryRoot(root, join('/repo', 'other', 'app.ts'))).toBe(false);+ expect(isWithinDiscoveryRoot(root, join('/repo', 'app.ts'))).toBe(false);+ // The root itself is not a file inside the root.+ expect(isWithinDiscoveryRoot(root, root)).toBe(false);+ // A sibling whose name merely starts with dots is outside, but a *child*+ // whose name does must stay in — the reason this compares segments.+ expect(isWithinDiscoveryRoot(root, join('/repo', '..shared', 'app.ts'))).toBe(false);+ expect(isWithinDiscoveryRoot(root, join(root, '..shared', 'app.ts'))).toBe(true);+ });+it('pre-scans every language whose index is parser-derived', () => {for (const filePath of ['src/Service.php',@@ -66,3 +91,115 @@ describe('dedupeDiscoveredFilePaths', () => {expect(needsIndexPrescan('src/service.py')).toBe(false);});});++describe('discovery symlink containment', () => {+ const scratch: string[] = [];++ afterEach(() => {+ for (const dir of scratch.splice(0)) rmSync(dir, { recursive: true, force: true });+ });++ function scratchDir(prefix: string): string {+ const dir = mkdtempSync(join(tmpdir(), prefix));+ scratch.push(dir);+ // realpath because macOS hands back /var/... for a /private/var/... temp dir,+ // and this whole guarantee is about comparing resolved paths.+ return realpathSync(dir);+ }++ it.skipIf(process.platform === 'win32')(+ 'drops a committed symlink that leaves the repository',+ () => {+ const repo = scratchDir('ix-repo-');+ const outside = scratchDir('ix-outside-');+ const secret = join(outside, 'secret.ts');+ writeFileSync(secret, 'export const token = "leaked";\n');++ mkdirSync(join(repo, 'src'), { recursive: true });+ writeFileSync(join(repo, 'src', 'real.ts'), 'export const ok = 1;\n');+ try {+ // A symlink with a source extension: git records it as an ordinary+ // entry, and the stat behind it describes the file it points at.+ symlinkSync(secret, join(repo, 'src', 'notes.ts'));+ execFileSync('git', ['init', '-q'], { cwd: repo });+ execFileSync('git', ['add', '-A'], { cwd: repo });+ } catch {+ return; // no git, or no permission to symlink — nothing to assert+ }++ const listed = tryGitLsFiles(repo, true);+ expect(listed).not.toBeNull();++ // The escape is real: discovery resolves the link and hands back a path+ // outside the repository. Without this the assertions below could pass+ // simply because nothing was ever discovered.+ expect(dedupeDiscoveredFilePaths(listed!)).toContain(secret);++ // The function runIngest actually calls, so removing the confinement+ // from it fails here rather than passing a re-implementation.+ const discovery = discoverIngestFilePaths(listed!, repo);++ expect(discovery.files).not.toContain(secret);+ expect(discovery.files).toEqual([join(repo, 'src', 'real.ts')]);+ expect(discovery.outsideRoot).toBe(1);+ },+ );++ it.skipIf(process.platform === 'win32')(+ 'still discovers everything when the root is reached through a symlink',+ () => {+ // Resolving the file but not the root drops every file and reports the+ // repository as empty. No CI runner here has a symlinked root (the matrix+ // is ubuntu + windows), so this test is the only thing holding it down.+ const real = scratchDir('ix-realroot-');+ const parent = scratchDir('ix-linkroot-');+ const link = join(parent, 'root');+ mkdirSync(join(real, 'src'), { recursive: true });+ writeFileSync(join(real, 'src', 'app.ts'), 'export const ok = 1;\n');+ try {+ symlinkSync(real, link, 'dir');+ } catch {+ return; // no permission to symlink — nothing to assert+ }++ const discovery = discoverIngestFilePaths([join(link, 'src', 'app.ts')], link);++ expect(discovery.files).toEqual([join(real, 'src', 'app.ts')]);+ expect(discovery.outsideRoot).toBe(0);+ },+ );++ it('leaves an explicitly named single file alone', () => {+ // No root: the user named the file, so there is nothing to escape from and+ // confinement must not reject it.+ const outside = '/elsewhere/app.ts';+ const discovery = discoverIngestFilePaths([outside], undefined, (p) => p);++ expect(discovery.files).toEqual([outside]);+ expect(discovery.outsideRoot).toBe(0);+ });++ it.skipIf(process.platform === 'win32')(+ 'keeps a symlink that stays inside the repository',+ () => {+ // Monorepos link packages around inside the tree. Confinement must not+ // turn into "no symlinks at all".+ const repo = scratchDir('ix-repo-');+ mkdirSync(join(repo, 'shared'), { recursive: true });+ mkdirSync(join(repo, 'src'), { recursive: true });+ writeFileSync(join(repo, 'shared', 'util.ts'), 'export const shared = 1;\n');+ try {+ symlinkSync(join(repo, 'shared', 'util.ts'), join(repo, 'src', 'util.ts'));+ execFileSync('git', ['init', '-q'], { cwd: repo });+ execFileSync('git', ['add', '-A'], { cwd: repo });+ } catch {+ return; // no git, or no permission to symlink — nothing to assert+ }++ const canonical = dedupeDiscoveredFilePaths(tryGitLsFiles(repo, true) ?? []);+ const kept = canonical.filter((candidate) => isWithinDiscoveryRoot(repo, candidate));++ expect(kept).toContain(join(repo, 'shared', 'util.ts'));+ },+ );+});diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 2c2a123a..905e0ba2 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -163,6 +163,28 @@ function canonicalizeDiscoveredFilePath(filePath: string): string {}}+/**+ * True when `candidate` lies inside `root`.+ *+ * Both arguments must ALREADY be canonical (realpath'd). This is pure path+ * arithmetic and so cannot see through a symlink itself — resolving first is+ * what gives it meaning, and resolving both sides is what keeps it from+ * rejecting everything when the root is itself reached through a link (macOS+ * `/var` -> `/private/var`, a home on a network mount, a `~/code` symlink).+ */+export function isWithinDiscoveryRoot(root: string, candidate: string): boolean {+ const relative = nodePath.relative(root, candidate);+ // Compare path *segments*: a bare `startsWith('..')` also rejects a+ // legitimate sibling whose name merely begins with dots, e.g. `..shared/`.+ // `isAbsolute` covers Windows, where `relative` across drives is absolute.+ return (+ relative !== '' &&+ relative !== '..' &&+ !relative.startsWith(`..${nodePath.sep}`) &&+ !nodePath.isAbsolute(relative)+ );+}+export function dedupeDiscoveredFilePaths(filePaths: string[],canonicalize: (filePath: string) => string = canonicalizeDiscoveredFilePath,@@ -178,7 +200,51 @@ export function dedupeDiscoveredFilePaths(return deduped;}-function tryGitLsFiles(dir: string, recursive: boolean): string[] | null {+/**+ * Canonicalize, de-duplicate, and confine discovered paths to `root`.+ *+ * Discovery resolves symlinks, so a repository can name files outside itself:+ * `git ls-files` reports a committed symlink as an ordinary entry and the+ * `statSync` behind it describes the *target*, so `src/notes.ts` ->+ * `~/.ssh/id_rsa` is discovered, read, parsed into the graph and sent to the+ * configured endpoint. `ix ingest --github owner/repo` runs this over+ * repositories nobody has read, which makes repository contents attacker input.+ *+ * (`walkFiles` is not the hole — readdir Dirents describe the link itself, so+ * `isFile()` is false for a symlink and it is skipped there. Git's list is+ * the gap.)+ *+ * A symlink *within* the tree still resolves and is kept, which is what+ * monorepos that link packages around depend on. `root` is `undefined` when a+ * single file was named explicitly: that IS the request, and there is nothing+ * for it to escape from.+ *+ * Kept as one function rather than a filter at the call site so the confinement+ * cannot be dropped without a test noticing.+ */+export function discoverIngestFilePaths(+ discovered: string[],+ root: string | undefined,+ canonicalize: (filePath: string) => string = canonicalizeDiscoveredFilePath,+): { files: string[]; outsideRoot: number } {+ const canonical = dedupeDiscoveredFilePaths(discovered, canonicalize);+ if (root === undefined) return { files: canonical, outsideRoot: 0 };+ // Canonicalize the root too. Comparing a resolved file against an unresolved+ // root drops everything whenever the root is itself reached through a link —+ // macOS `/var` -> `/private/var`, a home on a network mount, a `~/code`+ // symlink — which would look like "this repo has no source files".+ const canonicalRoot = canonicalize(root);+ const files = canonical.filter((candidate) => isWithinDiscoveryRoot(canonicalRoot, candidate));+ return { files, outsideRoot: canonical.length - files.length };+}++/**+ * Exported for the discovery tests: the symlink-containment guarantee is a+ * property of git-listing -> canonicalize -> confine *composed*, and each step+ * looks fine alone. A test that cannot run the real listing cannot show that a+ * committed symlink escapes in the first place.+ */+export function tryGitLsFiles(dir: string, recursive: boolean): string[] | null {try {const result = spawnSync('git',@@ -971,6 +1037,7 @@ export async function ingestFiles(let commitErrors = 0;let tooLarge = 0;let minifiedLikely = 0;+ let outsideRoot = 0;let latestRev = 0;let entitiesParsed = 0;let currentWorkLabel = '';@@ -1038,11 +1105,14 @@ export async function ingestFiles(throw new Error(`Path not found: ${resolvedPath}`);}const stat = fs.statSync(resolvedPath);- const filePaths: string[] = dedupeDiscoveredFilePaths(+ const discovery = discoverIngestFilePaths(stat.isFile()? (isSupportedSourceFile(resolvedPath) ? [resolvedPath] : []): (tryGitLsFiles(resolvedPath, opts.recursive ?? true) ?? Array.from(walkFiles(resolvedPath, opts.recursive ?? true))),+ stat.isFile() ? undefined : resolvedPath,);+ const filePaths: string[] = discovery.files;+ outsideRoot = discovery.outsideRoot;filesDiscovered = filePaths.length;const discovered = performance.now();@@ -1935,7 +2005,7 @@ export async function ingestFiles(filesSkipped,entitiesParsed,latestRev,- skipReasons: { unchanged: filesSkipped, emptyFile: 0, parseError: parseErrors, tooLarge, minifiedLikely },+ skipReasons: { unchanged: filesSkipped, emptyFile: 0, parseError: parseErrors, tooLarge, minifiedLikely, outsideRoot },commitErrors,elapsedSeconds: parseFloat(elapsed),timings: {@@ -1959,6 +2029,9 @@ export async function ingestFiles(if (commitErrors > 0) console.log(` ${chalk.red('commit errors:')} ${commitErrors}`);if (tooLarge > 0) console.log(` ${chalk.dim('skipped too large:')} ${tooLarge}`);if (minifiedLikely > 0) console.log(` ${chalk.dim('skipped minified:')} ${minifiedLikely}`);+ // Not dimmed like the others: these were dropped because the repo pointed+ // at files outside itself, which is worth a look rather than a shrug.+ if (outsideRoot > 0) console.log(` ${chalk.yellow('skipped outside root:')} ${outsideRoot} ${chalk.dim('(symlinks leaving the tree)')}`);console.log(` rev: ${latestRev}`);if (patchesApplied === 0 && filesDiscovered === 0) {