Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
refactor(ingest): remove dead commit-path code and cover fsmonitor on untracked discovery (IN-19B) (#839)
Ian Hock0e686b56fb5c
diff --git a/ix-cli/src/cli/__tests__/git-repo-config-commands.test.ts b/ix-cli/src/cli/__tests__/git-repo-config-commands.test.tsindex f46bd81e..d3504f60 100644--- a/ix-cli/src/cli/__tests__/git-repo-config-commands.test.ts+++ b/ix-cli/src/cli/__tests__/git-repo-config-commands.test.ts@@ -2,7 +2,7 @@import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";import { execFileSync } from "node:child_process";-import { chmodSync, existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { tryGitLsFiles } from "../commands/ingest.js";@@ -34,6 +34,9 @@ describe.skipIf(process.platform === "win32")("git calls do not run commands frowriteFileSync(join(repo, ".gitattributes"), "*.txt diff=conv\n");git("add", ".");git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "init");+ // Untracked under a SKIPPED_DIRS name: makes ingest's file listing also run gitUntracked.+ mkdirSync(join(repo, "build"));+ writeFileSync(join(repo, "build", "gen.ts"), "export const x = 1;\n");git("config", "core.fsmonitor", hook);git("config", "diff.conv.textconv", hook);writeFileSync(join(repo, "a.txt"), "needle\nmore\n");diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 48df4561..665d4d79 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -258,17 +258,6 @@ function isLikelyMinifiedSource(source: string): boolean {return maxLineLength >= MINIFIED_MAX_LINE_THRESHOLD || (lineCount <= 50 && averageLineLength >= MINIFIED_AVG_LINE_THRESHOLD);}-function stripChunkOps(patch: GraphPatchPayload): GraphPatchPayload {- return {- ...patch,- ops: patch.ops.filter(op => {- if (op.type === 'UpsertNode' && op['kind'] === 'chunk') return false;- if (op.type === 'UpsertEdge' && (op['predicate'] === 'CONTAINS_CHUNK' || op['predicate'] === 'NEXT')) return false;- return true;- }),- };-}-/*** Strip chunks and claims from patches — mapMode only needs nodes + edges.*@@ -2239,9 +2228,7 @@ export async function ingestFiles(const timings = {moduleLoadMs: 0, discoverMs: 0, hashMs: 0, parseMs: 0,goIndexMs: 0, // time to build Go global resolution index- parseOnlyMs: 0, // sum of just parseFile() call durationsresolveMs: 0, buildPatchMs: 0, commitMs: 0,- stripChunkOpsMs: 0, // sum of stripChunkOps() durations (mapMode only)bulkCommitMs: 0, // time in successful commitPatchBulk() callsfallbackCommitMs: 0, // time in per-file fallback commitPatch() calls};@@ -2261,7 +2248,6 @@ export async function ingestFiles(files: number,resolveEdgesMs: number,buildPatchMs: number,- stripMs: number,commitMs: number,totalSaveMs: number,nodes: number,@@ -2271,7 +2257,6 @@ export async function ingestFiles(if (!debug) return;process.stderr.write(`\n[ingest-save] files=${files} resolveEdgesMs=${resolveEdgesMs} buildPatchMs=${buildPatchMs}` +- (mapMode ? ` stripChunkOpsMs=${stripMs}` : '') +` commitMs=${commitMs} totalSaveMs=${totalSaveMs} nodes=${nodes} edges=${edges} claims=${claims}\n`);};@@ -2534,9 +2519,6 @@ export async function ingestFiles(// resolveEdges runs on that batch and patches are committed before the next// batch is parsed. This keeps heap usage at O(chunk) instead of O(total).//- // YIELD_EVERY yields the Node.js event loop so the progress bar timer can- // fire even during the synchronous tree-sitter parse calls.- //// Two paths:// A) Has baseline or mtime-changed files → pre-scan first, skip module load// entirely if nothing changed (common "ix map" re-run case).@@ -2687,11 +2669,10 @@ export async function ingestFiles(};let globalIndex: any = undefined;- // Larger chunks for big repos: fewer resolve+commit cycles, better edge resolution per batch.// Smaller chunks pipeline better: resolve(N) overlaps with commit(N-1) + parse(N+1).// With 2000-file chunks, resolve takes 68s and dominates. With 500, resolve ~17s// and overlaps well with commit (~5s) + parse (~5s).- const PARSE_STREAM_CHUNK = filePaths.length > 10_000 ? 500 : 500;+ const PARSE_STREAM_CHUNK = 500;const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 1000); // files per HTTP request to the backend// Ops per HTTP request; see DEFAULT_COMMIT_MAX_OPS.const COMMIT_MAX_OPS = parsePositiveIntEnv('IX_COMMIT_MAX_OPS', DEFAULT_COMMIT_MAX_OPS);@@ -2766,7 +2747,6 @@ export async function ingestFiles(baseRevRetries++;if (debug) process.stderr.write(`\n [base-rev race] another writer moved the rev; re-sending\n`);};- const YIELD_EVERY = 100; // yield event loop every N files during parseif (debug) {process.stderr.write(@@ -3574,7 +3554,6 @@ export async function ingestFiles(const saveStart = performance.now();let resolveEdgesMs = 0;let buildPatchMs = 0;- const localStripMs = 0;let commitMs = 0;let batchNodes = 0;let batchEdges = 0;@@ -3611,76 +3590,16 @@ export async function ingestFiles(const totalSaveMs = Math.round(performance.now() - saveStart);timings.resolveMs += resolveEdgesMs;timings.buildPatchMs += buildPatchMs;- timings.stripChunkOpsMs += localStripMs;timings.commitMs += commitMs;totalNodesEmitted += batchNodes;totalEdgesEmitted += batchEdges;totalClaimsEmitted += batchClaims;- logSaveTiming(batch.length, resolveEdgesMs, buildPatchMs, localStripMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);+ logSaveTiming(batch.length, resolveEdgesMs, buildPatchMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);progressPhase = 'Parsing';progressStart = performance.now();}};- /** Parse-all-first commit: resolve edges over full set, then commit in HTTP-sized batches. */- const flushAll = async (allParsed: ParsedFile[]): Promise<void> => {- if (allParsed.length === 0) return;- filesChanged += allParsed.length;- const previousPhase = progressPhase;- const saveStart = performance.now();- let resolveEdgesMs = 0;- let buildPatchMs = 0;- const localStripMs = 0;- let commitMs = 0;- let batchNodes = 0;- let batchEdges = 0;- let batchClaims = 0;- progressPhase = 'Saving';- progressTotal = allParsed.length;- progressCurrent = 0;- try {- setCurrentWork(`resolve ${allParsed.length} files`);- const resolveStart = performance.now();- const resolvedEdges = resolveEdgesFn!(allParsed.map(f => f.parsed), resolveStats, globalIndex, resolveOpts);- resolveEdgesMs = Math.round(performance.now() - resolveStart);- if (stitchEnabled) collectStitch(allParsed.map(f => f.parsed));- const edgesByFile = new Map<string, any[]>();- for (const edge of resolvedEdges) {- let arr = edgesByFile.get(edge.srcFilePath);- if (!arr) { arr = []; edgesByFile.set(edge.srcFilePath, arr); }- arr.push(edge);- }- setCurrentWork(`build commit payloads ending ${nodePath.basename(allParsed[allParsed.length - 1].filePath)}`);- const buildStart = performance.now();- const preparedPatches = await buildFilePatches(allParsed, edgesByFile);- buildPatchMs += Math.round(performance.now() - buildStart);- for (const pp of preparedPatches) {- for (const op of pp.patch.ops) {- if (op.type === 'UpsertNode') batchNodes++;- else if (op.type === 'UpsertEdge') batchEdges++;- else if (op.type === 'AssertClaim') batchClaims++;- }- }- if (preparedPatches.length === 0) {- progressCurrent = allParsed.length;- return;- }- commitMs += await commitPreparedPatches(preparedPatches, allParsed.length, { updateProgress: true, onCommitted: noteRebuilt });- } finally {- const totalSaveMs = Math.round(performance.now() - saveStart);- timings.resolveMs += resolveEdgesMs;- timings.buildPatchMs += buildPatchMs;- timings.stripChunkOpsMs += localStripMs;- timings.commitMs += commitMs;- totalNodesEmitted += batchNodes;- totalEdgesEmitted += batchEdges;- totalClaimsEmitted += batchClaims;- logSaveTiming(allParsed.length, resolveEdgesMs, buildPatchMs, localStripMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);- progressPhase = previousPhase;- progressStart = performance.now();- }- };-// A local baseline with mtimes is enough to take the incremental path, even// when the backend knows none of the changed files: adding one new file to a// mapped repo is exactly that, and it used to re-send the whole repository.@@ -4647,11 +4566,7 @@ export async function ingestFiles(stitchSkipped: stitchSkipped ?? null,stitchSkippedRule: stitchSkippedRule ?? null,elapsedSeconds: parseFloat(elapsed),- timings: {- ...timings,- parseOnlyMs: Math.round(timings.parseOnlyMs),- stripChunkOpsMs: Math.round(timings.stripChunkOpsMs),- },+ timings,emitted: { nodes: totalNodesEmitted, edges: totalEdgesEmitted, claims: totalClaimsEmitted },langParseMs: Object.fromEntries([...langParseMs.entries()].map(([lang, s]) => [lang, { ms: Math.round(s.ms), files: s.files }])@@ -4710,12 +4625,8 @@ export async function ingestFiles(console.log(chalk.dim(` hash: ${timings.hashMs}ms`));console.log(chalk.dim(` go index: ${timings.goIndexMs}ms`));console.log(chalk.dim(` parse total: ${timings.parseMs}ms`));- console.log(chalk.dim(` parse only: ${Math.round(timings.parseOnlyMs)}ms`));console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));console.log(chalk.dim(` build: ${timings.buildPatchMs}ms`));- if (mapMode) {- console.log(chalk.dim(` stripChunk: ${Math.round(timings.stripChunkOpsMs)}ms`));- }console.log(chalk.dim(` commit: ${timings.commitMs}ms`));console.log(chalk.dim(` bulk: ${timings.bulkCommitMs}ms`));console.log(chalk.dim(` fallback: ${timings.fallbackCommitMs}ms`));