Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

refactor(ingest): remove dead commit-path code and cover fsmonitor on untracked discovery (IN-19B) (#839)

Ian Hock0e686b56fb5c
diff --git a/ix-cli/src/cli/__tests__/git-repo-config-commands.test.ts b/ix-cli/src/cli/__tests__/git-repo-config-commands.test.ts
index f46bd81e..d3504f60 100644
--- a/ix-cli/src/cli/__tests__/git-repo-config-commands.test.ts
+++ b/ix-cli/src/cli/__tests__/git-repo-config-commands.test.ts
@@ -2,7 +2,7 @@
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
import { execFileSync } from "node:child_process";
-import { chmodSync, existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { tryGitLsFiles } from "../commands/ingest.js";
@@ -34,6 +34,9 @@ describe.skipIf(process.platform === "win32")("git calls do not run commands fro
writeFileSync(join(repo, ".gitattributes"), "*.txt diff=conv\n");
git("add", ".");
git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "init");
+ // Untracked under a SKIPPED_DIRS name: makes ingest's file listing also run gitUntracked.
+ mkdirSync(join(repo, "build"));
+ writeFileSync(join(repo, "build", "gen.ts"), "export const x = 1;\n");
git("config", "core.fsmonitor", hook);
git("config", "diff.conv.textconv", hook);
writeFileSync(join(repo, "a.txt"), "needle\nmore\n");
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 48df4561..665d4d79 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -258,17 +258,6 @@ function isLikelyMinifiedSource(source: string): boolean {
return maxLineLength >= MINIFIED_MAX_LINE_THRESHOLD || (lineCount <= 50 && averageLineLength >= MINIFIED_AVG_LINE_THRESHOLD);
}
-function stripChunkOps(patch: GraphPatchPayload): GraphPatchPayload {
- return {
- ...patch,
- ops: patch.ops.filter(op => {
- if (op.type === 'UpsertNode' && op['kind'] === 'chunk') return false;
- if (op.type === 'UpsertEdge' && (op['predicate'] === 'CONTAINS_CHUNK' || op['predicate'] === 'NEXT')) return false;
- return true;
- }),
- };
-}
-
/**
* Strip chunks and claims from patches — mapMode only needs nodes + edges.
*
@@ -2239,9 +2228,7 @@ export async function ingestFiles(
const timings = {
moduleLoadMs: 0, discoverMs: 0, hashMs: 0, parseMs: 0,
goIndexMs: 0, // time to build Go global resolution index
- parseOnlyMs: 0, // sum of just parseFile() call durations
resolveMs: 0, buildPatchMs: 0, commitMs: 0,
- stripChunkOpsMs: 0, // sum of stripChunkOps() durations (mapMode only)
bulkCommitMs: 0, // time in successful commitPatchBulk() calls
fallbackCommitMs: 0, // time in per-file fallback commitPatch() calls
};
@@ -2261,7 +2248,6 @@ export async function ingestFiles(
files: number,
resolveEdgesMs: number,
buildPatchMs: number,
- stripMs: number,
commitMs: number,
totalSaveMs: number,
nodes: number,
@@ -2271,7 +2257,6 @@ export async function ingestFiles(
if (!debug) return;
process.stderr.write(
`\n[ingest-save] files=${files} resolveEdgesMs=${resolveEdgesMs} buildPatchMs=${buildPatchMs}` +
- (mapMode ? ` stripChunkOpsMs=${stripMs}` : '') +
` commitMs=${commitMs} totalSaveMs=${totalSaveMs} nodes=${nodes} edges=${edges} claims=${claims}\n`
);
};
@@ -2534,9 +2519,6 @@ export async function ingestFiles(
// resolveEdges runs on that batch and patches are committed before the next
// batch is parsed. This keeps heap usage at O(chunk) instead of O(total).
//
- // YIELD_EVERY yields the Node.js event loop so the progress bar timer can
- // fire even during the synchronous tree-sitter parse calls.
- //
// Two paths:
// A) Has baseline or mtime-changed files → pre-scan first, skip module load
// entirely if nothing changed (common "ix map" re-run case).
@@ -2687,11 +2669,10 @@ export async function ingestFiles(
};
let globalIndex: any = undefined;
- // Larger chunks for big repos: fewer resolve+commit cycles, better edge resolution per batch.
// Smaller chunks pipeline better: resolve(N) overlaps with commit(N-1) + parse(N+1).
// With 2000-file chunks, resolve takes 68s and dominates. With 500, resolve ~17s
// and overlaps well with commit (~5s) + parse (~5s).
- const PARSE_STREAM_CHUNK = filePaths.length > 10_000 ? 500 : 500;
+ const PARSE_STREAM_CHUNK = 500;
const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 1000); // files per HTTP request to the backend
// Ops per HTTP request; see DEFAULT_COMMIT_MAX_OPS.
const COMMIT_MAX_OPS = parsePositiveIntEnv('IX_COMMIT_MAX_OPS', DEFAULT_COMMIT_MAX_OPS);
@@ -2766,7 +2747,6 @@ export async function ingestFiles(
baseRevRetries++;
if (debug) process.stderr.write(`\n [base-rev race] another writer moved the rev; re-sending\n`);
};
- const YIELD_EVERY = 100; // yield event loop every N files during parse
if (debug) {
process.stderr.write(
@@ -3574,7 +3554,6 @@ export async function ingestFiles(
const saveStart = performance.now();
let resolveEdgesMs = 0;
let buildPatchMs = 0;
- const localStripMs = 0;
let commitMs = 0;
let batchNodes = 0;
let batchEdges = 0;
@@ -3611,76 +3590,16 @@ export async function ingestFiles(
const totalSaveMs = Math.round(performance.now() - saveStart);
timings.resolveMs += resolveEdgesMs;
timings.buildPatchMs += buildPatchMs;
- timings.stripChunkOpsMs += localStripMs;
timings.commitMs += commitMs;
totalNodesEmitted += batchNodes;
totalEdgesEmitted += batchEdges;
totalClaimsEmitted += batchClaims;
- logSaveTiming(batch.length, resolveEdgesMs, buildPatchMs, localStripMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);
+ logSaveTiming(batch.length, resolveEdgesMs, buildPatchMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);
progressPhase = 'Parsing';
progressStart = performance.now();
}
};
- /** Parse-all-first commit: resolve edges over full set, then commit in HTTP-sized batches. */
- const flushAll = async (allParsed: ParsedFile[]): Promise<void> => {
- if (allParsed.length === 0) return;
- filesChanged += allParsed.length;
- const previousPhase = progressPhase;
- const saveStart = performance.now();
- let resolveEdgesMs = 0;
- let buildPatchMs = 0;
- const localStripMs = 0;
- let commitMs = 0;
- let batchNodes = 0;
- let batchEdges = 0;
- let batchClaims = 0;
- progressPhase = 'Saving';
- progressTotal = allParsed.length;
- progressCurrent = 0;
- try {
- setCurrentWork(`resolve ${allParsed.length} files`);
- const resolveStart = performance.now();
- const resolvedEdges = resolveEdgesFn!(allParsed.map(f => f.parsed), resolveStats, globalIndex, resolveOpts);
- resolveEdgesMs = Math.round(performance.now() - resolveStart);
- if (stitchEnabled) collectStitch(allParsed.map(f => f.parsed));
- const edgesByFile = new Map<string, any[]>();
- for (const edge of resolvedEdges) {
- let arr = edgesByFile.get(edge.srcFilePath);
- if (!arr) { arr = []; edgesByFile.set(edge.srcFilePath, arr); }
- arr.push(edge);
- }
- setCurrentWork(`build commit payloads ending ${nodePath.basename(allParsed[allParsed.length - 1].filePath)}`);
- const buildStart = performance.now();
- const preparedPatches = await buildFilePatches(allParsed, edgesByFile);
- buildPatchMs += Math.round(performance.now() - buildStart);
- for (const pp of preparedPatches) {
- for (const op of pp.patch.ops) {
- if (op.type === 'UpsertNode') batchNodes++;
- else if (op.type === 'UpsertEdge') batchEdges++;
- else if (op.type === 'AssertClaim') batchClaims++;
- }
- }
- if (preparedPatches.length === 0) {
- progressCurrent = allParsed.length;
- return;
- }
- commitMs += await commitPreparedPatches(preparedPatches, allParsed.length, { updateProgress: true, onCommitted: noteRebuilt });
- } finally {
- const totalSaveMs = Math.round(performance.now() - saveStart);
- timings.resolveMs += resolveEdgesMs;
- timings.buildPatchMs += buildPatchMs;
- timings.stripChunkOpsMs += localStripMs;
- timings.commitMs += commitMs;
- totalNodesEmitted += batchNodes;
- totalEdgesEmitted += batchEdges;
- totalClaimsEmitted += batchClaims;
- logSaveTiming(allParsed.length, resolveEdgesMs, buildPatchMs, localStripMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);
- progressPhase = previousPhase;
- progressStart = performance.now();
- }
- };
-
// A local baseline with mtimes is enough to take the incremental path, even
// when the backend knows none of the changed files: adding one new file to a
// mapped repo is exactly that, and it used to re-send the whole repository.
@@ -4647,11 +4566,7 @@ export async function ingestFiles(
stitchSkipped: stitchSkipped ?? null,
stitchSkippedRule: stitchSkippedRule ?? null,
elapsedSeconds: parseFloat(elapsed),
- timings: {
- ...timings,
- parseOnlyMs: Math.round(timings.parseOnlyMs),
- stripChunkOpsMs: Math.round(timings.stripChunkOpsMs),
- },
+ timings,
emitted: { nodes: totalNodesEmitted, edges: totalEdgesEmitted, claims: totalClaimsEmitted },
langParseMs: Object.fromEntries(
[...langParseMs.entries()].map(([lang, s]) => [lang, { ms: Math.round(s.ms), files: s.files }])
@@ -4710,12 +4625,8 @@ export async function ingestFiles(
console.log(chalk.dim(` hash: ${timings.hashMs}ms`));
console.log(chalk.dim(` go index: ${timings.goIndexMs}ms`));
console.log(chalk.dim(` parse total: ${timings.parseMs}ms`));
- console.log(chalk.dim(` parse only: ${Math.round(timings.parseOnlyMs)}ms`));
console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));
console.log(chalk.dim(` build: ${timings.buildPatchMs}ms`));
- if (mapMode) {
- console.log(chalk.dim(` stripChunk: ${Math.round(timings.stripChunkOpsMs)}ms`));
- }
console.log(chalk.dim(` commit: ${timings.commitMs}ms`));
console.log(chalk.dim(` bulk: ${timings.bulkCommitMs}ms`));
console.log(chalk.dim(` fallback: ${timings.fallbackCommitMs}ms`));