Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

perf(ingest): overlap file I/O with parse dispatch; raise COMMIT_CONCURRENCY to 8

Joseph Mikhail560001e996e7
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 15b491c4..d658c126 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -528,7 +528,7 @@ export async function ingestFiles(
const PARSE_STREAM_CHUNK = 1000; // streaming batch size for large repos
const SMALL_REPO_THRESHOLD = 5_000; // parse-all-first below this; streaming above
const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 200); // files per HTTP request to the backend
- const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', mapMode ? 2 : 4); // parallel HTTP save requests
+ const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', 8); // parallel HTTP save requests
const COMMIT_CONFLICT_RETRIES = parsePositiveIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6); // retry transient Arango lock conflicts
const YIELD_EVERY = 100; // yield event loop every N files during parse
@@ -852,20 +852,24 @@ export async function ingestFiles(
progressTotal = filePaths.length;
progressCurrent = 0;
- // Pipeline overlap: parse next chunk in worker pool while committing current chunk
+ // Pipeline overlap: parse next chunk in worker pool while committing current chunk.
+ // Each file is dispatched to the parse pool immediately after it is read so workers
+ // start processing as soon as the first file in the chunk is ready rather than
+ // waiting for all reads in the chunk to complete.
let pendingFlush: Promise<void> = Promise.resolve();
for (let i = 0; i < filePaths.length; i += PARSE_STREAM_CHUNK) {
const chunk = filePaths.slice(i, i + PARSE_STREAM_CHUNK);
type FileData = { filePath: string; source: string; hash: string; previousHash: string | undefined } | null;
const fileData: FileData[] = [];
+ const parsePromises: Promise<unknown>[] = [];
for (const filePath of chunk) {
try {
const fileSize = fs.statSync(filePath).size;
- if (fileSize === 0) { filesSkipped++; fileData.push(null); continue; }
- if (fileSize > MAX_FILE_BYTES) { tooLarge++; fileData.push(null); continue; }
+ if (fileSize === 0) { filesSkipped++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }
+ if (fileSize > MAX_FILE_BYTES) { tooLarge++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }
const bytes = fs.readFileSync(filePath);
const hash = sha256(bytes);
- if (!opts.force && knownHashes.get(filePath) === hash) { filesSkipped++; fileData.push(null); continue; }
+ if (!opts.force && knownHashes.get(filePath) === hash) { filesSkipped++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }
const previousHash = knownHashes.get(filePath);
const sourceText = bytes.toString('utf-8');
if (isLikelyMinifiedSource(sourceText)) {
@@ -873,22 +877,20 @@ export async function ingestFiles(
filesSkipped++;
if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
fileData.push(null);
+ parsePromises.push(Promise.resolve(null));
continue;
}
- fileData.push({ filePath, source: sourceText, hash, previousHash: previousHash !== hash ? previousHash : undefined });
+ const fd: NonNullable<FileData> = { filePath, source: sourceText, hash, previousHash: previousHash !== hash ? previousHash : undefined };
+ fileData.push(fd);
+ parsePromises.push(pool.parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; }));
} catch (err) {
parseErrors++;
fileData.push(null);
+ parsePromises.push(Promise.resolve(null));
process.stderr.write(`\n [read error] ${filePath}: ${err}\n`);
}
}
- const parseResults = await Promise.all(
- fileData.map(f =>
- f
- ? pool.parse(f.filePath, f.source).then(r => { progressCurrent++; return r; })
- : Promise.resolve(null),
- ),
- );
+ const parseResults = await Promise.all(parsePromises);
const batch: ParsedFile[] = [];
for (let j = 0; j < fileData.length; j++) {
const f = fileData[j];