Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
perf(ingest): overlap file I/O with parse dispatch; raise COMMIT_CONCURRENCY to 8
Joseph Mikhail560001e996e7
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 15b491c4..d658c126 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -528,7 +528,7 @@ export async function ingestFiles(const PARSE_STREAM_CHUNK = 1000; // streaming batch size for large reposconst SMALL_REPO_THRESHOLD = 5_000; // parse-all-first below this; streaming aboveconst COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 200); // files per HTTP request to the backend- const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', mapMode ? 2 : 4); // parallel HTTP save requests+ const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', 8); // parallel HTTP save requestsconst COMMIT_CONFLICT_RETRIES = parsePositiveIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6); // retry transient Arango lock conflictsconst YIELD_EVERY = 100; // yield event loop every N files during parse@@ -852,20 +852,24 @@ export async function ingestFiles(progressTotal = filePaths.length;progressCurrent = 0;- // Pipeline overlap: parse next chunk in worker pool while committing current chunk+ // Pipeline overlap: parse next chunk in worker pool while committing current chunk.+ // Each file is dispatched to the parse pool immediately after it is read so workers+ // start processing as soon as the first file in the chunk is ready rather than+ // waiting for all reads in the chunk to complete.let pendingFlush: Promise<void> = Promise.resolve();for (let i = 0; i < filePaths.length; i += PARSE_STREAM_CHUNK) {const chunk = filePaths.slice(i, i + PARSE_STREAM_CHUNK);type FileData = { filePath: string; source: string; hash: string; previousHash: string | undefined } | null;const fileData: FileData[] = [];+ const parsePromises: Promise<unknown>[] = [];for (const filePath of chunk) {try {const fileSize = fs.statSync(filePath).size;- if (fileSize === 0) { filesSkipped++; fileData.push(null); continue; }- if (fileSize > MAX_FILE_BYTES) { tooLarge++; fileData.push(null); continue; }+ if (fileSize === 0) { filesSkipped++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }+ if (fileSize > MAX_FILE_BYTES) { tooLarge++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }const bytes = fs.readFileSync(filePath);const hash = sha256(bytes);- if (!opts.force && knownHashes.get(filePath) === hash) { filesSkipped++; fileData.push(null); continue; }+ if (!opts.force && knownHashes.get(filePath) === hash) { filesSkipped++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }const previousHash = knownHashes.get(filePath);const sourceText = bytes.toString('utf-8');if (isLikelyMinifiedSource(sourceText)) {@@ -873,22 +877,20 @@ export async function ingestFiles(filesSkipped++;if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);fileData.push(null);+ parsePromises.push(Promise.resolve(null));continue;}- fileData.push({ filePath, source: sourceText, hash, previousHash: previousHash !== hash ? previousHash : undefined });+ const fd: NonNullable<FileData> = { filePath, source: sourceText, hash, previousHash: previousHash !== hash ? previousHash : undefined };+ fileData.push(fd);+ parsePromises.push(pool.parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; }));} catch (err) {parseErrors++;fileData.push(null);+ parsePromises.push(Promise.resolve(null));process.stderr.write(`\n [read error] ${filePath}: ${err}\n`);}}- const parseResults = await Promise.all(- fileData.map(f =>- f- ? pool.parse(f.filePath, f.source).then(r => { progressCurrent++; return r; })- : Promise.resolve(null),- ),- );+ const parseResults = await Promise.all(parsePromises);const batch: ParsedFile[] = [];for (let j = 0; j < fileData.length; j++) {const f = fileData[j];