Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat: speed up ingestion with change detection fix and bulk commits
Tanner Kocherb05716746a3e
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 206987d3..aa0c866a 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -69,10 +69,11 @@ export function registerIngestCommand(program: Command): void {.option('--force', 'Force re-ingest even if files are unchanged (useful after parser upgrades)').option('--format <fmt>', 'Output format (text|json)', 'text').option('--root <dir>', 'Workspace root directory')+ .option('--debug', 'Show phase timing breakdown', false).addHelpText('after', '\nExamples:\n ix ingest ./src\n ix ingest --path ./src --force\n ix ingest --github owner/repo\n ix ingest --github owner/repo --since 2026-01-01 --limit 20 --format json\n ix ingest --github owner/repo --token ghp_xxxx').action(async (positionalPath: string | undefined, opts: {path?: string; recursive?: boolean; force?: boolean; github?: string; token?: string;- since?: string; limit: string; format: string; root?: string;+ since?: string; limit: string; format: string; root?: string; debug?: boolean;}) => {const effectivePath = positionalPath ?? opts.path;if (opts.github) {@@ -92,13 +93,14 @@ export function registerIngestCommand(program: Command): void {export async function ingestFiles(path: string,- opts: { recursive?: boolean; force?: boolean; format: string; root?: string }+ opts: { recursive?: boolean; force?: boolean; format: string; root?: string; debug?: boolean }): Promise<void> {const [{ parseFile, resolveEdges, isGrammarSupported }, { buildPatchWithResolution }] = await loadIngestionModules();const resolvedPath = nodePath.isAbsolute(path)? path: nodePath.resolve(resolveWorkspaceRoot(opts.root), path);+ const debug = opts.debug || process.env.IX_DEBUG === '1';const client = new IxClient(getEndpoint());const start = performance.now();@@ -110,6 +112,7 @@ export async function ingestFiles(}, 200) : null;let filesDiscovered = 0;+ let filesChanged = 0;let patchesApplied = 0;let filesSkipped = 0;let parseErrors = 0;@@ -117,19 +120,26 @@ export async function ingestFiles(let latestRev = 0;let entitiesParsed = 0;+ // Phase timing marks+ const timings = { discoverMs: 0, hashMs: 0, parseMs: 0, resolveMs: 0, commitMs: 0 };+try {- // Collect files+ // Phase: discover filesconst stat = fs.statSync(resolvedPath);const filePaths: string[] = stat.isFile()? [resolvedPath]: Array.from(walkFiles(resolvedPath, opts.recursive ?? true, isGrammarSupported));filesDiscovered = filePaths.length;+ const discovered = performance.now();+ timings.discoverMs = Math.round(discovered - start);- // Load existing source hashes for change detection (unless --force)- const knownHashes = opts.force ? new Map<string, string>() : await loadExistingHashes(client, filePaths);+ // Phase: hash lookup+ const knownHashes = opts.force ? new Map<string, string>() : await loadExistingHashes(client, filePaths, debug);+ const hashed = performance.now();+ timings.hashMs = Math.round(hashed - discovered);- // Phase 1: parse all files+ // Phase: parse all filestype ParsedFile = { filePath: string; parsed: any; hash: string; previousHash: string | undefined };const parsedFiles: ParsedFile[] = [];@@ -164,21 +174,49 @@ export async function ingestFiles(}}- // Phase 2: cross-file CALLS + EXTENDS resolution over the full batch+ filesChanged = parsedFiles.length;+ const parsed = performance.now();+ timings.parseMs = Math.round(parsed - hashed);++ // Phase: cross-file CALLS + EXTENDS resolution over the full batchconst resolvedEdges = resolveEdges(parsedFiles.map(f => f.parsed));+ const resolved = performance.now();+ timings.resolveMs = Math.round(resolved - parsed);- // Phase 3: build and commit patches- for (const { parsed, hash, previousHash } of parsedFiles) {+ // Phase: build patches and bulk commit+ const patches: GraphPatchPayload[] = [];+ for (const { parsed: p, hash, previousHash } of parsedFiles) {try {- const patch = buildPatchWithResolution(parsed, hash, resolvedEdges, previousHash);- const result = await client.commitPatch(patch);- if (result.rev > latestRev) latestRev = result.rev;- patchesApplied++;+ patches.push(buildPatchWithResolution(p, hash, resolvedEdges, previousHash));} catch (err) {parseErrors++;- process.stderr.write(`\n [commit error] ${parsed.filePath}: ${err}\n`);+ process.stderr.write(`\n [patch build error] ${p.filePath}: ${err}\n`);+ }+ }++ if (patches.length > 0) {+ try {+ const result = await client.commitPatchBulk(patches);+ latestRev = result.rev;+ patchesApplied = patches.length;+ } catch (err) {+ // Fallback: try per-file commits if bulk endpoint fails+ if (debug) process.stderr.write(`\n [bulk commit failed, falling back to per-file] ${err}\n`);+ for (const patch of patches) {+ try {+ const result = await client.commitPatch(patch);+ if (result.rev > latestRev) latestRev = result.rev;+ patchesApplied++;+ } catch (commitErr) {+ parseErrors++;+ process.stderr.write(`\n [commit error] ${patch.source?.uri}: ${commitErr}\n`);+ }+ }}}++ const committed = performance.now();+ timings.commitMs = Math.round(committed - resolved);} finally {if (interval) {clearInterval(interval);@@ -191,17 +229,20 @@ export async function ingestFiles(if (opts.format === 'json') {console.log(JSON.stringify({filesProcessed: filesDiscovered,+ filesChanged,patchesApplied,filesSkipped,entitiesParsed,latestRev,skipReasons: { unchanged: filesSkipped, emptyFile: 0, parseError: parseErrors, tooLarge },elapsedSeconds: parseFloat(elapsed),+ timings,}, null, 2));} else {console.log(chalk.bold('\nIngest summary'));console.log(` processed: ${patchesApplied} files (${elapsed}s)`);console.log(` discovered: ${filesDiscovered} files`);+ console.log(` changed: ${filesChanged} files`);if (filesSkipped > 0) console.log(` ${chalk.dim('skipped unchanged:')} ${filesSkipped}`);if (parseErrors > 0) console.log(` ${chalk.red('parse errors:')} ${parseErrors}`);if (tooLarge > 0) console.log(` ${chalk.dim('skipped too large:')} ${tooLarge}`);@@ -212,6 +253,16 @@ export async function ingestFiles(} else if (patchesApplied === 0 && filesDiscovered > 0) {console.log(chalk.dim('\n All files unchanged since last ingest.'));}++ if (debug) {+ console.log(chalk.dim(`\n Phase timings:`));+ console.log(chalk.dim(` discover: ${timings.discoverMs}ms`));+ console.log(chalk.dim(` hash: ${timings.hashMs}ms`));+ console.log(chalk.dim(` parse: ${timings.parseMs}ms`));+ console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));+ console.log(chalk.dim(` commit: ${timings.commitMs}ms`));+ }+console.log();}}@@ -220,11 +271,11 @@ export async function ingestFiles(// Load existing hashes from the server for change detection// ----------------------------------------------------------------------------async function loadExistingHashes(client: IxClient, filePaths: string[]): Promise<Map<string, string>> {+async function loadExistingHashes(client: IxClient, filePaths: string[], debug = false): Promise<Map<string, string>> {try {- const hashes = await (client as any).getSourceHashes(filePaths);- return new Map(Object.entries(hashes ?? {}));- } catch {+ return await client.getSourceHashes(filePaths);+ } catch (err) {+ if (debug) process.stderr.write(`\n [hash lookup failed] ${err}\n`);return new Map();}}diff --git a/ix-cli/src/client/api.ts b/ix-cli/src/client/api.tsindex 0af60ea7..5c9e36ed 100644--- a/ix-cli/src/client/api.ts+++ b/ix-cli/src/client/api.ts@@ -174,6 +174,15 @@ export class IxClient {return this.post("/v1/patch", patch);}+ async getSourceHashes(filePaths: string[]): Promise<Map<string, string>> {+ const result = await this.post<Record<string, string>>('/v1/source-hashes', { uris: filePaths });+ return new Map(Object.entries(result));+ }++ async commitPatchBulk(patches: GraphPatchPayload[]): Promise<PatchCommitResult> {+ return this.post<PatchCommitResult>('/v1/patches/bulk', { patches });+ }+async map(): Promise<any> {return this.post("/v1/map", {});}diff --git a/memory-layer/src/main/scala/ix/memory/Main.scala b/memory-layer/src/main/scala/ix/memory/Main.scalaindex 11cd8b12..6e3366bc 100644--- a/memory-layer/src/main/scala/ix/memory/Main.scala+++ b/memory-layer/src/main/scala/ix/memory/Main.scala@@ -46,7 +46,7 @@ object Main extends IOApp.Simple {mapService = new MapService(client, queryApi, writeApi)// 4. HTTP routes- routes = Routes.all(contextService, ingestionService, bulkIngestionService, queryApi, writeApi, conflictService, client, mapService)+ routes = Routes.all(contextService, ingestionService, bulkIngestionService, queryApi, writeApi, conflictService, client, mapService, bulkWriteApi)// 5. Serverserver <- EmberServerBuilderdiff --git a/memory-layer/src/main/scala/ix/memory/api/BulkPatchCommitRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/BulkPatchCommitRoutes.scalanew file mode 100644index 00000000..3e1812e2--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/api/BulkPatchCommitRoutes.scala@@ -0,0 +1,55 @@+package ix.memory.api++import cats.effect.IO+import io.circe.{Decoder, Json}+import io.circe.generic.semiauto.deriveDecoder+import io.circe.syntax._+import org.http4s.HttpRoutes+import org.http4s.circe.CirceEntityCodec._+import org.http4s.dsl.io._++import ix.memory.db.{BulkWriteApi, FileBatch, GraphQueryApi}+import ix.memory.model.GraphPatch++class BulkPatchCommitRoutes(bulkWriteApi: BulkWriteApi, queryApi: GraphQueryApi) {++ private case class BulkPatchRequest(patches: List[GraphPatch])+ private implicit val reqDecoder: Decoder[BulkPatchRequest] = deriveDecoder++ private val mapper = new com.fasterxml.jackson.databind.ObjectMapper()++ val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {+ case req @ POST -> Root / "v1" / "patches" / "bulk" =>+ (for {+ body <- req.as[BulkPatchRequest]+ _ <- if (body.patches.isEmpty)+ IO.raiseError(new IllegalArgumentException("patches array must not be empty"))+ else IO.unit+ fileBatches = body.patches.map { patch =>+ FileBatch(+ filePath = patch.source.uri,+ sourceHash = patch.source.sourceHash,+ patch = patch,+ provenance = buildProvenanceMap(patch)+ )+ }.toVector+ latestRev <- queryApi.getLatestRev+ result <- bulkWriteApi.commitBatchChunked(fileBatches, latestRev.value)+ resp <- Ok(Json.obj(+ "status" -> result.status.toString.asJson,+ "rev" -> result.newRev.value.asJson+ ))+ } yield resp).handleErrorWith(ErrorHandler.handle(_))+ }++ private def buildProvenanceMap(patch: GraphPatch): java.util.Map[String, AnyRef] = {+ val json = Json.obj(+ "source_uri" -> Json.fromString(patch.source.uri),+ "source_hash" -> patch.source.sourceHash.fold(Json.Null)(Json.fromString),+ "extractor" -> Json.fromString(patch.source.extractor),+ "source_type" -> Json.fromString(patch.source.sourceType.asJson.asString.getOrElse("code")),+ "observed_at" -> Json.fromString(patch.timestamp.toString)+ )+ mapper.readValue(json.noSpaces, classOf[java.util.Map[String, AnyRef]])+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/api/Routes.scala b/memory-layer/src/main/scala/ix/memory/api/Routes.scalaindex 94ff863d..638b714f 100644--- a/memory-layer/src/main/scala/ix/memory/api/Routes.scala+++ b/memory-layer/src/main/scala/ix/memory/api/Routes.scala@@ -10,7 +10,7 @@ import org.http4s.dsl.io._import ix.memory.conflict.ConflictServiceimport ix.memory.context.ContextService-import ix.memory.db.{ArangoClient, GraphQueryApi, GraphWriteApi}+import ix.memory.db.{ArangoClient, BulkWriteApi, GraphQueryApi, GraphWriteApi}import ix.memory.ingestion.{BulkIngestionService, IngestionService}import ix.memory.map.MapService@@ -24,7 +24,8 @@ object Routes {writeApi: GraphWriteApi,conflictService: ConflictService,client: ArangoClient,- mapService: MapService+ mapService: MapService,+ bulkWriteApi: BulkWriteApi): HttpRoutes[IO] = {val health = HttpRoutes.of[IO] {@@ -47,13 +48,15 @@ object Routes {val statsRoutes = new StatsRoutes(client).routesval patchCommitRoutes = new PatchCommitRoutes(writeApi).routesval listRoutes = new ListRoutes(queryApi).routes- val mapRoutes = new MapRoutes(mapService).routes- val resetRoutes = new ResetRoutes(client).routes+ val mapRoutes = new MapRoutes(mapService).routes+ val resetRoutes = new ResetRoutes(client).routes+ val sourceHashRoutes = new SourceHashRoutes(queryApi).routes+ val bulkPatchCommitRoutes = new BulkPatchCommitRoutes(bulkWriteApi, queryApi).routeshealth <+> contextRoutes <+> ingestionRoutes <+> entityRoutes <+>diffRoutes <+> conflictRoutes <+> decideRoutes <+> searchRoutes <+>truthRoutes <+> patchRoutes <+> decisionListRoutes <+> expandRoutes <+>statsRoutes <+> patchCommitRoutes <+> listRoutes <+> goalRoutes <+>- mapRoutes <+> resetRoutes+ mapRoutes <+> resetRoutes <+> sourceHashRoutes <+> bulkPatchCommitRoutes}}diff --git a/memory-layer/src/main/scala/ix/memory/api/SourceHashRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/SourceHashRoutes.scalanew file mode 100644index 00000000..dd5e8ea5--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/api/SourceHashRoutes.scala@@ -0,0 +1,26 @@+package ix.memory.api++import cats.effect.IO+import io.circe.{Decoder, Json}+import io.circe.generic.semiauto.deriveDecoder+import io.circe.syntax._+import org.http4s.HttpRoutes+import org.http4s.circe.CirceEntityCodec._+import org.http4s.dsl.io._++import ix.memory.db.GraphQueryApi++class SourceHashRoutes(queryApi: GraphQueryApi) {++ private case class SourceHashRequest(uris: List[String])+ private implicit val reqDecoder: Decoder[SourceHashRequest] = deriveDecoder++ val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {+ case req @ POST -> Root / "v1" / "source-hashes" =>+ (for {+ body <- req.as[SourceHashRequest]+ hashes <- queryApi.getSourceHashes(body.uris)+ resp <- Ok(Json.fromFields(hashes.map { case (k, v) => k -> v.asJson }))+ } yield resp).handleErrorWith(ErrorHandler.handle(_))+ }+}