Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: speed up ingestion with change detection fix and bulk commits

Tanner Kocherb05716746a3e
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 206987d3..aa0c866a 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -69,10 +69,11 @@ export function registerIngestCommand(program: Command): void {
.option('--force', 'Force re-ingest even if files are unchanged (useful after parser upgrades)')
.option('--format <fmt>', 'Output format (text|json)', 'text')
.option('--root <dir>', 'Workspace root directory')
+ .option('--debug', 'Show phase timing breakdown', false)
.addHelpText('after', '\nExamples:\n ix ingest ./src\n ix ingest --path ./src --force\n ix ingest --github owner/repo\n ix ingest --github owner/repo --since 2026-01-01 --limit 20 --format json\n ix ingest --github owner/repo --token ghp_xxxx')
.action(async (positionalPath: string | undefined, opts: {
path?: string; recursive?: boolean; force?: boolean; github?: string; token?: string;
- since?: string; limit: string; format: string; root?: string;
+ since?: string; limit: string; format: string; root?: string; debug?: boolean;
}) => {
const effectivePath = positionalPath ?? opts.path;
if (opts.github) {
@@ -92,13 +93,14 @@ export function registerIngestCommand(program: Command): void {
export async function ingestFiles(
path: string,
- opts: { recursive?: boolean; force?: boolean; format: string; root?: string }
+ opts: { recursive?: boolean; force?: boolean; format: string; root?: string; debug?: boolean }
): Promise<void> {
const [{ parseFile, resolveEdges, isGrammarSupported }, { buildPatchWithResolution }] = await loadIngestionModules();
const resolvedPath = nodePath.isAbsolute(path)
? path
: nodePath.resolve(resolveWorkspaceRoot(opts.root), path);
+ const debug = opts.debug || process.env.IX_DEBUG === '1';
const client = new IxClient(getEndpoint());
const start = performance.now();
@@ -110,6 +112,7 @@ export async function ingestFiles(
}, 200) : null;
let filesDiscovered = 0;
+ let filesChanged = 0;
let patchesApplied = 0;
let filesSkipped = 0;
let parseErrors = 0;
@@ -117,19 +120,26 @@ export async function ingestFiles(
let latestRev = 0;
let entitiesParsed = 0;
+ // Phase timing marks
+ const timings = { discoverMs: 0, hashMs: 0, parseMs: 0, resolveMs: 0, commitMs: 0 };
+
try {
- // Collect files
+ // Phase: discover files
const stat = fs.statSync(resolvedPath);
const filePaths: string[] = stat.isFile()
? [resolvedPath]
: Array.from(walkFiles(resolvedPath, opts.recursive ?? true, isGrammarSupported));
filesDiscovered = filePaths.length;
+ const discovered = performance.now();
+ timings.discoverMs = Math.round(discovered - start);
- // Load existing source hashes for change detection (unless --force)
- const knownHashes = opts.force ? new Map<string, string>() : await loadExistingHashes(client, filePaths);
+ // Phase: hash lookup
+ const knownHashes = opts.force ? new Map<string, string>() : await loadExistingHashes(client, filePaths, debug);
+ const hashed = performance.now();
+ timings.hashMs = Math.round(hashed - discovered);
- // Phase 1: parse all files
+ // Phase: parse all files
type ParsedFile = { filePath: string; parsed: any; hash: string; previousHash: string | undefined };
const parsedFiles: ParsedFile[] = [];
@@ -164,21 +174,49 @@ export async function ingestFiles(
}
}
- // Phase 2: cross-file CALLS + EXTENDS resolution over the full batch
+ filesChanged = parsedFiles.length;
+ const parsed = performance.now();
+ timings.parseMs = Math.round(parsed - hashed);
+
+ // Phase: cross-file CALLS + EXTENDS resolution over the full batch
const resolvedEdges = resolveEdges(parsedFiles.map(f => f.parsed));
+ const resolved = performance.now();
+ timings.resolveMs = Math.round(resolved - parsed);
- // Phase 3: build and commit patches
- for (const { parsed, hash, previousHash } of parsedFiles) {
+ // Phase: build patches and bulk commit
+ const patches: GraphPatchPayload[] = [];
+ for (const { parsed: p, hash, previousHash } of parsedFiles) {
try {
- const patch = buildPatchWithResolution(parsed, hash, resolvedEdges, previousHash);
- const result = await client.commitPatch(patch);
- if (result.rev > latestRev) latestRev = result.rev;
- patchesApplied++;
+ patches.push(buildPatchWithResolution(p, hash, resolvedEdges, previousHash));
} catch (err) {
parseErrors++;
- process.stderr.write(`\n [commit error] ${parsed.filePath}: ${err}\n`);
+ process.stderr.write(`\n [patch build error] ${p.filePath}: ${err}\n`);
+ }
+ }
+
+ if (patches.length > 0) {
+ try {
+ const result = await client.commitPatchBulk(patches);
+ latestRev = result.rev;
+ patchesApplied = patches.length;
+ } catch (err) {
+ // Fallback: try per-file commits if bulk endpoint fails
+ if (debug) process.stderr.write(`\n [bulk commit failed, falling back to per-file] ${err}\n`);
+ for (const patch of patches) {
+ try {
+ const result = await client.commitPatch(patch);
+ if (result.rev > latestRev) latestRev = result.rev;
+ patchesApplied++;
+ } catch (commitErr) {
+ parseErrors++;
+ process.stderr.write(`\n [commit error] ${patch.source?.uri}: ${commitErr}\n`);
+ }
+ }
}
}
+
+ const committed = performance.now();
+ timings.commitMs = Math.round(committed - resolved);
} finally {
if (interval) {
clearInterval(interval);
@@ -191,17 +229,20 @@ export async function ingestFiles(
if (opts.format === 'json') {
console.log(JSON.stringify({
filesProcessed: filesDiscovered,
+ filesChanged,
patchesApplied,
filesSkipped,
entitiesParsed,
latestRev,
skipReasons: { unchanged: filesSkipped, emptyFile: 0, parseError: parseErrors, tooLarge },
elapsedSeconds: parseFloat(elapsed),
+ timings,
}, null, 2));
} else {
console.log(chalk.bold('\nIngest summary'));
console.log(` processed: ${patchesApplied} files (${elapsed}s)`);
console.log(` discovered: ${filesDiscovered} files`);
+ console.log(` changed: ${filesChanged} files`);
if (filesSkipped > 0) console.log(` ${chalk.dim('skipped unchanged:')} ${filesSkipped}`);
if (parseErrors > 0) console.log(` ${chalk.red('parse errors:')} ${parseErrors}`);
if (tooLarge > 0) console.log(` ${chalk.dim('skipped too large:')} ${tooLarge}`);
@@ -212,6 +253,16 @@ export async function ingestFiles(
} else if (patchesApplied === 0 && filesDiscovered > 0) {
console.log(chalk.dim('\n All files unchanged since last ingest.'));
}
+
+ if (debug) {
+ console.log(chalk.dim(`\n Phase timings:`));
+ console.log(chalk.dim(` discover: ${timings.discoverMs}ms`));
+ console.log(chalk.dim(` hash: ${timings.hashMs}ms`));
+ console.log(chalk.dim(` parse: ${timings.parseMs}ms`));
+ console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));
+ console.log(chalk.dim(` commit: ${timings.commitMs}ms`));
+ }
+
console.log();
}
}
@@ -220,11 +271,11 @@ export async function ingestFiles(
// Load existing hashes from the server for change detection
// ---------------------------------------------------------------------------
-async function loadExistingHashes(client: IxClient, filePaths: string[]): Promise<Map<string, string>> {
+async function loadExistingHashes(client: IxClient, filePaths: string[], debug = false): Promise<Map<string, string>> {
try {
- const hashes = await (client as any).getSourceHashes(filePaths);
- return new Map(Object.entries(hashes ?? {}));
- } catch {
+ return await client.getSourceHashes(filePaths);
+ } catch (err) {
+ if (debug) process.stderr.write(`\n [hash lookup failed] ${err}\n`);
return new Map();
}
}
diff --git a/ix-cli/src/client/api.ts b/ix-cli/src/client/api.ts
index 0af60ea7..5c9e36ed 100644
--- a/ix-cli/src/client/api.ts
+++ b/ix-cli/src/client/api.ts
@@ -174,6 +174,15 @@ export class IxClient {
return this.post("/v1/patch", patch);
}
+ async getSourceHashes(filePaths: string[]): Promise<Map<string, string>> {
+ const result = await this.post<Record<string, string>>('/v1/source-hashes', { uris: filePaths });
+ return new Map(Object.entries(result));
+ }
+
+ async commitPatchBulk(patches: GraphPatchPayload[]): Promise<PatchCommitResult> {
+ return this.post<PatchCommitResult>('/v1/patches/bulk', { patches });
+ }
+
async map(): Promise<any> {
return this.post("/v1/map", {});
}
diff --git a/memory-layer/src/main/scala/ix/memory/Main.scala b/memory-layer/src/main/scala/ix/memory/Main.scala
index 11cd8b12..6e3366bc 100644
--- a/memory-layer/src/main/scala/ix/memory/Main.scala
+++ b/memory-layer/src/main/scala/ix/memory/Main.scala
@@ -46,7 +46,7 @@ object Main extends IOApp.Simple {
mapService = new MapService(client, queryApi, writeApi)
// 4. HTTP routes
- routes = Routes.all(contextService, ingestionService, bulkIngestionService, queryApi, writeApi, conflictService, client, mapService)
+ routes = Routes.all(contextService, ingestionService, bulkIngestionService, queryApi, writeApi, conflictService, client, mapService, bulkWriteApi)
// 5. Server
server <- EmberServerBuilder
diff --git a/memory-layer/src/main/scala/ix/memory/api/BulkPatchCommitRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/BulkPatchCommitRoutes.scala
new file mode 100644
index 00000000..3e1812e2
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/api/BulkPatchCommitRoutes.scala
@@ -0,0 +1,55 @@
+package ix.memory.api
+
+import cats.effect.IO
+import io.circe.{Decoder, Json}
+import io.circe.generic.semiauto.deriveDecoder
+import io.circe.syntax._
+import org.http4s.HttpRoutes
+import org.http4s.circe.CirceEntityCodec._
+import org.http4s.dsl.io._
+
+import ix.memory.db.{BulkWriteApi, FileBatch, GraphQueryApi}
+import ix.memory.model.GraphPatch
+
+class BulkPatchCommitRoutes(bulkWriteApi: BulkWriteApi, queryApi: GraphQueryApi) {
+
+ private case class BulkPatchRequest(patches: List[GraphPatch])
+ private implicit val reqDecoder: Decoder[BulkPatchRequest] = deriveDecoder
+
+ private val mapper = new com.fasterxml.jackson.databind.ObjectMapper()
+
+ val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {
+ case req @ POST -> Root / "v1" / "patches" / "bulk" =>
+ (for {
+ body <- req.as[BulkPatchRequest]
+ _ <- if (body.patches.isEmpty)
+ IO.raiseError(new IllegalArgumentException("patches array must not be empty"))
+ else IO.unit
+ fileBatches = body.patches.map { patch =>
+ FileBatch(
+ filePath = patch.source.uri,
+ sourceHash = patch.source.sourceHash,
+ patch = patch,
+ provenance = buildProvenanceMap(patch)
+ )
+ }.toVector
+ latestRev <- queryApi.getLatestRev
+ result <- bulkWriteApi.commitBatchChunked(fileBatches, latestRev.value)
+ resp <- Ok(Json.obj(
+ "status" -> result.status.toString.asJson,
+ "rev" -> result.newRev.value.asJson
+ ))
+ } yield resp).handleErrorWith(ErrorHandler.handle(_))
+ }
+
+ private def buildProvenanceMap(patch: GraphPatch): java.util.Map[String, AnyRef] = {
+ val json = Json.obj(
+ "source_uri" -> Json.fromString(patch.source.uri),
+ "source_hash" -> patch.source.sourceHash.fold(Json.Null)(Json.fromString),
+ "extractor" -> Json.fromString(patch.source.extractor),
+ "source_type" -> Json.fromString(patch.source.sourceType.asJson.asString.getOrElse("code")),
+ "observed_at" -> Json.fromString(patch.timestamp.toString)
+ )
+ mapper.readValue(json.noSpaces, classOf[java.util.Map[String, AnyRef]])
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/api/Routes.scala b/memory-layer/src/main/scala/ix/memory/api/Routes.scala
index 94ff863d..638b714f 100644
--- a/memory-layer/src/main/scala/ix/memory/api/Routes.scala
+++ b/memory-layer/src/main/scala/ix/memory/api/Routes.scala
@@ -10,7 +10,7 @@ import org.http4s.dsl.io._
import ix.memory.conflict.ConflictService
import ix.memory.context.ContextService
-import ix.memory.db.{ArangoClient, GraphQueryApi, GraphWriteApi}
+import ix.memory.db.{ArangoClient, BulkWriteApi, GraphQueryApi, GraphWriteApi}
import ix.memory.ingestion.{BulkIngestionService, IngestionService}
import ix.memory.map.MapService
@@ -24,7 +24,8 @@ object Routes {
writeApi: GraphWriteApi,
conflictService: ConflictService,
client: ArangoClient,
- mapService: MapService
+ mapService: MapService,
+ bulkWriteApi: BulkWriteApi
): HttpRoutes[IO] = {
val health = HttpRoutes.of[IO] {
@@ -47,13 +48,15 @@ object Routes {
val statsRoutes = new StatsRoutes(client).routes
val patchCommitRoutes = new PatchCommitRoutes(writeApi).routes
val listRoutes = new ListRoutes(queryApi).routes
- val mapRoutes = new MapRoutes(mapService).routes
- val resetRoutes = new ResetRoutes(client).routes
+ val mapRoutes = new MapRoutes(mapService).routes
+ val resetRoutes = new ResetRoutes(client).routes
+ val sourceHashRoutes = new SourceHashRoutes(queryApi).routes
+ val bulkPatchCommitRoutes = new BulkPatchCommitRoutes(bulkWriteApi, queryApi).routes
health <+> contextRoutes <+> ingestionRoutes <+> entityRoutes <+>
diffRoutes <+> conflictRoutes <+> decideRoutes <+> searchRoutes <+>
truthRoutes <+> patchRoutes <+> decisionListRoutes <+> expandRoutes <+>
statsRoutes <+> patchCommitRoutes <+> listRoutes <+> goalRoutes <+>
- mapRoutes <+> resetRoutes
+ mapRoutes <+> resetRoutes <+> sourceHashRoutes <+> bulkPatchCommitRoutes
}
}
diff --git a/memory-layer/src/main/scala/ix/memory/api/SourceHashRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/SourceHashRoutes.scala
new file mode 100644
index 00000000..dd5e8ea5
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/api/SourceHashRoutes.scala
@@ -0,0 +1,26 @@
+package ix.memory.api
+
+import cats.effect.IO
+import io.circe.{Decoder, Json}
+import io.circe.generic.semiauto.deriveDecoder
+import io.circe.syntax._
+import org.http4s.HttpRoutes
+import org.http4s.circe.CirceEntityCodec._
+import org.http4s.dsl.io._
+
+import ix.memory.db.GraphQueryApi
+
+class SourceHashRoutes(queryApi: GraphQueryApi) {
+
+ private case class SourceHashRequest(uris: List[String])
+ private implicit val reqDecoder: Decoder[SourceHashRequest] = deriveDecoder
+
+ val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {
+ case req @ POST -> Root / "v1" / "source-hashes" =>
+ (for {
+ body <- req.as[SourceHashRequest]
+ hashes <- queryApi.getSourceHashes(body.uris)
+ resp <- Ok(Json.fromFields(hashes.map { case (k, v) => k -> v.asJson }))
+ } yield resp).handleErrorWith(ErrorHandler.handle(_))
+ }
+}