Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat: progress reporting during ingestion
riley02273480aff98e9a
diff --git a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scalaindex e7fde457..b2334c56 100644--- a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala+++ b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala@@ -9,7 +9,8 @@ import org.http4s.HttpRoutesimport org.http4s.circe.CirceEntityCodec._import org.http4s.dsl.io._-import ix.memory.ingestion.{BulkIngestionService, IngestionService}+import ix.memory.ingestion.{BulkIngestionService, IngestionProgress, IngestionService}+import org.typelevel.log4cats.slf4j.Slf4jLoggercase class IngestRequest(path: String, language: Option[String], recursive: Option[Boolean])@@ -35,6 +36,12 @@ object IngestResponse {class IngestionRoutes(ingestionService: IngestionService, bulkIngestionService: BulkIngestionService) {+ private val logger = Slf4jLogger.getLoggerFromName[IO]("ix.ingest.progress")++ private val progressLog: IngestionProgress => IO[Unit] = { p =>+ logger.info(s"discovered=${p.filesDiscovered} parsed=${p.filesParsed} chunks=${p.chunksWritten}/${p.totalChunks}")+ }+val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {case req @ POST -> Root / "v1" / "ingest" =>(for {@@ -43,7 +50,7 @@ class IngestionRoutes(ingestionService: IngestionService, bulkIngestionService:new IllegalArgumentException("Path traversal not allowed"))path = Paths.get(body.path)- result <- bulkIngestionService.ingestPath(path, body.language, body.recursive.getOrElse(false))+ result <- bulkIngestionService.ingestPath(path, body.language, body.recursive.getOrElse(false), progressLog)resp <- Ok(IngestResponse(filesProcessed = result.filesProcessed,patchesApplied = result.patchesApplied,diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scalaindex 9558ddd1..f4991319 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala@@ -41,10 +41,12 @@ class BulkIngestionService(def ingestPath(path: Path,language: Option[String],- recursive: Boolean+ recursive: Boolean,+ onProgress: IngestionProgress => IO[Unit] = _ => IO.unit): IO[IngestionResult] = {for {files <- discoverFiles(path, language, recursive)+ _ <- onProgress(IngestionProgress(files.size, 0, 0, 0))hashMap <- loadExistingHashes(files)outcomes <- files.parTraverseN(parallelism) { f =>parseFile(f, hashMap).handleErrorWith { err =>@@ -58,9 +60,11 @@ class BulkIngestionService(errorCount = outcomes.count { case Skipped("parseError") => true; case _ => false }tooLargeCount = outcomes.count { case Skipped("tooLarge") => true; case _ => false }skippedCount = unchangedCount + emptyCount + errorCount + tooLargeCount+ totalChunks = if (validBatches.isEmpty) 0 else (validBatches.size + 99) / 100+ _ <- onProgress(IngestionProgress(files.size, outcomes.size, 0, totalChunks))latestRev <- queryApi.getLatestRevresult <- if (validBatches.isEmpty) IO.pure(CommitResult(latestRev, CommitStatus.Ok))- else bulkWriteApi.commitBatchChunked(validBatches.toVector, latestRev.value)+ else commitChunkedWithProgress(validBatches.toVector, latestRev.value, onProgress, files.size, outcomes.size)} yield IngestionResult(filesProcessed = files.size,patchesApplied = validBatches.size,@@ -71,6 +75,27 @@ class BulkIngestionService()}+ private def commitChunkedWithProgress(+ batches: Vector[FileBatch],+ baseRev: Long,+ onProgress: IngestionProgress => IO[Unit],+ discovered: Int,+ parsed: Int+ ): IO[CommitResult] = {+ val chunkSize = 100+ val chunks = batches.grouped(chunkSize).toVector+ val totalChunks = chunks.size++ chunks.zipWithIndex.foldLeft(IO.pure(baseRev)) { case (revIO, (chunk, idx)) =>+ revIO.flatMap { currentRev =>+ bulkWriteApi.commitBatch(chunk, currentRev).flatMap { result =>+ onProgress(IngestionProgress(discovered, parsed, idx + 1, totalChunks)) *>+ IO.pure(result.newRev.value)+ }+ }+ }.map(finalRev => CommitResult(Rev(finalRev), CommitStatus.Ok))+ }+/*** Parse a single file into a FileBatch (or None if unchanged).*/diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scalaindex 3a349a7c..885f40cc 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala@@ -185,6 +185,13 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quer}}+case class IngestionProgress(+ filesDiscovered: Int,+ filesParsed: Int,+ chunksWritten: Int,+ totalChunks: Int+)+case class SkipReasons(unchanged: Int = 0,emptyFile: Int = 0,