Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: progress reporting during ingestion

riley02273480aff98e9a
diff --git a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
index e7fde457..b2334c56 100644
--- a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
+++ b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
@@ -9,7 +9,8 @@ import org.http4s.HttpRoutes
import org.http4s.circe.CirceEntityCodec._
import org.http4s.dsl.io._
-import ix.memory.ingestion.{BulkIngestionService, IngestionService}
+import ix.memory.ingestion.{BulkIngestionService, IngestionProgress, IngestionService}
+import org.typelevel.log4cats.slf4j.Slf4jLogger
case class IngestRequest(path: String, language: Option[String], recursive: Option[Boolean])
@@ -35,6 +36,12 @@ object IngestResponse {
class IngestionRoutes(ingestionService: IngestionService, bulkIngestionService: BulkIngestionService) {
+ private val logger = Slf4jLogger.getLoggerFromName[IO]("ix.ingest.progress")
+
+ private val progressLog: IngestionProgress => IO[Unit] = { p =>
+ logger.info(s"discovered=${p.filesDiscovered} parsed=${p.filesParsed} chunks=${p.chunksWritten}/${p.totalChunks}")
+ }
+
val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {
case req @ POST -> Root / "v1" / "ingest" =>
(for {
@@ -43,7 +50,7 @@ class IngestionRoutes(ingestionService: IngestionService, bulkIngestionService:
new IllegalArgumentException("Path traversal not allowed")
)
path = Paths.get(body.path)
- result <- bulkIngestionService.ingestPath(path, body.language, body.recursive.getOrElse(false))
+ result <- bulkIngestionService.ingestPath(path, body.language, body.recursive.getOrElse(false), progressLog)
resp <- Ok(IngestResponse(
filesProcessed = result.filesProcessed,
patchesApplied = result.patchesApplied,
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
index 9558ddd1..f4991319 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
@@ -41,10 +41,12 @@ class BulkIngestionService(
def ingestPath(
path: Path,
language: Option[String],
- recursive: Boolean
+ recursive: Boolean,
+ onProgress: IngestionProgress => IO[Unit] = _ => IO.unit
): IO[IngestionResult] = {
for {
files <- discoverFiles(path, language, recursive)
+ _ <- onProgress(IngestionProgress(files.size, 0, 0, 0))
hashMap <- loadExistingHashes(files)
outcomes <- files.parTraverseN(parallelism) { f =>
parseFile(f, hashMap).handleErrorWith { err =>
@@ -58,9 +60,11 @@ class BulkIngestionService(
errorCount = outcomes.count { case Skipped("parseError") => true; case _ => false }
tooLargeCount = outcomes.count { case Skipped("tooLarge") => true; case _ => false }
skippedCount = unchangedCount + emptyCount + errorCount + tooLargeCount
+ totalChunks = if (validBatches.isEmpty) 0 else (validBatches.size + 99) / 100
+ _ <- onProgress(IngestionProgress(files.size, outcomes.size, 0, totalChunks))
latestRev <- queryApi.getLatestRev
result <- if (validBatches.isEmpty) IO.pure(CommitResult(latestRev, CommitStatus.Ok))
- else bulkWriteApi.commitBatchChunked(validBatches.toVector, latestRev.value)
+ else commitChunkedWithProgress(validBatches.toVector, latestRev.value, onProgress, files.size, outcomes.size)
} yield IngestionResult(
filesProcessed = files.size,
patchesApplied = validBatches.size,
@@ -71,6 +75,27 @@ class BulkIngestionService(
)
}
+ private def commitChunkedWithProgress(
+ batches: Vector[FileBatch],
+ baseRev: Long,
+ onProgress: IngestionProgress => IO[Unit],
+ discovered: Int,
+ parsed: Int
+ ): IO[CommitResult] = {
+ val chunkSize = 100
+ val chunks = batches.grouped(chunkSize).toVector
+ val totalChunks = chunks.size
+
+ chunks.zipWithIndex.foldLeft(IO.pure(baseRev)) { case (revIO, (chunk, idx)) =>
+ revIO.flatMap { currentRev =>
+ bulkWriteApi.commitBatch(chunk, currentRev).flatMap { result =>
+ onProgress(IngestionProgress(discovered, parsed, idx + 1, totalChunks)) *>
+ IO.pure(result.newRev.value)
+ }
+ }
+ }.map(finalRev => CommitResult(Rev(finalRev), CommitStatus.Ok))
+ }
+
/**
* Parse a single file into a FileBatch (or None if unchanged).
*/
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
index 3a349a7c..885f40cc 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
@@ -185,6 +185,13 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quer
}
}
+case class IngestionProgress(
+ filesDiscovered: Int,
+ filesParsed: Int,
+ chunksWritten: Int,
+ totalChunks: Int
+)
+
case class SkipReasons(
unchanged: Int = 0,
emptyFile: Int = 0,