Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat: add file size guard to prevent OOM on large files
riley02270bf1c27bae4d
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 0ac46e19..8b2e8c17 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -25,6 +25,7 @@ export function registerIngestCommand(program: Command): void {if (sr.unchanged > 0) console.log(` ${chalk.dim("skipped unchanged:")} ${sr.unchanged}`);if (sr.emptyFile > 0) console.log(` ${chalk.dim("skipped empty:")} ${sr.emptyFile}`);if (sr.parseError > 0) console.log(` ${chalk.red("parse errors:")} ${sr.parseError}`);+ if (sr.tooLarge > 0) console.log(` ${chalk.dim("skipped too large:")} ${sr.tooLarge}`);} else if (result.filesSkipped) {console.log(` ${chalk.dim("skipped:")} ${result.filesSkipped} unchanged files`);}diff --git a/ix-cli/src/client/types.ts b/ix-cli/src/client/types.tsindex 10fd41b7..f2999e0d 100644--- a/ix-cli/src/client/types.ts+++ b/ix-cli/src/client/types.ts@@ -135,6 +135,7 @@ export interface SkipReasons {unchanged: number;emptyFile: number;parseError: number;+ tooLarge: number;}export interface IngestResult {diff --git a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scalaindex f4c66272..e7fde457 100644--- a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala+++ b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala@@ -18,7 +18,7 @@ object IngestRequest {implicit val encoder: Encoder[IngestRequest] = deriveEncoder[IngestRequest]}-case class SkipReasonsResponse(unchanged: Int, emptyFile: Int, parseError: Int)+case class SkipReasonsResponse(unchanged: Int, emptyFile: Int, parseError: Int, tooLarge: Int)object SkipReasonsResponse {implicit val encoder: Encoder[SkipReasonsResponse] = deriveEncoder[SkipReasonsResponse]@@ -53,7 +53,8 @@ class IngestionRoutes(ingestionService: IngestionService, bulkIngestionService:skipReasons = SkipReasonsResponse(result.skipReasons.unchanged,result.skipReasons.emptyFile,- result.skipReasons.parseError+ result.skipReasons.parseError,+ result.skipReasons.tooLarge)))} yield resp).handleErrorWith(ErrorHandler.handle(_))diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scalaindex f863e2ca..9dd1819d 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala@@ -29,6 +29,7 @@ class BulkIngestionService() {private val parallelism = Runtime.getRuntime.availableProcessors.max(2)+ private val MaxFileBytes: Long = 1024 * 1024 // 1 MBprivate val mapper = new com.fasterxml.jackson.databind.ObjectMapper()/**@@ -49,7 +50,8 @@ class BulkIngestionService(unchangedCount = batches.count { case Right(Skipped("unchanged")) => true; case _ => false }emptyCount = batches.count { case Right(Skipped("emptyFile")) => true; case _ => false }errorCount = batches.count { case Left(_) => true; case _ => false }- skippedCount = unchangedCount + emptyCount + errorCount+ tooLargeCount = batches.count { case Right(Skipped("tooLarge")) => true; case _ => false }+ skippedCount = unchangedCount + emptyCount + errorCount + tooLargeCountlatestRev <- queryApi.getLatestRevresult <- if (validBatches.isEmpty) IO.pure(CommitResult(latestRev, CommitStatus.Ok))else bulkWriteApi.commitBatch(validBatches.toVector, latestRev.value)@@ -59,7 +61,7 @@ class BulkIngestionService(filesSkipped = skippedCount,entitiesCreated = validBatches.flatMap(_.patch.ops.collect { case _: PatchOp.UpsertNode => 1 }).size,latestRev = result.newRev,- skipReasons = SkipReasons(unchanged = unchangedCount, emptyFile = emptyCount, parseError = errorCount)+ skipReasons = SkipReasons(unchanged = unchangedCount, emptyFile = emptyCount, parseError = errorCount, tooLarge = tooLargeCount))}@@ -68,30 +70,34 @@ class BulkIngestionService(*/private def parseFile(filePath: Path, hashMap: Map[String, String]): IO[ParseOutcome] = {for {- bytes <- IO.blocking(Files.readAllBytes(filePath))- result <- if (bytes.isEmpty) IO.pure(Skipped("emptyFile"): ParseOutcome)- else {- val hash = sha256Bytes(bytes)- hashMap.get(filePath.toString) match {- case Some(existingHash) if existingHash == hash =>- IO.pure(Skipped("unchanged"): ParseOutcome)- case _ =>- for {- source <- IO.blocking {- try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)- catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }- }- parserOpt = parserRouter.parserFor(filePath.toString)- parseResult = parserOpt match {- case Some(p) => p.parse(filePath.getFileName.toString, source)- case None => genericTextParse(filePath.getFileName.toString, source)- }- patch = GraphPatchBuilder.build(filePath.toString, Some(hash), parseResult)- _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))- provenance = buildProvenanceMap(patch)- } yield Parsed(FileBatch(filePath.toString, Some(hash), patch, provenance)): ParseOutcome+ fileSize <- IO.blocking(Files.size(filePath))+ result <- if (fileSize > MaxFileBytes) IO.pure(Skipped("tooLarge"): ParseOutcome)+ else if (fileSize == 0L) IO.pure(Skipped("emptyFile"): ParseOutcome)+ else {+ for {+ bytes <- IO.blocking(Files.readAllBytes(filePath))+ hash = sha256Bytes(bytes)+ outcome <- hashMap.get(filePath.toString) match {+ case Some(existingHash) if existingHash == hash =>+ IO.pure(Skipped("unchanged"): ParseOutcome)+ case _ =>+ for {+ source <- IO.blocking {+ try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)+ catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }+ }+ parserOpt = parserRouter.parserFor(filePath.toString)+ parseResult = parserOpt match {+ case Some(p) => p.parse(filePath.getFileName.toString, source)+ case None => genericTextParse(filePath.getFileName.toString, source)+ }+ patch = GraphPatchBuilder.build(filePath.toString, Some(hash), parseResult)+ _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))+ provenance = buildProvenanceMap(patch)+ } yield Parsed(FileBatch(filePath.toString, Some(hash), patch, provenance)): ParseOutcome+ }+ } yield outcome}- }} yield result}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scalaindex 49dce4f8..3a349a7c 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala@@ -188,7 +188,8 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quercase class SkipReasons(unchanged: Int = 0,emptyFile: Int = 0,- parseError: Int = 0+ parseError: Int = 0,+ tooLarge: Int = 0)case class IngestionResult(