Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: add file size guard to prevent OOM on large files

riley02270bf1c27bae4d
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 0ac46e19..8b2e8c17 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -25,6 +25,7 @@ export function registerIngestCommand(program: Command): void {
if (sr.unchanged > 0) console.log(` ${chalk.dim("skipped unchanged:")} ${sr.unchanged}`);
if (sr.emptyFile > 0) console.log(` ${chalk.dim("skipped empty:")} ${sr.emptyFile}`);
if (sr.parseError > 0) console.log(` ${chalk.red("parse errors:")} ${sr.parseError}`);
+ if (sr.tooLarge > 0) console.log(` ${chalk.dim("skipped too large:")} ${sr.tooLarge}`);
} else if (result.filesSkipped) {
console.log(` ${chalk.dim("skipped:")} ${result.filesSkipped} unchanged files`);
}
diff --git a/ix-cli/src/client/types.ts b/ix-cli/src/client/types.ts
index 10fd41b7..f2999e0d 100644
--- a/ix-cli/src/client/types.ts
+++ b/ix-cli/src/client/types.ts
@@ -135,6 +135,7 @@ export interface SkipReasons {
unchanged: number;
emptyFile: number;
parseError: number;
+ tooLarge: number;
}
export interface IngestResult {
diff --git a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
index f4c66272..e7fde457 100644
--- a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
+++ b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
@@ -18,7 +18,7 @@ object IngestRequest {
implicit val encoder: Encoder[IngestRequest] = deriveEncoder[IngestRequest]
}
-case class SkipReasonsResponse(unchanged: Int, emptyFile: Int, parseError: Int)
+case class SkipReasonsResponse(unchanged: Int, emptyFile: Int, parseError: Int, tooLarge: Int)
object SkipReasonsResponse {
implicit val encoder: Encoder[SkipReasonsResponse] = deriveEncoder[SkipReasonsResponse]
@@ -53,7 +53,8 @@ class IngestionRoutes(ingestionService: IngestionService, bulkIngestionService:
skipReasons = SkipReasonsResponse(
result.skipReasons.unchanged,
result.skipReasons.emptyFile,
- result.skipReasons.parseError
+ result.skipReasons.parseError,
+ result.skipReasons.tooLarge
)
))
} yield resp).handleErrorWith(ErrorHandler.handle(_))
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
index f863e2ca..9dd1819d 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
@@ -29,6 +29,7 @@ class BulkIngestionService(
) {
private val parallelism = Runtime.getRuntime.availableProcessors.max(2)
+ private val MaxFileBytes: Long = 1024 * 1024 // 1 MB
private val mapper = new com.fasterxml.jackson.databind.ObjectMapper()
/**
@@ -49,7 +50,8 @@ class BulkIngestionService(
unchangedCount = batches.count { case Right(Skipped("unchanged")) => true; case _ => false }
emptyCount = batches.count { case Right(Skipped("emptyFile")) => true; case _ => false }
errorCount = batches.count { case Left(_) => true; case _ => false }
- skippedCount = unchangedCount + emptyCount + errorCount
+ tooLargeCount = batches.count { case Right(Skipped("tooLarge")) => true; case _ => false }
+ skippedCount = unchangedCount + emptyCount + errorCount + tooLargeCount
latestRev <- queryApi.getLatestRev
result <- if (validBatches.isEmpty) IO.pure(CommitResult(latestRev, CommitStatus.Ok))
else bulkWriteApi.commitBatch(validBatches.toVector, latestRev.value)
@@ -59,7 +61,7 @@ class BulkIngestionService(
filesSkipped = skippedCount,
entitiesCreated = validBatches.flatMap(_.patch.ops.collect { case _: PatchOp.UpsertNode => 1 }).size,
latestRev = result.newRev,
- skipReasons = SkipReasons(unchanged = unchangedCount, emptyFile = emptyCount, parseError = errorCount)
+ skipReasons = SkipReasons(unchanged = unchangedCount, emptyFile = emptyCount, parseError = errorCount, tooLarge = tooLargeCount)
)
}
@@ -68,30 +70,34 @@ class BulkIngestionService(
*/
private def parseFile(filePath: Path, hashMap: Map[String, String]): IO[ParseOutcome] = {
for {
- bytes <- IO.blocking(Files.readAllBytes(filePath))
- result <- if (bytes.isEmpty) IO.pure(Skipped("emptyFile"): ParseOutcome)
- else {
- val hash = sha256Bytes(bytes)
- hashMap.get(filePath.toString) match {
- case Some(existingHash) if existingHash == hash =>
- IO.pure(Skipped("unchanged"): ParseOutcome)
- case _ =>
- for {
- source <- IO.blocking {
- try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)
- catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }
- }
- parserOpt = parserRouter.parserFor(filePath.toString)
- parseResult = parserOpt match {
- case Some(p) => p.parse(filePath.getFileName.toString, source)
- case None => genericTextParse(filePath.getFileName.toString, source)
- }
- patch = GraphPatchBuilder.build(filePath.toString, Some(hash), parseResult)
- _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))
- provenance = buildProvenanceMap(patch)
- } yield Parsed(FileBatch(filePath.toString, Some(hash), patch, provenance)): ParseOutcome
+ fileSize <- IO.blocking(Files.size(filePath))
+ result <- if (fileSize > MaxFileBytes) IO.pure(Skipped("tooLarge"): ParseOutcome)
+ else if (fileSize == 0L) IO.pure(Skipped("emptyFile"): ParseOutcome)
+ else {
+ for {
+ bytes <- IO.blocking(Files.readAllBytes(filePath))
+ hash = sha256Bytes(bytes)
+ outcome <- hashMap.get(filePath.toString) match {
+ case Some(existingHash) if existingHash == hash =>
+ IO.pure(Skipped("unchanged"): ParseOutcome)
+ case _ =>
+ for {
+ source <- IO.blocking {
+ try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)
+ catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }
+ }
+ parserOpt = parserRouter.parserFor(filePath.toString)
+ parseResult = parserOpt match {
+ case Some(p) => p.parse(filePath.getFileName.toString, source)
+ case None => genericTextParse(filePath.getFileName.toString, source)
+ }
+ patch = GraphPatchBuilder.build(filePath.toString, Some(hash), parseResult)
+ _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))
+ provenance = buildProvenanceMap(patch)
+ } yield Parsed(FileBatch(filePath.toString, Some(hash), patch, provenance)): ParseOutcome
+ }
+ } yield outcome
}
- }
} yield result
}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
index 49dce4f8..3a349a7c 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
@@ -188,7 +188,8 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quer
case class SkipReasons(
unchanged: Int = 0,
emptyFile: Int = 0,
- parseError: Int = 0
+ parseError: Int = 0,
+ tooLarge: Int = 0
)
case class IngestionResult(