Skip to content

A fast, helpful, and open-source document parser

feat(node): add bounded-memory batch parsing

Pierre-Loic doulcet3ee4ec376f0a
diff --git a/packages/node/README.md b/packages/node/README.md
index 1616631..4b376a8 100644
--- a/packages/node/README.md
+++ b/packages/node/README.md
@@ -26,6 +26,21 @@ for (const page of result.pages) {
}
```
+### Bounded-memory parsing
+
+For documents with many text items, consume page batches without retaining
+earlier results:
+
+```typescript
+const parser = new LiteParse();
+for await (const batch of parser.parseBatches('large.pdf', { batchSize: 20 })) {
+ await processPages(batch.result.pages);
+}
+```
+
+The document is reopened for each batch. Header/footer detection and image
+deduplication are therefore batch-local.
+
## Markdown Output
LiteParse can render documents directly to Markdown including headings, tables, lists,
diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts
index e130926..e854f1b 100644
--- a/packages/node/src/lib.ts
+++ b/packages/node/src/lib.ts
@@ -371,6 +371,21 @@ export interface ParseResult {
xfaPackets?: XfaPacket[];
}
+export interface ParseBatch {
+ /** First source page in this batch (1-indexed). */
+ startPage: number;
+ /** Last source page requested for this batch (1-indexed). */
+ endPage: number;
+ /** Total source-document pages, before the parser's `maxPages` cap. */
+ totalPages: number;
+ result: ParseResult;
+}
+
+export interface ParseBatchOptions {
+ /** Maximum pages materialized in one result. Default: 25. */
+ batchSize?: number;
+}
+
/** Provenance and tamper-analysis facts extracted from the source PDF. */
export interface DocumentMetadata {
creationDate?: string;
@@ -623,6 +638,66 @@ export class LiteParse {
};
}
+ /**
+ * Parse a document in bounded-memory page batches.
+ *
+ * Each yielded result is independent and becomes collectible when the
+ * caller advances the iterator. The document is reopened for each batch;
+ * this trades a small amount of repeated setup for a bounded JS object
+ * graph on documents with many text items. Do not retain prior results if
+ * bounded memory is required.
+ *
+ * Batch-local layout operations (including repeated header/footer removal
+ * and image deduplication) do not see pages outside their batch. A parser
+ * configured with `targetPages` is rejected because its selection would be
+ * ambiguous with the ranges generated here.
+ */
+ async *parseBatches(
+ input: LiteParseInput,
+ options: ParseBatchOptions = {},
+ ): AsyncGenerator<ParseBatch> {
+ if (this._config.targetPages != null) {
+ throw new Error(
+ "parseBatches cannot be used when targetPages is configured",
+ );
+ }
+ const batchSize = options.batchSize ?? 25;
+ if (
+ !Number.isSafeInteger(batchSize) ||
+ batchSize <= 0 ||
+ batchSize > 100_000
+ ) {
+ throw new RangeError("batchSize must be an integer between 1 and 100000");
+ }
+
+ let startPage = 1;
+ let pageLimit = this._config.maxPages;
+ while (startPage <= pageLimit) {
+ const requestedEndPage = Math.min(
+ startPage + batchSize - 1,
+ pageLimit,
+ );
+ const batchParser = new LiteParse({
+ ...this._config,
+ maxPages: batchSize,
+ targetPages: `${startPage}-${requestedEndPage}`,
+ });
+ const result = await batchParser.parse(input);
+ if (!Number.isSafeInteger(result.totalPages)) {
+ throw new Error(
+ "parseBatches requires a native LiteParse binary that exposes totalPages",
+ );
+ }
+ pageLimit = Math.min(result.totalPages, this._config.maxPages);
+ if (result.pages.length === 0) {
+ return;
+ }
+ const endPage = Math.min(requestedEndPage, pageLimit);
+ yield { startPage, endPage, totalPages: result.totalPages, result };
+ startPage = endPage + 1;
+ }
+ }
+
/**
* Parse from pre-extracted pages, skipping PDFium text extraction. Runs only
* grid projection + the configured output formatter, so the caller's own