Skip to content

A fast, helpful, and open-source document parser

add workers to page processing

Logan Markewichc6f5d33b2b12
diff --git a/cli/parse.ts b/cli/parse.ts
index c144092..043b728 100644
--- a/cli/parse.ts
+++ b/cli/parse.ts
@@ -1,6 +1,7 @@
import { Command } from "commander";
import fs from "fs/promises";
import { existsSync, readdirSync, statSync } from "fs";
+import os from "os";
import path from "path";
import { LiteParse } from "../src/core/parser.js";
import { LiteParseConfig, OutputFormat } from "../src/core/types.js";
@@ -19,6 +20,7 @@ interface ParseCommandOptions {
ocrServerUrl?: string;
ocr?: boolean;
ocrLanguage?: string;
+ numWorkers?: string;
maxPages?: string;
targetPages?: string;
dpi?: string;
@@ -43,6 +45,7 @@ interface BatchParseCommandOptions {
ocrServerUrl?: string;
ocr?: boolean;
ocrLanguage?: string;
+ numWorkers?: string;
maxPages?: string;
dpi?: string;
preciseBbox?: boolean;
@@ -67,6 +70,7 @@ program
.option("--ocr-server-url <url>", "HTTP OCR server URL (uses Tesseract if not provided)")
.option("--no-ocr", "Disable OCR")
.option("--ocr-language <lang>", "OCR language(s)", DEFAULT_LANGUAGE)
+ .option("--num-workers <n>", "Number of pages to OCR in parallel. Defaults to number of CPU cores minus one.")
.option("--max-pages <n>", "Max pages to parse", DEFAULT_MAX_PAGES.toString())
.option("--target-pages <pages>", 'Target pages (e.g., "1-5,10,15-20")')
.option("--dpi <dpi>", "DPI for rendering", DEFAULT_DPI.toString())
@@ -104,6 +108,7 @@ program
ocrEnabled: options.ocr !== false,
ocrServerUrl: options.ocrServerUrl,
ocrLanguage: options.ocrLanguage,
+ numWorkers: parseInt(options.numWorkers || (os.cpus().length - 1).toString()),
maxPages: parseInt(options.maxPages || DEFAULT_MAX_PAGES.toString()),
targetPages: options.targetPages,
dpi: parseInt(options.dpi || DEFAULT_DPI.toString()),
@@ -280,6 +285,7 @@ program
.option("--ocr-server-url <url>", "HTTP OCR server URL (uses Tesseract if not provided)")
.option("--no-ocr", "Disable OCR")
.option("--ocr-language <lang>", "OCR language(s)", DEFAULT_LANGUAGE)
+ .option("--num-workers <n>", "Number of pages to OCR in parallel. Defaults to number of CPU cores minus one.")
.option("--max-pages <n>", "Max pages to parse per file", DEFAULT_MAX_PAGES.toString())
.option("--dpi <dpi>", "DPI for rendering", DEFAULT_DPI.toString())
.option("--no-precise-bbox", "Disable precise bounding boxes")
@@ -339,6 +345,7 @@ program
ocrEnabled: options.ocr !== false,
ocrServerUrl: options.ocrServerUrl,
ocrLanguage: options.ocrLanguage,
+ numWorkers: parseInt(options.numWorkers || (os.cpus().length - 1).toString()),
maxPages: parseInt(options.maxPages || DEFAULT_MAX_PAGES.toString()),
dpi: parseInt(options.dpi || DEFAULT_DPI.toString()),
preciseBoundingBox: options.preciseBbox !== false,
diff --git a/package.json b/package.json
index 80ada6b..6268724 100644
--- a/package.json
+++ b/package.json
@@ -38,6 +38,7 @@
"canvas": "^3.0.0",
"commander": "^12.0.0",
"form-data": "^4.0.0",
+ "p-limit": "^7.3.0",
"sharp": "^0.34.5",
"tesseract.js": "^5.0.0",
"unified": "^11.0.0",
diff --git a/src/core/config.ts b/src/core/config.ts
index 094550e..f48b101 100644
--- a/src/core/config.ts
+++ b/src/core/config.ts
@@ -6,6 +6,7 @@ export const DEFAULT_CONFIG: LiteParseConfig = {
ocrLanguage: "en",
ocrEnabled: true,
ocrServerUrl: undefined, // If set, uses HTTP OCR; otherwise uses Tesseract
+ numWorkers: 4, // Number of pages to OCR in parallel
// Processing
maxPages: 1000,
diff --git a/src/core/parser.ts b/src/core/parser.ts
index 4600272..b3d80f3 100644
--- a/src/core/parser.ts
+++ b/src/core/parser.ts
@@ -1,3 +1,4 @@
+import pLimit from "p-limit";
import { LiteParseConfig, ParseResult, ScreenshotResult, TextItem } from "./types.js";
import { mergeConfig } from "./config.js";
import { PdfEngine, PdfDocument, PageData } from "../engines/pdf/interface.js";
@@ -29,7 +30,7 @@ export class LiteParse {
if (this.config.ocrServerUrl) {
this.ocrEngine = new HttpOcrEngine(this.config.ocrServerUrl);
} else {
- this.ocrEngine = new TesseractEngine();
+ this.ocrEngine = new TesseractEngine(this.config.numWorkers);
}
}
}
@@ -186,7 +187,7 @@ export class LiteParse {
}
/**
- * Run OCR on pages that need it
+ * Run OCR on pages that need it (in parallel with concurrency limit)
*/
private async runOCR(
doc: PdfDocument,
@@ -195,137 +196,150 @@ export class LiteParse {
): Promise<void> {
if (!this.ocrEngine) return;
- log("Running OCR on pages...");
+ log(`Running OCR on pages (concurrency: ${this.config.numWorkers})...`);
- for (let i = 0; i < pages.length; i++) {
- const page = pages[i];
+ const limit = pLimit(this.config.numWorkers);
- // Check if page has very little text (indicating need for OCR)
- const textLength = page.textItems.reduce(
- (sum: number, item: TextItem) => sum + item.str.length,
- 0
- );
+ await Promise.all(
+ pages.map((page) => limit(() => this.processPageOcr(doc, page, log)))
+ );
+ }
- // Determine if OCR is needed and what mode
- const hasGarbledRegions = page.garbledTextRegions && page.garbledTextRegions.length > 0;
- const needsFullOcr = textLength < 100 || page.images.length > 0;
+ /**
+ * Process OCR for a single page
+ */
+ private async processPageOcr(
+ doc: PdfDocument,
+ page: PageData,
+ log: (msg: string) => void
+ ): Promise<void> {
+ if (!this.ocrEngine) return;
- if (!needsFullOcr && !hasGarbledRegions) {
- continue;
- }
+ // Check if page has very little text (indicating need for OCR)
+ const textLength = page.textItems.reduce(
+ (sum: number, item: TextItem) => sum + item.str.length,
+ 0
+ );
- try {
- // Render page as image
- const imageBuffer = await this.pdfEngine.renderPageImage(
- doc,
- page.pageNum,
- this.config.dpi
- );
-
- // Save temporary image file
- const fs = await import("fs/promises");
- const path = await import("path");
- const os = await import("os");
- const tmpDir = os.tmpdir();
- const tmpImagePath = path.join(tmpDir, `page_${page.pageNum}_ocr.png`);
- await fs.writeFile(tmpImagePath, imageBuffer);
-
- // Run OCR
- log(` OCR on page ${page.pageNum}...`);
- const ocrResults = await this.ocrEngine.recognize(tmpImagePath, {
- language: this.config.ocrLanguage,
- correctRotation: true,
- });
+ // Determine if OCR is needed and what mode
+ const hasGarbledRegions = page.garbledTextRegions && page.garbledTextRegions.length > 0;
+ const needsFullOcr = textLength < 100 || page.images.length > 0;
- // Convert OCR results to text items and add to page
- if (ocrResults.length > 0) {
- // Scale factor to convert from OCR pixels to PDF points
- // OCR operates at config.dpi, PDF uses 72 points per inch (PDF spec constant)
- const scaleFactor = 72 / this.config.dpi;
-
- // Helper to check if an OCR result overlaps with garbled regions
- const overlapsGarbledRegion = (ocrBbox: number[]): boolean => {
- if (!page.garbledTextRegions) return false;
-
- const ocrX = ocrBbox[0] * scaleFactor;
- const ocrY = ocrBbox[1] * scaleFactor;
- const ocrW = (ocrBbox[2] - ocrBbox[0]) * scaleFactor;
- const ocrH = (ocrBbox[3] - ocrBbox[1]) * scaleFactor;
-
- // Check overlap with any garbled region (with some tolerance)
- const tolerance = 5; // PDF points
- for (const region of page.garbledTextRegions) {
- const overlapX =
- ocrX < region.x + region.width + tolerance && ocrX + ocrW > region.x - tolerance;
- const overlapY =
- ocrY < region.y + region.height + tolerance && ocrY + ocrH > region.y - tolerance;
- if (overlapX && overlapY) {
- return true;
- }
- }
- return false;
- };
-
- // Helper to check if an OCR result spatially overlaps with existing PDF text
- // This prevents duplicating text that PDF already extracted correctly
- const overlapsExistingText = (ocrBbox: number[]): boolean => {
- const ocrX = ocrBbox[0] * scaleFactor;
- const ocrY = ocrBbox[1] * scaleFactor;
- const ocrW = (ocrBbox[2] - ocrBbox[0]) * scaleFactor;
- const ocrH = (ocrBbox[3] - ocrBbox[1]) * scaleFactor;
-
- const tolerance = 2; // PDF points - tighter tolerance for existing text
- for (const item of page.textItems) {
- const itemRight = item.x + (item.width || item.w || 0);
- const itemBottom = item.y + (item.height || item.h || 0);
-
- const overlapX = ocrX < itemRight + tolerance && ocrX + ocrW > item.x - tolerance;
- const overlapY = ocrY < itemBottom + tolerance && ocrY + ocrH > item.y - tolerance;
-
- if (overlapX && overlapY) {
- return true;
- }
+ if (!needsFullOcr && !hasGarbledRegions) {
+ return;
+ }
+
+ try {
+ // Render page as image
+ const imageBuffer = await this.pdfEngine.renderPageImage(
+ doc,
+ page.pageNum,
+ this.config.dpi
+ );
+
+ // Save temporary image file
+ const fs = await import("fs/promises");
+ const path = await import("path");
+ const os = await import("os");
+ const tmpDir = os.tmpdir();
+ const tmpImagePath = path.join(tmpDir, `page_${page.pageNum}_ocr.png`);
+ await fs.writeFile(tmpImagePath, imageBuffer);
+
+ // Run OCR
+ log(` OCR on page ${page.pageNum}...`);
+ const ocrResults = await this.ocrEngine.recognize(tmpImagePath, {
+ language: this.config.ocrLanguage,
+ correctRotation: true,
+ });
+
+ // Convert OCR results to text items and add to page
+ if (ocrResults.length > 0) {
+ // Scale factor to convert from OCR pixels to PDF points
+ // OCR operates at config.dpi, PDF uses 72 points per inch (PDF spec constant)
+ const scaleFactor = 72 / this.config.dpi;
+
+ // Helper to check if an OCR result overlaps with garbled regions
+ const overlapsGarbledRegion = (ocrBbox: number[]): boolean => {
+ if (!page.garbledTextRegions) return false;
+
+ const ocrX = ocrBbox[0] * scaleFactor;
+ const ocrY = ocrBbox[1] * scaleFactor;
+ const ocrW = (ocrBbox[2] - ocrBbox[0]) * scaleFactor;
+ const ocrH = (ocrBbox[3] - ocrBbox[1]) * scaleFactor;
+
+ // Check overlap with any garbled region (with some tolerance)
+ const tolerance = 5; // PDF points
+ for (const region of page.garbledTextRegions) {
+ const overlapX =
+ ocrX < region.x + region.width + tolerance && ocrX + ocrW > region.x - tolerance;
+ const overlapY =
+ ocrY < region.y + region.height + tolerance && ocrY + ocrH > region.y - tolerance;
+ if (overlapX && overlapY) {
+ return true;
}
- return false;
- };
-
- const ocrTextItems: TextItem[] = ocrResults
- .filter((r) => r.confidence > 0.1) // Filter low confidence
- .filter((r) => {
- // For targeted OCR (garbled regions only), only include results that overlap
- if (hasGarbledRegions && !needsFullOcr) {
- return overlapsGarbledRegion(r.bbox);
- }
- // For full OCR, include all results
+ }
+ return false;
+ };
+
+ // Helper to check if an OCR result spatially overlaps with existing PDF text
+ // This prevents duplicating text that PDF already extracted correctly
+ const overlapsExistingText = (ocrBbox: number[]): boolean => {
+ const ocrX = ocrBbox[0] * scaleFactor;
+ const ocrY = ocrBbox[1] * scaleFactor;
+ const ocrW = (ocrBbox[2] - ocrBbox[0]) * scaleFactor;
+ const ocrH = (ocrBbox[3] - ocrBbox[1]) * scaleFactor;
+
+ const tolerance = 2; // PDF points - tighter tolerance for existing text
+ for (const item of page.textItems) {
+ const itemRight = item.x + (item.width || item.w || 0);
+ const itemBottom = item.y + (item.height || item.h || 0);
+
+ const overlapX = ocrX < itemRight + tolerance && ocrX + ocrW > item.x - tolerance;
+ const overlapY = ocrY < itemBottom + tolerance && ocrY + ocrH > item.y - tolerance;
+
+ if (overlapX && overlapY) {
return true;
- })
- .filter((r) => {
- // Skip OCR results that spatially overlap with existing PDF text
- // This prevents duplicating text that PDF already extracted correctly
- return !overlapsExistingText(r.bbox);
- })
- .map((r) => ({
- str: r.text,
- x: r.bbox[0] * scaleFactor,
- y: r.bbox[1] * scaleFactor,
- width: (r.bbox[2] - r.bbox[0]) * scaleFactor,
- height: (r.bbox[3] - r.bbox[1]) * scaleFactor,
- w: (r.bbox[2] - r.bbox[0]) * scaleFactor,
- h: (r.bbox[3] - r.bbox[1]) * scaleFactor,
- fontName: "OCR",
- fontSize: (r.bbox[3] - r.bbox[1]) * scaleFactor,
- }));
-
- // Add OCR text items directly to page textItems
- page.textItems.push(...ocrTextItems);
- log(` Found ${ocrTextItems.length} text items from OCR on page ${page.pageNum}`);
- }
-
- // Clean up temp file
- await fs.unlink(tmpImagePath).catch(() => {});
- } catch (error) {
- log(` OCR failed for page ${page.pageNum}: ${error}`);
+ }
+ }
+ return false;
+ };
+
+ const ocrTextItems: TextItem[] = ocrResults
+ .filter((r) => r.confidence > 0.1) // Filter low confidence
+ .filter((r) => {
+ // For targeted OCR (garbled regions only), only include results that overlap
+ if (hasGarbledRegions && !needsFullOcr) {
+ return overlapsGarbledRegion(r.bbox);
+ }
+ // For full OCR, include all results
+ return true;
+ })
+ .filter((r) => {
+ // Skip OCR results that spatially overlap with existing PDF text
+ // This prevents duplicating text that PDF already extracted correctly
+ return !overlapsExistingText(r.bbox);
+ })
+ .map((r) => ({
+ str: r.text,
+ x: r.bbox[0] * scaleFactor,
+ y: r.bbox[1] * scaleFactor,
+ width: (r.bbox[2] - r.bbox[0]) * scaleFactor,
+ height: (r.bbox[3] - r.bbox[1]) * scaleFactor,
+ w: (r.bbox[2] - r.bbox[0]) * scaleFactor,
+ h: (r.bbox[3] - r.bbox[1]) * scaleFactor,
+ fontName: "OCR",
+ fontSize: (r.bbox[3] - r.bbox[1]) * scaleFactor,
+ }));
+
+ // Add OCR text items directly to page textItems
+ page.textItems.push(...ocrTextItems);
+ log(` Found ${ocrTextItems.length} text items from OCR on page ${page.pageNum}`);
}
+
+ // Clean up temp file
+ await fs.unlink(tmpImagePath).catch(() => {});
+ } catch (error) {
+ log(` OCR failed for page ${page.pageNum}: ${error}`);
}
}
diff --git a/src/core/types.ts b/src/core/types.ts
index 56271df..10232d3 100644
--- a/src/core/types.ts
+++ b/src/core/types.ts
@@ -8,6 +8,9 @@ export interface LiteParseConfig {
// HTTP OCR Server (optional - if not provided, uses Tesseract)
ocrServerUrl?: string;
+ // page processing concurrency (number of pages to process in parallel per document)
+ numWorkers: number;
+
// Processing
maxPages: number;
targetPages?: string;
diff --git a/src/engines/ocr/tesseract.ts b/src/engines/ocr/tesseract.ts
index 793d76a..77b75fc 100644
--- a/src/engines/ocr/tesseract.ts
+++ b/src/engines/ocr/tesseract.ts
@@ -1,22 +1,35 @@
-import { createWorker, Worker } from "tesseract.js";
+import { createWorker, createScheduler, Scheduler, Worker } from "tesseract.js";
import { OcrEngine, OcrOptions, OcrResult } from "./interface.js";
export class TesseractEngine implements OcrEngine {
name = "tesseract";
- private worker?: Worker;
+ private scheduler?: Scheduler;
+ private workers: Worker[] = [];
private currentLanguage?: string;
+ private concurrency: number;
+
+ constructor(concurrency: number = 4) {
+ this.concurrency = concurrency;
+ }
async initialize(language: string = "eng"): Promise<void> {
- if (this.worker && this.currentLanguage === language) {
+ if (this.scheduler && this.currentLanguage === language) {
return; // Already initialized for this language
}
- // Clean up existing worker if language changed
- if (this.worker) {
- await this.worker.terminate();
+ // Clean up existing scheduler and workers if language changed
+ await this.terminate();
+
+ // Create scheduler
+ this.scheduler = createScheduler();
+
+ // Create worker pool
+ for (let i = 0; i < this.concurrency; i++) {
+ const worker = await createWorker(language, 1);
+ this.workers.push(worker);
+ this.scheduler.addWorker(worker);
}
- this.worker = await createWorker(language, 1);
this.currentLanguage = language;
}
@@ -26,18 +39,18 @@ export class TesseractEngine implements OcrEngine {
Array.isArray(options.language) ? options.language[0] : options.language
);
- // Initialize worker if needed
+ // Initialize scheduler if needed
await this.initialize(language);
- if (!this.worker) {
- throw new Error("Tesseract worker not initialized");
+ if (!this.scheduler) {
+ throw new Error("Tesseract scheduler not initialized");
}
try {
- // Recognize text from image
+ // Recognize text from image using scheduler
const {
data: { words },
- } = await this.worker.recognize(imagePath);
+ } = await this.scheduler.addJob("recognize", imagePath);
// Convert to our OcrResult format
const results: OcrResult[] = words.map((word) => ({
@@ -60,23 +73,33 @@ export class TesseractEngine implements OcrEngine {
}
async recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]> {
- const results: OcrResult[][] = [];
+ // Handle language
+ const language = this.normalizeLanguage(
+ Array.isArray(options.language) ? options.language[0] : options.language
+ );
+
+ // Initialize scheduler if needed
+ await this.initialize(language);
- for (let i = 0; i < imagePaths.length; i++) {
- const imagePath = imagePaths[i];
- const result = await this.recognize(imagePath, options);
- results.push(result);
+ if (!this.scheduler) {
+ throw new Error("Tesseract scheduler not initialized");
}
- return results;
+ // Process all images in parallel - scheduler handles distribution
+ const jobs = imagePaths.map((imagePath) =>
+ this.recognize(imagePath, options)
+ );
+
+ return Promise.all(jobs);
}
async terminate(): Promise<void> {
- if (this.worker) {
- await this.worker.terminate();
- this.worker = undefined;
- this.currentLanguage = undefined;
+ if (this.scheduler) {
+ await this.scheduler.terminate();
+ this.scheduler = undefined;
}
+ this.workers = [];
+ this.currentLanguage = undefined;
}
/**