A fast, helpful, and open-source document parser
add workers to page processing
Logan Markewichc6f5d33b2b12
diff --git a/cli/parse.ts b/cli/parse.tsindex c144092..043b728 100644--- a/cli/parse.ts+++ b/cli/parse.ts@@ -1,6 +1,7 @@import { Command } from "commander";import fs from "fs/promises";import { existsSync, readdirSync, statSync } from "fs";+import os from "os";import path from "path";import { LiteParse } from "../src/core/parser.js";import { LiteParseConfig, OutputFormat } from "../src/core/types.js";@@ -19,6 +20,7 @@ interface ParseCommandOptions {ocrServerUrl?: string;ocr?: boolean;ocrLanguage?: string;+ numWorkers?: string;maxPages?: string;targetPages?: string;dpi?: string;@@ -43,6 +45,7 @@ interface BatchParseCommandOptions {ocrServerUrl?: string;ocr?: boolean;ocrLanguage?: string;+ numWorkers?: string;maxPages?: string;dpi?: string;preciseBbox?: boolean;@@ -67,6 +70,7 @@ program.option("--ocr-server-url <url>", "HTTP OCR server URL (uses Tesseract if not provided)").option("--no-ocr", "Disable OCR").option("--ocr-language <lang>", "OCR language(s)", DEFAULT_LANGUAGE)+ .option("--num-workers <n>", "Number of pages to OCR in parallel. Defaults to number of CPU cores minus one.").option("--max-pages <n>", "Max pages to parse", DEFAULT_MAX_PAGES.toString()).option("--target-pages <pages>", 'Target pages (e.g., "1-5,10,15-20")').option("--dpi <dpi>", "DPI for rendering", DEFAULT_DPI.toString())@@ -104,6 +108,7 @@ programocrEnabled: options.ocr !== false,ocrServerUrl: options.ocrServerUrl,ocrLanguage: options.ocrLanguage,+ numWorkers: parseInt(options.numWorkers || (os.cpus().length - 1).toString()),maxPages: parseInt(options.maxPages || DEFAULT_MAX_PAGES.toString()),targetPages: options.targetPages,dpi: parseInt(options.dpi || DEFAULT_DPI.toString()),@@ -280,6 +285,7 @@ program.option("--ocr-server-url <url>", "HTTP OCR server URL (uses Tesseract if not provided)").option("--no-ocr", "Disable OCR").option("--ocr-language <lang>", "OCR language(s)", DEFAULT_LANGUAGE)+ .option("--num-workers <n>", "Number of pages to OCR in parallel. Defaults to number of CPU cores minus one.").option("--max-pages <n>", "Max pages to parse per file", DEFAULT_MAX_PAGES.toString()).option("--dpi <dpi>", "DPI for rendering", DEFAULT_DPI.toString()).option("--no-precise-bbox", "Disable precise bounding boxes")@@ -339,6 +345,7 @@ programocrEnabled: options.ocr !== false,ocrServerUrl: options.ocrServerUrl,ocrLanguage: options.ocrLanguage,+ numWorkers: parseInt(options.numWorkers || (os.cpus().length - 1).toString()),maxPages: parseInt(options.maxPages || DEFAULT_MAX_PAGES.toString()),dpi: parseInt(options.dpi || DEFAULT_DPI.toString()),preciseBoundingBox: options.preciseBbox !== false,diff --git a/package.json b/package.jsonindex 80ada6b..6268724 100644--- a/package.json+++ b/package.json@@ -38,6 +38,7 @@"canvas": "^3.0.0","commander": "^12.0.0","form-data": "^4.0.0",+ "p-limit": "^7.3.0","sharp": "^0.34.5","tesseract.js": "^5.0.0","unified": "^11.0.0",diff --git a/src/core/config.ts b/src/core/config.tsindex 094550e..f48b101 100644--- a/src/core/config.ts+++ b/src/core/config.ts@@ -6,6 +6,7 @@ export const DEFAULT_CONFIG: LiteParseConfig = {ocrLanguage: "en",ocrEnabled: true,ocrServerUrl: undefined, // If set, uses HTTP OCR; otherwise uses Tesseract+ numWorkers: 4, // Number of pages to OCR in parallel// ProcessingmaxPages: 1000,diff --git a/src/core/parser.ts b/src/core/parser.tsindex 4600272..b3d80f3 100644--- a/src/core/parser.ts+++ b/src/core/parser.ts@@ -1,3 +1,4 @@+import pLimit from "p-limit";import { LiteParseConfig, ParseResult, ScreenshotResult, TextItem } from "./types.js";import { mergeConfig } from "./config.js";import { PdfEngine, PdfDocument, PageData } from "../engines/pdf/interface.js";@@ -29,7 +30,7 @@ export class LiteParse {if (this.config.ocrServerUrl) {this.ocrEngine = new HttpOcrEngine(this.config.ocrServerUrl);} else {- this.ocrEngine = new TesseractEngine();+ this.ocrEngine = new TesseractEngine(this.config.numWorkers);}}}@@ -186,7 +187,7 @@ export class LiteParse {}/**- * Run OCR on pages that need it+ * Run OCR on pages that need it (in parallel with concurrency limit)*/private async runOCR(doc: PdfDocument,@@ -195,137 +196,150 @@ export class LiteParse {): Promise<void> {if (!this.ocrEngine) return;- log("Running OCR on pages...");+ log(`Running OCR on pages (concurrency: ${this.config.numWorkers})...`);- for (let i = 0; i < pages.length; i++) {- const page = pages[i];+ const limit = pLimit(this.config.numWorkers);- // Check if page has very little text (indicating need for OCR)- const textLength = page.textItems.reduce(- (sum: number, item: TextItem) => sum + item.str.length,- 0- );+ await Promise.all(+ pages.map((page) => limit(() => this.processPageOcr(doc, page, log)))+ );+ }- // Determine if OCR is needed and what mode- const hasGarbledRegions = page.garbledTextRegions && page.garbledTextRegions.length > 0;- const needsFullOcr = textLength < 100 || page.images.length > 0;+ /**+ * Process OCR for a single page+ */+ private async processPageOcr(+ doc: PdfDocument,+ page: PageData,+ log: (msg: string) => void+ ): Promise<void> {+ if (!this.ocrEngine) return;- if (!needsFullOcr && !hasGarbledRegions) {- continue;- }+ // Check if page has very little text (indicating need for OCR)+ const textLength = page.textItems.reduce(+ (sum: number, item: TextItem) => sum + item.str.length,+ 0+ );- try {- // Render page as image- const imageBuffer = await this.pdfEngine.renderPageImage(- doc,- page.pageNum,- this.config.dpi- );-- // Save temporary image file- const fs = await import("fs/promises");- const path = await import("path");- const os = await import("os");- const tmpDir = os.tmpdir();- const tmpImagePath = path.join(tmpDir, `page_${page.pageNum}_ocr.png`);- await fs.writeFile(tmpImagePath, imageBuffer);-- // Run OCR- log(` OCR on page ${page.pageNum}...`);- const ocrResults = await this.ocrEngine.recognize(tmpImagePath, {- language: this.config.ocrLanguage,- correctRotation: true,- });+ // Determine if OCR is needed and what mode+ const hasGarbledRegions = page.garbledTextRegions && page.garbledTextRegions.length > 0;+ const needsFullOcr = textLength < 100 || page.images.length > 0;- // Convert OCR results to text items and add to page- if (ocrResults.length > 0) {- // Scale factor to convert from OCR pixels to PDF points- // OCR operates at config.dpi, PDF uses 72 points per inch (PDF spec constant)- const scaleFactor = 72 / this.config.dpi;-- // Helper to check if an OCR result overlaps with garbled regions- const overlapsGarbledRegion = (ocrBbox: number[]): boolean => {- if (!page.garbledTextRegions) return false;-- const ocrX = ocrBbox[0] * scaleFactor;- const ocrY = ocrBbox[1] * scaleFactor;- const ocrW = (ocrBbox[2] - ocrBbox[0]) * scaleFactor;- const ocrH = (ocrBbox[3] - ocrBbox[1]) * scaleFactor;-- // Check overlap with any garbled region (with some tolerance)- const tolerance = 5; // PDF points- for (const region of page.garbledTextRegions) {- const overlapX =- ocrX < region.x + region.width + tolerance && ocrX + ocrW > region.x - tolerance;- const overlapY =- ocrY < region.y + region.height + tolerance && ocrY + ocrH > region.y - tolerance;- if (overlapX && overlapY) {- return true;- }- }- return false;- };-- // Helper to check if an OCR result spatially overlaps with existing PDF text- // This prevents duplicating text that PDF already extracted correctly- const overlapsExistingText = (ocrBbox: number[]): boolean => {- const ocrX = ocrBbox[0] * scaleFactor;- const ocrY = ocrBbox[1] * scaleFactor;- const ocrW = (ocrBbox[2] - ocrBbox[0]) * scaleFactor;- const ocrH = (ocrBbox[3] - ocrBbox[1]) * scaleFactor;-- const tolerance = 2; // PDF points - tighter tolerance for existing text- for (const item of page.textItems) {- const itemRight = item.x + (item.width || item.w || 0);- const itemBottom = item.y + (item.height || item.h || 0);-- const overlapX = ocrX < itemRight + tolerance && ocrX + ocrW > item.x - tolerance;- const overlapY = ocrY < itemBottom + tolerance && ocrY + ocrH > item.y - tolerance;-- if (overlapX && overlapY) {- return true;- }+ if (!needsFullOcr && !hasGarbledRegions) {+ return;+ }++ try {+ // Render page as image+ const imageBuffer = await this.pdfEngine.renderPageImage(+ doc,+ page.pageNum,+ this.config.dpi+ );++ // Save temporary image file+ const fs = await import("fs/promises");+ const path = await import("path");+ const os = await import("os");+ const tmpDir = os.tmpdir();+ const tmpImagePath = path.join(tmpDir, `page_${page.pageNum}_ocr.png`);+ await fs.writeFile(tmpImagePath, imageBuffer);++ // Run OCR+ log(` OCR on page ${page.pageNum}...`);+ const ocrResults = await this.ocrEngine.recognize(tmpImagePath, {+ language: this.config.ocrLanguage,+ correctRotation: true,+ });++ // Convert OCR results to text items and add to page+ if (ocrResults.length > 0) {+ // Scale factor to convert from OCR pixels to PDF points+ // OCR operates at config.dpi, PDF uses 72 points per inch (PDF spec constant)+ const scaleFactor = 72 / this.config.dpi;++ // Helper to check if an OCR result overlaps with garbled regions+ const overlapsGarbledRegion = (ocrBbox: number[]): boolean => {+ if (!page.garbledTextRegions) return false;++ const ocrX = ocrBbox[0] * scaleFactor;+ const ocrY = ocrBbox[1] * scaleFactor;+ const ocrW = (ocrBbox[2] - ocrBbox[0]) * scaleFactor;+ const ocrH = (ocrBbox[3] - ocrBbox[1]) * scaleFactor;++ // Check overlap with any garbled region (with some tolerance)+ const tolerance = 5; // PDF points+ for (const region of page.garbledTextRegions) {+ const overlapX =+ ocrX < region.x + region.width + tolerance && ocrX + ocrW > region.x - tolerance;+ const overlapY =+ ocrY < region.y + region.height + tolerance && ocrY + ocrH > region.y - tolerance;+ if (overlapX && overlapY) {+ return true;}- return false;- };-- const ocrTextItems: TextItem[] = ocrResults- .filter((r) => r.confidence > 0.1) // Filter low confidence- .filter((r) => {- // For targeted OCR (garbled regions only), only include results that overlap- if (hasGarbledRegions && !needsFullOcr) {- return overlapsGarbledRegion(r.bbox);- }- // For full OCR, include all results+ }+ return false;+ };++ // Helper to check if an OCR result spatially overlaps with existing PDF text+ // This prevents duplicating text that PDF already extracted correctly+ const overlapsExistingText = (ocrBbox: number[]): boolean => {+ const ocrX = ocrBbox[0] * scaleFactor;+ const ocrY = ocrBbox[1] * scaleFactor;+ const ocrW = (ocrBbox[2] - ocrBbox[0]) * scaleFactor;+ const ocrH = (ocrBbox[3] - ocrBbox[1]) * scaleFactor;++ const tolerance = 2; // PDF points - tighter tolerance for existing text+ for (const item of page.textItems) {+ const itemRight = item.x + (item.width || item.w || 0);+ const itemBottom = item.y + (item.height || item.h || 0);++ const overlapX = ocrX < itemRight + tolerance && ocrX + ocrW > item.x - tolerance;+ const overlapY = ocrY < itemBottom + tolerance && ocrY + ocrH > item.y - tolerance;++ if (overlapX && overlapY) {return true;- })- .filter((r) => {- // Skip OCR results that spatially overlap with existing PDF text- // This prevents duplicating text that PDF already extracted correctly- return !overlapsExistingText(r.bbox);- })- .map((r) => ({- str: r.text,- x: r.bbox[0] * scaleFactor,- y: r.bbox[1] * scaleFactor,- width: (r.bbox[2] - r.bbox[0]) * scaleFactor,- height: (r.bbox[3] - r.bbox[1]) * scaleFactor,- w: (r.bbox[2] - r.bbox[0]) * scaleFactor,- h: (r.bbox[3] - r.bbox[1]) * scaleFactor,- fontName: "OCR",- fontSize: (r.bbox[3] - r.bbox[1]) * scaleFactor,- }));-- // Add OCR text items directly to page textItems- page.textItems.push(...ocrTextItems);- log(` Found ${ocrTextItems.length} text items from OCR on page ${page.pageNum}`);- }-- // Clean up temp file- await fs.unlink(tmpImagePath).catch(() => {});- } catch (error) {- log(` OCR failed for page ${page.pageNum}: ${error}`);+ }+ }+ return false;+ };++ const ocrTextItems: TextItem[] = ocrResults+ .filter((r) => r.confidence > 0.1) // Filter low confidence+ .filter((r) => {+ // For targeted OCR (garbled regions only), only include results that overlap+ if (hasGarbledRegions && !needsFullOcr) {+ return overlapsGarbledRegion(r.bbox);+ }+ // For full OCR, include all results+ return true;+ })+ .filter((r) => {+ // Skip OCR results that spatially overlap with existing PDF text+ // This prevents duplicating text that PDF already extracted correctly+ return !overlapsExistingText(r.bbox);+ })+ .map((r) => ({+ str: r.text,+ x: r.bbox[0] * scaleFactor,+ y: r.bbox[1] * scaleFactor,+ width: (r.bbox[2] - r.bbox[0]) * scaleFactor,+ height: (r.bbox[3] - r.bbox[1]) * scaleFactor,+ w: (r.bbox[2] - r.bbox[0]) * scaleFactor,+ h: (r.bbox[3] - r.bbox[1]) * scaleFactor,+ fontName: "OCR",+ fontSize: (r.bbox[3] - r.bbox[1]) * scaleFactor,+ }));++ // Add OCR text items directly to page textItems+ page.textItems.push(...ocrTextItems);+ log(` Found ${ocrTextItems.length} text items from OCR on page ${page.pageNum}`);}++ // Clean up temp file+ await fs.unlink(tmpImagePath).catch(() => {});+ } catch (error) {+ log(` OCR failed for page ${page.pageNum}: ${error}`);}}diff --git a/src/core/types.ts b/src/core/types.tsindex 56271df..10232d3 100644--- a/src/core/types.ts+++ b/src/core/types.ts@@ -8,6 +8,9 @@ export interface LiteParseConfig {// HTTP OCR Server (optional - if not provided, uses Tesseract)ocrServerUrl?: string;+ // page processing concurrency (number of pages to process in parallel per document)+ numWorkers: number;+// ProcessingmaxPages: number;targetPages?: string;diff --git a/src/engines/ocr/tesseract.ts b/src/engines/ocr/tesseract.tsindex 793d76a..77b75fc 100644--- a/src/engines/ocr/tesseract.ts+++ b/src/engines/ocr/tesseract.ts@@ -1,22 +1,35 @@-import { createWorker, Worker } from "tesseract.js";+import { createWorker, createScheduler, Scheduler, Worker } from "tesseract.js";import { OcrEngine, OcrOptions, OcrResult } from "./interface.js";export class TesseractEngine implements OcrEngine {name = "tesseract";- private worker?: Worker;+ private scheduler?: Scheduler;+ private workers: Worker[] = [];private currentLanguage?: string;+ private concurrency: number;++ constructor(concurrency: number = 4) {+ this.concurrency = concurrency;+ }async initialize(language: string = "eng"): Promise<void> {- if (this.worker && this.currentLanguage === language) {+ if (this.scheduler && this.currentLanguage === language) {return; // Already initialized for this language}- // Clean up existing worker if language changed- if (this.worker) {- await this.worker.terminate();+ // Clean up existing scheduler and workers if language changed+ await this.terminate();++ // Create scheduler+ this.scheduler = createScheduler();++ // Create worker pool+ for (let i = 0; i < this.concurrency; i++) {+ const worker = await createWorker(language, 1);+ this.workers.push(worker);+ this.scheduler.addWorker(worker);}- this.worker = await createWorker(language, 1);this.currentLanguage = language;}@@ -26,18 +39,18 @@ export class TesseractEngine implements OcrEngine {Array.isArray(options.language) ? options.language[0] : options.language);- // Initialize worker if needed+ // Initialize scheduler if neededawait this.initialize(language);- if (!this.worker) {- throw new Error("Tesseract worker not initialized");+ if (!this.scheduler) {+ throw new Error("Tesseract scheduler not initialized");}try {- // Recognize text from image+ // Recognize text from image using schedulerconst {data: { words },- } = await this.worker.recognize(imagePath);+ } = await this.scheduler.addJob("recognize", imagePath);// Convert to our OcrResult formatconst results: OcrResult[] = words.map((word) => ({@@ -60,23 +73,33 @@ export class TesseractEngine implements OcrEngine {}async recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]> {- const results: OcrResult[][] = [];+ // Handle language+ const language = this.normalizeLanguage(+ Array.isArray(options.language) ? options.language[0] : options.language+ );++ // Initialize scheduler if needed+ await this.initialize(language);- for (let i = 0; i < imagePaths.length; i++) {- const imagePath = imagePaths[i];- const result = await this.recognize(imagePath, options);- results.push(result);+ if (!this.scheduler) {+ throw new Error("Tesseract scheduler not initialized");}- return results;+ // Process all images in parallel - scheduler handles distribution+ const jobs = imagePaths.map((imagePath) =>+ this.recognize(imagePath, options)+ );++ return Promise.all(jobs);}async terminate(): Promise<void> {- if (this.worker) {- await this.worker.terminate();- this.worker = undefined;- this.currentLanguage = undefined;+ if (this.scheduler) {+ await this.scheduler.terminate();+ this.scheduler = undefined;}+ this.workers = [];+ this.currentLanguage = undefined;}/**