A fast, helpful, and open-source document parser
support input buffers as escape hatch for URL handling
Logan Markewichc8c8b56a507d
diff --git a/README.md b/README.mdindex 8be716a..c7da9ac 100644--- a/README.md+++ b/README.md@@ -143,6 +143,32 @@ const result = await parser.parse('document.pdf');console.log(result.text);```+#### Buffer / Uint8Array Input++You can pass raw bytes directly instead of a file path. PDF buffers are parsed with **zero disk I/O** — no temp files are written:++```typescript+import { LiteParse } from '@llamaindex/liteparse';+import { readFile } from 'fs/promises';++const parser = new LiteParse();++// From a file read+const pdfBytes = await readFile('document.pdf');+const result = await parser.parse(pdfBytes);++// From an HTTP response+const response = await fetch('https://example.com/document.pdf');+const buffer = Buffer.from(await response.arrayBuffer());+const result2 = await parser.parse(buffer);+```++Non-PDF buffers (images, Office documents) are written to a temp directory for format conversion. Screenshots also work with buffer input:++```typescript+const screenshots = await parser.screenshot(pdfBytes, [1, 2, 3]);+```+### CLI Options#### Parse Command@@ -227,6 +253,19 @@ lit parse document.pdf --ocr-language fralit parse document.pdf --no-ocr```+By default, Tesseract.js downloads language data from the internet on first use. For offline or air-gapped environments, set the `TESSDATA_PREFIX` environment variable to a directory containing pre-downloaded `.traineddata` files:++```bash+export TESSDATA_PREFIX=/path/to/tessdata+lit parse document.pdf --ocr-language eng+```++You can also pass `tessdataPath` in the library config:++```typescript+const parser = new LiteParse({ tessdataPath: '/path/to/tessdata' });+```+### Optional: HTTP OCR ServersFor higher accuracy or better performance, you can use an HTTP OCR server. We provide ready-to-use example wrappers for popular OCR engines:@@ -287,6 +326,13 @@ apt-get install imagemagickchoco install imagemagick.app # might require admin permissions```+## Environment Variables++| Variable | Description |+|----------|-------------|+| `TESSDATA_PREFIX` | Path to a directory containing Tesseract `.traineddata` files. Used for offline/air-gapped environments where Tesseract.js cannot download language data from the internet. |+| `LITEPARSE_TMPDIR` | Override the temp directory used for format conversion and intermediate files. Defaults to the OS temp directory (`os.tmpdir()`). Useful in containerized or read-only filesystem environments. |+## ConfigurationYou can configure parsing options via CLI flags or a JSON config file. The config file allows you to set sensible defaults and override as needed.diff --git a/docs/src/content/docs/liteparse/guides/library-usage.md b/docs/src/content/docs/liteparse/guides/library-usage.mdindex 2738bc5..7c70b43 100644--- a/docs/src/content/docs/liteparse/guides/library-usage.md+++ b/docs/src/content/docs/liteparse/guides/library-usage.md@@ -62,9 +62,30 @@ const parser = new LiteParse({});```+### Buffer / Uint8Array input++You can pass raw bytes directly instead of a file path. PDF buffers are parsed with **zero disk I/O** — no temp files are written:++```typescript+import { readFile } from "fs/promises";++const parser = new LiteParse();++// From a file read+const pdfBytes = await readFile("document.pdf");+const result = await parser.parse(pdfBytes);++// From an HTTP response+const response = await fetch("https://example.com/document.pdf");+const buffer = Buffer.from(await response.arrayBuffer());+const result2 = await parser.parse(buffer);+```++Non-PDF buffers (images, Office documents) are written to a temp directory for format conversion. You can control the temp directory with the `LITEPARSE_TMPDIR` environment variable.+### Screenshots-Generate page images as buffers — useful for sending to LLMs or saving to disk:+Generate page images as buffers — useful for sending to LLMs or saving to disk. Accepts file paths, `Buffer`, or `Uint8Array`:```typescriptconst parser = new LiteParse();@@ -74,8 +95,19 @@ for (const shot of screenshots) {console.log(`Page ${shot.pageNum}: ${shot.width}x${shot.height}`);// shot.imageBuffer contains the raw PNG/JPG data}++// Also works with buffer input+const pdfBytes = await readFile("document.pdf");+const shots = await parser.screenshot(pdfBytes, [1, 2, 3]);```+### Environment variables++| Variable | Description |+|----------|-------------|+| `TESSDATA_PREFIX` | Path to a directory containing Tesseract `.traineddata` files. For offline/air-gapped environments. Also available as the `tessdataPath` config option. |+| `LITEPARSE_TMPDIR` | Override the temp directory for format conversion. Defaults to `os.tmpdir()`. |+See the [API reference](/liteparse/api/) for full type details.---diff --git a/src/conversion/README.md b/src/conversion/README.mdindex 514db2f..73e7bc1 100644--- a/src/conversion/README.md+++ b/src/conversion/README.md@@ -29,27 +29,39 @@ LiteParse's core parsing works on PDFs. This module extends support to 50+ forma**Key Functions:**-`convertToPdf(filePath)` - Main entry point+`convertToPdf(filePath)` - Main entry point for file path input- Returns `ConversionResult` with `pdfPath` and `originalExtension`- Returns `ConversionPassthrough` with `content` for text-based formats (e.g. .txt, .csv)- Returns `ConversionError` with `message` and `code` on failure- If already PDF, returns path unchanged+`convertBufferToPdf(data)` - Entry point for buffer input+- Detects format from magic bytes, writes to temp file, then delegates to `convertToPdf`+- Used when `LiteParse.parse()` receives a non-PDF `Buffer` or `Uint8Array`+`cleanupConversionFiles(pdfPath)` - Removes temp files after parsing-- Only deletes files in system temp directory+- Only deletes files in the configured temp directory (`LITEPARSE_TMPDIR` or OS default)- Called by `LiteParse.parse()` after processing+`getTmpDir()` - Returns the temp directory for LiteParse operations+- Respects the `LITEPARSE_TMPDIR` environment variable+- Falls back to `os.tmpdir()`+`guessFileExtension(filePath)` - Detects format from extension or magic bytes- Checks file extension first- Falls back to magic byte detection (PDF, PNG, JPEG, ZIP-based)+`guessExtensionFromBuffer(data)` - Detects format from raw bytes using magic bytes+- Supports PDF, PNG, JPEG, TIFF (both endians), and ZIP-based formats+- Used by `convertBufferToPdf` to determine the temp file extension+**Design Decisions:**1. **External tools over native parsers**: Using LibreOffice and ImageMagick provides broad format support with minimal code. Trade-off is external dependency requirement.2. **Subprocess-based**: Conversion runs as separate process to isolate crashes and memory issues.-3. **Temporary file management**: Converted PDFs go to system temp directory, cleaned up after parsing.+3. **Temporary file management**: Converted PDFs go to the configured temp directory (`LITEPARSE_TMPDIR` env var or OS default), cleaned up after parsing.4. **Timeout handling**: 2 minutes for LibreOffice, 1 minute for ImageMagick.diff --git a/src/conversion/convertToPdf.test.ts b/src/conversion/convertToPdf.test.tsindex 095538d..10ff56c 100644--- a/src/conversion/convertToPdf.test.ts+++ b/src/conversion/convertToPdf.test.ts@@ -1,4 +1,4 @@-import { vi, describe, it, expect } from "vitest";+import { vi, describe, it, expect, afterEach } from "vitest";import { EventEmitter } from "events";const mockFd = {@@ -52,11 +52,13 @@ vi.mock("fs", async () => {import {guessFileExtension,+ guessExtensionFromBuffer,findImageMagickCommand,findLibreOfficeCommand,convertOfficeDocument,convertImageToPdf,convertToPdf,+ getTmpDir,} from "./convertToPdf";describe("test guessFileExtension", () => {@@ -234,3 +236,68 @@ describe("test convertToPdf", () => {});});});++describe("test guessExtensionFromBuffer", () => {+ it("detects PDF from magic bytes", () => {+ const pdfBytes = Buffer.from("%PDF-1.4 some content");+ expect(guessExtensionFromBuffer(pdfBytes)).toBe(".pdf");+ });++ it("detects PNG from magic bytes", () => {+ const pngBytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);+ expect(guessExtensionFromBuffer(pngBytes)).toBe(".png");+ });++ it("detects JPEG from magic bytes", () => {+ const jpegBytes = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]);+ expect(guessExtensionFromBuffer(jpegBytes)).toBe(".jpg");+ });++ it("detects TIFF (little-endian) from magic bytes", () => {+ const tiffBytes = Buffer.from([0x49, 0x49, 0x2a, 0x00]);+ expect(guessExtensionFromBuffer(tiffBytes)).toBe(".tiff");+ });++ it("detects TIFF (big-endian) from magic bytes", () => {+ const tiffBytes = Buffer.from([0x4d, 0x4d, 0x00, 0x2a]);+ expect(guessExtensionFromBuffer(tiffBytes)).toBe(".tiff");+ });++ it("detects ZIP-based formats from magic bytes", () => {+ const zipBytes = Buffer.from([0x50, 0x4b, 0x03, 0x04]);+ expect(guessExtensionFromBuffer(zipBytes)).toBe(".docx");+ });++ it("defaults to .pdf for unknown bytes", () => {+ const unknownBytes = Buffer.from([0x00, 0x01, 0x02, 0x03]);+ expect(guessExtensionFromBuffer(unknownBytes)).toBe(".pdf");+ });++ it("works with Uint8Array input", () => {+ const pdfBytes = new Uint8Array(Buffer.from("%PDF-1.7"));+ expect(guessExtensionFromBuffer(pdfBytes)).toBe(".pdf");+ });+});++describe("test getTmpDir", () => {+ const originalEnv = process.env.LITEPARSE_TMPDIR;++ afterEach(() => {+ if (originalEnv === undefined) {+ delete process.env.LITEPARSE_TMPDIR;+ } else {+ process.env.LITEPARSE_TMPDIR = originalEnv;+ }+ });++ it("returns LITEPARSE_TMPDIR when set", () => {+ process.env.LITEPARSE_TMPDIR = "/custom/tmp";+ expect(getTmpDir()).toBe("/custom/tmp");+ });++ it("falls back to os.tmpdir() when LITEPARSE_TMPDIR is not set", () => {+ delete process.env.LITEPARSE_TMPDIR;+ const os = require("os");+ expect(getTmpDir()).toBe(os.tmpdir());+ });+});diff --git a/src/conversion/convertToPdf.ts b/src/conversion/convertToPdf.tsindex b72cf40..2743c88 100644--- a/src/conversion/convertToPdf.ts+++ b/src/conversion/convertToPdf.ts@@ -3,6 +3,14 @@ import { spawn } from "child_process";import path from "path";import os from "os";+/**+ * Returns the temp directory for LiteParse operations.+ * Respects the `LITEPARSE_TMPDIR` environment variable, falling back to the OS default.+ */+export function getTmpDir(): string {+ return process.env.LITEPARSE_TMPDIR || os.tmpdir();+}+export interface ConversionResult {pdfPath: string;originalExtension: string;@@ -368,7 +376,7 @@ export async function convertToPdf(}// Create temp directory for output- const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "liteparse-"));+ const tmpDir = await fs.mkdtemp(path.join(getTmpDir(), "liteparse-"));// Convert based on file typelet pdfPath: string;@@ -406,7 +414,7 @@ export async function convertToPdf(export async function cleanupConversionFiles(pdfPath: string): Promise<void> {try {// Only delete if in temp directory- if (pdfPath.includes(os.tmpdir())) {+ if (pdfPath.includes(getTmpDir())) {const dir = path.dirname(pdfPath);await fs.rm(dir, { recursive: true, force: true });}@@ -414,3 +422,51 @@ export async function cleanupConversionFiles(pdfPath: string): Promise<void> {// Ignore cleanup errors}}++/**+ * Guess file extension from raw bytes using magic-byte detection.+ */+export function guessExtensionFromBuffer(data: Buffer | Uint8Array): string {+ // PDF: %PDF+ if (data.length >= 4 && data[0] === 0x25 && data[1] === 0x50 && data[2] === 0x44 && data[3] === 0x46) {+ return ".pdf";+ }+ // PNG: 89 50 4E 47+ if (data.length >= 4 && data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) {+ return ".png";+ }+ // JPEG: FF D8 FF+ if (data.length >= 3 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {+ return ".jpg";+ }+ // TIFF: 49 49 2A 00 (little-endian) or 4D 4D 00 2A (big-endian)+ if (+ data.length >= 4 &&+ ((data[0] === 0x49 && data[1] === 0x49 && data[2] === 0x2a && data[3] === 0x00) ||+ (data[0] === 0x4d && data[1] === 0x4d && data[2] === 0x00 && data[3] === 0x2a))+ ) {+ return ".tiff";+ }+ // ZIP-based formats (docx, xlsx, pptx, etc.): PK+ if (data.length >= 2 && data[0] === 0x50 && data[1] === 0x4b) {+ return ".docx";+ }+ return ".pdf"; // Default assumption+}++/**+ * Convert a raw byte buffer to PDF by writing to a temp file and converting.+ * For PDF buffers, callers should skip this and pass data directly to the PDF engine.+ */+export async function convertBufferToPdf(+ data: Buffer | Uint8Array+): Promise<ConversionResult | ConversionPassthrough | ConversionError> {+ const ext = guessExtensionFromBuffer(data);++ // Write buffer to temp file with detected extension+ const tmpDir = await fs.mkdtemp(path.join(getTmpDir(), "liteparse-"));+ const tmpPath = path.join(tmpDir, `input${ext}`);+ await fs.writeFile(tmpPath, data);++ return convertToPdf(tmpPath);+}diff --git a/src/core/README.md b/src/core/README.mdindex 6a69975..9a3bdb3 100644--- a/src/core/README.md+++ b/src/core/README.md@@ -14,15 +14,15 @@ The core module contains the main orchestrator class, configuration management,- Handles format conversion for non-PDF inputs**Public Methods:**-- `parse(filePath, quiet?)` - Main entry point for document parsing-- `screenshot(filePath, pageNumbers?, quiet?)` - Generate page screenshots+- `parse(input, quiet?)` - Main entry point for document parsing. Accepts a file path (`string`), `Buffer`, or `Uint8Array`.+- `screenshot(input, pageNumbers?, quiet?)` - Generate page screenshots. Accepts the same input types as `parse()`.- `getConfig()` - Returns current configuration**Pipeline Flow (in `parse()`):**-1. Convert to PDF if needed (via `convertToPdf`)+1. **Input routing**: File paths go through `convertToPdf`; PDF buffers go directly to the engine (zero disk I/O); non-PDF buffers are written to temp for conversion via `convertBufferToPdf`2. Load PDF document with PDF engine3. Extract pages-4. Run OCR on text-sparse pages/embedded images+4. Run OCR on text-sparse pages/embedded images (passes image buffers directly to OCR engines — no temp files)5. Project pages to grid (spatial text reconstruction)6. Build bounding boxes (if enabled)7. Format output (JSON or text)@@ -41,6 +41,7 @@ The core module contains the main orchestrator class, configuration management,**Configuration Types:**- `LiteParseConfig` - All configuration options+- `LiteParseInput` - Accepted input types: `string | Buffer | Uint8Array`- `OutputFormat` - 'json' | 'text'**Data Types:**diff --git a/src/core/parser.ts b/src/core/parser.tsindex bc77a41..e8a118d 100644--- a/src/core/parser.ts+++ b/src/core/parser.ts@@ -1,5 +1,5 @@import pLimit from "p-limit";-import { LiteParseConfig, ParseResult, ScreenshotResult, TextItem } from "./types.js";+import { LiteParseConfig, LiteParseInput, ParseResult, ScreenshotResult, TextItem } from "./types.js";import { mergeConfig } from "./config.js";import { PdfEngine, PdfDocument, PageData } from "../engines/pdf/interface.js";import { PdfJsEngine } from "../engines/pdf/pdfjs.js";@@ -10,7 +10,12 @@ import { HttpOcrEngine } from "../engines/ocr/http-simple.js";import { projectPagesToGrid } from "../processing/grid.js";import { buildBoundingBoxes } from "../processing/bbox.js";import { formatJSON } from "../output/json.js";-import { convertToPdf, cleanupConversionFiles } from "../conversion/convertToPdf.js";+import {+ convertToPdf,+ convertBufferToPdf,+ cleanupConversionFiles,+ guessExtensionFromBuffer,+} from "../conversion/convertToPdf.js";import { cleanOcrTableArtifacts } from "../processing/textUtils.js";/**@@ -78,44 +83,72 @@ export class LiteParse {* Supports PDFs natively. Non-PDF formats (DOCX, XLSX, images, etc.) are automatically* converted to PDF before parsing if the required system tools are installed.*- * @param filePath - Path to the document file.+ * @param input - A file path, `Buffer`, or `Uint8Array` containing document bytes.+ * When given raw bytes, PDF data is parsed directly with zero disk I/O.+ * Non-PDF bytes are written to a temp file for format conversion.* @param quiet - If `true`, suppresses progress logging to stderr.* @returns Parsed document data including text, per-page info, and optional JSON.** @throws Error if the file cannot be found, converted, or parsed.*/- async parse(filePath: string, quiet = false): Promise<ParseResult> {+ async parse(input: LiteParseInput, quiet = false): Promise<ParseResult> {const log = (msg: string) => {if (!quiet) console.error(msg); // Progress goes to stderr};- log(`Processing file: ${filePath}`);- const conversionResult = await convertToPdf(filePath);+ let doc: PdfDocument;+ let needsCleanup = false;+ let cleanupPath: string | undefined;- if ("code" in conversionResult) {- // Conversion error- throw new Error(`Conversion failed: ${conversionResult.message}`);- }+ if (typeof input === "string") {+ log(`Processing file: ${input}`);+ const conversionResult = await convertToPdf(input);- // Return early for text-based passthrough formats- if ("content" in conversionResult) {- log(`File is a text-based format. Returning content directly.`);- return {- pages: [],- text: conversionResult.content,- };- }+ if ("code" in conversionResult) {+ throw new Error(`Conversion failed: ${conversionResult.message}`);+ }++ if ("content" in conversionResult) {+ log(`File is a text-based format. Returning content directly.`);+ return { pages: [], text: conversionResult.content };+ }- // Convert to PDF if needed- const pdfPath = conversionResult.pdfPath;- const needsCleanup = pdfPath !== filePath;+ const pdfPath = conversionResult.pdfPath;+ needsCleanup = pdfPath !== input;+ if (needsCleanup) {+ cleanupPath = pdfPath;+ log(`Converted ${conversionResult.originalExtension} to PDF`);+ }++ doc = await this.pdfEngine.loadDocument(pdfPath);+ } else {+ log(`Processing buffer input (${input.byteLength} bytes)`);+ const ext = guessExtensionFromBuffer(input);++ if (ext === ".pdf") {+ // Zero-disk path: pass bytes directly to the PDF engine+ const data = input instanceof Uint8Array ? input : new Uint8Array(input);+ doc = await this.pdfEngine.loadDocument(data);+ } else {+ // Non-PDF buffer: write to temp file for conversion+ const conversionResult = await convertBufferToPdf(input);- if (needsCleanup) {- log(`Converted ${conversionResult.originalExtension} to PDF`);+ if ("code" in conversionResult) {+ throw new Error(`Conversion failed: ${conversionResult.message}`);+ }++ if ("content" in conversionResult) {+ log(`Buffer is a text-based format. Returning content directly.`);+ return { pages: [], text: conversionResult.content };+ }++ needsCleanup = true;+ cleanupPath = conversionResult.pdfPath;+ log(`Converted ${conversionResult.originalExtension} buffer to PDF`);+ doc = await this.pdfEngine.loadDocument(conversionResult.pdfPath);+ }}- // Load PDF document- const doc = await this.pdfEngine.loadDocument(pdfPath);log(`Loaded PDF with ${doc.numPages} pages`);// Extract pages@@ -152,8 +185,8 @@ export class LiteParse {}// Cleanup temporary conversion files- if (needsCleanup) {- await cleanupConversionFiles(pdfPath);+ if (needsCleanup && cleanupPath) {+ await cleanupConversionFiles(cleanupPath);}const result: ParseResult = {@@ -180,13 +213,13 @@ export class LiteParse {* Uses PDFium for high-quality rendering. Each page is returned as a* {@link ScreenshotResult} with the raw image buffer and dimensions.*- * @param filePath - Path to the PDF file.+ * @param input - A file path, `Buffer`, or `Uint8Array` containing PDF bytes.* @param pageNumbers - 1-indexed page numbers to screenshot. If omitted, all pages are rendered.* @param quiet - If `true`, suppresses progress logging to stderr.* @returns Array of screenshot results, one per rendered page.*/async screenshot(- filePath: string,+ input: LiteParseInput,pageNumbers?: number[],quiet = false): Promise<ScreenshotResult[]> {@@ -194,12 +227,15 @@ export class LiteParse {if (!quiet) console.error(msg); // Progress goes to stderr};- log(`Generating screenshots for: ${filePath}`);+ log(`Generating screenshots for: ${typeof input === "string" ? input : "<buffer>"}`);// Load PDF document to get page count and dimensions- const doc = await this.pdfEngine.loadDocument(filePath);+ const doc = await this.pdfEngine.loadDocument(input as string | Uint8Array);const totalPages = doc.numPages;+ // Determine the input to pass to the renderer (file path or buffer)+ const rendererInput: string | Buffer | Uint8Array = typeof input === "string" ? input : input;+const results: ScreenshotResult[] = [];const pages = pageNumbers || Array.from({ length: totalPages }, (_, i) => i + 1);@@ -214,7 +250,7 @@ export class LiteParse {}log(`Rendering page ${pageNum}...`);- const imageBuffer = await renderer.renderPageToBuffer(filePath, pageNum, this.config.dpi);+ const imageBuffer = await renderer.renderPageToBuffer(rendererInput, pageNum, this.config.dpi);// Get page dimensionsconst pageData = await this.pdfEngine.extractPage(doc, pageNum);@@ -278,20 +314,12 @@ export class LiteParse {}try {- // Render page as image+ // Render page as image bufferconst imageBuffer = await this.pdfEngine.renderPageImage(doc, page.pageNum, this.config.dpi);- // Save temporary image file- const fs = await import("fs/promises");- const path = await import("path");- const os = await import("os");- const tmpDir = os.tmpdir();- const tmpImagePath = path.join(tmpDir, `page_${page.pageNum}_ocr.png`);- await fs.writeFile(tmpImagePath, imageBuffer);-- // Run OCR+ // Run OCR directly on the buffer (no temp file needed)log(` OCR on page ${page.pageNum}...`);- const ocrResults = await this.ocrEngine.recognize(tmpImagePath, {+ const ocrResults = await this.ocrEngine.recognize(imageBuffer, {language: this.config.ocrLanguage,correctRotation: true,});@@ -384,9 +412,6 @@ export class LiteParse {page.textItems.push(...ocrTextItems);log(` Found ${ocrTextItems.length} text items from OCR on page ${page.pageNum}`);}-- // Clean up temp file- await fs.unlink(tmpImagePath).catch(() => {});} catch (error) {log(` OCR failed for page ${page.pageNum}: ${error}`);}diff --git a/src/core/types.ts b/src/core/types.tsindex 6f85087..5a51d4b 100644--- a/src/core/types.ts+++ b/src/core/types.ts@@ -6,6 +6,15 @@*/export type OutputFormat = "json" | "text";+/**+ * Accepted input types for {@link LiteParse.parse} and {@link LiteParse.screenshot}.+ *+ * - `string` — A file path to a document on disk.+ * - `Buffer | Uint8Array` — Raw file bytes (PDF bytes go straight to the parser with zero disk I/O;+ * non-PDF bytes are written to a temp file for format conversion).+ */+export type LiteParseInput = string | Buffer | Uint8Array;+/*** Configuration options for the {@link LiteParse} parser.*diff --git a/src/engines/ocr/README.md b/src/engines/ocr/README.mdindex fa55bde..3ec0399 100644--- a/src/engines/ocr/README.md+++ b/src/engines/ocr/README.md@@ -10,8 +10,8 @@ OCR engines for extracting text from images.```typescriptinterface OcrEngine {name: string;- recognize(imagePath: string, options: OcrOptions): Promise<OcrResult[]>;- recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]>;+ recognize(image: string | Buffer, options: OcrOptions): Promise<OcrResult[]>;+ recognizeBatch(images: (string | Buffer)[], options: OcrOptions): Promise<OcrResult[][]>;}interface OcrOptions {@@ -35,9 +35,11 @@ This is the default OCR engine when no `ocrServerUrl` is configured.**Key Features:**- Worker-based processing (runs in background thread)+- Accepts file paths or `Buffer` input directly (no temp files needed)- Language caching (reuses worker for same language)- Automatic language code normalization (e.g., `en` → `eng`)- Low-confidence filtering (removes results < 30%)+- Supports offline usage via `TESSDATA_PREFIX` env var or `tessdataPath` config for pre-downloaded `.traineddata` files**Language Normalization:**Maps common ISO 639-1 codes to Tesseract's 3-letter codes:@@ -47,8 +49,8 @@ Maps common ISO 639-1 codes to Tesseract's 3-letter codes:**Lifecycle:**- `initialize(language)` - Create/recreate worker for language-- `recognize(imagePath, options)` - OCR single image-- `recognizeBatch(imagePaths, options)` - OCR multiple images sequentially+- `recognize(image, options)` - OCR single image (accepts file path or Buffer)+- `recognizeBatch(images, options)` - OCR multiple images sequentially- `terminate()` - Clean up worker (called by LiteParse after parsing)**Design Decisions:**diff --git a/src/engines/ocr/http-simple.ts b/src/engines/ocr/http-simple.tsindex 2b57db8..785a24a 100644--- a/src/engines/ocr/http-simple.ts+++ b/src/engines/ocr/http-simple.ts@@ -27,11 +27,15 @@ export class HttpOcrEngine implements OcrEngine {this.serverUrl = serverUrl;}- async recognize(imagePath: string, options: OcrOptions): Promise<OcrResult[]> {+ async recognize(image: string | Buffer, options: OcrOptions): Promise<OcrResult[]> {try {// Prepare requestconst formData = new FormData();- formData.append("file", fs.createReadStream(imagePath));+ if (typeof image === "string") {+ formData.append("file", fs.createReadStream(image));+ } else {+ formData.append("file", image, { filename: "image.png", contentType: "image/png" });+ }const language = Array.isArray(options.language) ? options.language[0] : options.language;formData.append("language", language);@@ -56,23 +60,24 @@ export class HttpOcrEngine implements OcrEngine {confidence: item.confidence,}));} catch (error) {+ const label = typeof image === "string" ? image : "<buffer>";if (axios.isAxiosError(error)) {console.error(- `OCR HTTP error for ${imagePath}:`,+ `OCR HTTP error for ${label}:`,error.response?.status,error.response?.data?.error || error.message);} else {- console.error(`OCR error for ${imagePath}:`, error);+ console.error(`OCR error for ${label}:`, error);}return [];}}- async recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]> {+ async recognizeBatch(images: (string | Buffer)[], options: OcrOptions): Promise<OcrResult[][]> {const results: OcrResult[][] = [];- for (const imagePath of imagePaths) {- const result = await this.recognize(imagePath, options);+ for (const image of images) {+ const result = await this.recognize(image, options);results.push(result);}return results;diff --git a/src/engines/ocr/interface.ts b/src/engines/ocr/interface.tsindex 5759084..a7d72ae 100644--- a/src/engines/ocr/interface.ts+++ b/src/engines/ocr/interface.ts@@ -1,7 +1,7 @@export interface OcrEngine {name: string;- recognize(imagePath: string, options: OcrOptions): Promise<OcrResult[]>;- recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]>;+ recognize(image: string | Buffer, options: OcrOptions): Promise<OcrResult[]>;+ recognizeBatch(images: (string | Buffer)[], options: OcrOptions): Promise<OcrResult[][]>;}export interface OcrOptions {diff --git a/src/engines/ocr/tesseract.ts b/src/engines/ocr/tesseract.tsindex 102f304..bd9a962 100644--- a/src/engines/ocr/tesseract.ts+++ b/src/engines/ocr/tesseract.ts@@ -94,7 +94,7 @@ export class TesseractEngine implements OcrEngine {this.currentLanguage = language;}- async recognize(imagePath: string, options: OcrOptions): Promise<OcrResult[]> {+ async recognize(image: string | Buffer, options: OcrOptions): Promise<OcrResult[]> {// Handle language - tesseract.js uses language codes like 'eng', 'fra', 'deu'const language = this.normalizeLanguage(Array.isArray(options.language) ? options.language[0] : options.language@@ -109,10 +109,11 @@ export class TesseractEngine implements OcrEngine {try {// Recognize text from image using scheduler+ // tesseract.js accepts string (path/URL) or Buffer/Uint8Array// In tesseract.js v6+, we need to enable blocks output to get word-level dataconst {data: { blocks },- } = await this.scheduler.addJob("recognize", imagePath, {}, { blocks: true });+ } = await this.scheduler.addJob("recognize", image, {}, { blocks: true });// Extract words from hierarchical blocks structure: blocks → paragraphs → lines → wordsconst results: OcrResult[] = [];@@ -138,12 +139,13 @@ export class TesseractEngine implements OcrEngine {// Filter out low confidence results (below 30%)return results.filter((r) => r.confidence > 0.3);} catch (error) {- console.error(`\nTesseract OCR error for ${imagePath}:`, error);+ const label = typeof image === "string" ? image : "<buffer>";+ console.error(`\nTesseract OCR error for ${label}:`, error);return [];}}- async recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]> {+ async recognizeBatch(images: (string | Buffer)[], options: OcrOptions): Promise<OcrResult[][]> {// Handle languageconst language = this.normalizeLanguage(Array.isArray(options.language) ? options.language[0] : options.language@@ -157,7 +159,7 @@ export class TesseractEngine implements OcrEngine {}// Process all images in parallel - scheduler handles distribution- const jobs = imagePaths.map((imagePath) => this.recognize(imagePath, options));+ const jobs = images.map((image) => this.recognize(image, options));return Promise.all(jobs);}diff --git a/src/engines/pdf/README.md b/src/engines/pdf/README.mdindex 6ee15db..c64cde9 100644--- a/src/engines/pdf/README.md+++ b/src/engines/pdf/README.md@@ -11,7 +11,7 @@ PDF parsing engines for loading documents and extracting content.```typescriptinterface PdfEngine {name: string;- loadDocument(filePath: string): Promise<PdfDocument>;+ loadDocument(input: string | Uint8Array): Promise<PdfDocument>;extractPage(doc: PdfDocument, pageNum: number): Promise<PageData>;extractAllPages(doc, maxPages?, targetPages?): Promise<PageData[]>;renderPageImage(doc, pageNum, dpi): Promise<Buffer>;@@ -19,6 +19,8 @@ interface PdfEngine {}```+`loadDocument` accepts either a file path or raw PDF bytes as a `Uint8Array`. When given bytes, the document is loaded with zero disk I/O.+**Key Data Types:**- `PdfDocument` - Loaded document with `numPages`, `data` (Uint8Array), `metadata`- `PageData` - Extracted page: `pageNum`, `width`, `height`, `textItems`, `images`, `annotations`@@ -69,9 +71,9 @@ When garbled text is detected:This allows targeted OCR replacement of only the corrupted text while preserving high-quality PDF text extraction elsewhere.**Design Decisions:**-- **Stores PDF path**: Needed for PDFium rendering (pdfjs.ts:95)-- **Filters off-page items**: Removes text with negative coords or beyond page bounds (pdfjs.ts:191-198)-- **Zero-size filtering**: Skips text items with 0 width/height (pdfjs.ts:125)+- **Stores PDF path and data**: Keeps both the file path (when available) and the raw `Uint8Array` data for PDFium rendering. Buffer input skips file reads entirely.+- **Filters off-page items**: Removes text with negative coords or beyond page bounds+- **Zero-size filtering**: Skips text items with 0 width/height---@@ -88,7 +90,7 @@ Used for generating page images for OCR and the `screenshot` command. Provides b**Methods:**- `init()` - Initialize PDFium library (called automatically)-- `renderPageToBuffer(pdfPath, pageNumber, dpi)` - Render page to PNG buffer+- `renderPageToBuffer(pdfInput, pageNumber, dpi)` - Render page to PNG buffer. Accepts a file path (`string`), `Buffer`, or `Uint8Array`.- `close()` - Cleanup PDFium resources**Design Decision:**diff --git a/src/engines/pdf/interface.ts b/src/engines/pdf/interface.tsindex 0d8dd30..79d04bf 100644--- a/src/engines/pdf/interface.ts+++ b/src/engines/pdf/interface.ts@@ -2,7 +2,7 @@ import { TextItem } from "../../core/types.js";export interface PdfEngine {name: string;- loadDocument(filePath: string): Promise<PdfDocument>;+ loadDocument(input: string | Uint8Array): Promise<PdfDocument>;extractPage(doc: PdfDocument, pageNum: number): Promise<PageData>;extractAllPages(doc: PdfDocument, maxPages?: number, targetPages?: string): Promise<PageData[]>;renderPageImage(doc: PdfDocument, pageNum: number, dpi: number): Promise<Buffer>;diff --git a/src/engines/pdf/pdfium-renderer.ts b/src/engines/pdf/pdfium-renderer.tsindex e5c0705..2974aa5 100644--- a/src/engines/pdf/pdfium-renderer.ts+++ b/src/engines/pdf/pdfium-renderer.ts@@ -16,7 +16,7 @@ export class PdfiumRenderer {}async renderPageToBuffer(- pdfPath: string,+ pdfInput: string | Buffer | Uint8Array,pageNumber: number,dpi: number = 150): Promise<Buffer> {@@ -26,8 +26,10 @@ export class PdfiumRenderer {throw new Error("PDFium not initialized");}- // Read PDF file- const pdfBuffer = await fs.readFile(pdfPath);+ // Accept file path or raw bytes+ const pdfBuffer = typeof pdfInput === "string"+ ? await fs.readFile(pdfInput)+ : Buffer.from(pdfInput);// Load documentconst document = await this.pdfium.loadDocument(pdfBuffer);diff --git a/src/engines/pdf/pdfjs.ts b/src/engines/pdf/pdfjs.tsindex 877ce6c..280cffd 100644--- a/src/engines/pdf/pdfjs.ts+++ b/src/engines/pdf/pdfjs.ts@@ -431,12 +431,21 @@ export class PdfJsEngine implements PdfEngine {name = "pdfjs";private pdfiumRenderer: PdfiumRenderer | null = null;private currentPdfPath: string | null = null;+ private currentPdfData: Uint8Array | null = null;- async loadDocument(filePath: string): Promise<PdfDocument> {- const data = new Uint8Array(await fs.readFile(filePath));+ async loadDocument(input: string | Uint8Array): Promise<PdfDocument> {+ let data: Uint8Array;+ if (typeof input === "string") {+ data = new Uint8Array(await fs.readFile(input));+ this.currentPdfPath = input;+ } else {+ // pdf.js requires a plain Uint8Array, not a Buffer subclass+ data = new Uint8Array(input.buffer, input.byteOffset, input.byteLength);+ this.currentPdfPath = null;+ }- // Store path for PDFium rendering- this.currentPdfPath = filePath;+ // Store data for buffer-based rendering+ this.currentPdfData = data;const loadingTask = getDocument({data,@@ -616,16 +625,17 @@ export class PdfJsEngine implements PdfEngine {}async renderPageImage(_doc: PdfDocument, pageNum: number, dpi: number): Promise<Buffer> {- // Use PDFium for rendering (more robust with inline images)- if (!this.currentPdfPath) {- throw new Error("PDF path not available for rendering");- }-if (!this.pdfiumRenderer) {this.pdfiumRenderer = new PdfiumRenderer();}- return await this.pdfiumRenderer.renderPageToBuffer(this.currentPdfPath, pageNum, dpi);+ // Use file path when available, otherwise pass buffer directly+ const pdfInput = this.currentPdfPath || this.currentPdfData;+ if (!pdfInput) {+ throw new Error("No PDF path or data available for rendering");+ }++ return await this.pdfiumRenderer.renderPageToBuffer(pdfInput, pageNum, dpi);}async close(doc: PdfDocument): Promise<void> {@@ -640,6 +650,7 @@ export class PdfJsEngine implements PdfEngine {this.pdfiumRenderer = null;}this.currentPdfPath = null;+ this.currentPdfData = null;}private parseTargetPages(targetPages: string, maxPages: number): number[] {diff --git a/src/lib.ts b/src/lib.tsindex 2ea7879..2484209 100644--- a/src/lib.ts+++ b/src/lib.ts@@ -15,6 +15,7 @@export { LiteParse } from "./core/parser.js";export type {LiteParseConfig,+ LiteParseInput,OutputFormat,ParseResult,ParseResultJson,