A fast, helpful, and open-source document parser
clean up OCR usage
Logan Markewichbbce765d25c2
diff --git a/.gitignore b/.gitignoreindex 3030534..4781b7f 100644--- a/.gitignore+++ b/.gitignore@@ -37,3 +37,10 @@ screenshots/# Temporary filestmp/temp/+*.txt+*.png+deno.json++# Experimental/development artifacts+ac/+examples/diff --git a/README.md b/README.mdindex 8e7d7ea..26505c8 100644--- a/README.md+++ b/README.md@@ -22,72 +22,85 @@ LiteParse is a standalone OSS PDF parsing tool focused exclusively on **fast and## Installation-### Option 1: Use as a Library+### CLI Tool-Install via npm/pnpm:+#### Option 1: Global Install (Recommended)++Install globally via npm to use the `liteparse` command anywhere:```bash-pnpm add liteparse+npm i -g liteparse```-```typescript-import { LiteParse } from 'liteparse';+Then use it:-const parser = new LiteParse({ ocrEnabled: true });-const result = await parser.parse('document.pdf');-console.log(result.text);+```bash+liteparse parse document.pdf+liteparse screenshot document.pdf```-### Option 2: Build from Source+#### Option 2: Use with npx-#### Prerequisites+Run directly without installing:-- Node.js 22.4.1 or higher-- pnpm package manager-- [Bun](https://bun.sh) (for building binaries)+```bash+npx liteparse parse document.pdf+npx liteparse screenshot document.pdf+```++#### Option 3: Homebrew (macOS/Linux)-#### Install Dependencies+Coming soon! Once published, you'll be able to install via Homebrew:```bash-cd liteparse-pnpm install+brew tap yourusername/liteparse+brew install liteparse```-#### Build TypeScript+### Library Usage++Install as a dependency in your project:```bash-pnpm build+npm install liteparse+# or+pnpm add liteparse```-#### Build Binary (Optional)+```typescript+import { LiteParse } from 'liteparse';-Build a standalone executable that you can run anywhere:+const parser = new LiteParse({ ocrEnabled: true });+const result = await parser.parse('document.pdf');+console.log(result.text);+```-```bash-# Install Bun if you haven't already-curl -fsSL https://bun.sh/install | bash+### Build from Source-# Build binary for your current platform-pnpm build:binary+If you want to contribute or build from source:-# Or build for specific platforms-pnpm build:bun:linux # Linux x64-pnpm build:bun:macos # macOS ARM64-pnpm build:bun:windows # Windows x64-pnpm build:bun:all # All platforms+#### Prerequisites-# The binary will be in ./bin/lp (or lp-{platform})-```+- Node.js 18.0.0 or higher+- npm or pnpm package manager-Once built, you can use the binary directly:+#### Steps```bash-# Add to your PATH-export PATH="$PATH:/path/to/liteparse/bin"+# Clone the repository+git clone https://github.com/run-llama/liteparse.git+cd liteparse++# Install dependencies+npm install+# or+pnpm install++# Build TypeScript+npm run build-# Now use it anywhere-lp parse document.pdf-lp screenshot document.pdf+# Use the CLI locally+node dist/src/index.js parse document.pdf```## Development Usage@@ -136,7 +149,7 @@ pnpm screenshot document.pdf --pages "1-10"#### Parse Command```-lp parse <file> [options]+liteparse parse <file> [options]Options:-o, --output <file> Output file path@@ -158,7 +171,7 @@ Options:#### Screenshot Command```-lp screenshot <file> [options]+liteparse screenshot <file> [options]Options:-o, --output-dir <dir> Output directory for screenshots (default: "./screenshots")diff --git a/cli/parse.ts b/cli/parse.tsindex 30681fb..32d4371 100644--- a/cli/parse.ts+++ b/cli/parse.ts@@ -7,7 +7,7 @@ import { LiteParseConfig, OutputFormat } from '../src/core/types.js';const program = new Command();program- .name('lp')+ .name('liteparse').description('OSS document parsing tool (supports PDF, DOCX, XLSX, images, and more)').version('1.0.0');@@ -21,7 +21,7 @@ program.option('--ocr-language <lang>', 'OCR language(s)', 'en').option('--max-pages <n>', 'Max pages to parse', '1000').option('--target-pages <pages>', 'Target pages (e.g., "1-5,10,15-20")')- .option('--dpi <dpi>', 'DPI for rendering', '150')+ .option('--dpi <dpi>', 'DPI for rendering', '300').option('--no-tables', 'Disable table detection').option('--no-precise-bbox', 'Disable precise bounding boxes').option('--skip-diagonal-text', 'Skip diagonal text')diff --git a/ocr/paddleocr/server.py b/ocr/paddleocr/server.pyindex feaa1ac..d3d15d9 100644--- a/ocr/paddleocr/server.py+++ b/ocr/paddleocr/server.py@@ -38,11 +38,12 @@ def ocr_endpoint():# Initialize OCR if needed or language changedif ocr is None or current_language != paddle_lang:print(f"Initializing PaddleOCR for language: {paddle_lang}")+ # PaddleOCR 3.x parametersocr = PaddleOCR(- use_angle_cls=True,lang=paddle_lang,- use_gpu=False,- show_log=False+ use_doc_orientation_classify=False,+ use_doc_unwarping=False,+ use_textline_orientation=True,)current_language = paddle_lang@@ -61,28 +62,70 @@ def ocr_endpoint():image_array = np.array(image)# Run OCR- # PaddleOCR returns: [[[bbox], (text, confidence)], ...]- # where bbox is [[x1,y1], [x2,y2], [x3,y3], [x4,y4]]- results = ocr.ocr(image_array, cls=True)+ # PaddleOCR 3.x returns: list of result dicts+ # Each result has: res['rec_texts'], res['rec_scores'], res['rec_boxes']+ results = ocr.predict(image_array)++ # Debug: print result structure+ print(f"Results type: {type(results)}")+ if results:+ print(f"First result type: {type(results[0])}")+ if hasattr(results[0], '__dict__'):+ print(f"First result attrs: {dir(results[0])}")+ if isinstance(results[0], dict):+ print(f"First result keys: {results[0].keys()}")# Format results according to LiteParse OCR API spec# Convert to: { text, bbox: [x1, y1, x2, y2], confidence }formatted = []- if results and results[0]:- for line in results[0]:- # line is [[[x1,y1], [x2,y2], [x3,y3], [x4,y4]], (text, confidence)]- coords, (text, confidence) = line-- # Convert polygon to axis-aligned bounding box- xs = [point[0] for point in coords]- ys = [point[1] for point in coords]- bbox = [min(xs), min(ys), max(xs), max(ys)]+ if results and len(results) > 0:+ # Get the first result+ result = results[0]++ res_data = result.get('res', result) if isinstance(result, dict) else result+ print(f"res_data type: {type(res_data)}, keys/attrs: {res_data.keys() if isinstance(res_data, dict) else dir(res_data)}")++ # Extract texts, scores, and boxes from the result+ if isinstance(res_data, dict):+ texts = res_data.get('rec_texts', [])+ scores = res_data.get('rec_scores', [])+ boxes = res_data.get('rec_boxes', [])+ else:+ # Fallback for result object with attributes+ texts = getattr(res_data, 'rec_texts', []) or []+ scores = getattr(res_data, 'rec_scores', []) or []+ boxes = getattr(res_data, 'rec_boxes', []) or []++ # Convert numpy arrays to lists if needed+ if hasattr(texts, 'tolist'):+ texts = texts.tolist()+ if hasattr(scores, 'tolist'):+ scores = scores.tolist()+ if hasattr(boxes, 'tolist'):+ boxes = boxes.tolist()++ # Combine them - they should be parallel arrays+ for i in range(len(texts)):+ text = texts[i]+ confidence = float(scores[i]) if i < len(scores) else 0.0++ # Get bounding box coordinates+ # rec_boxes format is typically [x_min, y_min, x_max, y_max]+ if i < len(boxes):+ box = boxes[i]+ # Convert to list and ensure 4 coordinates+ if hasattr(box, 'tolist'):+ bbox = box.tolist()+ else:+ bbox = list(box)+ else:+ bbox = [0, 0, 0, 0]formatted.append({'text': text,'bbox': bbox,- 'confidence': float(confidence)+ 'confidence': confidence})return jsonify({diff --git a/package.json b/package.jsonindex 2e0c10a..76eda30 100644--- a/package.json+++ b/package.json@@ -22,45 +22,44 @@"LICENSE"],"scripts": {- "preinstall": "npx only-allow pnpm","build": "tsc && npm run copy-vendor","copy-vendor": "mkdir -p dist/src/vendor && cp -r src/vendor/pdfjs dist/src/vendor/",- "build:binary": "npm run build && npm run build:bun",- "build:bun": "bun build ./dist/src/index.js --compile --target=bun --outfile=bin/lp",- "build:bun:all": "npm run build:bun:linux && npm run build:bun:macos && npm run build:bun:windows",- "build:bun:linux": "bun build ./dist/src/index.js --compile --target=bun-linux-x64 --outfile=bin/lp-linux-x64",- "build:bun:macos": "bun build ./dist/src/index.js --compile --target=bun-darwin-arm64 --outfile=bin/lp-macos-arm64",- "build:bun:windows": "bun build ./dist/src/index.js --compile --target=bun-windows-x64 --outfile=bin/lp-windows-x64.exe",- "dev": "tsc --watch",- "parse": "node dist/src/index.js parse",- "screenshot": "node dist/src/index.js screenshot",- "test-parse": "node dist/src/index.js parse test.pdf"+ "dev": "tsc --watch"},"dependencies": {"@hyzyla/pdfium": "^2.1.9",- "axios": "^1.6.0",- "canvas": "^2.11.2",- "commander": "^11.0.0",+ "axios": "^1.7.0",+ "canvas": "^3.0.0",+ "commander": "^12.0.0","form-data": "^4.0.0",- "sharp": "^0.33.0",+ "sharp": "^0.34.5","tesseract.js": "^5.0.0","unified": "^11.0.0",- "zod": "^3.22.0"+ "zod": "^3.23.0"},"devDependencies": {"@types/node": "^22.0.0",- "bun-types": "^1.1.0","typescript": "~5.9.2"},- "volta": {- "node": "22.4.1"+ "engines": {+ "node": ">=18.0.0"},- "pnpm": {- "onlyBuiltDependencies": [- "canvas"- ]+ "keywords": [+ "pdf",+ "parser",+ "ocr",+ "text-extraction",+ "pdf-to-text",+ "document-parsing"+ ],+ "repository": {+ "type": "git",+ "url": "https://github.com/run-llama/liteparse.git"+ },+ "bugs": {+ "url": "https://github.com/run-llama/liteparse/issues"},+ "homepage": "https://github.com/run-llama/liteparse#readme","author": "",- "license": "Apache-2.0",- "packageManager": "[email protected]"+ "license": "Apache-2.0"}\ No newline at end of filediff --git a/scripts/test-build.sh b/scripts/test-build.shdeleted file mode 100755index c2e7b21..0000000--- a/scripts/test-build.sh+++ /dev/null@@ -1,123 +0,0 @@-#!/bin/bash--# Test script to verify builds are working correctly-# This script tests both the TypeScript build and binary compilation--set -e # Exit on any error--echo "=== LiteParse Build Test ==="-echo--# Check if Bun is installed-if ! command -v bun &> /dev/null; then- echo "⚠️ Bun not found. Binary build will be skipped."- echo " Install Bun: curl -fsSL https://bun.sh/install | bash"- echo- SKIP_BINARY=true-else- echo "✅ Bun found: $(bun --version)"- SKIP_BINARY=false-fi--# Check Node.js version-NODE_VERSION=$(node --version)-echo "✅ Node.js: $NODE_VERSION"-echo--# Step 1: Test TypeScript build-echo "Step 1: Testing TypeScript build..."-pnpm build--if [ ! -f "dist/src/lib.js" ]; then- echo "❌ Library build failed: dist/src/lib.js not found"- exit 1-fi--if [ ! -f "dist/src/index.js" ]; then- echo "❌ CLI build failed: dist/src/index.js not found"- exit 1-fi--if [ ! -f "dist/src/lib.d.ts" ]; then- echo "❌ TypeScript declarations not generated"- exit 1-fi--echo "✅ TypeScript build successful"-echo--# Step 2: Test CLI with Node-echo "Step 2: Testing CLI with Node.js..."-node dist/src/index.js --version--if [ $? -eq 0 ]; then- echo "✅ CLI executable with Node.js works"-else- echo "❌ CLI failed to run"- exit 1-fi-echo--# Step 3: Test binary compilation (if Bun is available)-if [ "$SKIP_BINARY" = false ]; then- echo "Step 3: Testing binary compilation..."- pnpm build:binary-- if [ ! -f "bin/lp" ] && [ ! -f "bin/lp.exe" ]; then- echo "❌ Binary build failed: bin/lp not found"- exit 1- fi-- echo "✅ Binary compilation successful"- echo-- # Step 4: Test binary execution- echo "Step 4: Testing binary execution..."- if [ -f "bin/lp" ]; then- ./bin/lp --version- if [ $? -eq 0 ]; then- echo "✅ Binary executable works"- else- echo "❌ Binary failed to run"- exit 1- fi- elif [ -f "bin/lp.exe" ]; then- ./bin/lp.exe --version- if [ $? -eq 0 ]; then- echo "✅ Binary executable works"- else- echo "❌ Binary failed to run"- exit 1- fi- fi- echo-else- echo "Step 3: Skipping binary compilation (Bun not installed)"- echo "Step 4: Skipping binary test"- echo-fi--# Summary-echo "=== Build Test Summary ==="-echo "✅ TypeScript build: PASSED"-echo "✅ CLI with Node.js: PASSED"--if [ "$SKIP_BINARY" = false ]; then- echo "✅ Binary compilation: PASSED"- echo "✅ Binary execution: PASSED"- echo- echo "🎉 All tests passed! Your build system is working correctly."- echo- echo "Next steps:"- echo " • Use 'pnpm parse <file>' to test parsing"- echo " • Use './bin/lp parse <file>' to test the binary"- echo " • Run 'pnpm build:bun:all' to build for all platforms"-else- echo "⚠️ Binary build: SKIPPED (install Bun to enable)"- echo- echo "✅ TypeScript build is working correctly!"- echo- echo "Next steps:"- echo " • Install Bun to enable binary compilation: curl -fsSL https://bun.sh/install | bash"- echo " • Use 'pnpm parse <file>' to test parsing"-fidiff --git a/src/core/parser.ts b/src/core/parser.tsindex c137b2d..50da915 100644--- a/src/core/parser.ts+++ b/src/core/parser.ts@@ -80,7 +80,12 @@ export class LiteParse {);log(`Extracted ${pages.length} pages`);- // Process pages with complete grid projection+ // run BEFORE grid projection+ if (this.ocrEngine) {+ await this.runOCR(doc, pages, log);+ }++ // Process pages with complete grid projection (after OCR)const processedPages = projectPagesToGrid(pages, this.config);// Build bounding boxes if enabled@@ -99,11 +104,6 @@ export class LiteParse {}}- // OCR integration- if (this.ocrEngine) {- await this.runOCR(doc, pages, processedPages, log);- }-// Build final textconst fullText = processedPages.map((p) => p.text).join('\n\n');@@ -139,7 +139,7 @@ export class LiteParse {}/**- * Generate screenshots of PDF pages using PDFium+ * Generate screenshots of PDF pages*/async screenshot(filePath: string,@@ -170,15 +170,13 @@ export class LiteParse {}log(`Rendering page ${pageNum}...`);-- // Render page using PDFiumconst imageBuffer = await renderer.renderPageToBuffer(filePath,pageNum,this.config.dpi);- // Get page dimensions from pdfjs+ // Get page dimensionsconst pageData = await this.pdfEngine.extractPage(doc, pageNum);results.push({@@ -203,17 +201,15 @@ export class LiteParse {*/private async runOCR(doc: any,- originalPages: any[],- processedPages: any[],+ pages: any[],log: (msg: string) => void): Promise<void> {if (!this.ocrEngine) return;log('Running OCR on pages...');- for (let i = 0; i < originalPages.length; i++) {- const page = originalPages[i];- const processedPage = processedPages[i];+ for (let i = 0; i < pages.length; i++) {+ const page = pages[i];// Check if page has very little text (indicating need for OCR)const textLength = page.textItems.reduce(@@ -249,7 +245,7 @@ export class LiteParse {correctRotation: true,});- // Convert OCR results to text items and add to processed page+ // Convert OCR results to text items and add to pageif (ocrResults.length > 0) {const ocrTextItems = ocrResults.filter((r) => r.confidence > 0.1) // Filter low confidence@@ -264,8 +260,8 @@ export class LiteParse {fromOcr: true,}));- // Add OCR text to page (will be merged in grid projection if needed)- processedPage.ocrTextItems = ocrTextItems;+ // Add OCR text items directly to page textItems+ page.textItems.push(...ocrTextItems);log(` Found ${ocrTextItems.length} text items from OCR on page ${page.pageNum}`);diff --git a/src/engines/ocr/tesseract.ts b/src/engines/ocr/tesseract.tsindex 2c2778b..684822b 100644--- a/src/engines/ocr/tesseract.ts+++ b/src/engines/ocr/tesseract.ts@@ -17,15 +17,7 @@ export class TesseractEngine implements OcrEngine {await this.worker.terminate();}- console.log(`Initializing Tesseract OCR for language: ${language}`);- this.worker = await createWorker(language, 1, {- logger: (m) => {- // Only log progress, not every detail- if (m.status === 'recognizing text') {- process.stdout.write(`\rOCR Progress: ${Math.round(m.progress * 100)}%`);- }- },- });+ this.worker = await createWorker(language, 1);this.currentLanguage = language;}@@ -77,16 +69,12 @@ export class TesseractEngine implements OcrEngine {): Promise<OcrResult[][]> {const results: OcrResult[][] = [];- console.log(`\nProcessing ${imagePaths.length} images with Tesseract OCR...`);-for (let i = 0; i < imagePaths.length; i++) {const imagePath = imagePaths[i];- console.log(`\nProcessing image ${i + 1}/${imagePaths.length}: ${imagePath}`);const result = await this.recognize(imagePath, options);results.push(result);}- console.log('\n'); // Clear progress linereturn results;}diff --git a/src/engines/pdf/pdfium-renderer.ts b/src/engines/pdf/pdfium-renderer.tsindex 8ddecac..89d4719 100644--- a/src/engines/pdf/pdfium-renderer.ts+++ b/src/engines/pdf/pdfium-renderer.ts@@ -50,7 +50,12 @@ export class PdfiumRenderer {channels: 4, // RGBA},})- .png()+ .png({+ compressionLevel: 6,+ })+ .withMetadata({+ density: dpi,+ }).toBuffer();},});diff --git a/src/lib.ts b/src/lib.tsindex 38a2742..b324b42 100644--- a/src/lib.ts+++ b/src/lib.ts@@ -1,19 +1,3 @@-/**- * LiteParse - Programmatic API- *- * Use this entry point for importing LiteParse as a library in your projects.- *- * @example- * ```typescript- * import { LiteParse } from 'liteparse';- *- * const parser = new LiteParse({ ocrEnabled: true });- * const result = await parser.parse('document.pdf');- * console.log(result.text);- * ```- */--// Export main parser class and typesexport { LiteParse } from './core/parser.js';export type {LiteParseConfig,