Skip to content

A fast, helpful, and open-source document parser

clean up OCR usage

Logan Markewichbbce765d25c2
diff --git a/.gitignore b/.gitignore
index 3030534..4781b7f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,3 +37,10 @@ screenshots/
# Temporary files
tmp/
temp/
+*.txt
+*.png
+deno.json
+
+# Experimental/development artifacts
+ac/
+examples/
diff --git a/README.md b/README.md
index 8e7d7ea..26505c8 100644
--- a/README.md
+++ b/README.md
@@ -22,72 +22,85 @@ LiteParse is a standalone OSS PDF parsing tool focused exclusively on **fast and
## Installation
-### Option 1: Use as a Library
+### CLI Tool
-Install via npm/pnpm:
+#### Option 1: Global Install (Recommended)
+
+Install globally via npm to use the `liteparse` command anywhere:
```bash
-pnpm add liteparse
+npm i -g liteparse
```
-```typescript
-import { LiteParse } from 'liteparse';
+Then use it:
-const parser = new LiteParse({ ocrEnabled: true });
-const result = await parser.parse('document.pdf');
-console.log(result.text);
+```bash
+liteparse parse document.pdf
+liteparse screenshot document.pdf
```
-### Option 2: Build from Source
+#### Option 2: Use with npx
-#### Prerequisites
+Run directly without installing:
-- Node.js 22.4.1 or higher
-- pnpm package manager
-- [Bun](https://bun.sh) (for building binaries)
+```bash
+npx liteparse parse document.pdf
+npx liteparse screenshot document.pdf
+```
+
+#### Option 3: Homebrew (macOS/Linux)
-#### Install Dependencies
+Coming soon! Once published, you'll be able to install via Homebrew:
```bash
-cd liteparse
-pnpm install
+brew tap yourusername/liteparse
+brew install liteparse
```
-#### Build TypeScript
+### Library Usage
+
+Install as a dependency in your project:
```bash
-pnpm build
+npm install liteparse
+# or
+pnpm add liteparse
```
-#### Build Binary (Optional)
+```typescript
+import { LiteParse } from 'liteparse';
-Build a standalone executable that you can run anywhere:
+const parser = new LiteParse({ ocrEnabled: true });
+const result = await parser.parse('document.pdf');
+console.log(result.text);
+```
-```bash
-# Install Bun if you haven't already
-curl -fsSL https://bun.sh/install | bash
+### Build from Source
-# Build binary for your current platform
-pnpm build:binary
+If you want to contribute or build from source:
-# Or build for specific platforms
-pnpm build:bun:linux # Linux x64
-pnpm build:bun:macos # macOS ARM64
-pnpm build:bun:windows # Windows x64
-pnpm build:bun:all # All platforms
+#### Prerequisites
-# The binary will be in ./bin/lp (or lp-{platform})
-```
+- Node.js 18.0.0 or higher
+- npm or pnpm package manager
-Once built, you can use the binary directly:
+#### Steps
```bash
-# Add to your PATH
-export PATH="$PATH:/path/to/liteparse/bin"
+# Clone the repository
+git clone https://github.com/run-llama/liteparse.git
+cd liteparse
+
+# Install dependencies
+npm install
+# or
+pnpm install
+
+# Build TypeScript
+npm run build
-# Now use it anywhere
-lp parse document.pdf
-lp screenshot document.pdf
+# Use the CLI locally
+node dist/src/index.js parse document.pdf
```
## Development Usage
@@ -136,7 +149,7 @@ pnpm screenshot document.pdf --pages "1-10"
#### Parse Command
```
-lp parse <file> [options]
+liteparse parse <file> [options]
Options:
-o, --output <file> Output file path
@@ -158,7 +171,7 @@ Options:
#### Screenshot Command
```
-lp screenshot <file> [options]
+liteparse screenshot <file> [options]
Options:
-o, --output-dir <dir> Output directory for screenshots (default: "./screenshots")
diff --git a/cli/parse.ts b/cli/parse.ts
index 30681fb..32d4371 100644
--- a/cli/parse.ts
+++ b/cli/parse.ts
@@ -7,7 +7,7 @@ import { LiteParseConfig, OutputFormat } from '../src/core/types.js';
const program = new Command();
program
- .name('lp')
+ .name('liteparse')
.description('OSS document parsing tool (supports PDF, DOCX, XLSX, images, and more)')
.version('1.0.0');
@@ -21,7 +21,7 @@ program
.option('--ocr-language <lang>', 'OCR language(s)', 'en')
.option('--max-pages <n>', 'Max pages to parse', '1000')
.option('--target-pages <pages>', 'Target pages (e.g., "1-5,10,15-20")')
- .option('--dpi <dpi>', 'DPI for rendering', '150')
+ .option('--dpi <dpi>', 'DPI for rendering', '300')
.option('--no-tables', 'Disable table detection')
.option('--no-precise-bbox', 'Disable precise bounding boxes')
.option('--skip-diagonal-text', 'Skip diagonal text')
diff --git a/ocr/paddleocr/server.py b/ocr/paddleocr/server.py
index feaa1ac..d3d15d9 100644
--- a/ocr/paddleocr/server.py
+++ b/ocr/paddleocr/server.py
@@ -38,11 +38,12 @@ def ocr_endpoint():
# Initialize OCR if needed or language changed
if ocr is None or current_language != paddle_lang:
print(f"Initializing PaddleOCR for language: {paddle_lang}")
+ # PaddleOCR 3.x parameters
ocr = PaddleOCR(
- use_angle_cls=True,
lang=paddle_lang,
- use_gpu=False,
- show_log=False
+ use_doc_orientation_classify=False,
+ use_doc_unwarping=False,
+ use_textline_orientation=True,
)
current_language = paddle_lang
@@ -61,28 +62,70 @@ def ocr_endpoint():
image_array = np.array(image)
# Run OCR
- # PaddleOCR returns: [[[bbox], (text, confidence)], ...]
- # where bbox is [[x1,y1], [x2,y2], [x3,y3], [x4,y4]]
- results = ocr.ocr(image_array, cls=True)
+ # PaddleOCR 3.x returns: list of result dicts
+ # Each result has: res['rec_texts'], res['rec_scores'], res['rec_boxes']
+ results = ocr.predict(image_array)
+
+ # Debug: print result structure
+ print(f"Results type: {type(results)}")
+ if results:
+ print(f"First result type: {type(results[0])}")
+ if hasattr(results[0], '__dict__'):
+ print(f"First result attrs: {dir(results[0])}")
+ if isinstance(results[0], dict):
+ print(f"First result keys: {results[0].keys()}")
# Format results according to LiteParse OCR API spec
# Convert to: { text, bbox: [x1, y1, x2, y2], confidence }
formatted = []
- if results and results[0]:
- for line in results[0]:
- # line is [[[x1,y1], [x2,y2], [x3,y3], [x4,y4]], (text, confidence)]
- coords, (text, confidence) = line
-
- # Convert polygon to axis-aligned bounding box
- xs = [point[0] for point in coords]
- ys = [point[1] for point in coords]
- bbox = [min(xs), min(ys), max(xs), max(ys)]
+ if results and len(results) > 0:
+ # Get the first result
+ result = results[0]
+
+ res_data = result.get('res', result) if isinstance(result, dict) else result
+ print(f"res_data type: {type(res_data)}, keys/attrs: {res_data.keys() if isinstance(res_data, dict) else dir(res_data)}")
+
+ # Extract texts, scores, and boxes from the result
+ if isinstance(res_data, dict):
+ texts = res_data.get('rec_texts', [])
+ scores = res_data.get('rec_scores', [])
+ boxes = res_data.get('rec_boxes', [])
+ else:
+ # Fallback for result object with attributes
+ texts = getattr(res_data, 'rec_texts', []) or []
+ scores = getattr(res_data, 'rec_scores', []) or []
+ boxes = getattr(res_data, 'rec_boxes', []) or []
+
+ # Convert numpy arrays to lists if needed
+ if hasattr(texts, 'tolist'):
+ texts = texts.tolist()
+ if hasattr(scores, 'tolist'):
+ scores = scores.tolist()
+ if hasattr(boxes, 'tolist'):
+ boxes = boxes.tolist()
+
+ # Combine them - they should be parallel arrays
+ for i in range(len(texts)):
+ text = texts[i]
+ confidence = float(scores[i]) if i < len(scores) else 0.0
+
+ # Get bounding box coordinates
+ # rec_boxes format is typically [x_min, y_min, x_max, y_max]
+ if i < len(boxes):
+ box = boxes[i]
+ # Convert to list and ensure 4 coordinates
+ if hasattr(box, 'tolist'):
+ bbox = box.tolist()
+ else:
+ bbox = list(box)
+ else:
+ bbox = [0, 0, 0, 0]
formatted.append({
'text': text,
'bbox': bbox,
- 'confidence': float(confidence)
+ 'confidence': confidence
})
return jsonify({
diff --git a/package.json b/package.json
index 2e0c10a..76eda30 100644
--- a/package.json
+++ b/package.json
@@ -22,45 +22,44 @@
"LICENSE"
],
"scripts": {
- "preinstall": "npx only-allow pnpm",
"build": "tsc && npm run copy-vendor",
"copy-vendor": "mkdir -p dist/src/vendor && cp -r src/vendor/pdfjs dist/src/vendor/",
- "build:binary": "npm run build && npm run build:bun",
- "build:bun": "bun build ./dist/src/index.js --compile --target=bun --outfile=bin/lp",
- "build:bun:all": "npm run build:bun:linux && npm run build:bun:macos && npm run build:bun:windows",
- "build:bun:linux": "bun build ./dist/src/index.js --compile --target=bun-linux-x64 --outfile=bin/lp-linux-x64",
- "build:bun:macos": "bun build ./dist/src/index.js --compile --target=bun-darwin-arm64 --outfile=bin/lp-macos-arm64",
- "build:bun:windows": "bun build ./dist/src/index.js --compile --target=bun-windows-x64 --outfile=bin/lp-windows-x64.exe",
- "dev": "tsc --watch",
- "parse": "node dist/src/index.js parse",
- "screenshot": "node dist/src/index.js screenshot",
- "test-parse": "node dist/src/index.js parse test.pdf"
+ "dev": "tsc --watch"
},
"dependencies": {
"@hyzyla/pdfium": "^2.1.9",
- "axios": "^1.6.0",
- "canvas": "^2.11.2",
- "commander": "^11.0.0",
+ "axios": "^1.7.0",
+ "canvas": "^3.0.0",
+ "commander": "^12.0.0",
"form-data": "^4.0.0",
- "sharp": "^0.33.0",
+ "sharp": "^0.34.5",
"tesseract.js": "^5.0.0",
"unified": "^11.0.0",
- "zod": "^3.22.0"
+ "zod": "^3.23.0"
},
"devDependencies": {
"@types/node": "^22.0.0",
- "bun-types": "^1.1.0",
"typescript": "~5.9.2"
},
- "volta": {
- "node": "22.4.1"
+ "engines": {
+ "node": ">=18.0.0"
},
- "pnpm": {
- "onlyBuiltDependencies": [
- "canvas"
- ]
+ "keywords": [
+ "pdf",
+ "parser",
+ "ocr",
+ "text-extraction",
+ "pdf-to-text",
+ "document-parsing"
+ ],
+ "repository": {
+ "type": "git",
+ "url": "https://github.com/run-llama/liteparse.git"
+ },
+ "bugs": {
+ "url": "https://github.com/run-llama/liteparse/issues"
},
+ "homepage": "https://github.com/run-llama/liteparse#readme",
"author": "",
- "license": "Apache-2.0",
- "packageManager": "[email protected]"
+ "license": "Apache-2.0"
}
\ No newline at end of file
diff --git a/scripts/test-build.sh b/scripts/test-build.sh
deleted file mode 100755
index c2e7b21..0000000
--- a/scripts/test-build.sh
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/bin/bash
-
-# Test script to verify builds are working correctly
-# This script tests both the TypeScript build and binary compilation
-
-set -e # Exit on any error
-
-echo "=== LiteParse Build Test ==="
-echo
-
-# Check if Bun is installed
-if ! command -v bun &> /dev/null; then
- echo "⚠️ Bun not found. Binary build will be skipped."
- echo " Install Bun: curl -fsSL https://bun.sh/install | bash"
- echo
- SKIP_BINARY=true
-else
- echo "✅ Bun found: $(bun --version)"
- SKIP_BINARY=false
-fi
-
-# Check Node.js version
-NODE_VERSION=$(node --version)
-echo "✅ Node.js: $NODE_VERSION"
-echo
-
-# Step 1: Test TypeScript build
-echo "Step 1: Testing TypeScript build..."
-pnpm build
-
-if [ ! -f "dist/src/lib.js" ]; then
- echo "❌ Library build failed: dist/src/lib.js not found"
- exit 1
-fi
-
-if [ ! -f "dist/src/index.js" ]; then
- echo "❌ CLI build failed: dist/src/index.js not found"
- exit 1
-fi
-
-if [ ! -f "dist/src/lib.d.ts" ]; then
- echo "❌ TypeScript declarations not generated"
- exit 1
-fi
-
-echo "✅ TypeScript build successful"
-echo
-
-# Step 2: Test CLI with Node
-echo "Step 2: Testing CLI with Node.js..."
-node dist/src/index.js --version
-
-if [ $? -eq 0 ]; then
- echo "✅ CLI executable with Node.js works"
-else
- echo "❌ CLI failed to run"
- exit 1
-fi
-echo
-
-# Step 3: Test binary compilation (if Bun is available)
-if [ "$SKIP_BINARY" = false ]; then
- echo "Step 3: Testing binary compilation..."
- pnpm build:binary
-
- if [ ! -f "bin/lp" ] && [ ! -f "bin/lp.exe" ]; then
- echo "❌ Binary build failed: bin/lp not found"
- exit 1
- fi
-
- echo "✅ Binary compilation successful"
- echo
-
- # Step 4: Test binary execution
- echo "Step 4: Testing binary execution..."
- if [ -f "bin/lp" ]; then
- ./bin/lp --version
- if [ $? -eq 0 ]; then
- echo "✅ Binary executable works"
- else
- echo "❌ Binary failed to run"
- exit 1
- fi
- elif [ -f "bin/lp.exe" ]; then
- ./bin/lp.exe --version
- if [ $? -eq 0 ]; then
- echo "✅ Binary executable works"
- else
- echo "❌ Binary failed to run"
- exit 1
- fi
- fi
- echo
-else
- echo "Step 3: Skipping binary compilation (Bun not installed)"
- echo "Step 4: Skipping binary test"
- echo
-fi
-
-# Summary
-echo "=== Build Test Summary ==="
-echo "✅ TypeScript build: PASSED"
-echo "✅ CLI with Node.js: PASSED"
-
-if [ "$SKIP_BINARY" = false ]; then
- echo "✅ Binary compilation: PASSED"
- echo "✅ Binary execution: PASSED"
- echo
- echo "🎉 All tests passed! Your build system is working correctly."
- echo
- echo "Next steps:"
- echo " • Use 'pnpm parse <file>' to test parsing"
- echo " • Use './bin/lp parse <file>' to test the binary"
- echo " • Run 'pnpm build:bun:all' to build for all platforms"
-else
- echo "⚠️ Binary build: SKIPPED (install Bun to enable)"
- echo
- echo "✅ TypeScript build is working correctly!"
- echo
- echo "Next steps:"
- echo " • Install Bun to enable binary compilation: curl -fsSL https://bun.sh/install | bash"
- echo " • Use 'pnpm parse <file>' to test parsing"
-fi
diff --git a/src/core/parser.ts b/src/core/parser.ts
index c137b2d..50da915 100644
--- a/src/core/parser.ts
+++ b/src/core/parser.ts
@@ -80,7 +80,12 @@ export class LiteParse {
);
log(`Extracted ${pages.length} pages`);
- // Process pages with complete grid projection
+ // run BEFORE grid projection
+ if (this.ocrEngine) {
+ await this.runOCR(doc, pages, log);
+ }
+
+ // Process pages with complete grid projection (after OCR)
const processedPages = projectPagesToGrid(pages, this.config);
// Build bounding boxes if enabled
@@ -99,11 +104,6 @@ export class LiteParse {
}
}
- // OCR integration
- if (this.ocrEngine) {
- await this.runOCR(doc, pages, processedPages, log);
- }
-
// Build final text
const fullText = processedPages.map((p) => p.text).join('\n\n');
@@ -139,7 +139,7 @@ export class LiteParse {
}
/**
- * Generate screenshots of PDF pages using PDFium
+ * Generate screenshots of PDF pages
*/
async screenshot(
filePath: string,
@@ -170,15 +170,13 @@ export class LiteParse {
}
log(`Rendering page ${pageNum}...`);
-
- // Render page using PDFium
const imageBuffer = await renderer.renderPageToBuffer(
filePath,
pageNum,
this.config.dpi
);
- // Get page dimensions from pdfjs
+ // Get page dimensions
const pageData = await this.pdfEngine.extractPage(doc, pageNum);
results.push({
@@ -203,17 +201,15 @@ export class LiteParse {
*/
private async runOCR(
doc: any,
- originalPages: any[],
- processedPages: any[],
+ pages: any[],
log: (msg: string) => void
): Promise<void> {
if (!this.ocrEngine) return;
log('Running OCR on pages...');
- for (let i = 0; i < originalPages.length; i++) {
- const page = originalPages[i];
- const processedPage = processedPages[i];
+ for (let i = 0; i < pages.length; i++) {
+ const page = pages[i];
// Check if page has very little text (indicating need for OCR)
const textLength = page.textItems.reduce(
@@ -249,7 +245,7 @@ export class LiteParse {
correctRotation: true,
});
- // Convert OCR results to text items and add to processed page
+ // Convert OCR results to text items and add to page
if (ocrResults.length > 0) {
const ocrTextItems = ocrResults
.filter((r) => r.confidence > 0.1) // Filter low confidence
@@ -264,8 +260,8 @@ export class LiteParse {
fromOcr: true,
}));
- // Add OCR text to page (will be merged in grid projection if needed)
- processedPage.ocrTextItems = ocrTextItems;
+ // Add OCR text items directly to page textItems
+ page.textItems.push(...ocrTextItems);
log(
` Found ${ocrTextItems.length} text items from OCR on page ${page.pageNum}`
);
diff --git a/src/engines/ocr/tesseract.ts b/src/engines/ocr/tesseract.ts
index 2c2778b..684822b 100644
--- a/src/engines/ocr/tesseract.ts
+++ b/src/engines/ocr/tesseract.ts
@@ -17,15 +17,7 @@ export class TesseractEngine implements OcrEngine {
await this.worker.terminate();
}
- console.log(`Initializing Tesseract OCR for language: ${language}`);
- this.worker = await createWorker(language, 1, {
- logger: (m) => {
- // Only log progress, not every detail
- if (m.status === 'recognizing text') {
- process.stdout.write(`\rOCR Progress: ${Math.round(m.progress * 100)}%`);
- }
- },
- });
+ this.worker = await createWorker(language, 1);
this.currentLanguage = language;
}
@@ -77,16 +69,12 @@ export class TesseractEngine implements OcrEngine {
): Promise<OcrResult[][]> {
const results: OcrResult[][] = [];
- console.log(`\nProcessing ${imagePaths.length} images with Tesseract OCR...`);
-
for (let i = 0; i < imagePaths.length; i++) {
const imagePath = imagePaths[i];
- console.log(`\nProcessing image ${i + 1}/${imagePaths.length}: ${imagePath}`);
const result = await this.recognize(imagePath, options);
results.push(result);
}
- console.log('\n'); // Clear progress line
return results;
}
diff --git a/src/engines/pdf/pdfium-renderer.ts b/src/engines/pdf/pdfium-renderer.ts
index 8ddecac..89d4719 100644
--- a/src/engines/pdf/pdfium-renderer.ts
+++ b/src/engines/pdf/pdfium-renderer.ts
@@ -50,7 +50,12 @@ export class PdfiumRenderer {
channels: 4, // RGBA
},
})
- .png()
+ .png({
+ compressionLevel: 6,
+ })
+ .withMetadata({
+ density: dpi,
+ })
.toBuffer();
},
});
diff --git a/src/lib.ts b/src/lib.ts
index 38a2742..b324b42 100644
--- a/src/lib.ts
+++ b/src/lib.ts
@@ -1,19 +1,3 @@
-/**
- * LiteParse - Programmatic API
- *
- * Use this entry point for importing LiteParse as a library in your projects.
- *
- * @example
- * ```typescript
- * import { LiteParse } from 'liteparse';
- *
- * const parser = new LiteParse({ ocrEnabled: true });
- * const result = await parser.parse('document.pdf');
- * console.log(result.text);
- * ```
- */
-
-// Export main parser class and types
export { LiteParse } from './core/parser.js';
export type {
LiteParseConfig,