Skip to content

A fast, helpful, and open-source document parser

optimize some algos

Logan Markewich783cb39f4bed
diff --git a/cli/parse.ts b/cli/parse.ts
index 338402c..627e425 100644
--- a/cli/parse.ts
+++ b/cli/parse.ts
@@ -1,8 +1,10 @@
import { Command } from 'commander';
import fs from 'fs/promises';
-import { existsSync } from 'fs';
+import { existsSync, readdirSync, statSync } from 'fs';
+import path from 'path';
import { LiteParse } from '../src/core/parser.js';
import { LiteParseConfig, OutputFormat } from '../src/core/types.js';
+import { performance } from 'perf_hooks';
const program = new Command();
@@ -179,6 +181,197 @@ program
}
});
+// Supported file extensions for batch parsing
+const SUPPORTED_EXTENSIONS = new Set([
+ '.pdf',
+ '.doc', '.docx', '.docm', '.dot', '.dotm', '.dotx', '.odt', '.ott',
+ '.ppt', '.pptx', '.pptm', '.pot', '.potm', '.potx', '.odp', '.otp',
+ '.xls', '.xlsx', '.xlsm', '.xlsb', '.ods', '.ots', '.csv', '.tsv',
+ '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg',
+ '.rtf', '.pages', '.key', '.numbers',
+]);
+
+program
+ .command('batch-parse <input-dir> <output-dir>')
+ .description('Parse multiple documents in batch mode')
+ .option('--format <format>', 'Output format: json|text', 'text')
+ .option('--ocr-server-url <url>', 'HTTP OCR server URL (uses Tesseract if not provided)')
+ .option('--no-ocr', 'Disable OCR')
+ .option('--ocr-language <lang>', 'OCR language(s)', 'en')
+ .option('--max-pages <n>', 'Max pages to parse per file', '1000')
+ .option('--dpi <dpi>', 'DPI for rendering', '150')
+ .option('--no-tables', 'Disable table detection')
+ .option('--no-precise-bbox', 'Disable precise bounding boxes')
+ .option('--recursive', 'Recursively search input directory')
+ .option('--extension <ext>', 'Only process files with this extension (e.g., ".pdf")')
+ .option('--config <file>', 'Config file (JSON)')
+ .option('-q, --quiet', 'Suppress progress output')
+ .action(async (inputDir: string, outputDir: string, options: any) => {
+ try {
+ const quiet = options.quiet || false;
+ const startTime = performance.now();
+
+ // Validate input directory
+ if (!existsSync(inputDir)) {
+ console.error(`Error: Input directory not found: ${inputDir}`);
+ process.exit(1);
+ }
+
+ const inputStat = statSync(inputDir);
+ if (!inputStat.isDirectory()) {
+ console.error(`Error: Input path is not a directory: ${inputDir}`);
+ process.exit(1);
+ }
+
+ // Create output directory
+ if (!existsSync(outputDir)) {
+ await fs.mkdir(outputDir, { recursive: true });
+ }
+
+ // Find all files to process
+ const files = findFiles(inputDir, options.recursive, options.extension);
+
+ if (files.length === 0) {
+ console.error('No supported files found in input directory');
+ process.exit(1);
+ }
+
+ if (!quiet) {
+ console.error(`Found ${files.length} files to process`);
+ }
+
+ // Load config
+ let config: Partial<LiteParseConfig> = {};
+ if (options.config) {
+ if (!existsSync(options.config)) {
+ console.error(`Error: Config file not found: ${options.config}`);
+ process.exit(1);
+ }
+ const configData = await fs.readFile(options.config, 'utf-8');
+ config = JSON.parse(configData);
+ }
+
+ // Apply CLI options
+ config = {
+ ...config,
+ outputFormat: options.format as OutputFormat,
+ ocrEnabled: options.ocr !== false,
+ ocrServerUrl: options.ocrServerUrl,
+ ocrLanguage: options.ocrLanguage,
+ maxPages: parseInt(options.maxPages),
+ dpi: parseInt(options.dpi),
+ tableDetection: options.tables !== false,
+ preciseBoundingBox: options.preciseBbox !== false,
+ };
+
+ // Create a SINGLE parser instance for all files (key for batch efficiency)
+ const parser = new LiteParse(config);
+
+ // Process files
+ let successCount = 0;
+ let errorCount = 0;
+ const outputExt = options.format === 'json' ? '.json' : '.txt';
+
+ for (let i = 0; i < files.length; i++) {
+ const file = files[i];
+ const relativePath = path.relative(inputDir, file);
+ const outputPath = path.join(
+ outputDir,
+ relativePath.replace(/\.[^.]+$/, outputExt)
+ );
+
+ // Create output subdirectory if needed
+ const outputSubdir = path.dirname(outputPath);
+ if (!existsSync(outputSubdir)) {
+ await fs.mkdir(outputSubdir, { recursive: true });
+ }
+
+ try {
+ const fileStart = performance.now();
+ const result = await parser.parse(file, true); // Always quiet for individual files
+
+ // Format output
+ let output: string;
+ if (options.format === 'json') {
+ output = JSON.stringify(result.json, null, 2);
+ } else {
+ output = result.text;
+ }
+
+ await fs.writeFile(outputPath, output);
+ successCount++;
+
+ if (!quiet) {
+ const fileTime = (performance.now() - fileStart).toFixed(0);
+ console.error(`[${i + 1}/${files.length}] ✓ ${relativePath} (${result.pages.length} pages, ${fileTime}ms)`);
+ }
+ } catch (error: any) {
+ errorCount++;
+ if (!quiet) {
+ console.error(`[${i + 1}/${files.length}] ✗ ${relativePath}: ${error.message}`);
+ }
+ }
+ }
+
+ const totalTime = ((performance.now() - startTime) / 1000).toFixed(2);
+ const avgTime = files.length > 0 ? ((performance.now() - startTime) / files.length).toFixed(0) : 0;
+
+ if (!quiet) {
+ console.error('');
+ console.error(`Batch complete: ${successCount} succeeded, ${errorCount} failed`);
+ console.error(`Total time: ${totalTime}s (avg ${avgTime}ms/file)`);
+ console.error(`Output: ${outputDir}`);
+ }
+
+ if (errorCount > 0) {
+ process.exit(1);
+ }
+ } catch (error: any) {
+ console.error(`\nError: ${error.message}`);
+ if (error.stack) {
+ console.error(error.stack);
+ }
+ process.exit(1);
+ }
+ });
+
+/**
+ * Find all supported files in a directory
+ */
+function findFiles(dir: string, recursive: boolean, filterExt?: string): string[] {
+ const files: string[] = [];
+
+ function scan(currentDir: string) {
+ const entries = readdirSync(currentDir);
+
+ for (const entry of entries) {
+ const fullPath = path.join(currentDir, entry);
+ const stat = statSync(fullPath);
+
+ if (stat.isDirectory()) {
+ if (recursive) {
+ scan(fullPath);
+ }
+ } else if (stat.isFile()) {
+ const ext = path.extname(entry).toLowerCase();
+
+ // Filter by extension if specified
+ if (filterExt && ext !== filterExt.toLowerCase()) {
+ continue;
+ }
+
+ // Check if supported
+ if (SUPPORTED_EXTENSIONS.has(ext)) {
+ files.push(fullPath);
+ }
+ }
+ }
+ }
+
+ scan(dir);
+ return files.sort();
+}
+
/**
* Parse page numbers from string like "1,3,5" or "1-5,10"
*/
diff --git a/src/conversion/convertToPdf.ts b/src/conversion/convertToPdf.ts
index b79e84b..8d5ac4d 100644
--- a/src/conversion/convertToPdf.ts
+++ b/src/conversion/convertToPdf.ts
@@ -13,6 +13,10 @@ export interface ConversionError {
code: string;
}
+export interface ConversionPassthrough {
+ content: string
+}
+
// File extension categories
export const officeExtensions = [
'.doc',
@@ -242,7 +246,7 @@ async function convertImageToPdf(
*/
export async function convertToPdf(
filePath: string
-): Promise<ConversionResult | ConversionError> {
+): Promise<ConversionResult | ConversionPassthrough | ConversionError> {
try {
// Check if file exists
try {
@@ -280,15 +284,13 @@ export async function convertToPdf(
} else if (imageExtensions.includes(extension)) {
console.error(`Converting image: ${path.basename(filePath)}`);
pdfPath = await convertImageToPdf(filePath, tmpDir);
- } else if (htmlExtensions.includes(extension)) {
- return {
- message: `HTML conversion not yet supported. Please convert to PDF manually.`,
- code: 'UNSUPPORTED_FORMAT',
- };
- } else {
+ }
+ else {
+ // Unsupported format
+ // Assume its a text-based format and pass through text directly
+ const content = await fs.readFile(filePath, 'utf-8');
return {
- message: `Unsupported file format: ${extension}`,
- code: 'UNSUPPORTED_FORMAT',
+ content,
};
}
diff --git a/src/core/parser.ts b/src/core/parser.ts
index 50da915..6a89d8e 100644
--- a/src/core/parser.ts
+++ b/src/core/parser.ts
@@ -58,6 +58,15 @@ export class LiteParse {
throw new Error(`Conversion failed: ${conversionResult.message}`);
}
+ // Return early for text-based passthrough formats
+ if ('content' in conversionResult) {
+ log(`File is a text-based format. Returning content directly.`);
+ return {
+ pages: [],
+ text: conversionResult.content,
+ };
+ }
+
pdfPath = conversionResult.pdfPath;
needsCleanup = pdfPath !== filePath;
diff --git a/src/core/types.ts b/src/core/types.ts
index 2863c90..5b47aea 100644
--- a/src/core/types.ts
+++ b/src/core/types.ts
@@ -1,3 +1,5 @@
+import { DetectedTable } from '../processing/tables.js';
+
export type OutputFormat = 'json' | 'text';
export interface LiteParseConfig {
@@ -110,8 +112,7 @@ export interface ParsedPage {
text: string;
textItems: TextItem[];
boundingBoxes?: BoundingBox[];
- tables?: any[];
- images?: any[];
+ tables?: DetectedTable[];
}
export interface ParseResult {
diff --git a/src/engines/pdf/pdfjs.ts b/src/engines/pdf/pdfjs.ts
index 04d488a..83db1d2 100644
--- a/src/engines/pdf/pdfjs.ts
+++ b/src/engines/pdf/pdfjs.ts
@@ -83,9 +83,14 @@ function singularValueDecompose2dScale(m: number[]): { x: number; y: number } {
return { x: Math.sqrt(sx), y: Math.sqrt(sy) };
}
+// Pre-compiled regex patterns for string decoding (avoid recompilation per item)
+const BUGGY_FONT_MARKER_REGEX = /:->|>_(\d+)_\d+_<|<-:/g;
+const BUGGY_FONT_MARKER_CHECK = ':->|>';
+const PIPE_PATTERN_REGEX = /\s*\|([^|])\|\s*/g;
+
export class PdfJsEngine implements PdfEngine {
name = 'pdfjs';
- private pdfiumRenderer: PdfiumRenderer = new PdfiumRenderer();
+ private pdfiumRenderer: PdfiumRenderer | null = null;
private currentPdfPath: string | null = null;
async loadDocument(filePath: string): Promise<PdfDocument> {
@@ -121,96 +126,95 @@ export class PdfJsEngine implements PdfEngine {
// Extract text content
const textContent = await page.getTextContent();
- const textItems: TextItem[] = textContent.items
- .filter((item: any) => item.height !== 0 && item.width !== 0)
- .map((item: any) => {
- // Apply viewport transformation to convert PDF coordinates to screen coordinates
- // This properly handles Y-axis flip (PDF is bottom-up, screen is top-down)
- const cm = multiplyMatrices(viewport.transform, item.transform);
-
- // Get lower-left corner (text space origin)
- const ll = applyTransformation({ x: 0, y: 0 }, cm);
-
- // Get scale factors to properly size the bounding box
- const scale = singularValueDecompose2dScale(item.transform);
-
- // Get upper-right corner
- const ur = applyTransformation(
- { x: item.width / scale.x, y: item.height / scale.y },
- cm
- );
-
- // Calculate final bounding box in viewport space
- const left = Math.min(ll.x, ur.x);
- const right = Math.max(ll.x, ur.x);
- const top = Math.min(ll.y, ur.y);
- const bottom = Math.max(ll.y, ur.y);
-
- const width = right - left;
- const height = bottom - top;
-
- // Calculate rotation from combined transformation matrix
- let rotation = getRotation(cm);
- // Normalize to 0-360 range
- if (rotation < 0) {
- rotation += 360;
- }
+ const viewportWidth = viewport.width;
+ const viewportHeight = viewport.height;
+ const viewportTransform = viewport.transform;
+
+ const textItems: TextItem[] = [];
+ for (const item of textContent.items as any[]) {
+ // Skip items with zero dimensions
+ if (item.height === 0 || item.width === 0) continue;
+
+ // Apply viewport transformation to convert PDF coordinates to screen coordinates
+ // This properly handles Y-axis flip (PDF is bottom-up, screen is top-down)
+ const cm = multiplyMatrices(viewportTransform, item.transform);
+
+ // Get lower-left corner (text space origin)
+ const ll = applyTransformation({ x: 0, y: 0 }, cm);
+
+ // Get scale factors to properly size the bounding box
+ const scale = singularValueDecompose2dScale(item.transform);
+
+ // Get upper-right corner
+ const ur = applyTransformation(
+ { x: item.width / scale.x, y: item.height / scale.y },
+ cm
+ );
+
+ // Calculate final bounding box in viewport space
+ const left = Math.min(ll.x, ur.x);
+ const right = Math.max(ll.x, ur.x);
+ const top = Math.min(ll.y, ur.y);
+ const bottom = Math.max(ll.y, ur.y);
+
+ // Skip items that are off-page (negative coordinates or beyond page bounds)
+ if (top < 0 || left < 0 || top > viewportHeight || left > viewportWidth) continue;
+
+ const width = right - left;
+ const height = bottom - top;
+
+ // Calculate rotation from combined transformation matrix
+ let rotation = getRotation(cm);
+ // Normalize to 0-360 range
+ if (rotation < 0) {
+ rotation += 360;
+ }
- // Decode buggy font markers from PDF.js
- // Format: :->|>_<charCode>_<fontChar>_<|<-:
- let decodedStr = item.str.replace(
- /:->|>_(\d+)_\d+_<|<-:/g,
+ // Decode buggy font markers from PDF.js (only if marker is present)
+ // Format: :->|>_<charCode>_<fontChar>_<|<-:
+ let decodedStr = item.str;
+ if (decodedStr.includes(BUGGY_FONT_MARKER_CHECK)) {
+ BUGGY_FONT_MARKER_REGEX.lastIndex = 0; // Reset regex state
+ decodedStr = decodedStr.replace(
+ BUGGY_FONT_MARKER_REGEX,
(_: string, charCode: string) => String.fromCharCode(parseInt(charCode))
);
+ }
- // Handle pipe-separated characters: " |a| |r| |X| " -> "arX"
- // Some PDFs encode text with characters separated by pipes and spaces
- if (decodedStr.includes('|')) {
- const pipePattern = /\s*\|([^|])\|\s*/g;
- const matches = [...decodedStr.matchAll(pipePattern)];
- if (matches.length > 0) {
- decodedStr = matches.map(m => m[1]).join('');
- }
+ // Handle pipe-separated characters: " |a| |r| |X| " -> "arX"
+ // Some PDFs encode text with characters separated by pipes and spaces
+ if (decodedStr.includes('|')) {
+ PIPE_PATTERN_REGEX.lastIndex = 0; // Reset regex state
+ const matches = [...decodedStr.matchAll(PIPE_PATTERN_REGEX)];
+ if (matches.length > 0) {
+ decodedStr = matches.map(m => m[1]).join('');
}
+ }
- return {
- str: decodedStr,
- x: left,
- y: top,
- width,
- height,
- w: width,
- h: height,
- r: rotation,
- fontName: item.fontName,
- fontSize: Math.sqrt(
- item.transform[0] * item.transform[0] + item.transform[1] * item.transform[1]
- ),
- };
- })
- .filter((item: TextItem) => {
- // Filter out items that are off-page (negative coordinates or beyond page bounds)
- return (
- item.y >= 0 &&
- item.x >= 0 &&
- item.y <= viewport.height &&
- item.x <= viewport.width
- );
+ textItems.push({
+ str: decodedStr,
+ x: left,
+ y: top,
+ width,
+ height,
+ w: width,
+ h: height,
+ r: rotation,
+ fontName: item.fontName,
+ fontSize: Math.sqrt(
+ item.transform[0] * item.transform[0] + item.transform[1] * item.transform[1]
+ ),
});
-
- // Extract annotations (links, etc.)
- const rawAnnotations = await page.getAnnotations();
- const annotations: Annotation[] = rawAnnotations.map((ann: any) => ({
- type: ann.annotationType,
- subtype: ann.subtype,
- url: ann.url,
- rect: ann.rect,
- }));
+ }
// For now, paths and images are empty - we'll add extraction logic later
const paths: Path[] = [];
const images: Image[] = [];
+ // Skip annotation extraction - not currently used in processing pipeline
+ // Can be re-enabled if needed for link extraction, etc.
+ const annotations: Annotation[] = [];
+
await page.cleanup();
return {
@@ -265,6 +269,10 @@ export class PdfJsEngine implements PdfEngine {
throw new Error('PDF path not available for rendering');
}
+ if (!this.pdfiumRenderer) {
+ this.pdfiumRenderer = new PdfiumRenderer();
+ }
+
return await this.pdfiumRenderer.renderPageToBuffer(
this.currentPdfPath,
pageNum,
@@ -278,8 +286,11 @@ export class PdfJsEngine implements PdfEngine {
await pdfDocument.destroy();
}
- // Clean up PDFium renderer
- await this.pdfiumRenderer.close();
+ // Clean up PDFium renderer (only if it was initialized)
+ if (this.pdfiumRenderer) {
+ await this.pdfiumRenderer.close();
+ this.pdfiumRenderer = null;
+ }
this.currentPdfPath = null;
}
diff --git a/src/output/json.ts b/src/output/json.ts
index a112249..ad3973a 100644
--- a/src/output/json.ts
+++ b/src/output/json.ts
@@ -21,7 +21,6 @@ export function buildJSON(pages: ParsedPage[]): any {
})),
boundingBoxes: page.boundingBoxes || [],
tables: page.tables || [],
- images: page.images || [],
})),
};
}
diff --git a/src/processing/gridProjection.ts b/src/processing/gridProjection.ts
index 5fbf5f1..2d47b5a 100644
--- a/src/processing/gridProjection.ts
+++ b/src/processing/gridProjection.ts
@@ -1508,8 +1508,7 @@ export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig):
text,
textItems: page.textItems,
boundingBoxes: [],
- tables: [],
- images: []
+ tables: []
});
}