A fast, helpful, and open-source document parser
optimize some algos
Logan Markewich783cb39f4bed
diff --git a/cli/parse.ts b/cli/parse.tsindex 338402c..627e425 100644--- a/cli/parse.ts+++ b/cli/parse.ts@@ -1,8 +1,10 @@import { Command } from 'commander';import fs from 'fs/promises';-import { existsSync } from 'fs';+import { existsSync, readdirSync, statSync } from 'fs';+import path from 'path';import { LiteParse } from '../src/core/parser.js';import { LiteParseConfig, OutputFormat } from '../src/core/types.js';+import { performance } from 'perf_hooks';const program = new Command();@@ -179,6 +181,197 @@ program}});+// Supported file extensions for batch parsing+const SUPPORTED_EXTENSIONS = new Set([+ '.pdf',+ '.doc', '.docx', '.docm', '.dot', '.dotm', '.dotx', '.odt', '.ott',+ '.ppt', '.pptx', '.pptm', '.pot', '.potm', '.potx', '.odp', '.otp',+ '.xls', '.xlsx', '.xlsm', '.xlsb', '.ods', '.ots', '.csv', '.tsv',+ '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg',+ '.rtf', '.pages', '.key', '.numbers',+]);++program+ .command('batch-parse <input-dir> <output-dir>')+ .description('Parse multiple documents in batch mode')+ .option('--format <format>', 'Output format: json|text', 'text')+ .option('--ocr-server-url <url>', 'HTTP OCR server URL (uses Tesseract if not provided)')+ .option('--no-ocr', 'Disable OCR')+ .option('--ocr-language <lang>', 'OCR language(s)', 'en')+ .option('--max-pages <n>', 'Max pages to parse per file', '1000')+ .option('--dpi <dpi>', 'DPI for rendering', '150')+ .option('--no-tables', 'Disable table detection')+ .option('--no-precise-bbox', 'Disable precise bounding boxes')+ .option('--recursive', 'Recursively search input directory')+ .option('--extension <ext>', 'Only process files with this extension (e.g., ".pdf")')+ .option('--config <file>', 'Config file (JSON)')+ .option('-q, --quiet', 'Suppress progress output')+ .action(async (inputDir: string, outputDir: string, options: any) => {+ try {+ const quiet = options.quiet || false;+ const startTime = performance.now();++ // Validate input directory+ if (!existsSync(inputDir)) {+ console.error(`Error: Input directory not found: ${inputDir}`);+ process.exit(1);+ }++ const inputStat = statSync(inputDir);+ if (!inputStat.isDirectory()) {+ console.error(`Error: Input path is not a directory: ${inputDir}`);+ process.exit(1);+ }++ // Create output directory+ if (!existsSync(outputDir)) {+ await fs.mkdir(outputDir, { recursive: true });+ }++ // Find all files to process+ const files = findFiles(inputDir, options.recursive, options.extension);++ if (files.length === 0) {+ console.error('No supported files found in input directory');+ process.exit(1);+ }++ if (!quiet) {+ console.error(`Found ${files.length} files to process`);+ }++ // Load config+ let config: Partial<LiteParseConfig> = {};+ if (options.config) {+ if (!existsSync(options.config)) {+ console.error(`Error: Config file not found: ${options.config}`);+ process.exit(1);+ }+ const configData = await fs.readFile(options.config, 'utf-8');+ config = JSON.parse(configData);+ }++ // Apply CLI options+ config = {+ ...config,+ outputFormat: options.format as OutputFormat,+ ocrEnabled: options.ocr !== false,+ ocrServerUrl: options.ocrServerUrl,+ ocrLanguage: options.ocrLanguage,+ maxPages: parseInt(options.maxPages),+ dpi: parseInt(options.dpi),+ tableDetection: options.tables !== false,+ preciseBoundingBox: options.preciseBbox !== false,+ };++ // Create a SINGLE parser instance for all files (key for batch efficiency)+ const parser = new LiteParse(config);++ // Process files+ let successCount = 0;+ let errorCount = 0;+ const outputExt = options.format === 'json' ? '.json' : '.txt';++ for (let i = 0; i < files.length; i++) {+ const file = files[i];+ const relativePath = path.relative(inputDir, file);+ const outputPath = path.join(+ outputDir,+ relativePath.replace(/\.[^.]+$/, outputExt)+ );++ // Create output subdirectory if needed+ const outputSubdir = path.dirname(outputPath);+ if (!existsSync(outputSubdir)) {+ await fs.mkdir(outputSubdir, { recursive: true });+ }++ try {+ const fileStart = performance.now();+ const result = await parser.parse(file, true); // Always quiet for individual files++ // Format output+ let output: string;+ if (options.format === 'json') {+ output = JSON.stringify(result.json, null, 2);+ } else {+ output = result.text;+ }++ await fs.writeFile(outputPath, output);+ successCount++;++ if (!quiet) {+ const fileTime = (performance.now() - fileStart).toFixed(0);+ console.error(`[${i + 1}/${files.length}] ✓ ${relativePath} (${result.pages.length} pages, ${fileTime}ms)`);+ }+ } catch (error: any) {+ errorCount++;+ if (!quiet) {+ console.error(`[${i + 1}/${files.length}] ✗ ${relativePath}: ${error.message}`);+ }+ }+ }++ const totalTime = ((performance.now() - startTime) / 1000).toFixed(2);+ const avgTime = files.length > 0 ? ((performance.now() - startTime) / files.length).toFixed(0) : 0;++ if (!quiet) {+ console.error('');+ console.error(`Batch complete: ${successCount} succeeded, ${errorCount} failed`);+ console.error(`Total time: ${totalTime}s (avg ${avgTime}ms/file)`);+ console.error(`Output: ${outputDir}`);+ }++ if (errorCount > 0) {+ process.exit(1);+ }+ } catch (error: any) {+ console.error(`\nError: ${error.message}`);+ if (error.stack) {+ console.error(error.stack);+ }+ process.exit(1);+ }+ });++/**+ * Find all supported files in a directory+ */+function findFiles(dir: string, recursive: boolean, filterExt?: string): string[] {+ const files: string[] = [];++ function scan(currentDir: string) {+ const entries = readdirSync(currentDir);++ for (const entry of entries) {+ const fullPath = path.join(currentDir, entry);+ const stat = statSync(fullPath);++ if (stat.isDirectory()) {+ if (recursive) {+ scan(fullPath);+ }+ } else if (stat.isFile()) {+ const ext = path.extname(entry).toLowerCase();++ // Filter by extension if specified+ if (filterExt && ext !== filterExt.toLowerCase()) {+ continue;+ }++ // Check if supported+ if (SUPPORTED_EXTENSIONS.has(ext)) {+ files.push(fullPath);+ }+ }+ }+ }++ scan(dir);+ return files.sort();+}+/*** Parse page numbers from string like "1,3,5" or "1-5,10"*/diff --git a/src/conversion/convertToPdf.ts b/src/conversion/convertToPdf.tsindex b79e84b..8d5ac4d 100644--- a/src/conversion/convertToPdf.ts+++ b/src/conversion/convertToPdf.ts@@ -13,6 +13,10 @@ export interface ConversionError {code: string;}+export interface ConversionPassthrough {+ content: string+}+// File extension categoriesexport const officeExtensions = ['.doc',@@ -242,7 +246,7 @@ async function convertImageToPdf(*/export async function convertToPdf(filePath: string-): Promise<ConversionResult | ConversionError> {+): Promise<ConversionResult | ConversionPassthrough | ConversionError> {try {// Check if file existstry {@@ -280,15 +284,13 @@ export async function convertToPdf(} else if (imageExtensions.includes(extension)) {console.error(`Converting image: ${path.basename(filePath)}`);pdfPath = await convertImageToPdf(filePath, tmpDir);- } else if (htmlExtensions.includes(extension)) {- return {- message: `HTML conversion not yet supported. Please convert to PDF manually.`,- code: 'UNSUPPORTED_FORMAT',- };- } else {+ }+ else {+ // Unsupported format+ // Assume its a text-based format and pass through text directly+ const content = await fs.readFile(filePath, 'utf-8');return {- message: `Unsupported file format: ${extension}`,- code: 'UNSUPPORTED_FORMAT',+ content,};}diff --git a/src/core/parser.ts b/src/core/parser.tsindex 50da915..6a89d8e 100644--- a/src/core/parser.ts+++ b/src/core/parser.ts@@ -58,6 +58,15 @@ export class LiteParse {throw new Error(`Conversion failed: ${conversionResult.message}`);}+ // Return early for text-based passthrough formats+ if ('content' in conversionResult) {+ log(`File is a text-based format. Returning content directly.`);+ return {+ pages: [],+ text: conversionResult.content,+ };+ }+pdfPath = conversionResult.pdfPath;needsCleanup = pdfPath !== filePath;diff --git a/src/core/types.ts b/src/core/types.tsindex 2863c90..5b47aea 100644--- a/src/core/types.ts+++ b/src/core/types.ts@@ -1,3 +1,5 @@+import { DetectedTable } from '../processing/tables.js';+export type OutputFormat = 'json' | 'text';export interface LiteParseConfig {@@ -110,8 +112,7 @@ export interface ParsedPage {text: string;textItems: TextItem[];boundingBoxes?: BoundingBox[];- tables?: any[];- images?: any[];+ tables?: DetectedTable[];}export interface ParseResult {diff --git a/src/engines/pdf/pdfjs.ts b/src/engines/pdf/pdfjs.tsindex 04d488a..83db1d2 100644--- a/src/engines/pdf/pdfjs.ts+++ b/src/engines/pdf/pdfjs.ts@@ -83,9 +83,14 @@ function singularValueDecompose2dScale(m: number[]): { x: number; y: number } {return { x: Math.sqrt(sx), y: Math.sqrt(sy) };}+// Pre-compiled regex patterns for string decoding (avoid recompilation per item)+const BUGGY_FONT_MARKER_REGEX = /:->|>_(\d+)_\d+_<|<-:/g;+const BUGGY_FONT_MARKER_CHECK = ':->|>';+const PIPE_PATTERN_REGEX = /\s*\|([^|])\|\s*/g;+export class PdfJsEngine implements PdfEngine {name = 'pdfjs';- private pdfiumRenderer: PdfiumRenderer = new PdfiumRenderer();+ private pdfiumRenderer: PdfiumRenderer | null = null;private currentPdfPath: string | null = null;async loadDocument(filePath: string): Promise<PdfDocument> {@@ -121,96 +126,95 @@ export class PdfJsEngine implements PdfEngine {// Extract text contentconst textContent = await page.getTextContent();- const textItems: TextItem[] = textContent.items- .filter((item: any) => item.height !== 0 && item.width !== 0)- .map((item: any) => {- // Apply viewport transformation to convert PDF coordinates to screen coordinates- // This properly handles Y-axis flip (PDF is bottom-up, screen is top-down)- const cm = multiplyMatrices(viewport.transform, item.transform);-- // Get lower-left corner (text space origin)- const ll = applyTransformation({ x: 0, y: 0 }, cm);-- // Get scale factors to properly size the bounding box- const scale = singularValueDecompose2dScale(item.transform);-- // Get upper-right corner- const ur = applyTransformation(- { x: item.width / scale.x, y: item.height / scale.y },- cm- );-- // Calculate final bounding box in viewport space- const left = Math.min(ll.x, ur.x);- const right = Math.max(ll.x, ur.x);- const top = Math.min(ll.y, ur.y);- const bottom = Math.max(ll.y, ur.y);-- const width = right - left;- const height = bottom - top;-- // Calculate rotation from combined transformation matrix- let rotation = getRotation(cm);- // Normalize to 0-360 range- if (rotation < 0) {- rotation += 360;- }+ const viewportWidth = viewport.width;+ const viewportHeight = viewport.height;+ const viewportTransform = viewport.transform;++ const textItems: TextItem[] = [];+ for (const item of textContent.items as any[]) {+ // Skip items with zero dimensions+ if (item.height === 0 || item.width === 0) continue;++ // Apply viewport transformation to convert PDF coordinates to screen coordinates+ // This properly handles Y-axis flip (PDF is bottom-up, screen is top-down)+ const cm = multiplyMatrices(viewportTransform, item.transform);++ // Get lower-left corner (text space origin)+ const ll = applyTransformation({ x: 0, y: 0 }, cm);++ // Get scale factors to properly size the bounding box+ const scale = singularValueDecompose2dScale(item.transform);++ // Get upper-right corner+ const ur = applyTransformation(+ { x: item.width / scale.x, y: item.height / scale.y },+ cm+ );++ // Calculate final bounding box in viewport space+ const left = Math.min(ll.x, ur.x);+ const right = Math.max(ll.x, ur.x);+ const top = Math.min(ll.y, ur.y);+ const bottom = Math.max(ll.y, ur.y);++ // Skip items that are off-page (negative coordinates or beyond page bounds)+ if (top < 0 || left < 0 || top > viewportHeight || left > viewportWidth) continue;++ const width = right - left;+ const height = bottom - top;++ // Calculate rotation from combined transformation matrix+ let rotation = getRotation(cm);+ // Normalize to 0-360 range+ if (rotation < 0) {+ rotation += 360;+ }- // Decode buggy font markers from PDF.js- // Format: :->|>_<charCode>_<fontChar>_<|<-:- let decodedStr = item.str.replace(- /:->|>_(\d+)_\d+_<|<-:/g,+ // Decode buggy font markers from PDF.js (only if marker is present)+ // Format: :->|>_<charCode>_<fontChar>_<|<-:+ let decodedStr = item.str;+ if (decodedStr.includes(BUGGY_FONT_MARKER_CHECK)) {+ BUGGY_FONT_MARKER_REGEX.lastIndex = 0; // Reset regex state+ decodedStr = decodedStr.replace(+ BUGGY_FONT_MARKER_REGEX,(_: string, charCode: string) => String.fromCharCode(parseInt(charCode)));+ }- // Handle pipe-separated characters: " |a| |r| |X| " -> "arX"- // Some PDFs encode text with characters separated by pipes and spaces- if (decodedStr.includes('|')) {- const pipePattern = /\s*\|([^|])\|\s*/g;- const matches = [...decodedStr.matchAll(pipePattern)];- if (matches.length > 0) {- decodedStr = matches.map(m => m[1]).join('');- }+ // Handle pipe-separated characters: " |a| |r| |X| " -> "arX"+ // Some PDFs encode text with characters separated by pipes and spaces+ if (decodedStr.includes('|')) {+ PIPE_PATTERN_REGEX.lastIndex = 0; // Reset regex state+ const matches = [...decodedStr.matchAll(PIPE_PATTERN_REGEX)];+ if (matches.length > 0) {+ decodedStr = matches.map(m => m[1]).join('');}+ }- return {- str: decodedStr,- x: left,- y: top,- width,- height,- w: width,- h: height,- r: rotation,- fontName: item.fontName,- fontSize: Math.sqrt(- item.transform[0] * item.transform[0] + item.transform[1] * item.transform[1]- ),- };- })- .filter((item: TextItem) => {- // Filter out items that are off-page (negative coordinates or beyond page bounds)- return (- item.y >= 0 &&- item.x >= 0 &&- item.y <= viewport.height &&- item.x <= viewport.width- );+ textItems.push({+ str: decodedStr,+ x: left,+ y: top,+ width,+ height,+ w: width,+ h: height,+ r: rotation,+ fontName: item.fontName,+ fontSize: Math.sqrt(+ item.transform[0] * item.transform[0] + item.transform[1] * item.transform[1]+ ),});-- // Extract annotations (links, etc.)- const rawAnnotations = await page.getAnnotations();- const annotations: Annotation[] = rawAnnotations.map((ann: any) => ({- type: ann.annotationType,- subtype: ann.subtype,- url: ann.url,- rect: ann.rect,- }));+ }// For now, paths and images are empty - we'll add extraction logic laterconst paths: Path[] = [];const images: Image[] = [];+ // Skip annotation extraction - not currently used in processing pipeline+ // Can be re-enabled if needed for link extraction, etc.+ const annotations: Annotation[] = [];+await page.cleanup();return {@@ -265,6 +269,10 @@ export class PdfJsEngine implements PdfEngine {throw new Error('PDF path not available for rendering');}+ if (!this.pdfiumRenderer) {+ this.pdfiumRenderer = new PdfiumRenderer();+ }+return await this.pdfiumRenderer.renderPageToBuffer(this.currentPdfPath,pageNum,@@ -278,8 +286,11 @@ export class PdfJsEngine implements PdfEngine {await pdfDocument.destroy();}- // Clean up PDFium renderer- await this.pdfiumRenderer.close();+ // Clean up PDFium renderer (only if it was initialized)+ if (this.pdfiumRenderer) {+ await this.pdfiumRenderer.close();+ this.pdfiumRenderer = null;+ }this.currentPdfPath = null;}diff --git a/src/output/json.ts b/src/output/json.tsindex a112249..ad3973a 100644--- a/src/output/json.ts+++ b/src/output/json.ts@@ -21,7 +21,6 @@ export function buildJSON(pages: ParsedPage[]): any {})),boundingBoxes: page.boundingBoxes || [],tables: page.tables || [],- images: page.images || [],})),};}diff --git a/src/processing/gridProjection.ts b/src/processing/gridProjection.tsindex 5fbf5f1..2d47b5a 100644--- a/src/processing/gridProjection.ts+++ b/src/processing/gridProjection.ts@@ -1508,8 +1508,7 @@ export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig):text,textItems: page.textItems,boundingBoxes: [],- tables: [],- images: []+ tables: []});}