A fast, helpful, and open-source document parser
use rust cli as --experimental engine
Logan Markewich2567131efd39
diff --git a/.gitignore b/.gitignoreindex ccfc9a3..fcab6ef 100644--- a/.gitignore+++ b/.gitignore@@ -68,3 +68,7 @@ ltt/.docs-preview/docs/src/content/docs/liteparse/api.mddocs/src/content/docs/liteparse/.api-tmp/++# Rust+packages/rust/target/+packages/rust/vendor/\ No newline at end of filediff --git a/cli/parse.ts b/cli/parse.tsindex b5ee4c6..ff2a08a 100644--- a/cli/parse.ts+++ b/cli/parse.ts@@ -27,6 +27,7 @@ interface ParseCommandOptions {preciseBbox?: boolean;preserveSmallText?: boolean;password?: string;+ experimental?: boolean;config?: string;quiet?: boolean;}@@ -82,6 +83,7 @@ program.option("--no-precise-bbox", "Disable precise bounding boxes").option("--preserve-small-text", "Preserve very small text").option("--password <password>", "Password for encrypted/protected documents")+ .option("--experimental", "Use experimental Rust-based PDF engine").option("--config <file>", "Config file (JSON)").option("-q, --quiet", "Suppress progress output").action(async (file: string, options: ParseCommandOptions) => {@@ -126,6 +128,7 @@ programpreciseBoundingBox: options.preciseBbox !== false,preserveVerySmallText: options.preserveSmallText || false,password: options.password,+ experimental: options.experimental || false,};// Create parserdiff --git a/packages/rust/src/extract.rs b/packages/rust/src/extract.rsindex 02d9f90..eb5112a 100644--- a/packages/rust/src/extract.rs+++ b/packages/rust/src/extract.rs@@ -15,6 +15,8 @@ struct TextItem {#[derive(Debug, Serialize)]struct Page {page_number: usize,+ page_width: f32,+ page_height: f32,text_items: Vec<TextItem>,}@@ -36,6 +38,15 @@ pub fn extract(pdf_path: &str, page_num: Option<u32>) -> Result<(), Box<dyn std:}let mut text_items = Vec::new();+ let mut cur_item = TextItem {+ text: String::new(),+ x: 0.0,+ y: 0.0,+ width: 0.0,+ height: 0.0,+ font_name: None,+ font_size: None,+ };for object in page.objects().iter() {if let Some(object) = object.as_text_object() {@@ -53,6 +64,8 @@ pub fn extract(pdf_path: &str, page_num: Option<u32>) -> Result<(), Box<dyn std:let page_data = Page {page_number: page_index + 1,+ page_width: page.width().value,+ page_height: page.height().value,text_items,};diff --git a/src/core/config.ts b/src/core/config.tsindex a9bc5e8..43591b5 100644--- a/src/core/config.ts+++ b/src/core/config.ts@@ -21,6 +21,7 @@ export const DEFAULT_CONFIG: LiteParseConfig = {preserveVerySmallText: false,preserveLayoutAlignmentAcrossPages: false,password: undefined,+ experimental: false,};export function mergeConfig(userConfig: Partial<LiteParseConfig>): LiteParseConfig {diff --git a/src/core/parser.ts b/src/core/parser.tsindex 8b0cca6..cfb3bcf 100644--- a/src/core/parser.ts+++ b/src/core/parser.ts@@ -9,6 +9,7 @@ import {import { mergeConfig } from "./config.js";import { PdfEngine, PdfDocument, PageData } from "../engines/pdf/interface.js";import { PdfJsEngine } from "../engines/pdf/pdfjs.js";+import { LiteParseRsEngine } from "../engines/pdf/liteparse-rs.js";import { PdfiumRenderer } from "../engines/pdf/pdfium-renderer.js";import { OcrEngine } from "../engines/ocr/interface.js";import { TesseractEngine } from "../engines/ocr/tesseract.js";@@ -70,7 +71,9 @@ export class LiteParse {this.config = mergeConfig(userConfig);// Initialize PDF engine- this.pdfEngine = new PdfJsEngine();+ this.pdfEngine = this.config.experimental+ ? new LiteParseRsEngine()+ : new PdfJsEngine();// Initialize OCR engine// Auto-detect: use HTTP OCR if URL provided, otherwise use Tesseractdiff --git a/src/core/types.ts b/src/core/types.tsindex e9ea6cc..d67569c 100644--- a/src/core/types.ts+++ b/src/core/types.ts@@ -136,6 +136,15 @@ export interface LiteParseConfig {* @defaultValue `undefined`*/password?: string;++ /**+ * Use the experimental Rust-based PDF engine (`liteparse-rs`) instead of PDF.js.+ * Requires the `liteparse-rs` binary to be built. Set the `LITEPARSE_RS_BIN`+ * environment variable to override the default binary path.+ *+ * @defaultValue `false`+ */+ experimental?: boolean;}/**diff --git a/src/engines/pdf/liteparse-rs.ts b/src/engines/pdf/liteparse-rs.tsnew file mode 100644index 0000000..fbc4600--- /dev/null+++ b/src/engines/pdf/liteparse-rs.ts@@ -0,0 +1,238 @@+import { execFile } from "node:child_process";+import { promisify } from "node:util";+import fs from "node:fs/promises";+import { statSync } from "node:fs";+import path from "node:path";+import os from "node:os";+import { PdfEngine, PdfDocument, PageData, Image } from "./interface.js";+import { TextItem } from "../../core/types.js";++const execFileAsync = promisify(execFile);++/** JSON output shape from the Rust CLI for a single page */+interface RustPageOutput {+ page_number: number;+ page_width: number;+ page_height: number;+ text_items: Array<{+ text: string;+ x: number;+ y: number;+ width: number;+ height: number;+ font_name: string | null;+ font_size: number | null;+ }>;+}++/** Extended PdfDocument that stores the file path for CLI invocations */+interface RustPdfDocument extends PdfDocument {+ _filePath: string;+ /** Cached extraction results from loadDocument (to avoid re-running CLI) */+ _cachedPages?: RustPageOutput[];+}++/**+ * Resolve the path to the liteparse-rs binary.+ *+ * Priority:+ * 1. LITEPARSE_RS_BIN environment variable+ * 2. Release build at packages/rust/target/release/liteparse-rs+ * 3. Debug build at packages/rust/target/debug/liteparse-rs+ */+function resolveBinaryPath(): string {+ if (process.env.LITEPARSE_RS_BIN) {+ return process.env.LITEPARSE_RS_BIN;+ }++ const rustPkgDir = path.resolve(+ new URL(import.meta.url).pathname,+ "../../../../packages/rust"+ );+ const releaseBin = path.join(rustPkgDir, "target/release/liteparse-rs");+ const debugBin = path.join(rustPkgDir, "target/debug/liteparse-rs");++ // Prefer release, fall back to debug+ try {+ if (statSync(releaseBin).isFile()) return releaseBin;+ } catch { /* not found */ }+ try {+ if (statSync(debugBin).isFile()) return debugBin;+ } catch { /* not found */ }++ return releaseBin; // default — will surface a clear error on exec+}++const BINARY_PATH = resolveBinaryPath();++export class LiteParseRsEngine implements PdfEngine {+ name = "liteparse-rs";++ async loadDocument(input: string | Uint8Array, _password?: string): Promise<PdfDocument> {+ let filePath: string;++ if (typeof input === "string") {+ filePath = input;+ } else {+ // Write buffer to a temp file since the Rust CLI only accepts file paths+ const tempFile = path.join(os.tmpdir(), `liteparse-rs-${Date.now()}.pdf`);+ await fs.writeFile(tempFile, input);+ filePath = tempFile;+ }++ // Run a full extraction to discover page count (and cache results)+ const pages = await this.runExtract(filePath);++ // Read the raw bytes so the PdfDocument contract is satisfied+ const data = new Uint8Array(await fs.readFile(filePath));++ return {+ numPages: pages.length,+ data,+ _filePath: filePath,+ _cachedPages: pages,+ } as RustPdfDocument;+ }++ async extractPage(doc: PdfDocument, pageNum: number): Promise<PageData> {+ const rustDoc = doc as RustPdfDocument;++ // Use cached data if available+ const cached = rustDoc._cachedPages?.find((p) => p.page_number === pageNum);+ if (cached) {+ return this.convertPage(cached);+ }++ // Otherwise run CLI for just this page+ const pages = await this.runExtract(rustDoc._filePath, pageNum);+ if (pages.length === 0) {+ return { pageNum, width: 0, height: 0, textItems: [], images: [] };+ }+ return this.convertPage(pages[0]);+ }++ async extractAllPages(+ doc: PdfDocument,+ maxPages?: number,+ targetPages?: string+ ): Promise<PageData[]> {+ const rustDoc = doc as RustPdfDocument;++ // Use cached data from loadDocument when possible+ let rawPages = rustDoc._cachedPages;+ if (!rawPages) {+ rawPages = await this.runExtract(rustDoc._filePath);+ }+ // Clear cache after first use to free memory+ rustDoc._cachedPages = undefined;++ // Filter to target pages if specified+ if (targetPages) {+ const targetSet = this.parseTargetPages(targetPages, doc.numPages);+ rawPages = rawPages.filter((p) => targetSet.has(p.page_number));+ }++ // Apply maxPages limit+ if (maxPages && rawPages.length > maxPages) {+ rawPages = rawPages.slice(0, maxPages);+ }++ return rawPages.map((p) => this.convertPage(p));+ }++ async renderPageImage(+ _doc: PdfDocument,+ _pageNum: number,+ _dpi: number,+ _password?: string+ ): Promise<Buffer> {+ // TODO: implement page rendering in the Rust CLI+ throw new Error("renderPageImage is not yet implemented in liteparse-rs");+ }++ async close(_doc: PdfDocument): Promise<void> {+ // Nothing to clean up — each CLI invocation is stateless+ }++ // ── private helpers ──────────────────────────────────────────────++ /** Run the Rust CLI and parse JSON-line output */+ private async runExtract(filePath: string, pageNum?: number): Promise<RustPageOutput[]> {+ const args = ["extract", "--pdf-path", filePath];+ if (pageNum !== undefined) {+ args.push("--page-num", String(pageNum));+ }++ let stdout: string;+ try {+ const result = await execFileAsync(BINARY_PATH, args, {+ maxBuffer: 100 * 1024 * 1024, // 100 MB+ });+ stdout = result.stdout;+ } catch (err: unknown) {+ const message = err instanceof Error ? err.message : String(err);+ throw new Error(`liteparse-rs failed: ${message}`);+ }++ // Each line is a JSON object for one page+ const pages: RustPageOutput[] = [];+ for (const line of stdout.split("\n")) {+ const trimmed = line.trim();+ if (!trimmed) continue;+ pages.push(JSON.parse(trimmed) as RustPageOutput);+ }++ return pages;+ }++ /** Convert Rust CLI output to the PageData format */+ private convertPage(raw: RustPageOutput): PageData {+ const pageWidth = raw.page_width;+ const pageHeight = raw.page_height;++ const textItems: TextItem[] = raw.text_items.map((item) => {+ // pdfium uses bottom-left origin; LiteParse expects top-left origin.+ // Convert: y_top = page_height - y_bottom - item_height+ const y = pageHeight - item.y - item.height;++ return {+ str: item.text,+ x: item.x,+ y,+ width: item.width,+ height: item.height,+ w: item.width,+ h: item.height,+ fontName: item.font_name ?? undefined,+ fontSize: item.font_size ?? undefined,+ };+ });++ const images: Image[] = [];++ return {+ pageNum: raw.page_number,+ width: pageWidth,+ height: pageHeight,+ textItems,+ images,+ };+ }++ private parseTargetPages(targetPages: string, maxPages: number): Set<number> {+ const pages = new Set<number>();+ for (const part of targetPages.split(",")) {+ const trimmed = part.trim();+ if (trimmed.includes("-")) {+ const [start, end] = trimmed.split("-").map((n) => parseInt(n.trim()));+ for (let i = start; i <= Math.min(end, maxPages); i++) {+ if (i >= 1) pages.add(i);+ }+ } else {+ const num = parseInt(trimmed);+ if (num >= 1 && num <= maxPages) pages.add(num);+ }+ }+ return pages;+ }+}