Skip to content

A fast, helpful, and open-source document parser

use rust cli as --experimental engine

Logan Markewich2567131efd39
diff --git a/.gitignore b/.gitignore
index ccfc9a3..fcab6ef 100644
--- a/.gitignore
+++ b/.gitignore
@@ -68,3 +68,7 @@ ltt/
.docs-preview/
docs/src/content/docs/liteparse/api.md
docs/src/content/docs/liteparse/.api-tmp/
+
+# Rust
+packages/rust/target/
+packages/rust/vendor/
\ No newline at end of file
diff --git a/cli/parse.ts b/cli/parse.ts
index b5ee4c6..ff2a08a 100644
--- a/cli/parse.ts
+++ b/cli/parse.ts
@@ -27,6 +27,7 @@ interface ParseCommandOptions {
preciseBbox?: boolean;
preserveSmallText?: boolean;
password?: string;
+ experimental?: boolean;
config?: string;
quiet?: boolean;
}
@@ -82,6 +83,7 @@ program
.option("--no-precise-bbox", "Disable precise bounding boxes")
.option("--preserve-small-text", "Preserve very small text")
.option("--password <password>", "Password for encrypted/protected documents")
+ .option("--experimental", "Use experimental Rust-based PDF engine")
.option("--config <file>", "Config file (JSON)")
.option("-q, --quiet", "Suppress progress output")
.action(async (file: string, options: ParseCommandOptions) => {
@@ -126,6 +128,7 @@ program
preciseBoundingBox: options.preciseBbox !== false,
preserveVerySmallText: options.preserveSmallText || false,
password: options.password,
+ experimental: options.experimental || false,
};
// Create parser
diff --git a/packages/rust/src/extract.rs b/packages/rust/src/extract.rs
index 02d9f90..eb5112a 100644
--- a/packages/rust/src/extract.rs
+++ b/packages/rust/src/extract.rs
@@ -15,6 +15,8 @@ struct TextItem {
#[derive(Debug, Serialize)]
struct Page {
page_number: usize,
+ page_width: f32,
+ page_height: f32,
text_items: Vec<TextItem>,
}
@@ -36,6 +38,15 @@ pub fn extract(pdf_path: &str, page_num: Option<u32>) -> Result<(), Box<dyn std:
}
let mut text_items = Vec::new();
+ let mut cur_item = TextItem {
+ text: String::new(),
+ x: 0.0,
+ y: 0.0,
+ width: 0.0,
+ height: 0.0,
+ font_name: None,
+ font_size: None,
+ };
for object in page.objects().iter() {
if let Some(object) = object.as_text_object() {
@@ -53,6 +64,8 @@ pub fn extract(pdf_path: &str, page_num: Option<u32>) -> Result<(), Box<dyn std:
let page_data = Page {
page_number: page_index + 1,
+ page_width: page.width().value,
+ page_height: page.height().value,
text_items,
};
diff --git a/src/core/config.ts b/src/core/config.ts
index a9bc5e8..43591b5 100644
--- a/src/core/config.ts
+++ b/src/core/config.ts
@@ -21,6 +21,7 @@ export const DEFAULT_CONFIG: LiteParseConfig = {
preserveVerySmallText: false,
preserveLayoutAlignmentAcrossPages: false,
password: undefined,
+ experimental: false,
};
export function mergeConfig(userConfig: Partial<LiteParseConfig>): LiteParseConfig {
diff --git a/src/core/parser.ts b/src/core/parser.ts
index 8b0cca6..cfb3bcf 100644
--- a/src/core/parser.ts
+++ b/src/core/parser.ts
@@ -9,6 +9,7 @@ import {
import { mergeConfig } from "./config.js";
import { PdfEngine, PdfDocument, PageData } from "../engines/pdf/interface.js";
import { PdfJsEngine } from "../engines/pdf/pdfjs.js";
+import { LiteParseRsEngine } from "../engines/pdf/liteparse-rs.js";
import { PdfiumRenderer } from "../engines/pdf/pdfium-renderer.js";
import { OcrEngine } from "../engines/ocr/interface.js";
import { TesseractEngine } from "../engines/ocr/tesseract.js";
@@ -70,7 +71,9 @@ export class LiteParse {
this.config = mergeConfig(userConfig);
// Initialize PDF engine
- this.pdfEngine = new PdfJsEngine();
+ this.pdfEngine = this.config.experimental
+ ? new LiteParseRsEngine()
+ : new PdfJsEngine();
// Initialize OCR engine
// Auto-detect: use HTTP OCR if URL provided, otherwise use Tesseract
diff --git a/src/core/types.ts b/src/core/types.ts
index e9ea6cc..d67569c 100644
--- a/src/core/types.ts
+++ b/src/core/types.ts
@@ -136,6 +136,15 @@ export interface LiteParseConfig {
* @defaultValue `undefined`
*/
password?: string;
+
+ /**
+ * Use the experimental Rust-based PDF engine (`liteparse-rs`) instead of PDF.js.
+ * Requires the `liteparse-rs` binary to be built. Set the `LITEPARSE_RS_BIN`
+ * environment variable to override the default binary path.
+ *
+ * @defaultValue `false`
+ */
+ experimental?: boolean;
}
/**
diff --git a/src/engines/pdf/liteparse-rs.ts b/src/engines/pdf/liteparse-rs.ts
new file mode 100644
index 0000000..fbc4600
--- /dev/null
+++ b/src/engines/pdf/liteparse-rs.ts
@@ -0,0 +1,238 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import fs from "node:fs/promises";
+import { statSync } from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { PdfEngine, PdfDocument, PageData, Image } from "./interface.js";
+import { TextItem } from "../../core/types.js";
+
+const execFileAsync = promisify(execFile);
+
+/** JSON output shape from the Rust CLI for a single page */
+interface RustPageOutput {
+ page_number: number;
+ page_width: number;
+ page_height: number;
+ text_items: Array<{
+ text: string;
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+ font_name: string | null;
+ font_size: number | null;
+ }>;
+}
+
+/** Extended PdfDocument that stores the file path for CLI invocations */
+interface RustPdfDocument extends PdfDocument {
+ _filePath: string;
+ /** Cached extraction results from loadDocument (to avoid re-running CLI) */
+ _cachedPages?: RustPageOutput[];
+}
+
+/**
+ * Resolve the path to the liteparse-rs binary.
+ *
+ * Priority:
+ * 1. LITEPARSE_RS_BIN environment variable
+ * 2. Release build at packages/rust/target/release/liteparse-rs
+ * 3. Debug build at packages/rust/target/debug/liteparse-rs
+ */
+function resolveBinaryPath(): string {
+ if (process.env.LITEPARSE_RS_BIN) {
+ return process.env.LITEPARSE_RS_BIN;
+ }
+
+ const rustPkgDir = path.resolve(
+ new URL(import.meta.url).pathname,
+ "../../../../packages/rust"
+ );
+ const releaseBin = path.join(rustPkgDir, "target/release/liteparse-rs");
+ const debugBin = path.join(rustPkgDir, "target/debug/liteparse-rs");
+
+ // Prefer release, fall back to debug
+ try {
+ if (statSync(releaseBin).isFile()) return releaseBin;
+ } catch { /* not found */ }
+ try {
+ if (statSync(debugBin).isFile()) return debugBin;
+ } catch { /* not found */ }
+
+ return releaseBin; // default — will surface a clear error on exec
+}
+
+const BINARY_PATH = resolveBinaryPath();
+
+export class LiteParseRsEngine implements PdfEngine {
+ name = "liteparse-rs";
+
+ async loadDocument(input: string | Uint8Array, _password?: string): Promise<PdfDocument> {
+ let filePath: string;
+
+ if (typeof input === "string") {
+ filePath = input;
+ } else {
+ // Write buffer to a temp file since the Rust CLI only accepts file paths
+ const tempFile = path.join(os.tmpdir(), `liteparse-rs-${Date.now()}.pdf`);
+ await fs.writeFile(tempFile, input);
+ filePath = tempFile;
+ }
+
+ // Run a full extraction to discover page count (and cache results)
+ const pages = await this.runExtract(filePath);
+
+ // Read the raw bytes so the PdfDocument contract is satisfied
+ const data = new Uint8Array(await fs.readFile(filePath));
+
+ return {
+ numPages: pages.length,
+ data,
+ _filePath: filePath,
+ _cachedPages: pages,
+ } as RustPdfDocument;
+ }
+
+ async extractPage(doc: PdfDocument, pageNum: number): Promise<PageData> {
+ const rustDoc = doc as RustPdfDocument;
+
+ // Use cached data if available
+ const cached = rustDoc._cachedPages?.find((p) => p.page_number === pageNum);
+ if (cached) {
+ return this.convertPage(cached);
+ }
+
+ // Otherwise run CLI for just this page
+ const pages = await this.runExtract(rustDoc._filePath, pageNum);
+ if (pages.length === 0) {
+ return { pageNum, width: 0, height: 0, textItems: [], images: [] };
+ }
+ return this.convertPage(pages[0]);
+ }
+
+ async extractAllPages(
+ doc: PdfDocument,
+ maxPages?: number,
+ targetPages?: string
+ ): Promise<PageData[]> {
+ const rustDoc = doc as RustPdfDocument;
+
+ // Use cached data from loadDocument when possible
+ let rawPages = rustDoc._cachedPages;
+ if (!rawPages) {
+ rawPages = await this.runExtract(rustDoc._filePath);
+ }
+ // Clear cache after first use to free memory
+ rustDoc._cachedPages = undefined;
+
+ // Filter to target pages if specified
+ if (targetPages) {
+ const targetSet = this.parseTargetPages(targetPages, doc.numPages);
+ rawPages = rawPages.filter((p) => targetSet.has(p.page_number));
+ }
+
+ // Apply maxPages limit
+ if (maxPages && rawPages.length > maxPages) {
+ rawPages = rawPages.slice(0, maxPages);
+ }
+
+ return rawPages.map((p) => this.convertPage(p));
+ }
+
+ async renderPageImage(
+ _doc: PdfDocument,
+ _pageNum: number,
+ _dpi: number,
+ _password?: string
+ ): Promise<Buffer> {
+ // TODO: implement page rendering in the Rust CLI
+ throw new Error("renderPageImage is not yet implemented in liteparse-rs");
+ }
+
+ async close(_doc: PdfDocument): Promise<void> {
+ // Nothing to clean up — each CLI invocation is stateless
+ }
+
+ // ── private helpers ──────────────────────────────────────────────
+
+ /** Run the Rust CLI and parse JSON-line output */
+ private async runExtract(filePath: string, pageNum?: number): Promise<RustPageOutput[]> {
+ const args = ["extract", "--pdf-path", filePath];
+ if (pageNum !== undefined) {
+ args.push("--page-num", String(pageNum));
+ }
+
+ let stdout: string;
+ try {
+ const result = await execFileAsync(BINARY_PATH, args, {
+ maxBuffer: 100 * 1024 * 1024, // 100 MB
+ });
+ stdout = result.stdout;
+ } catch (err: unknown) {
+ const message = err instanceof Error ? err.message : String(err);
+ throw new Error(`liteparse-rs failed: ${message}`);
+ }
+
+ // Each line is a JSON object for one page
+ const pages: RustPageOutput[] = [];
+ for (const line of stdout.split("\n")) {
+ const trimmed = line.trim();
+ if (!trimmed) continue;
+ pages.push(JSON.parse(trimmed) as RustPageOutput);
+ }
+
+ return pages;
+ }
+
+ /** Convert Rust CLI output to the PageData format */
+ private convertPage(raw: RustPageOutput): PageData {
+ const pageWidth = raw.page_width;
+ const pageHeight = raw.page_height;
+
+ const textItems: TextItem[] = raw.text_items.map((item) => {
+ // pdfium uses bottom-left origin; LiteParse expects top-left origin.
+ // Convert: y_top = page_height - y_bottom - item_height
+ const y = pageHeight - item.y - item.height;
+
+ return {
+ str: item.text,
+ x: item.x,
+ y,
+ width: item.width,
+ height: item.height,
+ w: item.width,
+ h: item.height,
+ fontName: item.font_name ?? undefined,
+ fontSize: item.font_size ?? undefined,
+ };
+ });
+
+ const images: Image[] = [];
+
+ return {
+ pageNum: raw.page_number,
+ width: pageWidth,
+ height: pageHeight,
+ textItems,
+ images,
+ };
+ }
+
+ private parseTargetPages(targetPages: string, maxPages: number): Set<number> {
+ const pages = new Set<number>();
+ for (const part of targetPages.split(",")) {
+ const trimmed = part.trim();
+ if (trimmed.includes("-")) {
+ const [start, end] = trimmed.split("-").map((n) => parseInt(n.trim()));
+ for (let i = start; i <= Math.min(end, maxPages); i++) {
+ if (i >= 1) pages.add(i);
+ }
+ } else {
+ const num = parseInt(trimmed);
+ if (num >= 1 && num <= maxPages) pages.add(num);
+ }
+ }
+ return pages;
+ }
+}