A fast, helpful, and open-source document parser
wip
Logan Markewich3f0e86ad7fd4
diff --git a/crates/liteparse-wasm/build.rs b/crates/liteparse-wasm/build.rsnew file mode 100644index 0000000..ba700dd--- /dev/null+++ b/crates/liteparse-wasm/build.rs@@ -0,0 +1,15 @@+use std::env;++fn main() {+ let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default();++ // pdfium's wasm build links libsetjmp.a for __c_longjmp, but that archive's+ // single object also defines the __wasm_setjmp/__wasm_longjmp helpers that+ // Rust's own codegen already emits. The definitions are identical LLVM SjLj+ // helpers, so allow the linker to keep the first and resolve the otherwise+ // missing __c_longjmp from the same archive. Must live here (the final+ // cdylib) because rustc-link-arg is not propagated from dependency crates.+ if target_arch == "wasm32" {+ println!("cargo:rustc-link-arg=--allow-multiple-definition");+ }+}diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rsindex b649856..a9340c2 100644--- a/crates/liteparse/src/markdown_layout/classify.rs+++ b/crates/liteparse/src/markdown_layout/classify.rs@@ -13,7 +13,7 @@ use super::inline::{use super::lists::{LIST_INDENT_STEP_PT, parse_list_marker};use super::paragraphs::{ParaAccum, append_to_paragraph, collapse_whitespace, continues_heading, continues_paragraph,- ends_hyphenated, is_soft_hyphen_break,+ ends_hyphenated, ends_sentence_final, is_soft_hyphen_break,};use super::repetition::is_header_or_footer;use super::tables::{detect_ruled_tables, detect_tables, merge_table_runs};@@ -617,11 +617,16 @@ fn classify_region(outline_heading_level(line, page.page_height, outline, text)}});- let size_level = if is_caption_line(text) || toc_suppress {- None- } else {- heading_level_for(heading_size_of(line), heading_map)- };+ // Rotated lines (margin stamps like a sideways "arXiv:…" watermark) are+ // excluded from `build_heading_map`, but their size can still coincide+ // with a map entry built from horizontal text. Suppress font-size+ // promotion for them so they don't surface as spurious headings.+ let size_level =+ if is_caption_line(text) || toc_suppress || is_rotated_line(line) || line.in_figure {+ None+ } else {+ heading_level_for(heading_size_of(line), heading_map)+ };// Guard against height-jitter false headings: a line that flows from// the previous line (same paragraph) AND starts lowercase is a// mid-paragraph continuation, not a heading — even if its@@ -639,8 +644,21 @@ fn classify_region(// line is its continuation regardless of capitalization// ("…SOLAR 10.7 Billion-" / "Parameter Model: We have…").let prev_hyphen_wrap = prev.is_some_and(|p| ends_hyphenated(&p.text));- !((starts_lower || prev_hyphen_wrap)- && prev.is_some_and(|p| continues_paragraph(p, line)))+ // `prev` is the *live body paragraph* (or list line) immediately+ // above, so it only exists mid-paragraph — never right after a+ // heading. A lowercase line that *completes a sentence* (ends with+ // terminal punctuation) while its predecessor is left open (no+ // terminal punctuation) is a wrapped sentence tail, not a heading —+ // even when a centered short tail's large left-edge indent defeats+ // `continues_paragraph` ("…journalistic or" → "scholarly works.").+ // Requiring the tail to end a sentence keeps genuine lowercase+ // headings (which rarely end in `.`/`!`/`?`) promoted.+ let sentence_tail = starts_lower+ && ends_sentence_final(text)+ && prev.is_some_and(|p| !ends_sentence_final(&p.text));+ let is_continuation =+ prev.is_some_and(|p| continues_paragraph(p, line)) || sentence_tail;+ !((starts_lower || prev_hyphen_wrap) && is_continuation)});// Long-text guard: a real heading is a label, not a sentence with// multiple clauses. Footnotes / citations / captions that happen todiff --git a/crates/liteparse/src/markdown_layout/headings.rs b/crates/liteparse/src/markdown_layout/headings.rsindex 79ebfaf..97a1f83 100644--- a/crates/liteparse/src/markdown_layout/headings.rs+++ b/crates/liteparse/src/markdown_layout/headings.rs@@ -21,6 +21,28 @@ pub(super) const ESTIMATED_HEADING_SIZE_MARGIN: f32 = 1.5;/// Cap on heading levels (matches plan: H1..H6).pub(super) const MAX_HEADING_LEVELS: usize = 6;+/// A normalized line text that recurs at least this many times among+/// heading-candidate lines (size > body) is treated as chart/figure label+/// spam, not a heading, and is excluded from the heading-size map. Example:+/// the "Input-Input Layer5" attention-plot labels in academic papers render+/// at 23–28pt (far above body) and would otherwise hijack H1..H3, demoting the+/// real title and section headings. Genuine headings are not byte-identical+/// across the document, so this never drops a real heading size *unless* that+/// size is occupied solely by the repeated string — and the exclusion is+/// per-line, so a repeated string sharing a font size with real headings+/// leaves that level intact.+pub(super) const REPEATED_HEADING_LINE_MIN: usize = 3;++/// Normalize a line's text for repeat-detection: collapse internal whitespace+/// and lowercase, so trivially-different renderings of the same label string+/// still hash together.+fn normalize_repeat_key(text: &str) -> String {+ text.split_whitespace()+ .collect::<Vec<_>>()+ .join(" ")+ .to_lowercase()+}+/// Tighter tolerance for matching against the heading-size map. Keeps the/// heading detector strict so descender-induced height jitter doesn't promote/// regular body lines to headings.@@ -621,11 +643,34 @@ const MIN_HEADING_ALPHA_RATIO: f32 = 0.5;/// artifacts), sorted descending, mapped to levels 1..=MAX_HEADING_LEVELS.pub fn build_heading_map(pages: &[ParsedPage], body_size: f32) -> Vec<(f32, u8)> {use std::collections::HashMap;+ let heading_margin = |line: &ProjectedLine| {+ if line.font_size_is_estimated {+ ESTIMATED_HEADING_SIZE_MARGIN+ } else {+ HEADING_SIZE_EPSILON+ }+ };+ // First pass: frequency of each normalized candidate-line text. A string+ // that recurs ≥ REPEATED_HEADING_LINE_MIN times is chart/figure label spam+ // (e.g. repeated attention-plot labels at 23–28pt), not a heading.+ let mut text_freq: HashMap<String, usize> = HashMap::new();+ for page in pages {+ for line in &page.projected_lines {+ if is_rotated_line(line) || is_caption_line(&line.text) || line.in_figure {+ continue;+ }+ if heading_size_of(line) > body_size + heading_margin(line) {+ *text_freq+ .entry(normalize_repeat_key(&line.text))+ .or_insert(0) += 1;+ }+ }+ }// (size_key → (size, line_count, total_chars, alpha_chars))let mut sizes: HashMap<u32, (f32, usize, usize, usize)> = HashMap::new();for page in pages {for line in &page.projected_lines {- if is_rotated_line(line) {+ if is_rotated_line(line) || line.in_figure {continue;}// Captions ("Figure 7", "Table 3.") often render slightly larger@@ -633,12 +678,18 @@ pub fn build_heading_map(pages: &[ParsedPage], body_size: f32) -> Vec<(f32, u8)>if is_caption_line(&line.text) {continue;}+ // Exclude repeated chart/figure labels (per-line, so a repeated+ // string sharing a size with real headings still leaves that level).+ if text_freq+ .get(&normalize_repeat_key(&line.text))+ .copied()+ .unwrap_or(0)+ >= REPEATED_HEADING_LINE_MIN+ {+ continue;+ }let size = heading_size_of(line);- let margin = if line.font_size_is_estimated {- ESTIMATED_HEADING_SIZE_MARGIN- } else {- HEADING_SIZE_EPSILON- };+ let margin = heading_margin(line);if size > body_size + margin {let key = (size * 100.0).round() as u32;let entry = sizes.entry(key).or_insert((size, 0, 0, 0));diff --git a/crates/liteparse/src/markdown_layout/paragraphs.rs b/crates/liteparse/src/markdown_layout/paragraphs.rsindex 73bec51..20c60dc 100644--- a/crates/liteparse/src/markdown_layout/paragraphs.rs+++ b/crates/liteparse/src/markdown_layout/paragraphs.rs@@ -68,6 +68,21 @@ pub(super) fn ends_hyphenated(text: &str) -> bool {t.ends_with('-') && t.chars().rev().nth(1).is_some_and(|c| c.is_alphabetic())}+/// True when `text` ends a sentence: its last non-space character (ignoring a+/// trailing closing quote / bracket) is `.`, `!`, or `?`. The heading+/// lowercase-continuation guard uses the negation — a previous line that does+/// *not* end a sentence is an open clause that a following lowercase line+/// continues, even when its left-edge indent defeats `continues_paragraph`+/// (centered short tail lines like "…journalistic or" → "scholarly works.").+pub(super) fn ends_sentence_final(text: &str) -> bool {+ let t = text+ .trim_end()+ .trim_end_matches(|c| matches!(c, '"' | '\'' | ')' | ']' | '»' | '”' | '’'));+ t.chars()+ .next_back()+ .is_some_and(|c| matches!(c, '.' | '!' | '?'))+}+/// True when `prev` ends with a soft hyphen (see [`ends_hyphenated`]) and/// `next` begins with a lowercase letter — the signal that a single word was/// split across a line / block / region break and should be rejoined with thediff --git a/crates/liteparse/src/markdown_layout/test_helpers.rs b/crates/liteparse/src/markdown_layout/test_helpers.rsindex dc95741..c167eb7 100644--- a/crates/liteparse/src/markdown_layout/test_helpers.rs+++ b/crates/liteparse/src/markdown_layout/test_helpers.rs@@ -24,6 +24,7 @@ pub(crate) fn line(text: &str, x: f32, y: f32, h: f32, size: f32) -> ProjectedLispans: vec![TextItem::default()],region_path: Vec::new(),mcid: None,+ in_figure: false,}}@@ -95,6 +96,7 @@ pub(crate) fn line_with_spans(cells: &[(&str, f32)], y: f32, size: f32) -> Projespans,region_path: Vec::new(),mcid: None,+ in_figure: false,}}@@ -145,6 +147,7 @@ pub(crate) fn styled_line(spans: &[(&str, f32, Option<&str>)], y: f32, size: f32spans: items,region_path: Vec::new(),mcid: None,+ in_figure: false,}}diff --git a/crates/liteparse/src/output/markdown.rs b/crates/liteparse/src/output/markdown.rsindex ee3cb1d..53cd69c 100644--- a/crates/liteparse/src/output/markdown.rs+++ b/crates/liteparse/src/output/markdown.rs@@ -127,6 +127,7 @@ mod tests {spans: vec![TextItem::default()],region_path: Vec::new(),mcid: None,+ in_figure: false,}}diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rsindex b0e141f..28c1661 100644--- a/crates/liteparse/src/projection.rs+++ b/crates/liteparse/src/projection.rs@@ -3091,16 +3091,14 @@ fn xy_root_bbox(items: &[ProjectedTextItem], page_width: f32, page_height: f32)let y = min_y.max(0.0);let w = (max_x - x).max(1.0);let h = (max_y - y).max(1.0);- let w = if page_width > 0.0 {- w.min(page_width)- } else {- w- };- let h = if page_height > 0.0 {- h.min(page_height)- } else {- h- };+ // Span the actual item extent. The page dimensions are only used as a+ // fallback (no items). We must NOT clamp the bbox down to them:+ // `handle_rotation_reading_order` lays rotated marginalia (e.g. an arXiv+ // sidebar stamp) out on a virtual canvas taller/wider than the physical+ // page and pushes the non-rotated body below it. Clamping to page_height+ // would drop those displaced body items outside the density buckets, so no+ // horizontal valley between a full-width header and the 2-column body is+ // ever found and the column V-cut shreds the centered title/author band.Rect {x,y,@@ -4568,6 +4566,19 @@ pub(crate) fn build_projected_lines(let mut leaves: Vec<(Vec<u16>, Vec<usize>)> = Vec::new();xy_walk_leaves(®ion, &mut Vec::new(), &mut leaves);+ // Figures used for the per-line `in_figure` heading-exclusion flag. Drop+ // page-dominating figures: on posters / slides / infographics the whole+ // page is one large graphic and its big text *is* the heading content, so+ // excluding it would demote real headings. A chart on an academic page+ // covers only a fraction of the page — that's the pollution we want out of+ // the heading histogram.+ let page_area = (page_width.max(1.0)) * (page_height.max(1.0));+ let heading_excl_figures: Vec<Rect> = figures+ .iter()+ .filter(|f| f.width * f.height < page_area * 0.55)+ .cloned()+ .collect();+let mut out: Vec<ProjectedLine> = Vec::new();for (path, indices) in leaves {// Sort within the leaf by y, tie-break by x. `build_one_line` re-sorts@@ -4626,14 +4637,24 @@ pub(crate) fn build_projected_lines(current_y = current_y.min(y);current_h = current_h.max(h);} else {- out.push(build_one_line(items, ¤t, path.clone()));+ out.push(build_one_line(+ items,+ ¤t,+ path.clone(),+ &heading_excl_figures,+ ));current = vec![idx];current_y = y;current_h = h;}}if !current.is_empty() {- out.push(build_one_line(items, ¤t, path.clone()));+ out.push(build_one_line(+ items,+ ¤t,+ path.clone(),+ &heading_excl_figures,+ ));}}@@ -4673,6 +4694,7 @@ fn build_one_line(items: &[ProjectedTextItem],idxs: &[usize],region_path: Vec<u16>,+ figures: &[Rect],) -> ProjectedLine {// Sort by x so concatenation reads left→right even if reading order had// rotated insertions.@@ -4684,6 +4706,14 @@ fn build_one_line(let mut min_y = f32::INFINITY;let mut max_x = f32::NEG_INFINITY;let mut max_y = f32::NEG_INFINITY;+ // Original-coordinate bbox (pre-rotation-displacement). `figures` are in+ // original page space, so figure-overlap must be tested here, not against+ // the projected `bbox` below (which can be displaced onto a virtual canvas+ // by `handle_rotation_reading_order`).+ let mut omin_x = f32::INFINITY;+ let mut omin_y = f32::INFINITY;+ let mut omax_x = f32::NEG_INFINITY;+ let mut omax_y = f32::NEG_INFINITY;let mut size_weights: HashMap<u32, (f32, usize)> = HashMap::new();let mut height_weights: HashMap<u32, (f32, usize)> = HashMap::new();@@ -4703,7 +4733,15 @@ fn build_one_line(for (pos, &i) in sorted.iter().enumerate() {let proj = &items[i];let it = &proj.item;- spans.push(it.clone());+ // `handle_rotation_reading_order` zeroes `item.rotation` after it+ // reorders cardinal-rotated text into reading order, so the span would+ // otherwise look horizontal to `is_rotated_line`. Restore the original+ // rotation onto the span clone (only consumed by markdown heading/TOC+ // rotation guards) so a sideways margin stamp doesn't pollute the+ // heading-size map.+ let mut span = it.clone();+ span.rotation = proj.orig_rotation;+ spans.push(span);// Concatenate item text. Use existing num_spaces from projection only as// a hint — the markdown emitter re-collapses whitespace, so we just@@ -4718,6 +4756,11 @@ fn build_one_line(max_x = max_x.max(it.x + it.width);max_y = max_y.max(it.y + it.height);+ omin_x = omin_x.min(proj.orig_x);+ omin_y = omin_y.min(proj.orig_y);+ omax_x = omax_x.max(proj.orig_x + proj.orig_width);+ omax_y = omax_y.max(proj.orig_y + proj.orig_height);+let n = it.text.chars().count().max(1);total_chars += n;@@ -4841,6 +4884,27 @@ fn build_one_line(},};+ // Figure membership: ≥50% of the line's original-coordinate area inside a+ // detected figure that is also clearly taller than the line. The height+ // guard is what separates real chart pollution (axis labels, legends — set+ // inside tall plot regions) from a section heading sitting on a thin+ // colored background bar, which figure detection can mis-cluster as a+ // figure ~1 line tall. Only the former should lose heading candidacy.+ let line_h = (omax_y - omin_y).max(0.0);+ let in_figure = if omax_x > omin_x && line_h > 0.0 {+ let line_area = (omax_x - omin_x) * line_h;+ figures.iter().any(|f| {+ if f.height < line_h * 3.0 {+ return false;+ }+ let ix = (omax_x.min(f.x + f.width) - omin_x.max(f.x)).max(0.0);+ let iy = (omax_y.min(f.y + f.height) - omin_y.max(f.y)).max(0.0);+ line_area > 0.0 && (ix * iy) / line_area >= 0.5+ })+ } else {+ false+ };+ProjectedLine {text,bbox: bbox.clone(),@@ -4860,6 +4924,7 @@ fn build_one_line(spans,region_path,mcid,+ in_figure,}}diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rsindex 0067a98..5717189 100644--- a/crates/liteparse/src/types.rs+++ b/crates/liteparse/src/types.rs@@ -276,6 +276,12 @@ pub struct ProjectedLine {/// (banded splits with sub-columns) survive paragraph/table grouping.pub region_path: Vec<u16>,pub mcid: Option<i32>,+ /// True when the line's original-coordinate bbox falls inside a detected+ /// figure/chart region. Chart text (axis labels, legends, category titles)+ /// is often set in a font larger than real headings; if it reached the+ /// heading-size histogram it would hijack the top heading levels and get+ /// promoted itself. Heading detection skips these lines.+ pub in_figure: bool,}/// XY-cut region tree node. A page's root region recursively splits along H ordiff --git a/crates/pdfium-sys/build.rs b/crates/pdfium-sys/build.rsindex 1db5263..ecf6941 100644--- a/crates/pdfium-sys/build.rs+++ b/crates/pdfium-sys/build.rs@@ -41,6 +41,19 @@ fn main() {println!("cargo:rustc-link-lib=static=c++abi");println!("cargo:rustc-link-lib=static=wasi-emulated-mman");println!("cargo:rustc-link-lib=static=wasi-emulated-signal");+ // Newer pdfium-binaries releases ship the wasm setjmp/longjmp runtime+ // in a standalone libsetjmp.a, which pdfium's bundled libjpeg/freetype+ // reference for their __c_longjmp error handling. Older releases folded+ // these into libc.a, so only link it when the archive is present.+ //+ // NOTE: libsetjmp.a's single object also redefines the __wasm_setjmp/+ // __wasm_longjmp helpers that Rust's own codegen emits. The+ // --allow-multiple-definition link-arg needed to tolerate that lives in+ // the final cdylib's build script (crates/liteparse-wasm/build.rs),+ // because rustc-link-arg does not propagate to dependent crates.+ if lib_dir.join("libsetjmp.a").exists() {+ println!("cargo:rustc-link-lib=static=setjmp");+ }println!("cargo:lib_path={}", lib_dir.display());} else {// Non-wasm: pdfium is loaded at runtime via libloading (no link-timediff --git a/dataset_eval_utils/src/liteparse_eval/benchmark.py b/dataset_eval_utils/src/liteparse_eval/benchmark.pyindex 4dad8ab..392d3d8 100644--- a/dataset_eval_utils/src/liteparse_eval/benchmark.py+++ b/dataset_eval_utils/src/liteparse_eval/benchmark.py@@ -16,6 +16,7 @@ from liteparse_eval.providers import (LiteparseProvider,MarkItDownProvider,OpenDataLoaderProvider,+ PdfInspectorProvider,PdfToTextProvider,PyMuPDFProvider,PyMuPDF4LLMMarkdownProvider,@@ -32,6 +33,7 @@ ALL_PROVIDERS = ["pymupdf4llm-text","pymupdf4llm-md","opendataloader",+ "pdf-inspector",]PROVIDER_MAP = {@@ -43,6 +45,7 @@ PROVIDER_MAP = {"pymupdf4llm-text": PyMuPDF4LLMTextProvider,"pymupdf4llm-md": PyMuPDF4LLMMarkdownProvider,"opendataloader": OpenDataLoaderProvider,+ "pdf-inspector": PdfInspectorProvider,}@@ -51,6 +54,16 @@ def find_pdfs(directory: Path) -> list[Path]:return sorted(directory.glob("*.pdf"))+def count_pages(file_path: Path) -> Optional[int]:+ """Return the page count of a PDF, or None if it can't be read."""+ try:+ import pypdf++ return len(pypdf.PdfReader(str(file_path)).pages)+ except Exception:+ return None++def time_extraction(provider: ParserProvider, file_path: Path) -> tuple[float, int]:"""Time a single extraction, returning (seconds, text_length)."""start = time.perf_counter()@@ -64,10 +77,20 @@ def format_table(doc_names: list[str],# results[provider_name][doc_name] = (seconds, chars) | None (error)results: dict[str, dict[str, tuple[float, int] | None]],+ page_counts: dict[str, Optional[int]] | None = None,) -> str:"""Build a formatted table string."""+ page_counts = page_counts or {}++ # Annotate each document with its page count, e.g. "457_pages.pdf (457p)".+ def doc_label(doc: str) -> str:+ pages = page_counts.get(doc)+ return f"{doc} ({pages}p)" if pages else doc++ labels = {doc: doc_label(doc) for doc in doc_names}+# Column widths- doc_col_w = max(len("Document"), *(len(n) for n in doc_names)) + 2+ doc_col_w = max(len("Document"), *(len(labels[d]) for d in doc_names)) + 2prov_col_w = 14# Header@@ -80,7 +103,7 @@ def format_table(# Per-document rowsfor doc in doc_names:- row = f"{doc:<{doc_col_w}}"+ row = f"{labels[doc]:<{doc_col_w}}"for p in providers:entry = results[p].get(doc)if entry is None:@@ -108,7 +131,7 @@ def format_table(lines.append(row)# Average row- row = f"{'AVG':<{doc_col_w}}"+ row = f"{'AVG/doc':<{doc_col_w}}"for p in providers:times = [results[p][d][0] for d in doc_names if results[p].get(d) is not None]if times:@@ -119,6 +142,23 @@ def format_table(row += f" {cell:>{prov_col_w}}"lines.append(row)+ # Per-page row (aggregate: total time over successfully-parsed pages).+ total_known_pages = sum(p for p in page_counts.values() if p)+ if total_known_pages:+ row = f"{'MS/PAGE':<{doc_col_w}}"+ for p in providers:+ secs = 0.0+ pages = 0+ for doc in doc_names:+ entry = results[p].get(doc)+ pc = page_counts.get(doc)+ if entry is not None and pc:+ secs += entry[0]+ pages += pc+ cell = f"{secs / pages * 1000:.2f}ms" if pages else "N/A"+ row += f" {cell:>{prov_col_w}}"+ lines.append(row)+lines.append(sep)return "\n".join(lines)@@ -140,8 +180,10 @@ def run_benchmark(return {}doc_names = [f.name for f in pdf_files]+ page_counts = {f.name: count_pages(f) for f in pdf_files}+ total_pages = sum(p for p in page_counts.values() if p)- print(f"Found {len(pdf_files)} documents in {input_dir}")+ print(f"Found {len(pdf_files)} documents ({total_pages} pages) in {input_dir}")print(f"Providers: {', '.join(providers)}")print()@@ -181,13 +223,15 @@ def run_benchmark(print()# Print table- table = format_table(providers, doc_names, results)+ table = format_table(providers, doc_names, results, page_counts)print(table)# Build JSON outputoutput = {"input_dir": str(input_dir),"documents": doc_names,+ "page_counts": page_counts,+ "total_pages": total_pages,"providers": {},}for p in providers:@@ -197,15 +241,31 @@ def run_benchmark(if entry is None:provider_results[doc] = {"error": True}else:+ pc = page_counts.get(doc)provider_results[doc] = {"seconds": round(entry[0], 4),"text_length": entry[1],+ "ms_per_page": round(entry[0] / pc * 1000, 3) if pc else None,}times = [results[p][d][0] for d in doc_names if results[p].get(d) is not None]+ # Aggregate ms/page over successfully-parsed pages only.+ parsed_secs = sum(+ results[p][d][0]+ for d in doc_names+ if results[p].get(d) is not None and page_counts.get(d)+ )+ parsed_pages = sum(+ page_counts[d]+ for d in doc_names+ if results[p].get(d) is not None and page_counts.get(d)+ )output["providers"][p] = {"per_document": provider_results,"total_seconds": round(sum(times), 4) if times else None,"avg_seconds": round(sum(times) / len(times), 4) if times else None,+ "ms_per_page": round(parsed_secs / parsed_pages * 1000, 3)+ if parsed_pages+ else None,"num_success": len(times),"num_error": len(doc_names) - len(times),}diff --git a/dataset_eval_utils/src/liteparse_eval/providers/__init__.py b/dataset_eval_utils/src/liteparse_eval/providers/__init__.pyindex d50f47a..420f4b3 100644--- a/dataset_eval_utils/src/liteparse_eval/providers/__init__.py+++ b/dataset_eval_utils/src/liteparse_eval/providers/__init__.py@@ -4,6 +4,7 @@ from .parsers import (LiteparseProvider,MarkItDownProvider,OpenDataLoaderProvider,+ PdfInspectorProvider,PdfToTextProvider,PyMuPDFProvider,PyMuPDF4LLMMarkdownProvider,@@ -19,6 +20,7 @@ __all__ = ["LiteparseProvider","MarkItDownProvider","OpenDataLoaderProvider",+ "PdfInspectorProvider","PdfToTextProvider","PyMuPDFProvider","PyMuPDF4LLMMarkdownProvider",diff --git a/dataset_eval_utils/src/liteparse_eval/providers/parsers/__init__.py b/dataset_eval_utils/src/liteparse_eval/providers/parsers/__init__.pyindex aaa41f2..b6b23d7 100644--- a/dataset_eval_utils/src/liteparse_eval/providers/parsers/__init__.py+++ b/dataset_eval_utils/src/liteparse_eval/providers/parsers/__init__.py@@ -2,6 +2,7 @@ from .base import ParserProviderfrom .liteparse import LiteparseProviderfrom .markitdown import MarkItDownProviderfrom .opendataloader import OpenDataLoaderProvider+from .pdf_inspector import PdfInspectorProviderfrom .pdftotext import PdfToTextProviderfrom .pymupdf import PyMuPDFProviderfrom .pymupdf4llm_md import PyMuPDF4LLMMarkdownProvider@@ -13,6 +14,7 @@ __all__ = ["LiteparseProvider","MarkItDownProvider","OpenDataLoaderProvider",+ "PdfInspectorProvider","PdfToTextProvider","PyMuPDFProvider","PyMuPDF4LLMMarkdownProvider",diff --git a/dataset_eval_utils/src/liteparse_eval/providers/parsers/liteparse.py b/dataset_eval_utils/src/liteparse_eval/providers/parsers/liteparse.pyindex 306d504..89c1557 100644--- a/dataset_eval_utils/src/liteparse_eval/providers/parsers/liteparse.py+++ b/dataset_eval_utils/src/liteparse_eval/providers/parsers/liteparse.py@@ -41,10 +41,11 @@ class LiteparseProvider(ParserProvider):max_pages=max_pages,dpi=dpi,preserve_very_small_text=preserve_very_small_text,+ output_format="markdown",quiet=True,)def extract_text(self, file_path: Path) -> str:- """Extract text from a document using liteparse."""+ """Extract markdown from a document using liteparse."""result = self.parser.parse(file_path)return result.textdiff --git a/dataset_eval_utils/src/liteparse_eval/providers/parsers/opendataloader.py b/dataset_eval_utils/src/liteparse_eval/providers/parsers/opendataloader.pyindex 98ab320..cf47c51 100644--- a/dataset_eval_utils/src/liteparse_eval/providers/parsers/opendataloader.py+++ b/dataset_eval_utils/src/liteparse_eval/providers/parsers/opendataloader.py@@ -19,19 +19,20 @@ class OpenDataLoaderProvider(ParserProvider):passdef extract_text(self, file_path: Path) -> str:- """Extract text from a document using OpenDataLoader PDF."""+ """Extract markdown from a document using OpenDataLoader PDF."""with tempfile.TemporaryDirectory() as tmp_dir:opendataloader_pdf.convert(input_path=[str(file_path)],output_dir=tmp_dir,- format="text",+ format="markdown",+ quiet=True,)- # Read the output text file- output_file = Path(tmp_dir) / f"{file_path.stem}.txt"+ # Read the output markdown file+ output_file = Path(tmp_dir) / f"{file_path.stem}.md"if output_file.exists():return output_file.read_text(encoding="utf-8")- # Fallback: find any .txt file in the output dir- txt_files = list(Path(tmp_dir).glob("*.txt"))- if txt_files:- return txt_files[0].read_text(encoding="utf-8")+ # Fallback: find any .md file in the output dir+ md_files = list(Path(tmp_dir).glob("*.md"))+ if md_files:+ return md_files[0].read_text(encoding="utf-8")return ""diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.tomlindex 6d1da89..d668047 100644--- a/packages/python/pyproject.toml+++ b/packages/python/pyproject.toml@@ -7,7 +7,7 @@ name = "liteparse"version = "2.0.8"description = "Python bindings for LiteParse - fast, lightweight PDF and document parsing"readme = "README.md"-license = "MIT"+license = "Apache-2.0"requires-python = ">=3.10"authors = [{ name = "Logan Markewich", email = "[email protected]" },@@ -16,7 +16,7 @@ keywords = ["pdf", "parsing", "ocr", "document", "text-extraction"]classifiers = ["Development Status :: 4 - Beta","Intended Audience :: Developers",- "License :: OSI Approved :: MIT License",+ "License :: OSI Approved :: Apache Software License","Programming Language :: Python :: 3","Programming Language :: Python :: 3.10","Programming Language :: Python :: 3.11",diff --git a/packages/wasm/README.md b/packages/wasm/README.mdindex b067174..07f744e 100644--- a/packages/wasm/README.md+++ b/packages/wasm/README.md@@ -87,8 +87,6 @@ npm run build:nodejs # for node.jsOutput goes to `pkg/`.-> **Note:** A real build also needs a static `libpdfium.a` compiled for `wasm32-unknown-emscripten`/`wasm32-unknown-unknown` exposed via `PDFIUM_LIB_PATH`. See the project root `crates/WASM_PLAN.md` for details.-## LicenseApache-2.0diff --git a/packages/wasm/scripts/patch-wasi-imports.js b/packages/wasm/scripts/patch-wasi-imports.jsindex e5057b4..60781a8 100644--- a/packages/wasm/scripts/patch-wasi-imports.js+++ b/packages/wasm/scripts/patch-wasi-imports.js@@ -109,12 +109,29 @@ source = source.replace();// 3. In the return object of __wbg_get_imports, replace the env/wasi entries-// with our stubs objects.-// The generated code has duplicate "wasi_snapshot_preview1" keys and an "env" key.-// We replace all of them with single entries pointing to our stubs.+// with our stubs objects. The generated code emits a run of duplicate+// "wasi_snapshot_preview1": importN keys (and, depending on the pdfium build,+// an "env": importN key). The order and presence of these vary between+// wasm-bindgen versions, so collapse each independently rather than assuming+// a fixed env-then-wasi layout.+const wasiBefore = source;source = source.replace(- /("env": import\d+,\n(?:\s+"wasi_snapshot_preview1": import\d+,?\n)+)/,- `"env": __env_stubs,\n "wasi_snapshot_preview1": __wasi_stubs,\n`+ /(?:[ \t]*"wasi_snapshot_preview1": import\d+,\n)+/,+ ` "wasi_snapshot_preview1": __wasi_stubs,\n`+);+if (source === wasiBefore) {+ throw new Error(+ "patch-wasi-imports: failed to find wasi_snapshot_preview1 import entries — " ++ "the wasm-bindgen glue layout changed, update this script."+ );+}++// The "env" import only exists when a symbol (e.g. __c_longjmp) is left+// unresolved at link time. When pdfium's libsetjmp.a is linked it disappears,+// so this replacement is best-effort.+source = source.replace(+ /[ \t]*"env": import\d+,\n/,+ ` "env": __env_stubs,\n`);writeFileSync(GLUE_PATH, source, "utf-8");diff --git a/wasm-demo-site/index.html b/wasm-demo-site/index.htmlindex 5b4dd99..9897fbb 100644--- a/wasm-demo-site/index.html+++ b/wasm-demo-site/index.html@@ -210,6 +210,11 @@word-break: normal;}+ #output.wrap {+ white-space: pre-wrap;+ word-break: break-word;+ }+/* Stats bar */.stats {display: flex;@@ -367,10 +372,12 @@<div class="controls"><label for="format-select">Output:</label><select id="format-select">+ <option value="markdown">Markdown</option><option value="text">Plain Text</option><option value="json">JSON (with bounding boxes)</option></select><div class="spacer"></div>+ <button id="wrap-btn">Wrap: Off</button><button id="copy-btn">Copy</button></div><pre id="output"></pre>@@ -384,14 +391,14 @@<span class="cmt">// Initialize the WASM module</span><span class="kw">await</span> <span class="fn">init</span>();-<span class="cmt">// Create a parser instance</span>-<span class="kw">const</span> <span class="const">parser</span> = <span class="kw">new</span> <span class="fn">LiteParse</span>({ <span class="const">outputFormat</span>: <span class="str">"json"</span> });+<span class="cmt">// Create a parser instance (markdown output)</span>+<span class="kw">const</span> <span class="const">parser</span> = <span class="kw">new</span> <span class="fn">LiteParse</span>({ <span class="const">outputFormat</span>: <span class="str">"markdown"</span> });<span class="cmt">// Parse PDF bytes (Uint8Array from File, fetch, etc.)</span><span class="kw">const</span> <span class="const">bytes</span> = <span class="kw">new</span> <span class="fn">Uint8Array</span>(<span class="kw">await</span> file.<span class="fn">arrayBuffer</span>());<span class="kw">const</span> <span class="const">result</span> = <span class="kw">await</span> parser.<span class="fn">parse</span>(<span class="const">bytes</span>);-console.<span class="fn">log</span>(<span class="const">result</span>.text); <span class="cmt">// full document text</span>+console.<span class="fn">log</span>(<span class="const">result</span>.text); <span class="cmt">// rendered markdown</span>console.<span class="fn">log</span>(<span class="const">result</span>.pages[<span class="const">0</span>].textItems); <span class="cmt">// items with bounding boxes</span></pre></div></main>@@ -413,8 +420,11 @@ console.<span class="fn">log</span>(<span class="const">result</span>.pages[<spaconst fileInput = document.getElementById("pdf-input");const formatSelect = document.getElementById("format-select");const copyBtn = document.getElementById("copy-btn");+ const wrapBtn = document.getElementById("wrap-btn");- let lastResult = null;+ let currentFile = null; // raw bytes of the loaded PDF+ let currentName = "";+ const resultCache = {}; // keyed by output format: "markdown" | "text" | "json"function setStatus(msg, type) {status.textContent = msg;@@ -426,12 +436,8 @@ console.<span class="fn">log</span>(<span class="const">result</span>.pages[<spalet LiteParse;try {- const mod = await import(- "https://cdn.jsdelivr.net/npm/@llamaindex/liteparse-wasm@2/pkg/liteparse_wasm.js"- );- await mod.default(- "https://cdn.jsdelivr.net/npm/@llamaindex/liteparse-wasm@2/pkg/liteparse_wasm_bg.wasm"- );+ const mod = await import("./pkg/liteparse_wasm.js");+ await mod.default("./pkg/liteparse_wasm_bg.wasm");LiteParse = mod.LiteParse;setStatus("Ready. Drop a PDF to parse.", "success");} catch (err) {@@ -465,38 +471,56 @@ console.<span class="fn">log</span>(<span class="const">result</span>.pages[<spaif (file) parseFile(file);});- // --- Parse ---+ // --- Load a file (store bytes, then render the selected format) ---async function parseFile(file) {if (!LiteParse) {setStatus("WASM module not loaded yet.", "error");return;}-- setStatus(`Parsing "${file.name}"...`, "loading");+ currentFile = new Uint8Array(await file.arrayBuffer());+ currentName = file.name;+ for (const k in resultCache) delete resultCache[k];outputContainer.classList.remove("visible");-+ await renderOutput();+ }++ // Parse with a given output format, caching the result per format.+ async function parseAs(format) {+ if (resultCache[format]) return resultCache[format];+ const t0 = performance.now();+ const parser = new LiteParse({+ ocrEnabled: false,+ outputFormat: format,+ quiet: true,+ });+ const result = await parser.parse(currentFile);+ const elapsed = (performance.now() - t0) / 1000;+ resultCache[format] = { result, elapsed };+ return resultCache[format];+ }++ // --- Render output for the selected format ---+ async function renderOutput() {+ if (!currentFile) return;+ const format = formatSelect.value;+ setStatus(`Parsing "${currentName}"...`, "loading");try {- const t0 = performance.now();- const bytes = new Uint8Array(await file.arrayBuffer());- const parser = new LiteParse({- ocrEnabled: false,- outputFormat: "json",- quiet: true,- });- const result = await parser.parse(bytes);- const elapsed = ((performance.now() - t0) / 1000).toFixed(2);-- lastResult = result;- setStatus(`Parsed ${result.pages.length} page(s) in ${elapsed}s`, "success");- renderOutput();+ const { result, elapsed } = await parseAs(format);++ if (format === "json") {+ output.textContent = JSON.stringify(result, null, 2);+ } else {+ // For "markdown" and "text", result.text holds the rendered output.+ output.textContent = result.text;+ }+ setStatus(`Parsed ${result.pages.length} page(s) in ${elapsed.toFixed(2)}s`, "success");statsEl.innerHTML = [`<span class="stat"><strong>${result.pages.length}</strong> pages</span>`,`<span class="stat"><strong>${result.text.length.toLocaleString()}</strong> chars</span>`,- `<span class="stat"><strong>${elapsed}s</strong> elapsed</span>`,- `<span class="stat"><strong>${(bytes.length / 1024).toFixed(0)} KB</strong> file size</span>`,+ `<span class="stat"><strong>${elapsed.toFixed(2)}s</strong> elapsed</span>`,+ `<span class="stat"><strong>${(currentFile.length / 1024).toFixed(0)} KB</strong> file size</span>`,].join("");-outputContainer.classList.add("visible");} catch (err) {setStatus("Parse error: " + err.toString(), "error");@@ -504,18 +528,13 @@ console.<span class="fn">log</span>(<span class="const">result</span>.pages[<spa}}- // --- Render output ---- function renderOutput() {- if (!lastResult) return;- if (formatSelect.value === "text") {- output.textContent = lastResult.text;- } else {- output.textContent = JSON.stringify(lastResult, null, 2);- }- }-formatSelect.addEventListener("change", renderOutput);+ wrapBtn.addEventListener("click", () => {+ const wrapped = output.classList.toggle("wrap");+ wrapBtn.textContent = wrapped ? "Wrap: On" : "Wrap: Off";+ });+copyBtn.addEventListener("click", () => {navigator.clipboard.writeText(output.textContent);copyBtn.textContent = "Copied!";