Skip to content

A fast, helpful, and open-source document parser

wip

Logan Markewich3f0e86ad7fd4
diff --git a/crates/liteparse-wasm/build.rs b/crates/liteparse-wasm/build.rs
new file mode 100644
index 0000000..ba700dd
--- /dev/null
+++ b/crates/liteparse-wasm/build.rs
@@ -0,0 +1,15 @@
+use std::env;
+
+fn main() {
+ let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default();
+
+ // pdfium's wasm build links libsetjmp.a for __c_longjmp, but that archive's
+ // single object also defines the __wasm_setjmp/__wasm_longjmp helpers that
+ // Rust's own codegen already emits. The definitions are identical LLVM SjLj
+ // helpers, so allow the linker to keep the first and resolve the otherwise
+ // missing __c_longjmp from the same archive. Must live here (the final
+ // cdylib) because rustc-link-arg is not propagated from dependency crates.
+ if target_arch == "wasm32" {
+ println!("cargo:rustc-link-arg=--allow-multiple-definition");
+ }
+}
diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rs
index b649856..a9340c2 100644
--- a/crates/liteparse/src/markdown_layout/classify.rs
+++ b/crates/liteparse/src/markdown_layout/classify.rs
@@ -13,7 +13,7 @@ use super::inline::{
use super::lists::{LIST_INDENT_STEP_PT, parse_list_marker};
use super::paragraphs::{
ParaAccum, append_to_paragraph, collapse_whitespace, continues_heading, continues_paragraph,
- ends_hyphenated, is_soft_hyphen_break,
+ ends_hyphenated, ends_sentence_final, is_soft_hyphen_break,
};
use super::repetition::is_header_or_footer;
use super::tables::{detect_ruled_tables, detect_tables, merge_table_runs};
@@ -617,11 +617,16 @@ fn classify_region(
outline_heading_level(line, page.page_height, outline, text)
}
});
- let size_level = if is_caption_line(text) || toc_suppress {
- None
- } else {
- heading_level_for(heading_size_of(line), heading_map)
- };
+ // Rotated lines (margin stamps like a sideways "arXiv:…" watermark) are
+ // excluded from `build_heading_map`, but their size can still coincide
+ // with a map entry built from horizontal text. Suppress font-size
+ // promotion for them so they don't surface as spurious headings.
+ let size_level =
+ if is_caption_line(text) || toc_suppress || is_rotated_line(line) || line.in_figure {
+ None
+ } else {
+ heading_level_for(heading_size_of(line), heading_map)
+ };
// Guard against height-jitter false headings: a line that flows from
// the previous line (same paragraph) AND starts lowercase is a
// mid-paragraph continuation, not a heading — even if its
@@ -639,8 +644,21 @@ fn classify_region(
// line is its continuation regardless of capitalization
// ("…SOLAR 10.7 Billion-" / "Parameter Model: We have…").
let prev_hyphen_wrap = prev.is_some_and(|p| ends_hyphenated(&p.text));
- !((starts_lower || prev_hyphen_wrap)
- && prev.is_some_and(|p| continues_paragraph(p, line)))
+ // `prev` is the *live body paragraph* (or list line) immediately
+ // above, so it only exists mid-paragraph — never right after a
+ // heading. A lowercase line that *completes a sentence* (ends with
+ // terminal punctuation) while its predecessor is left open (no
+ // terminal punctuation) is a wrapped sentence tail, not a heading —
+ // even when a centered short tail's large left-edge indent defeats
+ // `continues_paragraph` ("…journalistic or" → "scholarly works.").
+ // Requiring the tail to end a sentence keeps genuine lowercase
+ // headings (which rarely end in `.`/`!`/`?`) promoted.
+ let sentence_tail = starts_lower
+ && ends_sentence_final(text)
+ && prev.is_some_and(|p| !ends_sentence_final(&p.text));
+ let is_continuation =
+ prev.is_some_and(|p| continues_paragraph(p, line)) || sentence_tail;
+ !((starts_lower || prev_hyphen_wrap) && is_continuation)
});
// Long-text guard: a real heading is a label, not a sentence with
// multiple clauses. Footnotes / citations / captions that happen to
diff --git a/crates/liteparse/src/markdown_layout/headings.rs b/crates/liteparse/src/markdown_layout/headings.rs
index 79ebfaf..97a1f83 100644
--- a/crates/liteparse/src/markdown_layout/headings.rs
+++ b/crates/liteparse/src/markdown_layout/headings.rs
@@ -21,6 +21,28 @@ pub(super) const ESTIMATED_HEADING_SIZE_MARGIN: f32 = 1.5;
/// Cap on heading levels (matches plan: H1..H6).
pub(super) const MAX_HEADING_LEVELS: usize = 6;
+/// A normalized line text that recurs at least this many times among
+/// heading-candidate lines (size > body) is treated as chart/figure label
+/// spam, not a heading, and is excluded from the heading-size map. Example:
+/// the "Input-Input Layer5" attention-plot labels in academic papers render
+/// at 23–28pt (far above body) and would otherwise hijack H1..H3, demoting the
+/// real title and section headings. Genuine headings are not byte-identical
+/// across the document, so this never drops a real heading size *unless* that
+/// size is occupied solely by the repeated string — and the exclusion is
+/// per-line, so a repeated string sharing a font size with real headings
+/// leaves that level intact.
+pub(super) const REPEATED_HEADING_LINE_MIN: usize = 3;
+
+/// Normalize a line's text for repeat-detection: collapse internal whitespace
+/// and lowercase, so trivially-different renderings of the same label string
+/// still hash together.
+fn normalize_repeat_key(text: &str) -> String {
+ text.split_whitespace()
+ .collect::<Vec<_>>()
+ .join(" ")
+ .to_lowercase()
+}
+
/// Tighter tolerance for matching against the heading-size map. Keeps the
/// heading detector strict so descender-induced height jitter doesn't promote
/// regular body lines to headings.
@@ -621,11 +643,34 @@ const MIN_HEADING_ALPHA_RATIO: f32 = 0.5;
/// artifacts), sorted descending, mapped to levels 1..=MAX_HEADING_LEVELS.
pub fn build_heading_map(pages: &[ParsedPage], body_size: f32) -> Vec<(f32, u8)> {
use std::collections::HashMap;
+ let heading_margin = |line: &ProjectedLine| {
+ if line.font_size_is_estimated {
+ ESTIMATED_HEADING_SIZE_MARGIN
+ } else {
+ HEADING_SIZE_EPSILON
+ }
+ };
+ // First pass: frequency of each normalized candidate-line text. A string
+ // that recurs ≥ REPEATED_HEADING_LINE_MIN times is chart/figure label spam
+ // (e.g. repeated attention-plot labels at 23–28pt), not a heading.
+ let mut text_freq: HashMap<String, usize> = HashMap::new();
+ for page in pages {
+ for line in &page.projected_lines {
+ if is_rotated_line(line) || is_caption_line(&line.text) || line.in_figure {
+ continue;
+ }
+ if heading_size_of(line) > body_size + heading_margin(line) {
+ *text_freq
+ .entry(normalize_repeat_key(&line.text))
+ .or_insert(0) += 1;
+ }
+ }
+ }
// (size_key → (size, line_count, total_chars, alpha_chars))
let mut sizes: HashMap<u32, (f32, usize, usize, usize)> = HashMap::new();
for page in pages {
for line in &page.projected_lines {
- if is_rotated_line(line) {
+ if is_rotated_line(line) || line.in_figure {
continue;
}
// Captions ("Figure 7", "Table 3.") often render slightly larger
@@ -633,12 +678,18 @@ pub fn build_heading_map(pages: &[ParsedPage], body_size: f32) -> Vec<(f32, u8)>
if is_caption_line(&line.text) {
continue;
}
+ // Exclude repeated chart/figure labels (per-line, so a repeated
+ // string sharing a size with real headings still leaves that level).
+ if text_freq
+ .get(&normalize_repeat_key(&line.text))
+ .copied()
+ .unwrap_or(0)
+ >= REPEATED_HEADING_LINE_MIN
+ {
+ continue;
+ }
let size = heading_size_of(line);
- let margin = if line.font_size_is_estimated {
- ESTIMATED_HEADING_SIZE_MARGIN
- } else {
- HEADING_SIZE_EPSILON
- };
+ let margin = heading_margin(line);
if size > body_size + margin {
let key = (size * 100.0).round() as u32;
let entry = sizes.entry(key).or_insert((size, 0, 0, 0));
diff --git a/crates/liteparse/src/markdown_layout/paragraphs.rs b/crates/liteparse/src/markdown_layout/paragraphs.rs
index 73bec51..20c60dc 100644
--- a/crates/liteparse/src/markdown_layout/paragraphs.rs
+++ b/crates/liteparse/src/markdown_layout/paragraphs.rs
@@ -68,6 +68,21 @@ pub(super) fn ends_hyphenated(text: &str) -> bool {
t.ends_with('-') && t.chars().rev().nth(1).is_some_and(|c| c.is_alphabetic())
}
+/// True when `text` ends a sentence: its last non-space character (ignoring a
+/// trailing closing quote / bracket) is `.`, `!`, or `?`. The heading
+/// lowercase-continuation guard uses the negation — a previous line that does
+/// *not* end a sentence is an open clause that a following lowercase line
+/// continues, even when its left-edge indent defeats `continues_paragraph`
+/// (centered short tail lines like "…journalistic or" → "scholarly works.").
+pub(super) fn ends_sentence_final(text: &str) -> bool {
+ let t = text
+ .trim_end()
+ .trim_end_matches(|c| matches!(c, '"' | '\'' | ')' | ']' | '»' | '”' | '’'));
+ t.chars()
+ .next_back()
+ .is_some_and(|c| matches!(c, '.' | '!' | '?'))
+}
+
/// True when `prev` ends with a soft hyphen (see [`ends_hyphenated`]) and
/// `next` begins with a lowercase letter — the signal that a single word was
/// split across a line / block / region break and should be rejoined with the
diff --git a/crates/liteparse/src/markdown_layout/test_helpers.rs b/crates/liteparse/src/markdown_layout/test_helpers.rs
index dc95741..c167eb7 100644
--- a/crates/liteparse/src/markdown_layout/test_helpers.rs
+++ b/crates/liteparse/src/markdown_layout/test_helpers.rs
@@ -24,6 +24,7 @@ pub(crate) fn line(text: &str, x: f32, y: f32, h: f32, size: f32) -> ProjectedLi
spans: vec![TextItem::default()],
region_path: Vec::new(),
mcid: None,
+ in_figure: false,
}
}
@@ -95,6 +96,7 @@ pub(crate) fn line_with_spans(cells: &[(&str, f32)], y: f32, size: f32) -> Proje
spans,
region_path: Vec::new(),
mcid: None,
+ in_figure: false,
}
}
@@ -145,6 +147,7 @@ pub(crate) fn styled_line(spans: &[(&str, f32, Option<&str>)], y: f32, size: f32
spans: items,
region_path: Vec::new(),
mcid: None,
+ in_figure: false,
}
}
diff --git a/crates/liteparse/src/output/markdown.rs b/crates/liteparse/src/output/markdown.rs
index ee3cb1d..53cd69c 100644
--- a/crates/liteparse/src/output/markdown.rs
+++ b/crates/liteparse/src/output/markdown.rs
@@ -127,6 +127,7 @@ mod tests {
spans: vec![TextItem::default()],
region_path: Vec::new(),
mcid: None,
+ in_figure: false,
}
}
diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rs
index b0e141f..28c1661 100644
--- a/crates/liteparse/src/projection.rs
+++ b/crates/liteparse/src/projection.rs
@@ -3091,16 +3091,14 @@ fn xy_root_bbox(items: &[ProjectedTextItem], page_width: f32, page_height: f32)
let y = min_y.max(0.0);
let w = (max_x - x).max(1.0);
let h = (max_y - y).max(1.0);
- let w = if page_width > 0.0 {
- w.min(page_width)
- } else {
- w
- };
- let h = if page_height > 0.0 {
- h.min(page_height)
- } else {
- h
- };
+ // Span the actual item extent. The page dimensions are only used as a
+ // fallback (no items). We must NOT clamp the bbox down to them:
+ // `handle_rotation_reading_order` lays rotated marginalia (e.g. an arXiv
+ // sidebar stamp) out on a virtual canvas taller/wider than the physical
+ // page and pushes the non-rotated body below it. Clamping to page_height
+ // would drop those displaced body items outside the density buckets, so no
+ // horizontal valley between a full-width header and the 2-column body is
+ // ever found and the column V-cut shreds the centered title/author band.
Rect {
x,
y,
@@ -4568,6 +4566,19 @@ pub(crate) fn build_projected_lines(
let mut leaves: Vec<(Vec<u16>, Vec<usize>)> = Vec::new();
xy_walk_leaves(&region, &mut Vec::new(), &mut leaves);
+ // Figures used for the per-line `in_figure` heading-exclusion flag. Drop
+ // page-dominating figures: on posters / slides / infographics the whole
+ // page is one large graphic and its big text *is* the heading content, so
+ // excluding it would demote real headings. A chart on an academic page
+ // covers only a fraction of the page — that's the pollution we want out of
+ // the heading histogram.
+ let page_area = (page_width.max(1.0)) * (page_height.max(1.0));
+ let heading_excl_figures: Vec<Rect> = figures
+ .iter()
+ .filter(|f| f.width * f.height < page_area * 0.55)
+ .cloned()
+ .collect();
+
let mut out: Vec<ProjectedLine> = Vec::new();
for (path, indices) in leaves {
// Sort within the leaf by y, tie-break by x. `build_one_line` re-sorts
@@ -4626,14 +4637,24 @@ pub(crate) fn build_projected_lines(
current_y = current_y.min(y);
current_h = current_h.max(h);
} else {
- out.push(build_one_line(items, &current, path.clone()));
+ out.push(build_one_line(
+ items,
+ &current,
+ path.clone(),
+ &heading_excl_figures,
+ ));
current = vec![idx];
current_y = y;
current_h = h;
}
}
if !current.is_empty() {
- out.push(build_one_line(items, &current, path.clone()));
+ out.push(build_one_line(
+ items,
+ &current,
+ path.clone(),
+ &heading_excl_figures,
+ ));
}
}
@@ -4673,6 +4694,7 @@ fn build_one_line(
items: &[ProjectedTextItem],
idxs: &[usize],
region_path: Vec<u16>,
+ figures: &[Rect],
) -> ProjectedLine {
// Sort by x so concatenation reads left→right even if reading order had
// rotated insertions.
@@ -4684,6 +4706,14 @@ fn build_one_line(
let mut min_y = f32::INFINITY;
let mut max_x = f32::NEG_INFINITY;
let mut max_y = f32::NEG_INFINITY;
+ // Original-coordinate bbox (pre-rotation-displacement). `figures` are in
+ // original page space, so figure-overlap must be tested here, not against
+ // the projected `bbox` below (which can be displaced onto a virtual canvas
+ // by `handle_rotation_reading_order`).
+ let mut omin_x = f32::INFINITY;
+ let mut omin_y = f32::INFINITY;
+ let mut omax_x = f32::NEG_INFINITY;
+ let mut omax_y = f32::NEG_INFINITY;
let mut size_weights: HashMap<u32, (f32, usize)> = HashMap::new();
let mut height_weights: HashMap<u32, (f32, usize)> = HashMap::new();
@@ -4703,7 +4733,15 @@ fn build_one_line(
for (pos, &i) in sorted.iter().enumerate() {
let proj = &items[i];
let it = &proj.item;
- spans.push(it.clone());
+ // `handle_rotation_reading_order` zeroes `item.rotation` after it
+ // reorders cardinal-rotated text into reading order, so the span would
+ // otherwise look horizontal to `is_rotated_line`. Restore the original
+ // rotation onto the span clone (only consumed by markdown heading/TOC
+ // rotation guards) so a sideways margin stamp doesn't pollute the
+ // heading-size map.
+ let mut span = it.clone();
+ span.rotation = proj.orig_rotation;
+ spans.push(span);
// Concatenate item text. Use existing num_spaces from projection only as
// a hint — the markdown emitter re-collapses whitespace, so we just
@@ -4718,6 +4756,11 @@ fn build_one_line(
max_x = max_x.max(it.x + it.width);
max_y = max_y.max(it.y + it.height);
+ omin_x = omin_x.min(proj.orig_x);
+ omin_y = omin_y.min(proj.orig_y);
+ omax_x = omax_x.max(proj.orig_x + proj.orig_width);
+ omax_y = omax_y.max(proj.orig_y + proj.orig_height);
+
let n = it.text.chars().count().max(1);
total_chars += n;
@@ -4841,6 +4884,27 @@ fn build_one_line(
},
};
+ // Figure membership: ≥50% of the line's original-coordinate area inside a
+ // detected figure that is also clearly taller than the line. The height
+ // guard is what separates real chart pollution (axis labels, legends — set
+ // inside tall plot regions) from a section heading sitting on a thin
+ // colored background bar, which figure detection can mis-cluster as a
+ // figure ~1 line tall. Only the former should lose heading candidacy.
+ let line_h = (omax_y - omin_y).max(0.0);
+ let in_figure = if omax_x > omin_x && line_h > 0.0 {
+ let line_area = (omax_x - omin_x) * line_h;
+ figures.iter().any(|f| {
+ if f.height < line_h * 3.0 {
+ return false;
+ }
+ let ix = (omax_x.min(f.x + f.width) - omin_x.max(f.x)).max(0.0);
+ let iy = (omax_y.min(f.y + f.height) - omin_y.max(f.y)).max(0.0);
+ line_area > 0.0 && (ix * iy) / line_area >= 0.5
+ })
+ } else {
+ false
+ };
+
ProjectedLine {
text,
bbox: bbox.clone(),
@@ -4860,6 +4924,7 @@ fn build_one_line(
spans,
region_path,
mcid,
+ in_figure,
}
}
diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rs
index 0067a98..5717189 100644
--- a/crates/liteparse/src/types.rs
+++ b/crates/liteparse/src/types.rs
@@ -276,6 +276,12 @@ pub struct ProjectedLine {
/// (banded splits with sub-columns) survive paragraph/table grouping.
pub region_path: Vec<u16>,
pub mcid: Option<i32>,
+ /// True when the line's original-coordinate bbox falls inside a detected
+ /// figure/chart region. Chart text (axis labels, legends, category titles)
+ /// is often set in a font larger than real headings; if it reached the
+ /// heading-size histogram it would hijack the top heading levels and get
+ /// promoted itself. Heading detection skips these lines.
+ pub in_figure: bool,
}
/// XY-cut region tree node. A page's root region recursively splits along H or
diff --git a/crates/pdfium-sys/build.rs b/crates/pdfium-sys/build.rs
index 1db5263..ecf6941 100644
--- a/crates/pdfium-sys/build.rs
+++ b/crates/pdfium-sys/build.rs
@@ -41,6 +41,19 @@ fn main() {
println!("cargo:rustc-link-lib=static=c++abi");
println!("cargo:rustc-link-lib=static=wasi-emulated-mman");
println!("cargo:rustc-link-lib=static=wasi-emulated-signal");
+ // Newer pdfium-binaries releases ship the wasm setjmp/longjmp runtime
+ // in a standalone libsetjmp.a, which pdfium's bundled libjpeg/freetype
+ // reference for their __c_longjmp error handling. Older releases folded
+ // these into libc.a, so only link it when the archive is present.
+ //
+ // NOTE: libsetjmp.a's single object also redefines the __wasm_setjmp/
+ // __wasm_longjmp helpers that Rust's own codegen emits. The
+ // --allow-multiple-definition link-arg needed to tolerate that lives in
+ // the final cdylib's build script (crates/liteparse-wasm/build.rs),
+ // because rustc-link-arg does not propagate to dependent crates.
+ if lib_dir.join("libsetjmp.a").exists() {
+ println!("cargo:rustc-link-lib=static=setjmp");
+ }
println!("cargo:lib_path={}", lib_dir.display());
} else {
// Non-wasm: pdfium is loaded at runtime via libloading (no link-time
diff --git a/dataset_eval_utils/src/liteparse_eval/benchmark.py b/dataset_eval_utils/src/liteparse_eval/benchmark.py
index 4dad8ab..392d3d8 100644
--- a/dataset_eval_utils/src/liteparse_eval/benchmark.py
+++ b/dataset_eval_utils/src/liteparse_eval/benchmark.py
@@ -16,6 +16,7 @@ from liteparse_eval.providers import (
LiteparseProvider,
MarkItDownProvider,
OpenDataLoaderProvider,
+ PdfInspectorProvider,
PdfToTextProvider,
PyMuPDFProvider,
PyMuPDF4LLMMarkdownProvider,
@@ -32,6 +33,7 @@ ALL_PROVIDERS = [
"pymupdf4llm-text",
"pymupdf4llm-md",
"opendataloader",
+ "pdf-inspector",
]
PROVIDER_MAP = {
@@ -43,6 +45,7 @@ PROVIDER_MAP = {
"pymupdf4llm-text": PyMuPDF4LLMTextProvider,
"pymupdf4llm-md": PyMuPDF4LLMMarkdownProvider,
"opendataloader": OpenDataLoaderProvider,
+ "pdf-inspector": PdfInspectorProvider,
}
@@ -51,6 +54,16 @@ def find_pdfs(directory: Path) -> list[Path]:
return sorted(directory.glob("*.pdf"))
+def count_pages(file_path: Path) -> Optional[int]:
+ """Return the page count of a PDF, or None if it can't be read."""
+ try:
+ import pypdf
+
+ return len(pypdf.PdfReader(str(file_path)).pages)
+ except Exception:
+ return None
+
+
def time_extraction(provider: ParserProvider, file_path: Path) -> tuple[float, int]:
"""Time a single extraction, returning (seconds, text_length)."""
start = time.perf_counter()
@@ -64,10 +77,20 @@ def format_table(
doc_names: list[str],
# results[provider_name][doc_name] = (seconds, chars) | None (error)
results: dict[str, dict[str, tuple[float, int] | None]],
+ page_counts: dict[str, Optional[int]] | None = None,
) -> str:
"""Build a formatted table string."""
+ page_counts = page_counts or {}
+
+ # Annotate each document with its page count, e.g. "457_pages.pdf (457p)".
+ def doc_label(doc: str) -> str:
+ pages = page_counts.get(doc)
+ return f"{doc} ({pages}p)" if pages else doc
+
+ labels = {doc: doc_label(doc) for doc in doc_names}
+
# Column widths
- doc_col_w = max(len("Document"), *(len(n) for n in doc_names)) + 2
+ doc_col_w = max(len("Document"), *(len(labels[d]) for d in doc_names)) + 2
prov_col_w = 14
# Header
@@ -80,7 +103,7 @@ def format_table(
# Per-document rows
for doc in doc_names:
- row = f"{doc:<{doc_col_w}}"
+ row = f"{labels[doc]:<{doc_col_w}}"
for p in providers:
entry = results[p].get(doc)
if entry is None:
@@ -108,7 +131,7 @@ def format_table(
lines.append(row)
# Average row
- row = f"{'AVG':<{doc_col_w}}"
+ row = f"{'AVG/doc':<{doc_col_w}}"
for p in providers:
times = [results[p][d][0] for d in doc_names if results[p].get(d) is not None]
if times:
@@ -119,6 +142,23 @@ def format_table(
row += f" {cell:>{prov_col_w}}"
lines.append(row)
+ # Per-page row (aggregate: total time over successfully-parsed pages).
+ total_known_pages = sum(p for p in page_counts.values() if p)
+ if total_known_pages:
+ row = f"{'MS/PAGE':<{doc_col_w}}"
+ for p in providers:
+ secs = 0.0
+ pages = 0
+ for doc in doc_names:
+ entry = results[p].get(doc)
+ pc = page_counts.get(doc)
+ if entry is not None and pc:
+ secs += entry[0]
+ pages += pc
+ cell = f"{secs / pages * 1000:.2f}ms" if pages else "N/A"
+ row += f" {cell:>{prov_col_w}}"
+ lines.append(row)
+
lines.append(sep)
return "\n".join(lines)
@@ -140,8 +180,10 @@ def run_benchmark(
return {}
doc_names = [f.name for f in pdf_files]
+ page_counts = {f.name: count_pages(f) for f in pdf_files}
+ total_pages = sum(p for p in page_counts.values() if p)
- print(f"Found {len(pdf_files)} documents in {input_dir}")
+ print(f"Found {len(pdf_files)} documents ({total_pages} pages) in {input_dir}")
print(f"Providers: {', '.join(providers)}")
print()
@@ -181,13 +223,15 @@ def run_benchmark(
print()
# Print table
- table = format_table(providers, doc_names, results)
+ table = format_table(providers, doc_names, results, page_counts)
print(table)
# Build JSON output
output = {
"input_dir": str(input_dir),
"documents": doc_names,
+ "page_counts": page_counts,
+ "total_pages": total_pages,
"providers": {},
}
for p in providers:
@@ -197,15 +241,31 @@ def run_benchmark(
if entry is None:
provider_results[doc] = {"error": True}
else:
+ pc = page_counts.get(doc)
provider_results[doc] = {
"seconds": round(entry[0], 4),
"text_length": entry[1],
+ "ms_per_page": round(entry[0] / pc * 1000, 3) if pc else None,
}
times = [results[p][d][0] for d in doc_names if results[p].get(d) is not None]
+ # Aggregate ms/page over successfully-parsed pages only.
+ parsed_secs = sum(
+ results[p][d][0]
+ for d in doc_names
+ if results[p].get(d) is not None and page_counts.get(d)
+ )
+ parsed_pages = sum(
+ page_counts[d]
+ for d in doc_names
+ if results[p].get(d) is not None and page_counts.get(d)
+ )
output["providers"][p] = {
"per_document": provider_results,
"total_seconds": round(sum(times), 4) if times else None,
"avg_seconds": round(sum(times) / len(times), 4) if times else None,
+ "ms_per_page": round(parsed_secs / parsed_pages * 1000, 3)
+ if parsed_pages
+ else None,
"num_success": len(times),
"num_error": len(doc_names) - len(times),
}
diff --git a/dataset_eval_utils/src/liteparse_eval/providers/__init__.py b/dataset_eval_utils/src/liteparse_eval/providers/__init__.py
index d50f47a..420f4b3 100644
--- a/dataset_eval_utils/src/liteparse_eval/providers/__init__.py
+++ b/dataset_eval_utils/src/liteparse_eval/providers/__init__.py
@@ -4,6 +4,7 @@ from .parsers import (
LiteparseProvider,
MarkItDownProvider,
OpenDataLoaderProvider,
+ PdfInspectorProvider,
PdfToTextProvider,
PyMuPDFProvider,
PyMuPDF4LLMMarkdownProvider,
@@ -19,6 +20,7 @@ __all__ = [
"LiteparseProvider",
"MarkItDownProvider",
"OpenDataLoaderProvider",
+ "PdfInspectorProvider",
"PdfToTextProvider",
"PyMuPDFProvider",
"PyMuPDF4LLMMarkdownProvider",
diff --git a/dataset_eval_utils/src/liteparse_eval/providers/parsers/__init__.py b/dataset_eval_utils/src/liteparse_eval/providers/parsers/__init__.py
index aaa41f2..b6b23d7 100644
--- a/dataset_eval_utils/src/liteparse_eval/providers/parsers/__init__.py
+++ b/dataset_eval_utils/src/liteparse_eval/providers/parsers/__init__.py
@@ -2,6 +2,7 @@ from .base import ParserProvider
from .liteparse import LiteparseProvider
from .markitdown import MarkItDownProvider
from .opendataloader import OpenDataLoaderProvider
+from .pdf_inspector import PdfInspectorProvider
from .pdftotext import PdfToTextProvider
from .pymupdf import PyMuPDFProvider
from .pymupdf4llm_md import PyMuPDF4LLMMarkdownProvider
@@ -13,6 +14,7 @@ __all__ = [
"LiteparseProvider",
"MarkItDownProvider",
"OpenDataLoaderProvider",
+ "PdfInspectorProvider",
"PdfToTextProvider",
"PyMuPDFProvider",
"PyMuPDF4LLMMarkdownProvider",
diff --git a/dataset_eval_utils/src/liteparse_eval/providers/parsers/liteparse.py b/dataset_eval_utils/src/liteparse_eval/providers/parsers/liteparse.py
index 306d504..89c1557 100644
--- a/dataset_eval_utils/src/liteparse_eval/providers/parsers/liteparse.py
+++ b/dataset_eval_utils/src/liteparse_eval/providers/parsers/liteparse.py
@@ -41,10 +41,11 @@ class LiteparseProvider(ParserProvider):
max_pages=max_pages,
dpi=dpi,
preserve_very_small_text=preserve_very_small_text,
+ output_format="markdown",
quiet=True,
)
def extract_text(self, file_path: Path) -> str:
- """Extract text from a document using liteparse."""
+ """Extract markdown from a document using liteparse."""
result = self.parser.parse(file_path)
return result.text
diff --git a/dataset_eval_utils/src/liteparse_eval/providers/parsers/opendataloader.py b/dataset_eval_utils/src/liteparse_eval/providers/parsers/opendataloader.py
index 98ab320..cf47c51 100644
--- a/dataset_eval_utils/src/liteparse_eval/providers/parsers/opendataloader.py
+++ b/dataset_eval_utils/src/liteparse_eval/providers/parsers/opendataloader.py
@@ -19,19 +19,20 @@ class OpenDataLoaderProvider(ParserProvider):
pass
def extract_text(self, file_path: Path) -> str:
- """Extract text from a document using OpenDataLoader PDF."""
+ """Extract markdown from a document using OpenDataLoader PDF."""
with tempfile.TemporaryDirectory() as tmp_dir:
opendataloader_pdf.convert(
input_path=[str(file_path)],
output_dir=tmp_dir,
- format="text",
+ format="markdown",
+ quiet=True,
)
- # Read the output text file
- output_file = Path(tmp_dir) / f"{file_path.stem}.txt"
+ # Read the output markdown file
+ output_file = Path(tmp_dir) / f"{file_path.stem}.md"
if output_file.exists():
return output_file.read_text(encoding="utf-8")
- # Fallback: find any .txt file in the output dir
- txt_files = list(Path(tmp_dir).glob("*.txt"))
- if txt_files:
- return txt_files[0].read_text(encoding="utf-8")
+ # Fallback: find any .md file in the output dir
+ md_files = list(Path(tmp_dir).glob("*.md"))
+ if md_files:
+ return md_files[0].read_text(encoding="utf-8")
return ""
diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml
index 6d1da89..d668047 100644
--- a/packages/python/pyproject.toml
+++ b/packages/python/pyproject.toml
@@ -7,7 +7,7 @@ name = "liteparse"
version = "2.0.8"
description = "Python bindings for LiteParse - fast, lightweight PDF and document parsing"
readme = "README.md"
-license = "MIT"
+license = "Apache-2.0"
requires-python = ">=3.10"
authors = [
{ name = "Logan Markewich", email = "[email protected]" },
@@ -16,7 +16,7 @@ keywords = ["pdf", "parsing", "ocr", "document", "text-extraction"]
classifiers = [
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
- "License :: OSI Approved :: MIT License",
+ "License :: OSI Approved :: Apache Software License",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
diff --git a/packages/wasm/README.md b/packages/wasm/README.md
index b067174..07f744e 100644
--- a/packages/wasm/README.md
+++ b/packages/wasm/README.md
@@ -87,8 +87,6 @@ npm run build:nodejs # for node.js
Output goes to `pkg/`.
-> **Note:** A real build also needs a static `libpdfium.a` compiled for `wasm32-unknown-emscripten`/`wasm32-unknown-unknown` exposed via `PDFIUM_LIB_PATH`. See the project root `crates/WASM_PLAN.md` for details.
-
## License
Apache-2.0
diff --git a/packages/wasm/scripts/patch-wasi-imports.js b/packages/wasm/scripts/patch-wasi-imports.js
index e5057b4..60781a8 100644
--- a/packages/wasm/scripts/patch-wasi-imports.js
+++ b/packages/wasm/scripts/patch-wasi-imports.js
@@ -109,12 +109,29 @@ source = source.replace(
);
// 3. In the return object of __wbg_get_imports, replace the env/wasi entries
-// with our stubs objects.
-// The generated code has duplicate "wasi_snapshot_preview1" keys and an "env" key.
-// We replace all of them with single entries pointing to our stubs.
+// with our stubs objects. The generated code emits a run of duplicate
+// "wasi_snapshot_preview1": importN keys (and, depending on the pdfium build,
+// an "env": importN key). The order and presence of these vary between
+// wasm-bindgen versions, so collapse each independently rather than assuming
+// a fixed env-then-wasi layout.
+const wasiBefore = source;
source = source.replace(
- /("env": import\d+,\n(?:\s+"wasi_snapshot_preview1": import\d+,?\n)+)/,
- `"env": __env_stubs,\n "wasi_snapshot_preview1": __wasi_stubs,\n`
+ /(?:[ \t]*"wasi_snapshot_preview1": import\d+,\n)+/,
+ ` "wasi_snapshot_preview1": __wasi_stubs,\n`
+);
+if (source === wasiBefore) {
+ throw new Error(
+ "patch-wasi-imports: failed to find wasi_snapshot_preview1 import entries — " +
+ "the wasm-bindgen glue layout changed, update this script."
+ );
+}
+
+// The "env" import only exists when a symbol (e.g. __c_longjmp) is left
+// unresolved at link time. When pdfium's libsetjmp.a is linked it disappears,
+// so this replacement is best-effort.
+source = source.replace(
+ /[ \t]*"env": import\d+,\n/,
+ ` "env": __env_stubs,\n`
);
writeFileSync(GLUE_PATH, source, "utf-8");
diff --git a/wasm-demo-site/index.html b/wasm-demo-site/index.html
index 5b4dd99..9897fbb 100644
--- a/wasm-demo-site/index.html
+++ b/wasm-demo-site/index.html
@@ -210,6 +210,11 @@
word-break: normal;
}
+ #output.wrap {
+ white-space: pre-wrap;
+ word-break: break-word;
+ }
+
/* Stats bar */
.stats {
display: flex;
@@ -367,10 +372,12 @@
<div class="controls">
<label for="format-select">Output:</label>
<select id="format-select">
+ <option value="markdown">Markdown</option>
<option value="text">Plain Text</option>
<option value="json">JSON (with bounding boxes)</option>
</select>
<div class="spacer"></div>
+ <button id="wrap-btn">Wrap: Off</button>
<button id="copy-btn">Copy</button>
</div>
<pre id="output"></pre>
@@ -384,14 +391,14 @@
<span class="cmt">// Initialize the WASM module</span>
<span class="kw">await</span> <span class="fn">init</span>();
-<span class="cmt">// Create a parser instance</span>
-<span class="kw">const</span> <span class="const">parser</span> = <span class="kw">new</span> <span class="fn">LiteParse</span>({ <span class="const">outputFormat</span>: <span class="str">"json"</span> });
+<span class="cmt">// Create a parser instance (markdown output)</span>
+<span class="kw">const</span> <span class="const">parser</span> = <span class="kw">new</span> <span class="fn">LiteParse</span>({ <span class="const">outputFormat</span>: <span class="str">"markdown"</span> });
<span class="cmt">// Parse PDF bytes (Uint8Array from File, fetch, etc.)</span>
<span class="kw">const</span> <span class="const">bytes</span> = <span class="kw">new</span> <span class="fn">Uint8Array</span>(<span class="kw">await</span> file.<span class="fn">arrayBuffer</span>());
<span class="kw">const</span> <span class="const">result</span> = <span class="kw">await</span> parser.<span class="fn">parse</span>(<span class="const">bytes</span>);
-console.<span class="fn">log</span>(<span class="const">result</span>.text); <span class="cmt">// full document text</span>
+console.<span class="fn">log</span>(<span class="const">result</span>.text); <span class="cmt">// rendered markdown</span>
console.<span class="fn">log</span>(<span class="const">result</span>.pages[<span class="const">0</span>].textItems); <span class="cmt">// items with bounding boxes</span></pre>
</div>
</main>
@@ -413,8 +420,11 @@ console.<span class="fn">log</span>(<span class="const">result</span>.pages[<spa
const fileInput = document.getElementById("pdf-input");
const formatSelect = document.getElementById("format-select");
const copyBtn = document.getElementById("copy-btn");
+ const wrapBtn = document.getElementById("wrap-btn");
- let lastResult = null;
+ let currentFile = null; // raw bytes of the loaded PDF
+ let currentName = "";
+ const resultCache = {}; // keyed by output format: "markdown" | "text" | "json"
function setStatus(msg, type) {
status.textContent = msg;
@@ -426,12 +436,8 @@ console.<span class="fn">log</span>(<span class="const">result</span>.pages[<spa
let LiteParse;
try {
- const mod = await import(
- "https://cdn.jsdelivr.net/npm/@llamaindex/liteparse-wasm@2/pkg/liteparse_wasm.js"
- );
- await mod.default(
- "https://cdn.jsdelivr.net/npm/@llamaindex/liteparse-wasm@2/pkg/liteparse_wasm_bg.wasm"
- );
+ const mod = await import("./pkg/liteparse_wasm.js");
+ await mod.default("./pkg/liteparse_wasm_bg.wasm");
LiteParse = mod.LiteParse;
setStatus("Ready. Drop a PDF to parse.", "success");
} catch (err) {
@@ -465,38 +471,56 @@ console.<span class="fn">log</span>(<span class="const">result</span>.pages[<spa
if (file) parseFile(file);
});
- // --- Parse ---
+ // --- Load a file (store bytes, then render the selected format) ---
async function parseFile(file) {
if (!LiteParse) {
setStatus("WASM module not loaded yet.", "error");
return;
}
-
- setStatus(`Parsing "${file.name}"...`, "loading");
+ currentFile = new Uint8Array(await file.arrayBuffer());
+ currentName = file.name;
+ for (const k in resultCache) delete resultCache[k];
outputContainer.classList.remove("visible");
-
+ await renderOutput();
+ }
+
+ // Parse with a given output format, caching the result per format.
+ async function parseAs(format) {
+ if (resultCache[format]) return resultCache[format];
+ const t0 = performance.now();
+ const parser = new LiteParse({
+ ocrEnabled: false,
+ outputFormat: format,
+ quiet: true,
+ });
+ const result = await parser.parse(currentFile);
+ const elapsed = (performance.now() - t0) / 1000;
+ resultCache[format] = { result, elapsed };
+ return resultCache[format];
+ }
+
+ // --- Render output for the selected format ---
+ async function renderOutput() {
+ if (!currentFile) return;
+ const format = formatSelect.value;
+ setStatus(`Parsing "${currentName}"...`, "loading");
try {
- const t0 = performance.now();
- const bytes = new Uint8Array(await file.arrayBuffer());
- const parser = new LiteParse({
- ocrEnabled: false,
- outputFormat: "json",
- quiet: true,
- });
- const result = await parser.parse(bytes);
- const elapsed = ((performance.now() - t0) / 1000).toFixed(2);
-
- lastResult = result;
- setStatus(`Parsed ${result.pages.length} page(s) in ${elapsed}s`, "success");
- renderOutput();
+ const { result, elapsed } = await parseAs(format);
+
+ if (format === "json") {
+ output.textContent = JSON.stringify(result, null, 2);
+ } else {
+ // For "markdown" and "text", result.text holds the rendered output.
+ output.textContent = result.text;
+ }
+ setStatus(`Parsed ${result.pages.length} page(s) in ${elapsed.toFixed(2)}s`, "success");
statsEl.innerHTML = [
`<span class="stat"><strong>${result.pages.length}</strong> pages</span>`,
`<span class="stat"><strong>${result.text.length.toLocaleString()}</strong> chars</span>`,
- `<span class="stat"><strong>${elapsed}s</strong> elapsed</span>`,
- `<span class="stat"><strong>${(bytes.length / 1024).toFixed(0)} KB</strong> file size</span>`,
+ `<span class="stat"><strong>${elapsed.toFixed(2)}s</strong> elapsed</span>`,
+ `<span class="stat"><strong>${(currentFile.length / 1024).toFixed(0)} KB</strong> file size</span>`,
].join("");
-
outputContainer.classList.add("visible");
} catch (err) {
setStatus("Parse error: " + err.toString(), "error");
@@ -504,18 +528,13 @@ console.<span class="fn">log</span>(<span class="const">result</span>.pages[<spa
}
}
- // --- Render output ---
- function renderOutput() {
- if (!lastResult) return;
- if (formatSelect.value === "text") {
- output.textContent = lastResult.text;
- } else {
- output.textContent = JSON.stringify(lastResult, null, 2);
- }
- }
-
formatSelect.addEventListener("change", renderOutput);
+ wrapBtn.addEventListener("click", () => {
+ const wrapped = output.classList.toggle("wrap");
+ wrapBtn.textContent = wrapped ? "Wrap: On" : "Wrap: Off";
+ });
+
copyBtn.addEventListener("click", () => {
navigator.clipboard.writeText(output.textContent);
copyBtn.textContent = "Copied!";