A fast, helpful, and open-source document parser
add run script
Logan Markewich9820afadde44
diff --git a/run_benchmarks.sh b/run_benchmarks.shnew file mode 100755index 0000000..a451aed--- /dev/null+++ b/run_benchmarks.sh@@ -0,0 +1,240 @@+#!/usr/bin/env bash+#+# run_benchmarks.sh — reproduce the LiteParse markdown benchmark sweep.+#+# Runs LiteParse (and, by default, the model-free competitors) across the three+# tracked benchmarks — olmOCR-bench, opendataloader-bench, ParseBench — and prints+# a comparison table per benchmark. See LAUNCH_BENCHMARKS.md for the published run.+#+# Usage:+# ./run_benchmarks.sh [options]+#+# Options:+# --liteparse-only Only run LiteParse (skip all competitor tools).+# --competitors-only Only run the model-free competitors (keep existing LiteParse outputs).+# --benches=LIST Comma-separated subset of: olmocr,opendataloader,parsebench+# (default: all three).+# --skip-build Don't rebuild the lit binary / Python binding.+# --out=DIR Where to write logs + the summary (default: bench_results/latest).+# --ocr=MODE LiteParse OCR mode: none (default), tesseract, or paddle.+# paddle = the HTTP PaddleOCR server (ocr/paddleocr, port 8829;+# override with LITEPARSE_PADDLE_OCR_URL). OCR modes run with+# num_workers=4 and low harness concurrency — OCR is resource-hungry.+# Outputs land under mode-suffixed names (liteparse_tesseract, …) so+# all modes coexist and show up together in SUMMARY.md.+# -h, --help Show this help.+#+# Competitors (model-free): pymupdf4llm, markitdown, opendataloader, pdf_inspector.+# - pymupdf4llm >= 1.28 bundles the pymupdf-layout GNN model (activated on import) and, in+# ParseBench's canonical pipeline, RapidOCR (`pip install rapidocr`). The extra+# pymupdf4llm_nolayout / pymupdf4llm_markdown_nolayout rows run it with the layout model+# off (and no OCR) as the model-free reference.+# - opendataloader needs Java 11+ (the script auto-detects Homebrew openjdk@11/openjdk).+# - pdf_inspector needs the `pdf2md` binary (cargo install pdf-inspector).+# - nutrient is commercial and is NOT re-run here; see LAUNCH_BENCHMARKS.md for its+# preserved opendataloader-bench numbers.+set -uo pipefail++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+cd "$ROOT"++LITEPARSE_ONLY=0+COMPETITORS_ONLY=0+SKIP_BUILD=0+BENCHES="olmocr opendataloader parsebench"+OUT="$ROOT/bench_results/latest"+OCR_MODE="none"++for arg in "$@"; do+ case "$arg" in+ --liteparse-only) LITEPARSE_ONLY=1 ;;+ --competitors-only) COMPETITORS_ONLY=1 ;;+ --skip-build) SKIP_BUILD=1 ;;+ --benches=*) BENCHES="${arg#*=}"; BENCHES="${BENCHES//,/ }" ;;+ --out=*) OUT="${arg#*=}" ;;+ --ocr=*) OCR_MODE="${arg#*=}" ;;+ -h|--help) sed -n '2,30p' "${BASH_SOURCE[0]}"; exit 0 ;;+ *) echo "Unknown option: $arg" >&2; exit 2 ;;+ esac+done++mkdir -p "$OUT"+PY="$ROOT/.venv/bin/python"++# --- OCR mode → per-harness LiteParse engine/pipeline names ---------------+case "$OCR_MODE" in+ none)+ OLM_LP="liteparse"; OLM_LP_NAME="liteparse"; ODL_LP="liteparse"; PB_LP="liteparse_markdown"; PB_CONC=6 ;;+ tesseract)+ OLM_LP="liteparse:ocr_engine=tesseract:num_workers=4:name=liteparse_tesseract"; OLM_LP_NAME="liteparse_tesseract"+ ODL_LP="liteparse-tesseract"; PB_LP="liteparse_markdown_tesseract"; PB_CONC=3 ;;+ paddle)+ OLM_LP="liteparse:ocr_engine=paddle:num_workers=4:name=liteparse_paddle"; OLM_LP_NAME="liteparse_paddle"+ ODL_LP="liteparse-paddle"; PB_LP="liteparse_markdown_paddle"; PB_CONC=3+ PURL="${LITEPARSE_PADDLE_OCR_URL:-http://localhost:8829/ocr}"+ curl -s -m 5 -o /dev/null "${PURL%/ocr}/" || echo "WARNING: no PaddleOCR server answering at $PURL (start: cd ocr/paddleocr && uv run server.py)" >&2 ;;+ *) echo "Unknown --ocr mode: $OCR_MODE (none|tesseract|paddle)" >&2; exit 2 ;;+esac+TIMING="$OUT/timings.txt"+stamp() { echo "$(date +%s) $*" >> "$TIMING"; }++# --- Java 11+ detection (needed by the opendataloader competitor) -------------+JAVA11_BIN=""+for cand in /opt/homebrew/opt/openjdk@11/bin /opt/homebrew/opt/openjdk/bin /opt/homebrew/opt/openjdk@25/bin; do+ if [ -x "$cand/java" ]; then JAVA11_BIN="$cand"; break; fi+done+if [ -n "$JAVA11_BIN" ]; then export PATH="$JAVA11_BIN:$PATH"; fi++run_engines() { # echoes the engine list for a benchmark, honoring --liteparse-only / --competitors-only+ if [ "$LITEPARSE_ONLY" = "1" ]; then echo "$1"; return; fi+ shift+ if [ "$COMPETITORS_ONLY" = "1" ]; then shift; fi+ echo "$@"+}++# === 1. Build artifacts =======================================================+if [ "$COMPETITORS_ONLY" = "1" ]; then SKIP_BUILD=1; fi+if [ "$SKIP_BUILD" = "0" ]; then+ echo "### Building lit binary + Python binding ..."+ cargo build --release --bin lit || { echo "cargo build failed"; exit 1; }+ ( cd packages/python && maturin develop --release ) || { echo "maturin build failed"; exit 1; }+else+ echo "### Skipping build (--skip-build)"+fi++# === 2. olmOCR-bench ==========================================================+if [[ " $BENCHES " == *" olmocr "* ]]; then+ echo "### olmOCR-bench ..."+ DIR="$ROOT/olmocr/olmOCR-bench/bench_data"+ for E in $(run_engines "$OLM_LP" "$OLM_LP" pymupdf4llm "pymupdf4llm:use_layout=False:name=pymupdf4llm_nolayout" markitdown opendataloader pdf_inspector); do+ N="$E"; [ "$E" = "$OLM_LP" ] && N="$OLM_LP_NAME"+ [[ "$E" == *:name=* ]] && N="${E##*:name=}"+ echo " convert $N"; stamp "olmocr_convert_$N start"+ $PY -m olmocr.bench.convert "$E" --dir "$DIR" --parallel 1 --force \+ > "$OUT/olmocr_convert_$N.log" 2>&1+ stamp "olmocr_convert_$N end"+ echo " benchmark $N"+ $PY -m olmocr.bench.benchmark --dir "$DIR" --candidate "$N" \+ > "$OUT/olmocr_bench_$N.log" 2>&1+ done+fi++# === 3. opendataloader-bench ==================================================+if [[ " $BENCHES " == *" opendataloader "* ]]; then+ echo "### opendataloader-bench ..."+ for E in $(run_engines "$ODL_LP" "$ODL_LP" opendataloader markitdown); do+ echo " run $E"; stamp "odl_$E start"+ ( cd opendataloader-bench && \+ LITEPARSE_BIN="$ROOT/target/release/lit" \+ ../.venv/bin/python src/run.py --engine "$E" --force ) \+ > "$OUT/odl_$E.log" 2>&1+ stamp "odl_$E end"+ done+fi++# === 4. ParseBench (all 5 groups, rule-based, no API key) =====================+if [[ " $BENCHES " == *" parsebench "* ]]; then+ echo "### ParseBench ..."+ for P in $(run_engines "$PB_LP" "$PB_LP" pymupdf4llm_markdown pymupdf4llm_markdown_nolayout markitdown opendataloader_markdown pdf_inspector); do+ echo " run $P"; stamp "parsebench_$P start"+ EXTRA="--force"; [[ "$P" == liteparse* ]] && EXTRA="--force -m $PB_CONC"+ ( cd ParseBench && LITEPARSE_BIN="$ROOT/target/release/lit" \+ ../.venv/bin/parse-bench run "$P" --open_report=False $EXTRA ) \+ > "$OUT/parsebench_$P.log" 2>&1+ stamp "parsebench_$P end"+ done+fi++# === 5. Aggregate + print tables =============================================+echo+echo "### Results (also written to $OUT/SUMMARY.md)"+ROOT="$ROOT" OUT="$OUT" BENCHES="$BENCHES" "$PY" - <<'PYEOF' | tee "$OUT/SUMMARY.md"+import json, os, re, glob+ROOT=os.environ["ROOT"]; OUT=os.environ["OUT"]; BENCHES=os.environ["BENCHES"].split()++def table(rows, hdr):+ out=['| '+' | '.join(hdr)+' |', '|'+'|'.join(['---']*len(hdr))+'|']+ for r in rows: out.append('| '+' | '.join(r)+' |')+ return '\n'.join(out)++print(f"# Benchmark results\n")++# --- olmOCR: parse the benchmark logs ---+if 'olmocr' in BENCHES:+ cats=['baseline','headers_footers','multi_column','table_tests','long_tiny_text',+ 'old_scans','arxiv_math','old_scans_math']+ data={}+ for lf in sorted(glob.glob(f"{OUT}/olmocr_bench_*.log")):+ eng=re.search(r'olmocr_bench_(.+)\.log',lf).group(1)+ txt=open(lf,encoding='utf-8',errors='ignore').read()+ m=re.search(r'Average Score:\s*([\d.]+)%\s*(?:±|\()',txt)+ overall=m.group(1) if m else 'NA'+ per={}+ for c in cats:+ mm=re.search(rf'{re.escape(c)}(?:\.jsonl)?\s*:\s*([\d.]+)%\s*\(',txt)+ per[c]=mm.group(1) if mm else 'NA'+ data[eng]={'Overall':overall, **per}+ order=[e for e in ['liteparse','liteparse_tesseract','liteparse_paddle','pymupdf4llm','pymupdf4llm_nolayout','markitdown','opendataloader','pdf_inspector'] if e in data]+ if order:+ hdr=['Engine','Overall']+cats+ rows=[[e]+[data[e]['Overall']]+[data[e][c] for c in cats] for e in order]+ print("## olmOCR-bench (% pass)\n"); print(table(rows,hdr)); print()++# --- opendataloader: read evaluation.json ---+if 'opendataloader' in BENCHES:+ rows=[]+ for e in ['liteparse','liteparse-tesseract','liteparse-paddle','nutrient','opendataloader','pymupdf4llm','markitdown']:+ f=f"{ROOT}/opendataloader-bench/prediction/{e}/evaluation.json"+ if not os.path.exists(f): continue+ s=json.load(open(f))['metrics']['score']+ rows.append([e,'%.4f'%s['overall_mean'],'%.4f'%s['nid_mean'],'%.4f'%s['teds_mean'],'%.4f'%s['mhs_mean']])+ if rows:+ print("## opendataloader-bench\n")+ print(table(rows,['Engine','Overall','NID','TEDS','MHS']))+ print("\n*nutrient/pymupdf4llm = preserved data (not re-run here).*\n")++# --- ParseBench: per-category canonical metric (matches ParseBench leaderboard) ---+if 'parsebench' in BENCHES:+ cats=[('table',['avg_grits_trm_composite'],'Tables'),+ ('chart',['avg_rule_pass_rate'],'Charts'),+ ('text_content',['avg_content_faithfulness','avg_rule_pass_rate'],'Content_Faithfulness'),+ ('text_formatting',['avg_semantic_formatting','avg_rule_pass_rate'],'Semantic_Formatting'),+ ('layout',['avg_layout_element_rule_pass_rate','avg_rule_pass_rate'],'Visual_Grounding')]+ pipes=[('liteparse_markdown','LiteParse'),('liteparse_markdown_tesseract','LiteParse+tesseract'),+ ('liteparse_markdown_paddle','LiteParse+paddle'),('pymupdf4llm_markdown','pymupdf4llm'),+ ('pymupdf4llm_markdown_nolayout','pymupdf4llm (no layout model)'),+ ('markitdown','markitdown'),('opendataloader_markdown','opendataloader'),+ ('pdf_inspector','pdf-inspector')]+ rows=[]; lay_n=None+ for p,disp in pipes:+ vals={}+ for grp,keys,col in cats:+ f=f"{ROOT}/ParseBench/output/{p}/{grp}/_evaluation_report.json"+ if os.path.exists(f):+ d=json.load(open(f)); m=d['aggregate_metrics']+ vals[col]=next((m[k] for k in keys if k in m),None)+ if grp=='layout':+ # The harness averages only over scored docs; a doc it *skipped* (no+ # layout data emitted) is excluded, while a *failed* one counts as 0.+ # Count skipped docs as 0 too, so rows with different skip counts are+ # comparable: element rules over docs that have element GT, otherwise+ # reading-order rules over every doc.+ per=d['per_example_results']+ le=[x['value'] for r in per for x in (r.get('metrics') or []) if x['metric_name']=='layout_element_rule_pass_rate']+ rp=[x['value'] for r in per for x in (r.get('metrics') or []) if x['metric_name']=='rule_pass_rate']+ if le: vals[col]=sum(le)/(len(le)+d['skipped'])+ elif rp: vals[col]=sum(rp)/d['total_examples']+ if lay_n is None: lay_n=(len(le)+d['skipped'] if le else d['total_examples'], d['total_examples'])+ else: vals[col]=None+ present=[v for v in vals.values() if v is not None]+ if not present: continue+ overall=sum(present)/len(present)+ rows.append([disp,'%.4f'%overall]+['%.4f'%vals[c[2]] if vals[c[2]] is not None else 'NA' for c in cats])+ if rows:+ hdr=['Pipeline','Overall']+[c[2] for c in cats]+ print("## ParseBench (rule-based; per-category canonical metric)\n")+ print(table(rows,hdr))+ print("\n*Visual_Grounding: layout-element rules over the docs that have element ground truth (436/500) when the pipeline emits layout data, else reading-order rules over all 500; a doc with no output counts as 0.*")+PYEOF+echo+echo "### Done. Logs + SUMMARY.md in $OUT"