Skip to content

A fast, helpful, and open-source document parser

11.4 KB
Raw Blame
#!/usr/bin/env bash
#
# run_benchmarks.sh — reproduce the LiteParse markdown benchmark sweep.
#
# Runs LiteParse (and, by default, the model-free competitors) across the three
# tracked benchmarks — olmOCR-bench, opendataloader-bench, ParseBench — and prints
# a comparison table per benchmark. See LAUNCH_BENCHMARKS.md for the published run.
#
# Usage:
#   ./run_benchmarks.sh [options]
#
# Options:
#   --liteparse-only        Only run LiteParse (skip all competitor tools).
#   --competitors-only      Only run the model-free competitors (keep existing LiteParse outputs).
#   --benches=LIST          Comma-separated subset of: olmocr,opendataloader,parsebench
#                           (default: all three).
#   --skip-build            Don't rebuild the lit binary / Python binding.
#   --out=DIR               Where to write logs + the summary (default: bench_results/latest).
#   --ocr=MODE              LiteParse OCR mode: none (default), tesseract, or paddle.
#                           paddle = the HTTP PaddleOCR server (ocr/paddleocr, port 8829;
#                           override with LITEPARSE_PADDLE_OCR_URL). OCR modes run with
#                           num_workers=4 and low harness concurrency — OCR is resource-hungry.
#                           Outputs land under mode-suffixed names (liteparse_tesseract, …) so
#                           all modes coexist and show up together in SUMMARY.md.
#   -h, --help              Show this help.
#
# Competitors (model-free, commercial): markitdown, opendataloader, pdf_inspector.
#   - opendataloader needs Java 11+ (the script auto-detects Homebrew openjdk@11/openjdk).
#   - pdf_inspector needs the `pdf2md` binary (cargo install pdf-inspector).
#   - nutrient is commercial and is NOT re-run here; see LAUNCH_BENCHMARKS.md for its
#     preserved opendataloader-bench numbers.
set -uo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$ROOT"

LITEPARSE_ONLY=0
COMPETITORS_ONLY=0
SKIP_BUILD=0
BENCHES="olmocr opendataloader parsebench"
OUT="$ROOT/bench_results/latest"
OCR_MODE="none"

for arg in "$@"; do
  case "$arg" in
    --liteparse-only) LITEPARSE_ONLY=1 ;;
    --competitors-only) COMPETITORS_ONLY=1 ;;
    --skip-build)     SKIP_BUILD=1 ;;
    --benches=*)      BENCHES="${arg#*=}"; BENCHES="${BENCHES//,/ }" ;;
    --out=*)          OUT="${arg#*=}" ;;
    --ocr=*)          OCR_MODE="${arg#*=}" ;;
    -h|--help)        sed -n '2,30p' "${BASH_SOURCE[0]}"; exit 0 ;;
    *) echo "Unknown option: $arg" >&2; exit 2 ;;
  esac
done

mkdir -p "$OUT"
PY="$ROOT/.venv/bin/python"

# --- OCR mode → per-harness LiteParse engine/pipeline names ---------------
case "$OCR_MODE" in
  none)
    OLM_LP="liteparse"; OLM_LP_NAME="liteparse"; ODL_LP="liteparse"; PB_LP="liteparse_markdown"; PB_CONC=6 ;;
  tesseract)
    OLM_LP="liteparse:ocr_engine=tesseract:num_workers=4:name=liteparse_tesseract"; OLM_LP_NAME="liteparse_tesseract"
    ODL_LP="liteparse-tesseract"; PB_LP="liteparse_markdown_tesseract"; PB_CONC=3 ;;
  paddle)
    OLM_LP="liteparse:ocr_engine=paddle:num_workers=4:name=liteparse_paddle"; OLM_LP_NAME="liteparse_paddle"
    ODL_LP="liteparse-paddle"; PB_LP="liteparse_markdown_paddle"; PB_CONC=3
    PURL="${LITEPARSE_PADDLE_OCR_URL:-http://localhost:8829/ocr}"
    curl -s -m 5 -o /dev/null "${PURL%/ocr}/" || echo "WARNING: no PaddleOCR server answering at $PURL (start: cd ocr/paddleocr && uv run server.py)" >&2 ;;
  *) echo "Unknown --ocr mode: $OCR_MODE (none|tesseract|paddle)" >&2; exit 2 ;;
esac
TIMING="$OUT/timings.txt"
stamp() { echo "$(date +%s) $*" >> "$TIMING"; }

# --- Java 11+ detection (needed by the opendataloader competitor) -------------
JAVA11_BIN=""
for cand in /opt/homebrew/opt/openjdk@11/bin /opt/homebrew/opt/openjdk/bin /opt/homebrew/opt/openjdk@25/bin; do
  if [ -x "$cand/java" ]; then JAVA11_BIN="$cand"; break; fi
done
if [ -n "$JAVA11_BIN" ]; then export PATH="$JAVA11_BIN:$PATH"; fi

run_engines() {  # echoes the engine list for a benchmark, honoring --liteparse-only / --competitors-only
  if [ "$LITEPARSE_ONLY" = "1" ]; then echo "$1"; return; fi
  shift
  if [ "$COMPETITORS_ONLY" = "1" ]; then shift; fi
  echo "$@"
}

# === 1. Build artifacts =======================================================
if [ "$COMPETITORS_ONLY" = "1" ]; then SKIP_BUILD=1; fi
if [ "$SKIP_BUILD" = "0" ]; then
  echo "### Building lit binary + Python binding ..."
  cargo build --release --bin lit || { echo "cargo build failed"; exit 1; }
  ( cd packages/python && maturin develop --release ) || { echo "maturin build failed"; exit 1; }
else
  echo "### Skipping build (--skip-build)"
fi

# === 2. olmOCR-bench ==========================================================
if [[ " $BENCHES " == *" olmocr "* ]]; then
  echo "### olmOCR-bench ..."
  DIR="$ROOT/olmocr/olmOCR-bench/bench_data"
  for E in $(run_engines "$OLM_LP" "$OLM_LP"  markitdown opendataloader pdf_inspector); do
    N="$E"; [ "$E" = "$OLM_LP" ] && N="$OLM_LP_NAME"
    [[ "$E" == *:name=* ]] && N="${E##*:name=}"
    echo "  convert $N"; stamp "olmocr_convert_$N start"
    $PY -m olmocr.bench.convert "$E" --dir "$DIR" --parallel 1 --force \
        > "$OUT/olmocr_convert_$N.log" 2>&1
    stamp "olmocr_convert_$N end"
    echo "  benchmark $N"
    $PY -m olmocr.bench.benchmark --dir "$DIR" --candidate "$N" \
        > "$OUT/olmocr_bench_$N.log" 2>&1
  done
fi

# === 3. opendataloader-bench ==================================================
if [[ " $BENCHES " == *" opendataloader "* ]]; then
  echo "### opendataloader-bench ..."
  for E in $(run_engines "$ODL_LP" "$ODL_LP" opendataloader markitdown); do
    echo "  run $E"; stamp "odl_$E start"
    ( cd opendataloader-bench && \
      LITEPARSE_BIN="$ROOT/target/release/lit" \
      ../.venv/bin/python src/run.py --engine "$E" --force ) \
        > "$OUT/odl_$E.log" 2>&1
    stamp "odl_$E end"
  done
fi

# === 4. ParseBench (all 5 groups, rule-based, no API key) =====================
if [[ " $BENCHES " == *" parsebench "* ]]; then
  echo "### ParseBench ..."
  for P in $(run_engines "$PB_LP" "$PB_LP" markitdown opendataloader_markdown pdf_inspector); do
    echo "  run $P"; stamp "parsebench_$P start"
    EXTRA="--force"; [[ "$P" == liteparse* ]] && EXTRA="--force -m $PB_CONC"
    ( cd ParseBench && LITEPARSE_BIN="$ROOT/target/release/lit" \
      ../.venv/bin/parse-bench run "$P" --open_report=False $EXTRA ) \
        > "$OUT/parsebench_$P.log" 2>&1
    stamp "parsebench_$P end"
  done
fi

# === 5. Aggregate + print tables =============================================
echo
echo "### Results (also written to $OUT/SUMMARY.md)"
ROOT="$ROOT" OUT="$OUT" BENCHES="$BENCHES" "$PY" - <<'PYEOF' | tee "$OUT/SUMMARY.md"
import json, os, re, glob
ROOT=os.environ["ROOT"]; OUT=os.environ["OUT"]; BENCHES=os.environ["BENCHES"].split()

def table(rows, hdr):
    out=['| '+' | '.join(hdr)+' |', '|'+'|'.join(['---']*len(hdr))+'|']
    for r in rows: out.append('| '+' | '.join(r)+' |')
    return '\n'.join(out)

print(f"# Benchmark results\n")

# --- olmOCR: parse the benchmark logs ---
if 'olmocr' in BENCHES:
    cats=['baseline','headers_footers','multi_column','table_tests','long_tiny_text',
          'old_scans','arxiv_math','old_scans_math']
    data={}
    for lf in sorted(glob.glob(f"{OUT}/olmocr_bench_*.log")):
        eng=re.search(r'olmocr_bench_(.+)\.log',lf).group(1)
        txt=open(lf,encoding='utf-8',errors='ignore').read()
        m=re.search(r'Average Score:\s*([\d.]+)%\s*(?:±|\()',txt)
        overall=m.group(1) if m else 'NA'
        per={}
        for c in cats:
            mm=re.search(rf'{re.escape(c)}(?:\.jsonl)?\s*:\s*([\d.]+)%\s*\(',txt)
            per[c]=mm.group(1) if mm else 'NA'
        data[eng]={'Overall':overall, **per}
    order=[e for e in ['liteparse','liteparse_tesseract','liteparse_paddle','markitdown','opendataloader','pdf_inspector'] if e in data]
    if order:
        hdr=['Engine','Overall']+cats
        rows=[[e]+[data[e]['Overall']]+[data[e][c] for c in cats] for e in order]
        print("## olmOCR-bench (% pass)\n"); print(table(rows,hdr)); print()

# --- opendataloader: read evaluation.json ---
if 'opendataloader' in BENCHES:
    rows=[]
    for e in ['liteparse','liteparse-tesseract','liteparse-paddle','nutrient','opendataloader','markitdown']:
        f=f"{ROOT}/opendataloader-bench/prediction/{e}/evaluation.json"
        if not os.path.exists(f): continue
        s=json.load(open(f))['metrics']['score']
        rows.append([e,'%.4f'%s['overall_mean'],'%.4f'%s['nid_mean'],'%.4f'%s['teds_mean'],'%.4f'%s['mhs_mean']])
    if rows:
        print("## opendataloader-bench\n")
        print(table(rows,['Engine','Overall','NID','TEDS','MHS']))
        print("\n*nutrient = preserved data (not re-run here).*\n")

# --- ParseBench: per-category canonical metric (matches ParseBench leaderboard) ---
if 'parsebench' in BENCHES:
    cats=[('table',['avg_grits_trm_composite'],'Tables'),
          ('chart',['avg_rule_pass_rate'],'Charts'),
          ('text_content',['avg_content_faithfulness','avg_rule_pass_rate'],'Content_Faithfulness'),
          ('text_formatting',['avg_semantic_formatting','avg_rule_pass_rate'],'Semantic_Formatting'),
          ('layout',['avg_layout_element_rule_pass_rate','avg_rule_pass_rate'],'Visual_Grounding')]
    pipes=[('liteparse_markdown','LiteParse'),('liteparse_markdown_tesseract','LiteParse+tesseract'),
           ('liteparse_markdown_paddle','LiteParse+paddle'),
           ('markitdown','markitdown'),('opendataloader_markdown','opendataloader'),
           ('pdf_inspector','pdf-inspector')]
    rows=[]; lay_n=None
    for p,disp in pipes:
        vals={}
        for grp,keys,col in cats:
            f=f"{ROOT}/ParseBench/output/{p}/{grp}/_evaluation_report.json"
            if os.path.exists(f):
                d=json.load(open(f)); m=d['aggregate_metrics']
                vals[col]=next((m[k] for k in keys if k in m),None)
                if grp=='layout':
                    # The harness averages only over scored docs; a doc it *skipped* (no
                    # layout data emitted) is excluded, while a *failed* one counts as 0.
                    # Count skipped docs as 0 too, so rows with different skip counts are
                    # comparable: element rules over docs that have element GT, otherwise
                    # reading-order rules over every doc.
                    per=d['per_example_results']
                    le=[x['value'] for r in per for x in (r.get('metrics') or []) if x['metric_name']=='layout_element_rule_pass_rate']
                    rp=[x['value'] for r in per for x in (r.get('metrics') or []) if x['metric_name']=='rule_pass_rate']
                    if le: vals[col]=sum(le)/(len(le)+d['skipped'])
                    elif rp: vals[col]=sum(rp)/d['total_examples']
                    if lay_n is None: lay_n=(len(le)+d['skipped'] if le else d['total_examples'], d['total_examples'])
            else: vals[col]=None
        present=[v for v in vals.values() if v is not None]
        if not present: continue
        overall=sum(present)/len(present)
        rows.append([disp,'%.4f'%overall]+['%.4f'%vals[c[2]] if vals[c[2]] is not None else 'NA' for c in cats])
    if rows:
        hdr=['Pipeline','Overall']+[c[2] for c in cats]
        print("## ParseBench (rule-based; per-category canonical metric)\n")
        print(table(rows,hdr))
        print("\n*Visual_Grounding: layout-element rules over the docs that have element ground truth (436/500) when the pipeline emits layout data, else reading-order rules over all 500; a doc with no output counts as 0.*")
PYEOF
echo
echo "### Done. Logs + SUMMARY.md in $OUT"