A fast, helpful, and open-source document parser
nit
Logan Markewichb751f608471c
diff --git a/run_benchmarks.sh b/run_benchmarks.shindex a451aed..e14d57f 100755--- a/run_benchmarks.sh+++ b/run_benchmarks.sh@@ -24,11 +24,7 @@# all modes coexist and show up together in SUMMARY.md.# -h, --help Show this help.#-# Competitors (model-free): pymupdf4llm, markitdown, opendataloader, pdf_inspector.-# - pymupdf4llm >= 1.28 bundles the pymupdf-layout GNN model (activated on import) and, in-# ParseBench's canonical pipeline, RapidOCR (`pip install rapidocr`). The extra-# pymupdf4llm_nolayout / pymupdf4llm_markdown_nolayout rows run it with the layout model-# off (and no OCR) as the model-free reference.+# Competitors (model-free, commercial): markitdown, opendataloader, pdf_inspector.# - opendataloader needs Java 11+ (the script auto-detects Homebrew openjdk@11/openjdk).# - pdf_inspector needs the `pdf2md` binary (cargo install pdf-inspector).# - nutrient is commercial and is NOT re-run here; see LAUNCH_BENCHMARKS.md for its@@ -106,7 +102,7 @@ fiif [[ " $BENCHES " == *" olmocr "* ]]; thenecho "### olmOCR-bench ..."DIR="$ROOT/olmocr/olmOCR-bench/bench_data"- for E in $(run_engines "$OLM_LP" "$OLM_LP" pymupdf4llm "pymupdf4llm:use_layout=False:name=pymupdf4llm_nolayout" markitdown opendataloader pdf_inspector); do+ for E in $(run_engines "$OLM_LP" "$OLM_LP" markitdown opendataloader pdf_inspector); doN="$E"; [ "$E" = "$OLM_LP" ] && N="$OLM_LP_NAME"[[ "$E" == *:name=* ]] && N="${E##*:name=}"echo " convert $N"; stamp "olmocr_convert_$N start"@@ -135,7 +131,7 @@ fi# === 4. ParseBench (all 5 groups, rule-based, no API key) =====================if [[ " $BENCHES " == *" parsebench "* ]]; thenecho "### ParseBench ..."- for P in $(run_engines "$PB_LP" "$PB_LP" pymupdf4llm_markdown pymupdf4llm_markdown_nolayout markitdown opendataloader_markdown pdf_inspector); do+ for P in $(run_engines "$PB_LP" "$PB_LP" markitdown opendataloader_markdown pdf_inspector); doecho " run $P"; stamp "parsebench_$P start"EXTRA="--force"; [[ "$P" == liteparse* ]] && EXTRA="--force -m $PB_CONC"( cd ParseBench && LITEPARSE_BIN="$ROOT/target/release/lit" \@@ -174,7 +170,7 @@ if 'olmocr' in BENCHES:mm=re.search(rf'{re.escape(c)}(?:\.jsonl)?\s*:\s*([\d.]+)%\s*\(',txt)per[c]=mm.group(1) if mm else 'NA'data[eng]={'Overall':overall, **per}- order=[e for e in ['liteparse','liteparse_tesseract','liteparse_paddle','pymupdf4llm','pymupdf4llm_nolayout','markitdown','opendataloader','pdf_inspector'] if e in data]+ order=[e for e in ['liteparse','liteparse_tesseract','liteparse_paddle','markitdown','opendataloader','pdf_inspector'] if e in data]if order:hdr=['Engine','Overall']+catsrows=[[e]+[data[e]['Overall']]+[data[e][c] for c in cats] for e in order]@@ -183,7 +179,7 @@ if 'olmocr' in BENCHES:# --- opendataloader: read evaluation.json ---if 'opendataloader' in BENCHES:rows=[]- for e in ['liteparse','liteparse-tesseract','liteparse-paddle','nutrient','opendataloader','pymupdf4llm','markitdown']:+ for e in ['liteparse','liteparse-tesseract','liteparse-paddle','nutrient','opendataloader','markitdown']:f=f"{ROOT}/opendataloader-bench/prediction/{e}/evaluation.json"if not os.path.exists(f): continues=json.load(open(f))['metrics']['score']@@ -191,7 +187,7 @@ if 'opendataloader' in BENCHES:if rows:print("## opendataloader-bench\n")print(table(rows,['Engine','Overall','NID','TEDS','MHS']))- print("\n*nutrient/pymupdf4llm = preserved data (not re-run here).*\n")+ print("\n*nutrient = preserved data (not re-run here).*\n")# --- ParseBench: per-category canonical metric (matches ParseBench leaderboard) ---if 'parsebench' in BENCHES:@@ -201,8 +197,7 @@ if 'parsebench' in BENCHES:('text_formatting',['avg_semantic_formatting','avg_rule_pass_rate'],'Semantic_Formatting'),('layout',['avg_layout_element_rule_pass_rate','avg_rule_pass_rate'],'Visual_Grounding')]pipes=[('liteparse_markdown','LiteParse'),('liteparse_markdown_tesseract','LiteParse+tesseract'),- ('liteparse_markdown_paddle','LiteParse+paddle'),('pymupdf4llm_markdown','pymupdf4llm'),- ('pymupdf4llm_markdown_nolayout','pymupdf4llm (no layout model)'),+ ('liteparse_markdown_paddle','LiteParse+paddle'),('markitdown','markitdown'),('opendataloader_markdown','opendataloader'),('pdf_inspector','pdf-inspector')]rows=[]; lay_n=None