A fast, helpful, and open-source document parser
clean up part 1
Logan Markewich2b643b8d96dc
diff --git a/crates/liteparse/src/markdown_layout/blocks.rs b/crates/liteparse/src/markdown_layout/blocks.rsindex f836b26..2e289c7 100644--- a/crates/liteparse/src/markdown_layout/blocks.rs+++ b/crates/liteparse/src/markdown_layout/blocks.rs@@ -49,7 +49,6 @@ pub enum Block {/// ``. Suppressed entirely when `ImageMode::Off`.Figure {id: String,- bbox: crate::types::Rect,},}diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rsindex fc97ff4..3977c25 100644--- a/crates/liteparse/src/markdown_layout/classify.rs+++ b/crates/liteparse/src/markdown_layout/classify.rs@@ -74,7 +74,7 @@ pub fn classify_page_with_filters(image_mode: crate::config::ImageMode,chrome_indices: &std::collections::HashSet<usize>,) -> Vec<Block> {- let debug = std::env::var("LITEPARSE_DEBUG_MD").is_ok();+ let debug = *super::flags::DEBUG_MD;// TOC suppression: when ≥3 lines on this page look like TOC entries// (alpha body + trailing page-number), demote heading promotion so each@@ -160,7 +160,7 @@ pub fn classify_page_with_filters(let mut global_ruled_tables: Vec<(f32, Block)> = Vec::new();let mut global_ruled_consumed: std::collections::HashSet<usize> =std::collections::HashSet::new();- if std::env::var("LITEPARSE_DISABLE_GLOBAL_RULED").is_err() {+ if !*super::flags::DISABLE_GLOBAL_RULED {for (run, consumed) in super::tables::detect_ruled_tables_global(lines,&page.graphics,@@ -324,10 +324,7 @@ pub fn classify_page_with_filters(let push_interruption = |blocks: &mut Vec<Block>, kind: Interruption| {blocks.push(match kind {Interruption::Hr => Block::HorizontalRule,- Interruption::Figure(r) => Block::Figure {- id: r.id,- bbox: r.bbox,- },+ Interruption::Figure(r) => Block::Figure { id: r.id },Interruption::Table(b) => b,});};@@ -377,7 +374,6 @@ pub fn classify_page_with_filters(page,heading_map,outline,- image_mode,toc_page,debug,precomputed_tables,@@ -395,7 +391,7 @@ pub fn classify_page_with_filters(interrupt_cursor += 1;}- return stitch_regions(blocks, ®ion_boundaries);+ stitch_regions(blocks, ®ion_boundaries)}/// Classify the lines of a single xy_cut leaf into a sequence of blocks. All@@ -411,12 +407,10 @@ fn classify_region(page: &ParsedPage,heading_map: &[(f32, u8)],outline: &[OutlineTarget],- image_mode: crate::config::ImageMode,toc_page: bool,debug: bool,precomputed_tables: Option<Vec<super::tables::TableRun>>,) -> Vec<Block> {- let _ = image_mode; // interruption emission already gated upstreamlet mut blocks: Vec<Block> = Vec::new();let mut paragraph: Option<ParaAccum> = None;let mut code: Option<Vec<String>> = None;@@ -502,10 +496,7 @@ fn classify_region(*last_line = None;blocks.push(match kind {Interruption::Hr => Block::HorizontalRule,- Interruption::Figure(r) => Block::Figure {- id: r.id,- bbox: r.bbox,- },+ Interruption::Figure(r) => Block::Figure { id: r.id },Interruption::Table(b) => b,});}@@ -629,7 +620,7 @@ fn classify_region(// A previous line ending in a mid-word hyphen wrap means this// line is its continuation regardless of capitalization// ("…SOLAR 10.7 Billion-" / "Parameter Model: We have…").- let prev_hyphen_wrap = std::env::var("LITEPARSE_DISABLE_HEADING_GUARDS").is_err()+ let prev_hyphen_wrap = !*super::flags::DISABLE_HEADING_GUARDS&& prev.is_some_and(|p| {let t = p.text.trim_end();t.ends_with('-') && t.chars().rev().nth(1).is_some_and(|c| c.is_alphanumeric())diff --git a/crates/liteparse/src/markdown_layout/headings.rs b/crates/liteparse/src/markdown_layout/headings.rsindex 266e4b7..7fccdd7 100644--- a/crates/liteparse/src/markdown_layout/headings.rs+++ b/crates/liteparse/src/markdown_layout/headings.rs@@ -383,7 +383,7 @@ pub(super) fn looks_like_bold_heading(// - ≥3 space-separated words after the break// - line ends with mid-word "-" (wrap continuation) OR is >50 chars// - first char after the break is uppercase ASCII letter- let colon_ok = std::env::var("LITEPARSE_DISABLE_HEADING_GUARDS").is_err();+ let colon_ok = !*super::flags::DISABLE_HEADING_GUARDS;let run_in_break = text.find(". ").map(|p| (p, 2))@@ -404,7 +404,7 @@ pub(super) fn looks_like_bold_heading(&& starts_upper&& ((word_count >= 2 && ends_hyphen) || (word_count >= 3 && text.chars().count() > 50)){- if std::env::var("LITEPARSE_DEBUG_MD").is_ok() {+ if *super::flags::DEBUG_MD {eprintln!("[MD bold-heading REJECT run-in] '{}' (pos={} word_count={} ends_hyphen={} len={})",text.chars().take(80).collect::<String>(),diff --git a/crates/liteparse/src/markdown_layout/inline.rs b/crates/liteparse/src/markdown_layout/inline.rsindex dc2dd6d..32a9813 100644--- a/crates/liteparse/src/markdown_layout/inline.rs+++ b/crates/liteparse/src/markdown_layout/inline.rs@@ -3,14 +3,8 @@ use crate::types::{ProjectedLine, TextItem};use super::paragraphs::{collapse_whitespace, dehyphenate_join};-/// Lightweight bold detection on a raw `TextItem`. Delegates to-/// `projection::is_bold_item` so per-span and per-line logic stay in sync.-pub(super) fn is_bold_span(s: &TextItem) -> bool {- is_bold_item(s)-}-/// Per-span style flags used by the inline-emphasis renderer.-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]+#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub(super) struct SpanStyle {pub(super) bold: bool,pub(super) italic: bool,@@ -52,7 +46,7 @@ pub(super) fn escape_inline(s: &str) -> String {/// Wrap `inner` with the markdown markers for `style`. Mono wins over bold/italic:/// inline code (`` `…` ``) doesn't compose with emphasis in CommonMark, so when/// a span is mono we drop the `**/*` wrap. Bold + italic → `***…***`.-pub(super) fn apply_style(inner: &str, style: SpanStyle) -> String {+fn apply_style(inner: &str, style: SpanStyle) -> String {if style.mono {// Use backticks; if inner already contains backticks, switch to a// longer fence (pair of backticks plus a space buffer) per CommonMark.@@ -176,7 +170,7 @@ pub(super) fn render_list_item_text(line: &ProjectedLine, marker: &str, rest: &s/// Try to strip a leading list marker (optionally wrapped in emphasis markers)/// off `s`. Recognizes `***MARK*** `, `**MARK** `, `*MARK* `, `` `MARK` ``,/// and bare `MARK `. Returns the suffix on a match.-pub(super) fn strip_leading_marker_from_inline(s: &str, marker: &str) -> Option<String> {+fn strip_leading_marker_from_inline(s: &str, marker: &str) -> Option<String> {for wrap in ["***", "**", "*", "`"] {let prefix = format!("{wrap}{marker}{wrap} ");if let Some(rest) = s.strip_prefix(&prefix) {diff --git a/crates/liteparse/src/markdown_layout/mod.rs b/crates/liteparse/src/markdown_layout/mod.rsindex ec5bf09..97c1ef4 100644--- a/crates/liteparse/src/markdown_layout/mod.rs+++ b/crates/liteparse/src/markdown_layout/mod.rs@@ -9,6 +9,7 @@mod blocks;mod classify;mod cross_region;+mod flags;mod headings;mod hr;mod inline;diff --git a/crates/liteparse/src/markdown_layout/paragraphs.rs b/crates/liteparse/src/markdown_layout/paragraphs.rsindex c3035e8..b749806 100644--- a/crates/liteparse/src/markdown_layout/paragraphs.rs+++ b/crates/liteparse/src/markdown_layout/paragraphs.rs@@ -3,23 +3,23 @@ use crate::types::{Anchor, ProjectedLine};use super::inline::{SpanStyle, line_all_bold, line_uniform_style, render_line_inline};/// Multiplier on line height used as the paragraph-break threshold.-pub(super) const PARAGRAPH_GAP_MULTIPLIER: f32 = 1.5;+const PARAGRAPH_GAP_MULTIPLIER: f32 = 1.5;/// Tolerance for treating two font sizes as "the same" when grouping/// paragraph lines, when at least one side falls back to a bbox-height/// estimate. Set above descender-driven jitter (~1pt) but below a/// 12→14pt section-heading step that often follows a smaller caption/// line with the same bold style.-pub(super) const FONT_SIZE_PARAGRAPH_TOLERANCE: f32 = 1.5;+const FONT_SIZE_PARAGRAPH_TOLERANCE: f32 = 1.5;/// Tolerance when both lines report real Tf-set sizes. Tight, so a/// 12pt-bold caption line directly above a 14pt-bold section heading/// doesn't accidentally merge them into a single paragraph that swallows/// the heading.-pub(super) const FONT_SIZE_PARAGRAPH_TOLERANCE_REAL: f32 = 0.5;+const FONT_SIZE_PARAGRAPH_TOLERANCE_REAL: f32 = 0.5;/// Pick the right size-equality tolerance depending on whether either/// side has an estimated (jitter-prone) font size.-pub(super) fn font_size_paragraph_tolerance(prev: &ProjectedLine, cur: &ProjectedLine) -> f32 {+fn font_size_paragraph_tolerance(prev: &ProjectedLine, cur: &ProjectedLine) -> f32 {if prev.font_size_is_estimated || cur.font_size_is_estimated {FONT_SIZE_PARAGRAPH_TOLERANCE} else {@@ -28,7 +28,7 @@ pub(super) fn font_size_paragraph_tolerance(prev: &ProjectedLine, cur: &Projecte}/// Tolerance in points for treating two indent positions as "the same column".-pub(super) const INDENT_TOLERANCE: f32 = 6.0;+const INDENT_TOLERANCE: f32 = 6.0;/// Collapse runs of whitespace into single spaces. The projected text from/// `projection.rs` pads with column-alignment spaces (e.g. `for instance`)diff --git a/crates/liteparse/src/markdown_layout/repetition.rs b/crates/liteparse/src/markdown_layout/repetition.rsindex 22e3080..5e11bdb 100644--- a/crates/liteparse/src/markdown_layout/repetition.rs+++ b/crates/liteparse/src/markdown_layout/repetition.rs@@ -5,10 +5,10 @@ use super::paragraphs::collapse_whitespace;/// Fraction of page height treated as the "top band" for header detection./// Most running headers sit within the top 8–12% of a page; 12% gives some/// slack for two-line headers without sweeping in body text.-pub(super) const HEADER_BAND_FRACTION: f32 = 0.12;+const HEADER_BAND_FRACTION: f32 = 0.12;/// Fraction of page height treated as the "bottom band" for footer detection.-pub(super) const FOOTER_BAND_FRACTION: f32 = 0.12;+const FOOTER_BAND_FRACTION: f32 = 0.12;/// Fraction of pages on which a normalized line must appear (in the same/// band) to be classified as a running header/footer.@@ -116,7 +116,7 @@ const SP_ISOLATION_GAP_RATIO: f32 = 1.0;////// Returns true only for unambiguous chrome patterns — false-positive/// recall on body prose is the main risk to guard.-pub(super) fn matches_chrome_pattern(text: &str) -> bool {+fn matches_chrome_pattern(text: &str) -> bool {let t = text.trim();if t.is_empty() {return false;@@ -267,9 +267,6 @@ pub fn detect_single_page_chrome(}let top_cutoff = h * SP_TOP_BAND_FRACTION;let bottom_cutoff = h * (1.0 - SP_BOTTOM_BAND_FRACTION);- // Use body size as the gap reference when known; otherwise fall back to- // the line's own height.- let body_gap_ref = if body_size > 0.0 { body_size } else { 0.0 };// Cache line top/bottom so isolation checks are straightforward.let tops: Vec<f32> = page.projected_lines.iter().map(|l| l.bbox.y).collect();@@ -292,9 +289,10 @@ pub fn detect_single_page_chrome(// Isolation: gap to the nearest non-band neighbor must be ≥ the// configured ratio of body line height. For a top header we look- // *down*; for a bottom footer we look *up*.- let gap_ref = if body_gap_ref > 0.0 {- body_gap_ref+ // *down*; for a bottom footer we look *up*. Use body size as the gap+ // reference when known; otherwise fall back to the line's own height.+ let gap_ref = if body_size > 0.0 {+ body_size} else {line.bbox.height.max(1.0)};@@ -312,7 +310,7 @@ pub fn detect_single_page_chrome(*j != idx && tops[*j] > bots[idx] && !matches_chrome_pattern(l.text.trim())}).map(|(j, _)| tops[j] - bots[idx])- .min_by(|a, b| a.partial_cmp(b).unwrap())+ .min_by(|a, b| a.total_cmp(b)).map(|gap| gap >= required_gap).unwrap_or(true)} else {@@ -323,7 +321,7 @@ pub fn detect_single_page_chrome(*j != idx && bots[*j] < tops[idx] && !matches_chrome_pattern(l.text.trim())}).map(|(j, _)| tops[idx] - bots[j])- .min_by(|a, b| a.partial_cmp(b).unwrap())+ .min_by(|a, b| a.total_cmp(b)).map(|gap| gap >= required_gap).unwrap_or(true)};diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rsindex 025e495..784a343 100644--- a/crates/liteparse/src/markdown_layout/tables.rs+++ b/crates/liteparse/src/markdown_layout/tables.rs@@ -1,8 +1,8 @@use crate::types::{GraphicPrimitive, ProjectedLine, Rect, TextItem};use super::blocks::Block;-use super::inline::is_bold_span;use super::paragraphs::collapse_whitespace;+use crate::projection::is_bold_item;/// Minimum cells per row for a region to qualify as a table.pub(super) const TABLE_MIN_COLUMNS: usize = 3;@@ -112,7 +112,7 @@ pub(super) fn split_cells(line: &ProjectedLine) -> Vec<TableCell> {current_text.push_str(&span.text);let n = span.text.chars().count();current_total_chars += n;- if is_bold_span(span) {+ if is_bold_item(span) {current_bold_chars += n;}prev_right = span.x + span.width.max(0.0);@@ -137,10 +137,7 @@ pub(super) fn split_cells(line: &ProjectedLine) -> Vec<TableCell> {////// Returns the patched cells if every short cell could be cleanly split to/// recover `tracks.len()` cells total; otherwise `None`.-pub(super) fn recover_merged_cell(- mut cells: Vec<TableCell>,- tracks: &[f32],-) -> Option<Vec<TableCell>> {+fn recover_merged_cell(mut cells: Vec<TableCell>, tracks: &[f32]) -> Option<Vec<TableCell>> {let target = tracks.len();if cells.len() >= target {return None;@@ -450,13 +447,6 @@ fn cells_from_raw_items_with_tracks(if spans.len() < 2 {return None;}- // Reject rows that contain a span with implausibly narrow reported- // width-per-character (a known PDFium quirk on some encodings where the- // bbox shrinks but the text payload is full-length). Without this, the- // narrow x-range tricks track inference into treating the next visible- // span as a "second column", manufacturing a fake table from prose.- // Real text averages 4-10pt/char at common font sizes; 2pt/char is a- // generous lower bound that only flags genuinely degenerate widths.let tol = TABLE_TRACK_TOLERANCE_PT;let mut cells: Vec<TableCell> = tracks.iter()@@ -504,13 +494,13 @@ fn cells_from_raw_items_with_tracks(let idx = covered[0];push_text(&mut cells[idx].text, &span.text);cells[idx].end_x = cells[idx].end_x.max(x1);- if is_bold_span(span) {+ if is_bold_item(span) {cells[idx].bold = true;}}_ => {let pieces = split_span_at_anchors(span, &covered, tracks)?;- let bold = is_bold_span(span);+ let bold = is_bold_item(span);for (idx, piece) in covered.iter().zip(pieces.iter()) {if piece.is_empty() {return None;@@ -629,7 +619,7 @@ fn try_detect_table_inferred(start_idx: usize,floor: usize,) -> Option<TableRun> {- let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();+ let dbgt = *super::flags::DEBUG_TABLE;let seed_txt: String = lines[start_idx].spans.iter()@@ -1013,7 +1003,7 @@ fn try_detect_table(lines: &[ProjectedLine], start_idx: usize, floor: usize) ->return None;}- if std::env::var("LITEPARSE_DEBUG_TABLE").is_ok() {+ if *super::flags::DEBUG_TABLE {eprintln!("[tbl-detect @{start_idx}..{end}] cols={column_count} header={header:?} rows={}",body_rows.len()@@ -1041,7 +1031,7 @@ fn absorb_header_lines(column_count: usize,floor: usize,) -> Option<(usize, Vec<String>)> {- let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();+ let dbgt = *super::flags::DEBUG_TABLE;let mut absorbed: Vec<Vec<TableCell>> = Vec::new();let mut j = start_idx;while j > floor {@@ -1570,10 +1560,10 @@ const TABLE_CLUSTER_MAX_FAILED_ROW_FRAC: f32 = 0.3;const TABLE_CLUSTER_MAX_HEADER_LINES: usize = 4;fn merge_fragmented_table_clusters(runs: Vec<TableRun>, lines: &[ProjectedLine]) -> Vec<TableRun> {- if runs.len() < 2 || std::env::var("LITEPARSE_DISABLE_CLUSTER_MERGE").is_ok() {+ if runs.len() < 2 || *super::flags::DISABLE_CLUSTER_MERGE {return runs;}- let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();+ let dbgt = *super::flags::DEBUG_TABLE;let mut out: Vec<TableRun> = Vec::with_capacity(runs.len());let mut i = 0;while i < runs.len() {@@ -1644,7 +1634,7 @@ fn cluster_adjacent(a: &TableRun, b: &TableRun, lines: &[ProjectedLine]) -> bool// a 4-col transaction table) do not. Without this gate the union merge// fuses them and shreds both. Require ≥75% of the narrower run's tracks// to align to the wider run's tracks.- let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();+ let dbgt = *super::flags::DEBUG_TABLE;let (Some(a_tracks), Some(b_tracks)) = (run_body_tracks(a, lines), run_body_tracks(b, lines))else {// Inferred-path runs often have no line whose split_cells count@@ -1877,7 +1867,7 @@ fn build_union_table(lines: &[ProjectedLine],floor: usize,) -> Option<TableRun> {- let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();+ let dbgt = *super::flags::DEBUG_TABLE;let window_start = cluster.first()?.body_start;let window_end = cluster.last()?.end.min(lines.len());if window_start >= window_end {@@ -1951,8 +1941,7 @@ fn build_union_table(// boundaries, or label-column row anchoring).let mut rows: Vec<Vec<String>> = Vec::new();let mut failed_count = 0usize;- for idx in window_start..window_end {- let line = &lines[idx];+ for line in &lines[window_start..window_end] {if let Some(cells) = cells_from_raw_items_with_tracks(line, &tracks) {if cells.iter().any(|c| !c.text.is_empty()) {rows.push(cells.into_iter().map(|c| c.text).collect());@@ -2559,7 +2548,7 @@ fn build_ruled_table(page_height: f32,global: bool,) -> Option<(TableRun, Vec<usize>)> {- let dbg = std::env::var("LITEPARSE_DEBUG_RULED").is_ok();+ let dbg = *super::flags::DEBUG_RULED;let mut xs: Vec<f32> = v_indices.iter().map(|&i| vs[i].x).collect();xs.sort_by(|a, b| a.total_cmp(b));// Coarser, mean-centered clustering for column boundaries: cell-border@@ -2671,11 +2660,11 @@ fn build_ruled_table(let mut span_total = 0usize;let mut span_straddle = 0usize;- let mut push_cell = |cells: &mut Vec<Vec<String>>,- cell_has_text: &mut Vec<Vec<bool>>,- row: usize,- col: usize,- txt: &str| {+ let push_cell = |cells: &mut Vec<Vec<String>>,+ cell_has_text: &mut Vec<Vec<bool>>,+ row: usize,+ col: usize,+ txt: &str| {let txt = txt.trim();if txt.is_empty() {return;@@ -2836,10 +2825,7 @@ fn build_ruled_table(if dbg {eprintln!("[ruled] straddle {span_straddle}/{span_total} = {straddle_frac:.2}");}- if span_total >= 6- && straddle_frac > 0.45- && std::env::var("LITEPARSE_DISABLE_STRADDLE_GUARD").is_err()- {+ if span_total >= 6 && straddle_frac > 0.45 && !*super::flags::DISABLE_STRADDLE_GUARD {if dbg {eprintln!("[ruled] REJECT straddle-frac {straddle_frac:.2}");}@@ -3359,9 +3345,16 @@ fn detect_ruled_tables_impl(let components = find_grid_components(&hs, &vs);let mut out = Vec::new();for (h_idx, v_idx) in components {- if let Some(run) =- build_ruled_table(&hs, &vs, &h_idx, &v_idx, lines, page_width, page_height, global)- {+ if let Some(run) = build_ruled_table(+ &hs,+ &vs,+ &h_idx,+ &v_idx,+ lines,+ page_width,+ page_height,+ global,+ ) {out.push(run);}}diff --git a/crates/liteparse/src/markdown_layout/test_helpers.rs b/crates/liteparse/src/markdown_layout/test_helpers.rsindex 7cc10f9..d733fc9 100644--- a/crates/liteparse/src/markdown_layout/test_helpers.rs+++ b/crates/liteparse/src/markdown_layout/test_helpers.rs@@ -167,14 +167,11 @@ pub(crate) fn page_with_graphics(pub(crate) fn header_footer_page(n: usize, header: &str, footer: &str, body: &str) -> ParsedPage {// Page height 100 → header band ≤12pt, footer band ≥88pt.- let mut lines = vec![+ let lines = vec![line(header, 50.0, 5.0, 8.0, 8.0),line(body, 50.0, 50.0, 10.0, 10.0),line(footer, 50.0, 92.0, 6.0, 6.0),];- for l in &mut lines {- l.region_path = Vec::new();- }ParsedPage {page_number: n,page_width: 612.0,