Skip to content

A fast, helpful, and open-source document parser

clean up part 1

Logan Markewich2b643b8d96dc
diff --git a/crates/liteparse/src/markdown_layout/blocks.rs b/crates/liteparse/src/markdown_layout/blocks.rs
index f836b26..2e289c7 100644
--- a/crates/liteparse/src/markdown_layout/blocks.rs
+++ b/crates/liteparse/src/markdown_layout/blocks.rs
@@ -49,7 +49,6 @@ pub enum Block {
/// `![](image_{id}.png)`. Suppressed entirely when `ImageMode::Off`.
Figure {
id: String,
- bbox: crate::types::Rect,
},
}
diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rs
index fc97ff4..3977c25 100644
--- a/crates/liteparse/src/markdown_layout/classify.rs
+++ b/crates/liteparse/src/markdown_layout/classify.rs
@@ -74,7 +74,7 @@ pub fn classify_page_with_filters(
image_mode: crate::config::ImageMode,
chrome_indices: &std::collections::HashSet<usize>,
) -> Vec<Block> {
- let debug = std::env::var("LITEPARSE_DEBUG_MD").is_ok();
+ let debug = *super::flags::DEBUG_MD;
// TOC suppression: when ≥3 lines on this page look like TOC entries
// (alpha body + trailing page-number), demote heading promotion so each
@@ -160,7 +160,7 @@ pub fn classify_page_with_filters(
let mut global_ruled_tables: Vec<(f32, Block)> = Vec::new();
let mut global_ruled_consumed: std::collections::HashSet<usize> =
std::collections::HashSet::new();
- if std::env::var("LITEPARSE_DISABLE_GLOBAL_RULED").is_err() {
+ if !*super::flags::DISABLE_GLOBAL_RULED {
for (run, consumed) in super::tables::detect_ruled_tables_global(
lines,
&page.graphics,
@@ -324,10 +324,7 @@ pub fn classify_page_with_filters(
let push_interruption = |blocks: &mut Vec<Block>, kind: Interruption| {
blocks.push(match kind {
Interruption::Hr => Block::HorizontalRule,
- Interruption::Figure(r) => Block::Figure {
- id: r.id,
- bbox: r.bbox,
- },
+ Interruption::Figure(r) => Block::Figure { id: r.id },
Interruption::Table(b) => b,
});
};
@@ -377,7 +374,6 @@ pub fn classify_page_with_filters(
page,
heading_map,
outline,
- image_mode,
toc_page,
debug,
precomputed_tables,
@@ -395,7 +391,7 @@ pub fn classify_page_with_filters(
interrupt_cursor += 1;
}
- return stitch_regions(blocks, &region_boundaries);
+ stitch_regions(blocks, &region_boundaries)
}
/// Classify the lines of a single xy_cut leaf into a sequence of blocks. All
@@ -411,12 +407,10 @@ fn classify_region(
page: &ParsedPage,
heading_map: &[(f32, u8)],
outline: &[OutlineTarget],
- image_mode: crate::config::ImageMode,
toc_page: bool,
debug: bool,
precomputed_tables: Option<Vec<super::tables::TableRun>>,
) -> Vec<Block> {
- let _ = image_mode; // interruption emission already gated upstream
let mut blocks: Vec<Block> = Vec::new();
let mut paragraph: Option<ParaAccum> = None;
let mut code: Option<Vec<String>> = None;
@@ -502,10 +496,7 @@ fn classify_region(
*last_line = None;
blocks.push(match kind {
Interruption::Hr => Block::HorizontalRule,
- Interruption::Figure(r) => Block::Figure {
- id: r.id,
- bbox: r.bbox,
- },
+ Interruption::Figure(r) => Block::Figure { id: r.id },
Interruption::Table(b) => b,
});
}
@@ -629,7 +620,7 @@ fn classify_region(
// A previous line ending in a mid-word hyphen wrap means this
// line is its continuation regardless of capitalization
// ("…SOLAR 10.7 Billion-" / "Parameter Model: We have…").
- let prev_hyphen_wrap = std::env::var("LITEPARSE_DISABLE_HEADING_GUARDS").is_err()
+ let prev_hyphen_wrap = !*super::flags::DISABLE_HEADING_GUARDS
&& prev.is_some_and(|p| {
let t = p.text.trim_end();
t.ends_with('-') && t.chars().rev().nth(1).is_some_and(|c| c.is_alphanumeric())
diff --git a/crates/liteparse/src/markdown_layout/headings.rs b/crates/liteparse/src/markdown_layout/headings.rs
index 266e4b7..7fccdd7 100644
--- a/crates/liteparse/src/markdown_layout/headings.rs
+++ b/crates/liteparse/src/markdown_layout/headings.rs
@@ -383,7 +383,7 @@ pub(super) fn looks_like_bold_heading(
// - ≥3 space-separated words after the break
// - line ends with mid-word "-" (wrap continuation) OR is >50 chars
// - first char after the break is uppercase ASCII letter
- let colon_ok = std::env::var("LITEPARSE_DISABLE_HEADING_GUARDS").is_err();
+ let colon_ok = !*super::flags::DISABLE_HEADING_GUARDS;
let run_in_break = text
.find(". ")
.map(|p| (p, 2))
@@ -404,7 +404,7 @@ pub(super) fn looks_like_bold_heading(
&& starts_upper
&& ((word_count >= 2 && ends_hyphen) || (word_count >= 3 && text.chars().count() > 50))
{
- if std::env::var("LITEPARSE_DEBUG_MD").is_ok() {
+ if *super::flags::DEBUG_MD {
eprintln!(
"[MD bold-heading REJECT run-in] '{}' (pos={} word_count={} ends_hyphen={} len={})",
text.chars().take(80).collect::<String>(),
diff --git a/crates/liteparse/src/markdown_layout/inline.rs b/crates/liteparse/src/markdown_layout/inline.rs
index dc2dd6d..32a9813 100644
--- a/crates/liteparse/src/markdown_layout/inline.rs
+++ b/crates/liteparse/src/markdown_layout/inline.rs
@@ -3,14 +3,8 @@ use crate::types::{ProjectedLine, TextItem};
use super::paragraphs::{collapse_whitespace, dehyphenate_join};
-/// Lightweight bold detection on a raw `TextItem`. Delegates to
-/// `projection::is_bold_item` so per-span and per-line logic stay in sync.
-pub(super) fn is_bold_span(s: &TextItem) -> bool {
- is_bold_item(s)
-}
-
/// Per-span style flags used by the inline-emphasis renderer.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct SpanStyle {
pub(super) bold: bool,
pub(super) italic: bool,
@@ -52,7 +46,7 @@ pub(super) fn escape_inline(s: &str) -> String {
/// Wrap `inner` with the markdown markers for `style`. Mono wins over bold/italic:
/// inline code (`` `…` ``) doesn't compose with emphasis in CommonMark, so when
/// a span is mono we drop the `**/*` wrap. Bold + italic → `***…***`.
-pub(super) fn apply_style(inner: &str, style: SpanStyle) -> String {
+fn apply_style(inner: &str, style: SpanStyle) -> String {
if style.mono {
// Use backticks; if inner already contains backticks, switch to a
// longer fence (pair of backticks plus a space buffer) per CommonMark.
@@ -176,7 +170,7 @@ pub(super) fn render_list_item_text(line: &ProjectedLine, marker: &str, rest: &s
/// Try to strip a leading list marker (optionally wrapped in emphasis markers)
/// off `s`. Recognizes `***MARK*** `, `**MARK** `, `*MARK* `, `` `MARK` ``,
/// and bare `MARK `. Returns the suffix on a match.
-pub(super) fn strip_leading_marker_from_inline(s: &str, marker: &str) -> Option<String> {
+fn strip_leading_marker_from_inline(s: &str, marker: &str) -> Option<String> {
for wrap in ["***", "**", "*", "`"] {
let prefix = format!("{wrap}{marker}{wrap} ");
if let Some(rest) = s.strip_prefix(&prefix) {
diff --git a/crates/liteparse/src/markdown_layout/mod.rs b/crates/liteparse/src/markdown_layout/mod.rs
index ec5bf09..97c1ef4 100644
--- a/crates/liteparse/src/markdown_layout/mod.rs
+++ b/crates/liteparse/src/markdown_layout/mod.rs
@@ -9,6 +9,7 @@
mod blocks;
mod classify;
mod cross_region;
+mod flags;
mod headings;
mod hr;
mod inline;
diff --git a/crates/liteparse/src/markdown_layout/paragraphs.rs b/crates/liteparse/src/markdown_layout/paragraphs.rs
index c3035e8..b749806 100644
--- a/crates/liteparse/src/markdown_layout/paragraphs.rs
+++ b/crates/liteparse/src/markdown_layout/paragraphs.rs
@@ -3,23 +3,23 @@ use crate::types::{Anchor, ProjectedLine};
use super::inline::{SpanStyle, line_all_bold, line_uniform_style, render_line_inline};
/// Multiplier on line height used as the paragraph-break threshold.
-pub(super) const PARAGRAPH_GAP_MULTIPLIER: f32 = 1.5;
+const PARAGRAPH_GAP_MULTIPLIER: f32 = 1.5;
/// Tolerance for treating two font sizes as "the same" when grouping
/// paragraph lines, when at least one side falls back to a bbox-height
/// estimate. Set above descender-driven jitter (~1pt) but below a
/// 12→14pt section-heading step that often follows a smaller caption
/// line with the same bold style.
-pub(super) const FONT_SIZE_PARAGRAPH_TOLERANCE: f32 = 1.5;
+const FONT_SIZE_PARAGRAPH_TOLERANCE: f32 = 1.5;
/// Tolerance when both lines report real Tf-set sizes. Tight, so a
/// 12pt-bold caption line directly above a 14pt-bold section heading
/// doesn't accidentally merge them into a single paragraph that swallows
/// the heading.
-pub(super) const FONT_SIZE_PARAGRAPH_TOLERANCE_REAL: f32 = 0.5;
+const FONT_SIZE_PARAGRAPH_TOLERANCE_REAL: f32 = 0.5;
/// Pick the right size-equality tolerance depending on whether either
/// side has an estimated (jitter-prone) font size.
-pub(super) fn font_size_paragraph_tolerance(prev: &ProjectedLine, cur: &ProjectedLine) -> f32 {
+fn font_size_paragraph_tolerance(prev: &ProjectedLine, cur: &ProjectedLine) -> f32 {
if prev.font_size_is_estimated || cur.font_size_is_estimated {
FONT_SIZE_PARAGRAPH_TOLERANCE
} else {
@@ -28,7 +28,7 @@ pub(super) fn font_size_paragraph_tolerance(prev: &ProjectedLine, cur: &Projecte
}
/// Tolerance in points for treating two indent positions as "the same column".
-pub(super) const INDENT_TOLERANCE: f32 = 6.0;
+const INDENT_TOLERANCE: f32 = 6.0;
/// Collapse runs of whitespace into single spaces. The projected text from
/// `projection.rs` pads with column-alignment spaces (e.g. `for instance`)
diff --git a/crates/liteparse/src/markdown_layout/repetition.rs b/crates/liteparse/src/markdown_layout/repetition.rs
index 22e3080..5e11bdb 100644
--- a/crates/liteparse/src/markdown_layout/repetition.rs
+++ b/crates/liteparse/src/markdown_layout/repetition.rs
@@ -5,10 +5,10 @@ use super::paragraphs::collapse_whitespace;
/// Fraction of page height treated as the "top band" for header detection.
/// Most running headers sit within the top 8–12% of a page; 12% gives some
/// slack for two-line headers without sweeping in body text.
-pub(super) const HEADER_BAND_FRACTION: f32 = 0.12;
+const HEADER_BAND_FRACTION: f32 = 0.12;
/// Fraction of page height treated as the "bottom band" for footer detection.
-pub(super) const FOOTER_BAND_FRACTION: f32 = 0.12;
+const FOOTER_BAND_FRACTION: f32 = 0.12;
/// Fraction of pages on which a normalized line must appear (in the same
/// band) to be classified as a running header/footer.
@@ -116,7 +116,7 @@ const SP_ISOLATION_GAP_RATIO: f32 = 1.0;
///
/// Returns true only for unambiguous chrome patterns — false-positive
/// recall on body prose is the main risk to guard.
-pub(super) fn matches_chrome_pattern(text: &str) -> bool {
+fn matches_chrome_pattern(text: &str) -> bool {
let t = text.trim();
if t.is_empty() {
return false;
@@ -267,9 +267,6 @@ pub fn detect_single_page_chrome(
}
let top_cutoff = h * SP_TOP_BAND_FRACTION;
let bottom_cutoff = h * (1.0 - SP_BOTTOM_BAND_FRACTION);
- // Use body size as the gap reference when known; otherwise fall back to
- // the line's own height.
- let body_gap_ref = if body_size > 0.0 { body_size } else { 0.0 };
// Cache line top/bottom so isolation checks are straightforward.
let tops: Vec<f32> = page.projected_lines.iter().map(|l| l.bbox.y).collect();
@@ -292,9 +289,10 @@ pub fn detect_single_page_chrome(
// Isolation: gap to the nearest non-band neighbor must be ≥ the
// configured ratio of body line height. For a top header we look
- // *down*; for a bottom footer we look *up*.
- let gap_ref = if body_gap_ref > 0.0 {
- body_gap_ref
+ // *down*; for a bottom footer we look *up*. Use body size as the gap
+ // reference when known; otherwise fall back to the line's own height.
+ let gap_ref = if body_size > 0.0 {
+ body_size
} else {
line.bbox.height.max(1.0)
};
@@ -312,7 +310,7 @@ pub fn detect_single_page_chrome(
*j != idx && tops[*j] > bots[idx] && !matches_chrome_pattern(l.text.trim())
})
.map(|(j, _)| tops[j] - bots[idx])
- .min_by(|a, b| a.partial_cmp(b).unwrap())
+ .min_by(|a, b| a.total_cmp(b))
.map(|gap| gap >= required_gap)
.unwrap_or(true)
} else {
@@ -323,7 +321,7 @@ pub fn detect_single_page_chrome(
*j != idx && bots[*j] < tops[idx] && !matches_chrome_pattern(l.text.trim())
})
.map(|(j, _)| tops[idx] - bots[j])
- .min_by(|a, b| a.partial_cmp(b).unwrap())
+ .min_by(|a, b| a.total_cmp(b))
.map(|gap| gap >= required_gap)
.unwrap_or(true)
};
diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rs
index 025e495..784a343 100644
--- a/crates/liteparse/src/markdown_layout/tables.rs
+++ b/crates/liteparse/src/markdown_layout/tables.rs
@@ -1,8 +1,8 @@
use crate::types::{GraphicPrimitive, ProjectedLine, Rect, TextItem};
use super::blocks::Block;
-use super::inline::is_bold_span;
use super::paragraphs::collapse_whitespace;
+use crate::projection::is_bold_item;
/// Minimum cells per row for a region to qualify as a table.
pub(super) const TABLE_MIN_COLUMNS: usize = 3;
@@ -112,7 +112,7 @@ pub(super) fn split_cells(line: &ProjectedLine) -> Vec<TableCell> {
current_text.push_str(&span.text);
let n = span.text.chars().count();
current_total_chars += n;
- if is_bold_span(span) {
+ if is_bold_item(span) {
current_bold_chars += n;
}
prev_right = span.x + span.width.max(0.0);
@@ -137,10 +137,7 @@ pub(super) fn split_cells(line: &ProjectedLine) -> Vec<TableCell> {
///
/// Returns the patched cells if every short cell could be cleanly split to
/// recover `tracks.len()` cells total; otherwise `None`.
-pub(super) fn recover_merged_cell(
- mut cells: Vec<TableCell>,
- tracks: &[f32],
-) -> Option<Vec<TableCell>> {
+fn recover_merged_cell(mut cells: Vec<TableCell>, tracks: &[f32]) -> Option<Vec<TableCell>> {
let target = tracks.len();
if cells.len() >= target {
return None;
@@ -450,13 +447,6 @@ fn cells_from_raw_items_with_tracks(
if spans.len() < 2 {
return None;
}
- // Reject rows that contain a span with implausibly narrow reported
- // width-per-character (a known PDFium quirk on some encodings where the
- // bbox shrinks but the text payload is full-length). Without this, the
- // narrow x-range tricks track inference into treating the next visible
- // span as a "second column", manufacturing a fake table from prose.
- // Real text averages 4-10pt/char at common font sizes; 2pt/char is a
- // generous lower bound that only flags genuinely degenerate widths.
let tol = TABLE_TRACK_TOLERANCE_PT;
let mut cells: Vec<TableCell> = tracks
.iter()
@@ -504,13 +494,13 @@ fn cells_from_raw_items_with_tracks(
let idx = covered[0];
push_text(&mut cells[idx].text, &span.text);
cells[idx].end_x = cells[idx].end_x.max(x1);
- if is_bold_span(span) {
+ if is_bold_item(span) {
cells[idx].bold = true;
}
}
_ => {
let pieces = split_span_at_anchors(span, &covered, tracks)?;
- let bold = is_bold_span(span);
+ let bold = is_bold_item(span);
for (idx, piece) in covered.iter().zip(pieces.iter()) {
if piece.is_empty() {
return None;
@@ -629,7 +619,7 @@ fn try_detect_table_inferred(
start_idx: usize,
floor: usize,
) -> Option<TableRun> {
- let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();
+ let dbgt = *super::flags::DEBUG_TABLE;
let seed_txt: String = lines[start_idx]
.spans
.iter()
@@ -1013,7 +1003,7 @@ fn try_detect_table(lines: &[ProjectedLine], start_idx: usize, floor: usize) ->
return None;
}
- if std::env::var("LITEPARSE_DEBUG_TABLE").is_ok() {
+ if *super::flags::DEBUG_TABLE {
eprintln!(
"[tbl-detect @{start_idx}..{end}] cols={column_count} header={header:?} rows={}",
body_rows.len()
@@ -1041,7 +1031,7 @@ fn absorb_header_lines(
column_count: usize,
floor: usize,
) -> Option<(usize, Vec<String>)> {
- let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();
+ let dbgt = *super::flags::DEBUG_TABLE;
let mut absorbed: Vec<Vec<TableCell>> = Vec::new();
let mut j = start_idx;
while j > floor {
@@ -1570,10 +1560,10 @@ const TABLE_CLUSTER_MAX_FAILED_ROW_FRAC: f32 = 0.3;
const TABLE_CLUSTER_MAX_HEADER_LINES: usize = 4;
fn merge_fragmented_table_clusters(runs: Vec<TableRun>, lines: &[ProjectedLine]) -> Vec<TableRun> {
- if runs.len() < 2 || std::env::var("LITEPARSE_DISABLE_CLUSTER_MERGE").is_ok() {
+ if runs.len() < 2 || *super::flags::DISABLE_CLUSTER_MERGE {
return runs;
}
- let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();
+ let dbgt = *super::flags::DEBUG_TABLE;
let mut out: Vec<TableRun> = Vec::with_capacity(runs.len());
let mut i = 0;
while i < runs.len() {
@@ -1644,7 +1634,7 @@ fn cluster_adjacent(a: &TableRun, b: &TableRun, lines: &[ProjectedLine]) -> bool
// a 4-col transaction table) do not. Without this gate the union merge
// fuses them and shreds both. Require ≥75% of the narrower run's tracks
// to align to the wider run's tracks.
- let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();
+ let dbgt = *super::flags::DEBUG_TABLE;
let (Some(a_tracks), Some(b_tracks)) = (run_body_tracks(a, lines), run_body_tracks(b, lines))
else {
// Inferred-path runs often have no line whose split_cells count
@@ -1877,7 +1867,7 @@ fn build_union_table(
lines: &[ProjectedLine],
floor: usize,
) -> Option<TableRun> {
- let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();
+ let dbgt = *super::flags::DEBUG_TABLE;
let window_start = cluster.first()?.body_start;
let window_end = cluster.last()?.end.min(lines.len());
if window_start >= window_end {
@@ -1951,8 +1941,7 @@ fn build_union_table(
// boundaries, or label-column row anchoring).
let mut rows: Vec<Vec<String>> = Vec::new();
let mut failed_count = 0usize;
- for idx in window_start..window_end {
- let line = &lines[idx];
+ for line in &lines[window_start..window_end] {
if let Some(cells) = cells_from_raw_items_with_tracks(line, &tracks) {
if cells.iter().any(|c| !c.text.is_empty()) {
rows.push(cells.into_iter().map(|c| c.text).collect());
@@ -2559,7 +2548,7 @@ fn build_ruled_table(
page_height: f32,
global: bool,
) -> Option<(TableRun, Vec<usize>)> {
- let dbg = std::env::var("LITEPARSE_DEBUG_RULED").is_ok();
+ let dbg = *super::flags::DEBUG_RULED;
let mut xs: Vec<f32> = v_indices.iter().map(|&i| vs[i].x).collect();
xs.sort_by(|a, b| a.total_cmp(b));
// Coarser, mean-centered clustering for column boundaries: cell-border
@@ -2671,11 +2660,11 @@ fn build_ruled_table(
let mut span_total = 0usize;
let mut span_straddle = 0usize;
- let mut push_cell = |cells: &mut Vec<Vec<String>>,
- cell_has_text: &mut Vec<Vec<bool>>,
- row: usize,
- col: usize,
- txt: &str| {
+ let push_cell = |cells: &mut Vec<Vec<String>>,
+ cell_has_text: &mut Vec<Vec<bool>>,
+ row: usize,
+ col: usize,
+ txt: &str| {
let txt = txt.trim();
if txt.is_empty() {
return;
@@ -2836,10 +2825,7 @@ fn build_ruled_table(
if dbg {
eprintln!("[ruled] straddle {span_straddle}/{span_total} = {straddle_frac:.2}");
}
- if span_total >= 6
- && straddle_frac > 0.45
- && std::env::var("LITEPARSE_DISABLE_STRADDLE_GUARD").is_err()
- {
+ if span_total >= 6 && straddle_frac > 0.45 && !*super::flags::DISABLE_STRADDLE_GUARD {
if dbg {
eprintln!("[ruled] REJECT straddle-frac {straddle_frac:.2}");
}
@@ -3359,9 +3345,16 @@ fn detect_ruled_tables_impl(
let components = find_grid_components(&hs, &vs);
let mut out = Vec::new();
for (h_idx, v_idx) in components {
- if let Some(run) =
- build_ruled_table(&hs, &vs, &h_idx, &v_idx, lines, page_width, page_height, global)
- {
+ if let Some(run) = build_ruled_table(
+ &hs,
+ &vs,
+ &h_idx,
+ &v_idx,
+ lines,
+ page_width,
+ page_height,
+ global,
+ ) {
out.push(run);
}
}
diff --git a/crates/liteparse/src/markdown_layout/test_helpers.rs b/crates/liteparse/src/markdown_layout/test_helpers.rs
index 7cc10f9..d733fc9 100644
--- a/crates/liteparse/src/markdown_layout/test_helpers.rs
+++ b/crates/liteparse/src/markdown_layout/test_helpers.rs
@@ -167,14 +167,11 @@ pub(crate) fn page_with_graphics(
pub(crate) fn header_footer_page(n: usize, header: &str, footer: &str, body: &str) -> ParsedPage {
// Page height 100 → header band ≤12pt, footer band ≥88pt.
- let mut lines = vec![
+ let lines = vec![
line(header, 50.0, 5.0, 8.0, 8.0),
line(body, 50.0, 50.0, 10.0, 10.0),
line(footer, 50.0, 92.0, 6.0, 6.0),
];
- for l in &mut lines {
- l.region_path = Vec::new();
- }
ParsedPage {
page_number: n,
page_width: 612.0,