A fast, helpful, and open-source document parser
fix: add proper RTL and LTR text detection
Logan Markewich3ffc614ed4cb
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex e67d39c..c6cdf16 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -1,3 +1,4 @@+use crate::bidi::is_rtl_char;use crate::error::LiteParseError;use crate::glyph_names::resolve_glyph_name;use crate::types::{@@ -1341,22 +1342,26 @@ fn extract_page_text_items(let y_overlap = vp_loose.top < seg.vp_bottom + y_tolerance&& vp_loose.bottom > seg.vp_top - y_tolerance;- let gap = vp_strict.left - seg.last_char_right;+ // Gaps are measured along the segment's writing direction, so a+ // right-to-left run reads exactly like a left-to-right one: positive+ // means "further along the line", negative means "doubled back".+ let gap = seg.gap_to(&vp_strict, c);// Detect line change using complementary checks:// 1. Strict vertical separation: char's strict top is well below last char's strict bottom- // 2. Line wrap: char goes back leftward AND strict top is below last char's strict bottom- // (even slightly), indicating text wrapped to a new line within the same text object- // 3. Very large leftward jump: if the char jumps back by more than the current+ // 2. Line wrap: char goes back against the writing direction AND strict top is below+ // last char's strict bottom (even slightly), indicating text wrapped to a new line+ // within the same text object+ // 3. Very large backward jump: if the char jumps back by more than the current// segment width, it's definitely a new line (handles OCR text with tall bounding// boxes that overlap vertically between lines)let strict_below = vp_strict.top > seg.last_char_bottom;- let large_leftward_jump = gap < -5.0;+ let large_backward_jump = gap < -5.0;let seg_width = seg.vp_right - seg.vp_left;- let very_large_leftward_jump = seg_width > 20.0 && gap < -(seg_width * 0.5);+ let very_large_backward_jump = seg_width > 20.0 && gap < -(seg_width * 0.5);let line_changed = vp_strict.top > seg.last_char_bottom + y_tolerance- || (strict_below && large_leftward_jump)- || very_large_leftward_jump;+ || (strict_below && large_backward_jump)+ || very_large_backward_jump;// Dot leader detection: break at the boundary between dots and non-dots.// This prevents items like "Total . . . . 330,100" from merging.@@ -1384,7 +1389,7 @@ fn extract_page_text_items(};let split = gap >= MAX_INLINE_GAP|| (seg.pending_space && gap > seg.avg_char_width() * 2.2);- let loose_gap = vp_strict.left - seg.last_char_loose_right;+ let loose_gap = seg.loose_gap_to(&vp_strict, c);let em_vp = (vp_loose.bottom - vp_loose.top).abs();let space_w = ch.font_space_width().map(|w| w * em_vp).unwrap_or(-1.0);eprintln!(@@ -1429,7 +1434,7 @@ fn extract_page_text_items(.is_some_and(|p| p.is_ascii_alphanumeric());if prev_alnum && c.is_ascii_alphanumeric() {let em_vp = (vp_loose.bottom - vp_loose.top).abs();- let loose_gap = vp_strict.left - seg.last_char_loose_right;+ let loose_gap = seg.loose_gap_to(&vp_strict, c);if em_vp > 0.0 && loose_gap > 0.0 {let s = font_space_cal.entry(fk.clone()).or_default();if s.len() < 512 {@@ -1454,7 +1459,7 @@ fn extract_page_text_items(// of the rendered em height as the space estimate.let em_vp = (vp_loose.bottom - vp_loose.top).abs();let space_w = ch.font_space_width().map(|w| w * em_vp).unwrap_or(0.0);- let loose_gap = vp_strict.left - seg.last_char_loose_right;+ let loose_gap = seg.loose_gap_to(&vp_strict, c);let both_alnum = c.is_ascii_alphanumeric()&& seg.text@@ -2102,6 +2107,16 @@ struct SegmentBuilder {last_char_right: f32,// Right edge of last char LOOSE bounds (advance-relative gap calculation)last_char_loose_right: f32,+ // Left edges of the same boxes. Right-to-left runs (Hebrew, Arabic) advance+ // leftward, so their trailing edge is the LEFT one; `gap_to` picks the pair+ // that matches the segment's writing direction.+ last_char_left: f32,+ last_char_loose_left: f32,+ // Writing direction of the segment, latched from its first strong-direction+ // character. Stays `None` through leading neutrals (digits, punctuation) so+ // a segment opening with "(" still picks up RTL from the Hebrew/Arabic+ // letter that follows.+ dir_rtl: Option<bool>,// Bottom of last char strict bounds (for line-change detection)last_char_bottom: f32,// Count of non-space characters (for avg width calculation)@@ -2156,6 +2171,9 @@ impl SegmentBuilder {vp_bottom: f32::MIN,last_char_right: f32::MIN,last_char_loose_right: f32::MIN,+ last_char_left: f32::MAX,+ last_char_loose_left: f32::MAX,+ dir_rtl: None,last_char_bottom: f32::MIN,char_count: 0,unmapped_char_count: 0,@@ -2233,6 +2251,48 @@ impl SegmentBuilder {self.word_has = false;}+ /// Gap from the segment's last character to the incoming one, measured+ /// along the writing direction against the previous char's *trailing* edge+ /// (its right edge for left-to-right text, its left edge for right-to-left).+ /// Positive means the incoming char sits further along the line, negative+ /// means it doubled back — the same sign convention in both directions, so+ /// every threshold downstream stays direction-agnostic.+ fn gap_to(&self, vp_strict: &RectF, c: char) -> f32 {+ if self.dir_is_rtl(c) {+ self.last_char_left - vp_strict.right+ } else {+ vp_strict.left - self.last_char_right+ }+ }++ /// As [`gap_to`](Self::gap_to), but against the previous char's LOOSE box,+ /// which subtracts out intra-word kerning/overhang. This is the+ /// advance-relative gap the missing-space recovery compares to the font's+ /// space width.+ fn loose_gap_to(&self, vp_strict: &RectF, c: char) -> f32 {+ if self.dir_is_rtl(c) {+ self.last_char_loose_left - vp_strict.right+ } else {+ vp_strict.left - self.last_char_loose_right+ }+ }++ /// Writing direction to use for the pair (segment tail, `c`). Falls back to+ /// the incoming character while the segment has only seen neutrals.+ fn dir_is_rtl(&self, c: char) -> bool {+ self.dir_rtl.unwrap_or_else(|| is_rtl_char(c))+ }++ /// Record `c`'s contribution to the segment's writing direction. Neutral+ /// characters (digits, punctuation, spaces) leave it untouched.+ fn note_direction(&mut self, c: char) {+ if is_rtl_char(c) {+ self.dir_rtl = Some(true);+ } else if c.is_alphabetic() {+ self.dir_rtl = Some(false);+ }+ }+/// Average width of non-space characters in the current segment./// Prefers actual glyph widths (text_width) over bbox width, since bbox/// includes inter-character gaps that inflate the average and cause@@ -2266,7 +2326,11 @@ impl SegmentBuilder {self.vp_bottom = vp_loose.bottom;self.last_char_right = vp_strict.right;self.last_char_loose_right = vp_loose.right;+ self.last_char_left = vp_strict.left;+ self.last_char_loose_left = vp_loose.left;self.last_char_bottom = vp_strict.bottom;+ self.dir_rtl = None;+ self.note_direction(c);self.char_count = 1;self.unmapped_char_count = if counts_as_unmapped(recovered, ch.has_unicode_map_error()) {1@@ -2384,7 +2448,10 @@ impl SegmentBuilder {self.vp_bottom = self.vp_bottom.max(vp_loose.bottom);self.last_char_right = vp_strict.right;self.last_char_loose_right = vp_loose.right;+ self.last_char_left = vp_strict.left;+ self.last_char_loose_left = vp_loose.left;self.last_char_bottom = vp_strict.bottom;+ self.note_direction(c);self.char_count += 1;if self.extract_text_metadata {self.char_codes.push(ch.char_code());diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rsindex 56dd89b..56416da 100644--- a/crates/liteparse/src/lib.rs+++ b/crates/liteparse/src/lib.rs@@ -25,6 +25,7 @@ pub mod types;// ── Internal modules (available for binding crates, hidden from docs) ──#[cfg(not(target_arch = "wasm32"))]mod acroform_repair;+mod bidi;#[cfg(not(target_arch = "wasm32"))]#[doc(hidden)]pub mod conversion;diff --git a/crates/liteparse/src/markdown_layout/cross_region.rs b/crates/liteparse/src/markdown_layout/cross_region.rsindex a0472ce..77246c6 100644--- a/crates/liteparse/src/markdown_layout/cross_region.rs+++ b/crates/liteparse/src/markdown_layout/cross_region.rs@@ -228,14 +228,28 @@ fn cluster_rows<'a>(set: &[(&'a ProjectedLine, bool)], tol: f32) -> Vec<Cluster</// with gap-proportional spacing, styles AND together.fn fuse_cluster(cluster: &Cluster, path: &[u16]) -> ProjectedLine {let mut members: Vec<&ProjectedLine> = cluster.members().copied().collect();- members.sort_by(|a, b| a.bbox.x.total_cmp(&b.bbox.x));+ // Fuse in reading order: a right-to-left row starts at its highest x, so+ // walking left-to-right would emit its cells backwards. `fused.spans` is+ // re-sorted x-ascending at the end either way, keeping cell geometry intact.+ let rtl = crate::bidi::is_rtl_pieces(members.iter().map(|m| m.text.as_str()));+ if rtl {+ members.sort_by(|a, b| b.bbox.x.total_cmp(&a.bbox.x));+ } else {+ members.sort_by(|a, b| a.bbox.x.total_cmp(&b.bbox.x));+ }let first = members[0];let mut fused = first.clone();+ fused.rtl = rtl;fused.region_path = path.to_vec();for m in &members[1..] {- let prev_right = fused.bbox.x + fused.bbox.width;- let gap = (m.bbox.x - prev_right).max(0.0);+ // Gap to the previously-fused run, measured along the reading+ // direction so the space count stays proportional in both.+ let gap = if rtl {+ (fused.bbox.x - (m.bbox.x + m.bbox.width)).max(0.0)+ } else {+ (m.bbox.x - (fused.bbox.x + fused.bbox.width)).max(0.0)+ };let approx_char_w = (fused.dominant_font_size * 0.5).max(2.0);let n_spaces = ((gap / approx_char_w) as usize).clamp(2, 24);fused.text.push_str(&" ".repeat(n_spaces));diff --git a/crates/liteparse/src/markdown_layout/inline.rs b/crates/liteparse/src/markdown_layout/inline.rsindex e0c1e0d..e3751a5 100644--- a/crates/liteparse/src/markdown_layout/inline.rs+++ b/crates/liteparse/src/markdown_layout/inline.rs@@ -105,10 +105,17 @@ pub(super) fn render_line_inline(line: &ProjectedLine) -> String {return collapse_whitespace(&line.text);}- // Sort spans by x so we render in visual reading order regardless of- // extraction order. Stable so equal-x spans keep their original sequence.+ // Sort spans into reading order regardless of extraction order. Stable so+ // equal-x spans keep their original sequence. A right-to-left line reads+ // from the highest x down, matching the join `build_one_line` used for+ // `line.text` — the uniform-style shortcut below falls back on that string,+ // so the two orderings have to agree.let mut spans = spans;- spans.sort_by(|a, b| a.x.total_cmp(&b.x));+ if line.rtl {+ spans.sort_by(|a, b| b.x.total_cmp(&a.x));+ } else {+ spans.sort_by(|a, b| a.x.total_cmp(&b.x));+ }let styles: Vec<SpanStyle> = spans.iter().map(|s| SpanStyle::from_item(s)).collect();let links: Vec<Option<&str>> = spans.iter().map(|s| s.link.as_deref()).collect();diff --git a/crates/liteparse/src/markdown_layout/test_helpers.rs b/crates/liteparse/src/markdown_layout/test_helpers.rsindex e350f25..9f6874b 100644--- a/crates/liteparse/src/markdown_layout/test_helpers.rs+++ b/crates/liteparse/src/markdown_layout/test_helpers.rs@@ -5,6 +5,7 @@ use crate::types::{Anchor, GraphicPrimitive, ParsedPage, ProjectedLine, Rect, Tepub(crate) fn line(text: &str, x: f32, y: f32, h: f32, size: f32) -> ProjectedLine {ProjectedLine {text: text.into(),+ rtl: crate::bidi::is_rtl_text(text),bbox: Rect {x,y,@@ -84,6 +85,7 @@ pub(crate) fn line_with_spans(cells: &[(&str, f32)], y: f32, size: f32) -> Proje.map(|(t, _)| *t).collect::<Vec<_>>().join(" "),+ rtl: false,bbox: Rect {x: min_x,y,@@ -134,6 +136,7 @@ pub(crate) fn styled_line(spans: &[(&str, f32, Option<&str>)], y: f32, size: f32.map(|s| s.x + s.width).fold(f32::NEG_INFINITY, f32::max);ProjectedLine {+ rtl: crate::bidi::is_rtl_text(&joined),text: joined,bbox: Rect {x: min_x,diff --git a/crates/liteparse/src/output/markdown.rs b/crates/liteparse/src/output/markdown.rsindex 8d69805..79899d0 100644--- a/crates/liteparse/src/output/markdown.rs+++ b/crates/liteparse/src/output/markdown.rs@@ -134,6 +134,7 @@ mod tests {fn line(text: &str, x: f32, y: f32, h: f32, size: f32) -> ProjectedLine {ProjectedLine {text: text.into(),+ rtl: crate::bidi::is_rtl_text(text),bbox: Rect {x,y,diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rsindex 6c22e02..2690dca 100644--- a/crates/liteparse/src/projection.rs+++ b/crates/liteparse/src/projection.rs@@ -4737,11 +4737,15 @@ fn build_one_line(figures: &[Rect],) -> ProjectedLine {// Sort by x so concatenation reads left→right even if reading order had- // rotated insertions.+ // rotated insertions. `spans` stays in this x-ascending order — the table+ // cell splitter and the inline emphasis renderer both read geometry off it+ // — while `text` below is joined in *reading* order, which for a+ // right-to-left line runs the other way across the page.let mut sorted: Vec<usize> = idxs.to_vec();sorted.sort_by(|a, b| items[*a].item.x.total_cmp(&items[*b].item.x));- let mut text = String::new();+ // Item texts in x order; joined once the line's base direction is known.+ let mut piece_texts: Vec<&str> = Vec::with_capacity(sorted.len());let mut min_x = f32::INFINITY;let mut min_y = f32::INFINITY;let mut max_x = f32::NEG_INFINITY;@@ -4770,7 +4774,7 @@ fn build_one_line(let mut mcid: Option<i32> = None;let mut spans: Vec<TextItem> = Vec::with_capacity(sorted.len());- for (pos, &i) in sorted.iter().enumerate() {+ for &i in sorted.iter() {let proj = &items[i];let it = &proj.item;// `handle_rotation_reading_order` zeroes `item.rotation` after it@@ -4783,13 +4787,10 @@ fn build_one_line(span.rotation = proj.orig_rotation;spans.push(span);- // Concatenate item text. Use existing num_spaces from projection only as+ // Collect item text. Use existing num_spaces from projection only as// a hint — the markdown emitter re-collapses whitespace, so we just// ensure there's *some* separation between adjacent items.- if pos > 0 && !text.ends_with(' ') {- text.push(' ');- }- text.push_str(&it.text);+ piece_texts.push(it.text.as_str());min_x = min_x.min(it.x);min_y = min_y.min(it.y);@@ -4849,6 +4850,25 @@ fn build_one_line(}}+ // Join the line in reading order. PDFium already hands back each item's+ // characters in logical order, so a right-to-left line only needs its+ // *items* walked right-to-left across the page — otherwise an invoice line+ // like "المجموع الفرعي: 75.00 SAR" comes out as "SAR 75.00 المجموع الفرعي:",+ // detaching every label from its value. Direction is decided from the+ // assembled line, so a neutral-only line (pure digits) stays LTR and every+ // left-to-right document takes exactly the path it took before.+ let rtl = crate::bidi::is_rtl_pieces(piece_texts.iter().copied());+ if rtl {+ piece_texts.reverse();+ }+ let mut text = String::new();+ for piece in &piece_texts {+ if !text.is_empty() && !text.ends_with(' ') {+ text.push(' ');+ }+ text.push_str(piece);+ }+// NOTE on tie-breaks: `max_by_key` over a HashMap returns the *last* max it// iterates, and HashMap iteration order is randomized per process. Ties on// char-weight would therefore pick a different winner every run, making the@@ -4947,6 +4967,7 @@ fn build_one_line(ProjectedLine {text,+ rtl,bbox: bbox.clone(),anchor,// Real column detection is deferred (carry-forward in MARKDOWN_PROGRESS).diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rsindex aacaee4..df40150 100644--- a/crates/liteparse/src/types.rs+++ b/crates/liteparse/src/types.rs@@ -584,6 +584,11 @@ pub struct ProjectedLine {pub all_mono: bool,pub all_strike: bool,pub spans: Vec<TextItem>,+ /// True when the line's base direction is right-to-left (strong RTL+ /// characters outnumber strong LTR ones). `spans` is always in x-ascending+ /// order; consumers that rebuild the line's *text* must walk it in reverse+ /// when this is set, or labels detach from their values.+ pub rtl: bool,/// Path from the page's region-tree root to the leaf containing this line./// Equality means "same leaf"; prefix relationship means "one contains the/// other". Replaces the prior flat `column_id` scheme so nested layouts