Skip to content

A fast, helpful, and open-source document parser

fix: add proper RTL and LTR text detection

Logan Markewich3ffc614ed4cb
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index e67d39c..c6cdf16 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -1,3 +1,4 @@
+use crate::bidi::is_rtl_char;
use crate::error::LiteParseError;
use crate::glyph_names::resolve_glyph_name;
use crate::types::{
@@ -1341,22 +1342,26 @@ fn extract_page_text_items(
let y_overlap = vp_loose.top < seg.vp_bottom + y_tolerance
&& vp_loose.bottom > seg.vp_top - y_tolerance;
- let gap = vp_strict.left - seg.last_char_right;
+ // Gaps are measured along the segment's writing direction, so a
+ // right-to-left run reads exactly like a left-to-right one: positive
+ // means "further along the line", negative means "doubled back".
+ let gap = seg.gap_to(&vp_strict, c);
// Detect line change using complementary checks:
// 1. Strict vertical separation: char's strict top is well below last char's strict bottom
- // 2. Line wrap: char goes back leftward AND strict top is below last char's strict bottom
- // (even slightly), indicating text wrapped to a new line within the same text object
- // 3. Very large leftward jump: if the char jumps back by more than the current
+ // 2. Line wrap: char goes back against the writing direction AND strict top is below
+ // last char's strict bottom (even slightly), indicating text wrapped to a new line
+ // within the same text object
+ // 3. Very large backward jump: if the char jumps back by more than the current
// segment width, it's definitely a new line (handles OCR text with tall bounding
// boxes that overlap vertically between lines)
let strict_below = vp_strict.top > seg.last_char_bottom;
- let large_leftward_jump = gap < -5.0;
+ let large_backward_jump = gap < -5.0;
let seg_width = seg.vp_right - seg.vp_left;
- let very_large_leftward_jump = seg_width > 20.0 && gap < -(seg_width * 0.5);
+ let very_large_backward_jump = seg_width > 20.0 && gap < -(seg_width * 0.5);
let line_changed = vp_strict.top > seg.last_char_bottom + y_tolerance
- || (strict_below && large_leftward_jump)
- || very_large_leftward_jump;
+ || (strict_below && large_backward_jump)
+ || very_large_backward_jump;
// Dot leader detection: break at the boundary between dots and non-dots.
// This prevents items like "Total . . . . 330,100" from merging.
@@ -1384,7 +1389,7 @@ fn extract_page_text_items(
};
let split = gap >= MAX_INLINE_GAP
|| (seg.pending_space && gap > seg.avg_char_width() * 2.2);
- let loose_gap = vp_strict.left - seg.last_char_loose_right;
+ let loose_gap = seg.loose_gap_to(&vp_strict, c);
let em_vp = (vp_loose.bottom - vp_loose.top).abs();
let space_w = ch.font_space_width().map(|w| w * em_vp).unwrap_or(-1.0);
eprintln!(
@@ -1429,7 +1434,7 @@ fn extract_page_text_items(
.is_some_and(|p| p.is_ascii_alphanumeric());
if prev_alnum && c.is_ascii_alphanumeric() {
let em_vp = (vp_loose.bottom - vp_loose.top).abs();
- let loose_gap = vp_strict.left - seg.last_char_loose_right;
+ let loose_gap = seg.loose_gap_to(&vp_strict, c);
if em_vp > 0.0 && loose_gap > 0.0 {
let s = font_space_cal.entry(fk.clone()).or_default();
if s.len() < 512 {
@@ -1454,7 +1459,7 @@ fn extract_page_text_items(
// of the rendered em height as the space estimate.
let em_vp = (vp_loose.bottom - vp_loose.top).abs();
let space_w = ch.font_space_width().map(|w| w * em_vp).unwrap_or(0.0);
- let loose_gap = vp_strict.left - seg.last_char_loose_right;
+ let loose_gap = seg.loose_gap_to(&vp_strict, c);
let both_alnum = c.is_ascii_alphanumeric()
&& seg
.text
@@ -2102,6 +2107,16 @@ struct SegmentBuilder {
last_char_right: f32,
// Right edge of last char LOOSE bounds (advance-relative gap calculation)
last_char_loose_right: f32,
+ // Left edges of the same boxes. Right-to-left runs (Hebrew, Arabic) advance
+ // leftward, so their trailing edge is the LEFT one; `gap_to` picks the pair
+ // that matches the segment's writing direction.
+ last_char_left: f32,
+ last_char_loose_left: f32,
+ // Writing direction of the segment, latched from its first strong-direction
+ // character. Stays `None` through leading neutrals (digits, punctuation) so
+ // a segment opening with "(" still picks up RTL from the Hebrew/Arabic
+ // letter that follows.
+ dir_rtl: Option<bool>,
// Bottom of last char strict bounds (for line-change detection)
last_char_bottom: f32,
// Count of non-space characters (for avg width calculation)
@@ -2156,6 +2171,9 @@ impl SegmentBuilder {
vp_bottom: f32::MIN,
last_char_right: f32::MIN,
last_char_loose_right: f32::MIN,
+ last_char_left: f32::MAX,
+ last_char_loose_left: f32::MAX,
+ dir_rtl: None,
last_char_bottom: f32::MIN,
char_count: 0,
unmapped_char_count: 0,
@@ -2233,6 +2251,48 @@ impl SegmentBuilder {
self.word_has = false;
}
+ /// Gap from the segment's last character to the incoming one, measured
+ /// along the writing direction against the previous char's *trailing* edge
+ /// (its right edge for left-to-right text, its left edge for right-to-left).
+ /// Positive means the incoming char sits further along the line, negative
+ /// means it doubled back — the same sign convention in both directions, so
+ /// every threshold downstream stays direction-agnostic.
+ fn gap_to(&self, vp_strict: &RectF, c: char) -> f32 {
+ if self.dir_is_rtl(c) {
+ self.last_char_left - vp_strict.right
+ } else {
+ vp_strict.left - self.last_char_right
+ }
+ }
+
+ /// As [`gap_to`](Self::gap_to), but against the previous char's LOOSE box,
+ /// which subtracts out intra-word kerning/overhang. This is the
+ /// advance-relative gap the missing-space recovery compares to the font's
+ /// space width.
+ fn loose_gap_to(&self, vp_strict: &RectF, c: char) -> f32 {
+ if self.dir_is_rtl(c) {
+ self.last_char_loose_left - vp_strict.right
+ } else {
+ vp_strict.left - self.last_char_loose_right
+ }
+ }
+
+ /// Writing direction to use for the pair (segment tail, `c`). Falls back to
+ /// the incoming character while the segment has only seen neutrals.
+ fn dir_is_rtl(&self, c: char) -> bool {
+ self.dir_rtl.unwrap_or_else(|| is_rtl_char(c))
+ }
+
+ /// Record `c`'s contribution to the segment's writing direction. Neutral
+ /// characters (digits, punctuation, spaces) leave it untouched.
+ fn note_direction(&mut self, c: char) {
+ if is_rtl_char(c) {
+ self.dir_rtl = Some(true);
+ } else if c.is_alphabetic() {
+ self.dir_rtl = Some(false);
+ }
+ }
+
/// Average width of non-space characters in the current segment.
/// Prefers actual glyph widths (text_width) over bbox width, since bbox
/// includes inter-character gaps that inflate the average and cause
@@ -2266,7 +2326,11 @@ impl SegmentBuilder {
self.vp_bottom = vp_loose.bottom;
self.last_char_right = vp_strict.right;
self.last_char_loose_right = vp_loose.right;
+ self.last_char_left = vp_strict.left;
+ self.last_char_loose_left = vp_loose.left;
self.last_char_bottom = vp_strict.bottom;
+ self.dir_rtl = None;
+ self.note_direction(c);
self.char_count = 1;
self.unmapped_char_count = if counts_as_unmapped(recovered, ch.has_unicode_map_error()) {
1
@@ -2384,7 +2448,10 @@ impl SegmentBuilder {
self.vp_bottom = self.vp_bottom.max(vp_loose.bottom);
self.last_char_right = vp_strict.right;
self.last_char_loose_right = vp_loose.right;
+ self.last_char_left = vp_strict.left;
+ self.last_char_loose_left = vp_loose.left;
self.last_char_bottom = vp_strict.bottom;
+ self.note_direction(c);
self.char_count += 1;
if self.extract_text_metadata {
self.char_codes.push(ch.char_code());
diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rs
index 56dd89b..56416da 100644
--- a/crates/liteparse/src/lib.rs
+++ b/crates/liteparse/src/lib.rs
@@ -25,6 +25,7 @@ pub mod types;
// ── Internal modules (available for binding crates, hidden from docs) ──
#[cfg(not(target_arch = "wasm32"))]
mod acroform_repair;
+mod bidi;
#[cfg(not(target_arch = "wasm32"))]
#[doc(hidden)]
pub mod conversion;
diff --git a/crates/liteparse/src/markdown_layout/cross_region.rs b/crates/liteparse/src/markdown_layout/cross_region.rs
index a0472ce..77246c6 100644
--- a/crates/liteparse/src/markdown_layout/cross_region.rs
+++ b/crates/liteparse/src/markdown_layout/cross_region.rs
@@ -228,14 +228,28 @@ fn cluster_rows<'a>(set: &[(&'a ProjectedLine, bool)], tol: f32) -> Vec<Cluster<
/// with gap-proportional spacing, styles AND together.
fn fuse_cluster(cluster: &Cluster, path: &[u16]) -> ProjectedLine {
let mut members: Vec<&ProjectedLine> = cluster.members().copied().collect();
- members.sort_by(|a, b| a.bbox.x.total_cmp(&b.bbox.x));
+ // Fuse in reading order: a right-to-left row starts at its highest x, so
+ // walking left-to-right would emit its cells backwards. `fused.spans` is
+ // re-sorted x-ascending at the end either way, keeping cell geometry intact.
+ let rtl = crate::bidi::is_rtl_pieces(members.iter().map(|m| m.text.as_str()));
+ if rtl {
+ members.sort_by(|a, b| b.bbox.x.total_cmp(&a.bbox.x));
+ } else {
+ members.sort_by(|a, b| a.bbox.x.total_cmp(&b.bbox.x));
+ }
let first = members[0];
let mut fused = first.clone();
+ fused.rtl = rtl;
fused.region_path = path.to_vec();
for m in &members[1..] {
- let prev_right = fused.bbox.x + fused.bbox.width;
- let gap = (m.bbox.x - prev_right).max(0.0);
+ // Gap to the previously-fused run, measured along the reading
+ // direction so the space count stays proportional in both.
+ let gap = if rtl {
+ (fused.bbox.x - (m.bbox.x + m.bbox.width)).max(0.0)
+ } else {
+ (m.bbox.x - (fused.bbox.x + fused.bbox.width)).max(0.0)
+ };
let approx_char_w = (fused.dominant_font_size * 0.5).max(2.0);
let n_spaces = ((gap / approx_char_w) as usize).clamp(2, 24);
fused.text.push_str(&" ".repeat(n_spaces));
diff --git a/crates/liteparse/src/markdown_layout/inline.rs b/crates/liteparse/src/markdown_layout/inline.rs
index e0c1e0d..e3751a5 100644
--- a/crates/liteparse/src/markdown_layout/inline.rs
+++ b/crates/liteparse/src/markdown_layout/inline.rs
@@ -105,10 +105,17 @@ pub(super) fn render_line_inline(line: &ProjectedLine) -> String {
return collapse_whitespace(&line.text);
}
- // Sort spans by x so we render in visual reading order regardless of
- // extraction order. Stable so equal-x spans keep their original sequence.
+ // Sort spans into reading order regardless of extraction order. Stable so
+ // equal-x spans keep their original sequence. A right-to-left line reads
+ // from the highest x down, matching the join `build_one_line` used for
+ // `line.text` — the uniform-style shortcut below falls back on that string,
+ // so the two orderings have to agree.
let mut spans = spans;
- spans.sort_by(|a, b| a.x.total_cmp(&b.x));
+ if line.rtl {
+ spans.sort_by(|a, b| b.x.total_cmp(&a.x));
+ } else {
+ spans.sort_by(|a, b| a.x.total_cmp(&b.x));
+ }
let styles: Vec<SpanStyle> = spans.iter().map(|s| SpanStyle::from_item(s)).collect();
let links: Vec<Option<&str>> = spans.iter().map(|s| s.link.as_deref()).collect();
diff --git a/crates/liteparse/src/markdown_layout/test_helpers.rs b/crates/liteparse/src/markdown_layout/test_helpers.rs
index e350f25..9f6874b 100644
--- a/crates/liteparse/src/markdown_layout/test_helpers.rs
+++ b/crates/liteparse/src/markdown_layout/test_helpers.rs
@@ -5,6 +5,7 @@ use crate::types::{Anchor, GraphicPrimitive, ParsedPage, ProjectedLine, Rect, Te
pub(crate) fn line(text: &str, x: f32, y: f32, h: f32, size: f32) -> ProjectedLine {
ProjectedLine {
text: text.into(),
+ rtl: crate::bidi::is_rtl_text(text),
bbox: Rect {
x,
y,
@@ -84,6 +85,7 @@ pub(crate) fn line_with_spans(cells: &[(&str, f32)], y: f32, size: f32) -> Proje
.map(|(t, _)| *t)
.collect::<Vec<_>>()
.join(" "),
+ rtl: false,
bbox: Rect {
x: min_x,
y,
@@ -134,6 +136,7 @@ pub(crate) fn styled_line(spans: &[(&str, f32, Option<&str>)], y: f32, size: f32
.map(|s| s.x + s.width)
.fold(f32::NEG_INFINITY, f32::max);
ProjectedLine {
+ rtl: crate::bidi::is_rtl_text(&joined),
text: joined,
bbox: Rect {
x: min_x,
diff --git a/crates/liteparse/src/output/markdown.rs b/crates/liteparse/src/output/markdown.rs
index 8d69805..79899d0 100644
--- a/crates/liteparse/src/output/markdown.rs
+++ b/crates/liteparse/src/output/markdown.rs
@@ -134,6 +134,7 @@ mod tests {
fn line(text: &str, x: f32, y: f32, h: f32, size: f32) -> ProjectedLine {
ProjectedLine {
text: text.into(),
+ rtl: crate::bidi::is_rtl_text(text),
bbox: Rect {
x,
y,
diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rs
index 6c22e02..2690dca 100644
--- a/crates/liteparse/src/projection.rs
+++ b/crates/liteparse/src/projection.rs
@@ -4737,11 +4737,15 @@ fn build_one_line(
figures: &[Rect],
) -> ProjectedLine {
// Sort by x so concatenation reads left→right even if reading order had
- // rotated insertions.
+ // rotated insertions. `spans` stays in this x-ascending order — the table
+ // cell splitter and the inline emphasis renderer both read geometry off it
+ // — while `text` below is joined in *reading* order, which for a
+ // right-to-left line runs the other way across the page.
let mut sorted: Vec<usize> = idxs.to_vec();
sorted.sort_by(|a, b| items[*a].item.x.total_cmp(&items[*b].item.x));
- let mut text = String::new();
+ // Item texts in x order; joined once the line's base direction is known.
+ let mut piece_texts: Vec<&str> = Vec::with_capacity(sorted.len());
let mut min_x = f32::INFINITY;
let mut min_y = f32::INFINITY;
let mut max_x = f32::NEG_INFINITY;
@@ -4770,7 +4774,7 @@ fn build_one_line(
let mut mcid: Option<i32> = None;
let mut spans: Vec<TextItem> = Vec::with_capacity(sorted.len());
- for (pos, &i) in sorted.iter().enumerate() {
+ for &i in sorted.iter() {
let proj = &items[i];
let it = &proj.item;
// `handle_rotation_reading_order` zeroes `item.rotation` after it
@@ -4783,13 +4787,10 @@ fn build_one_line(
span.rotation = proj.orig_rotation;
spans.push(span);
- // Concatenate item text. Use existing num_spaces from projection only as
+ // Collect item text. Use existing num_spaces from projection only as
// a hint — the markdown emitter re-collapses whitespace, so we just
// ensure there's *some* separation between adjacent items.
- if pos > 0 && !text.ends_with(' ') {
- text.push(' ');
- }
- text.push_str(&it.text);
+ piece_texts.push(it.text.as_str());
min_x = min_x.min(it.x);
min_y = min_y.min(it.y);
@@ -4849,6 +4850,25 @@ fn build_one_line(
}
}
+ // Join the line in reading order. PDFium already hands back each item's
+ // characters in logical order, so a right-to-left line only needs its
+ // *items* walked right-to-left across the page — otherwise an invoice line
+ // like "المجموع الفرعي: 75.00 SAR" comes out as "SAR 75.00 المجموع الفرعي:",
+ // detaching every label from its value. Direction is decided from the
+ // assembled line, so a neutral-only line (pure digits) stays LTR and every
+ // left-to-right document takes exactly the path it took before.
+ let rtl = crate::bidi::is_rtl_pieces(piece_texts.iter().copied());
+ if rtl {
+ piece_texts.reverse();
+ }
+ let mut text = String::new();
+ for piece in &piece_texts {
+ if !text.is_empty() && !text.ends_with(' ') {
+ text.push(' ');
+ }
+ text.push_str(piece);
+ }
+
// NOTE on tie-breaks: `max_by_key` over a HashMap returns the *last* max it
// iterates, and HashMap iteration order is randomized per process. Ties on
// char-weight would therefore pick a different winner every run, making the
@@ -4947,6 +4967,7 @@ fn build_one_line(
ProjectedLine {
text,
+ rtl,
bbox: bbox.clone(),
anchor,
// Real column detection is deferred (carry-forward in MARKDOWN_PROGRESS).
diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rs
index aacaee4..df40150 100644
--- a/crates/liteparse/src/types.rs
+++ b/crates/liteparse/src/types.rs
@@ -584,6 +584,11 @@ pub struct ProjectedLine {
pub all_mono: bool,
pub all_strike: bool,
pub spans: Vec<TextItem>,
+ /// True when the line's base direction is right-to-left (strong RTL
+ /// characters outnumber strong LTR ones). `spans` is always in x-ascending
+ /// order; consumers that rebuild the line's *text* must walk it in reverse
+ /// when this is set, or labels detach from their values.
+ pub rtl: bool,
/// Path from the page's region-tree root to the leaf containing this line.
/// Equality means "same leaf"; prefix relationship means "one contains the
/// other". Replaces the prior flat `column_id` scheme so nested layouts