Skip to content

A fast, helpful, and open-source document parser

Keep plain text and JSON output as on main: emphasis and super/subscript splitting only for markdown and layout blocks

Logan Markewich55b50d88cb0b
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index 2ce87eb..4fb1e44 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -362,6 +362,7 @@ fn extract_single_page(
glyph_resolver,
output_options.emit_word_boxes,
output_options.extract_text_metadata,
+ output_options.split_scripts,
&consumed,
)?;
items.extend(emitted.into_iter().map(math_item));
@@ -506,6 +507,9 @@ pub struct ExtractionOutputOptions {
pub extract_form_fields: bool,
pub extract_structure_tree: bool,
pub emit_word_boxes: bool,
+ /// Split items at a superscript / subscript run (a size step with a baseline shift), so
+ /// markdown can mark it. Plain text keeps such runs in their word.
+ pub split_scripts: bool,
}
fn non_empty(value: &Option<String>) -> Option<String> {
@@ -1586,6 +1590,7 @@ fn extract_page_text_items(
glyph_resolver: Option<&dyn crate::GlyphResolver>,
emit_word_boxes: bool,
extract_text_metadata: bool,
+ split_scripts: bool,
math_chars: &std::collections::HashSet<i32>,
) -> Result<Vec<TextItem>, LiteParseError> {
let char_count = text_page.char_count();
@@ -1883,23 +1888,24 @@ fn extract_page_text_items(
// text it follows. A declared size change alone is no script: producers draw the same
// visible text with different size / matrix pairs ("290-300." after a reference's
// "Nat. Energy 2018, 3,"), and splitting those let projection misplace the piece.
- let size_break = seg.font_height.is_some_and(|sh| {
- let m = obj_meta.meta_for(&ch, cv.text_object());
- let fh = if m.font_size > 0.0 {
- m.scale_y.map(|sy| m.font_size * sy)
- } else {
- None
- };
- let shifted = (vp_strict.bottom - seg.last_char_bottom).abs()
- > 0.15
- * (seg.vp_bottom - seg.vp_top)
- .abs()
- .max(vp_strict.bottom - vp_strict.top);
- shifted
- && fh.is_some_and(|fh| {
- sh > 0.0 && fh > 0.0 && (fh < sh * 0.78 || sh < fh * 0.78)
- })
- });
+ let size_break = split_scripts
+ && seg.font_height.is_some_and(|sh| {
+ let m = obj_meta.meta_for(&ch, cv.text_object());
+ let fh = if m.font_size > 0.0 {
+ m.scale_y.map(|sy| m.font_size * sy)
+ } else {
+ None
+ };
+ let shifted = (vp_strict.bottom - seg.last_char_bottom).abs()
+ > 0.15
+ * (seg.vp_bottom - seg.vp_top)
+ .abs()
+ .max(vp_strict.bottom - vp_strict.top);
+ shifted
+ && fh.is_some_and(|fh| {
+ sh > 0.0 && fh > 0.0 && (fh < sh * 0.78 || sh < fh * 0.78)
+ })
+ });
if !y_overlap || line_changed || gap >= MAX_INLINE_GAP || dot_leader_break || size_break
{
seg.flush(&mut items);
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index 6cf7a9f..129e8fd 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -375,6 +375,7 @@ impl LiteParse {
extract_annotations: self.config.extract_annotations,
extract_form_fields: self.config.extract_form_fields,
extract_structure_tree: self.config.extract_structure_tree,
+ split_scripts: self.style_runs(),
},
}
}
@@ -901,7 +902,7 @@ impl LiteParse {
.collect();
// Grid projection
- let mut parsed_pages = stages::project(pages);
+ let mut parsed_pages = self.project(pages);
// Attach per-page complexity signals, including the layout signals
// that need the projected page (same as `is_complex()` reports).
@@ -984,9 +985,26 @@ impl LiteParse {
/// configured output formatter, so it touches neither PDFium nor OCR and
/// is fully synchronous. Used when an external extractor (e.g. with its
/// own font-recovery pipeline) owns text extraction.
+ /// Markdown output and layout blocks keep emphasis and super/subscript runs apart; plain
+ /// text and JSON keep such runs inside their word, as the text grid was always built.
+ /// Blocks follow markdown so `extract_blocks` reports the same blocks under every format.
+ fn style_runs(&self) -> bool {
+ self.config.output_format == crate::config::OutputFormat::Markdown
+ || self.config.extract_blocks
+ }
+
+ /// Grid projection (see [`Self::style_runs`]).
+ fn project(&self, pages: Vec<Page>) -> Vec<ParsedPage> {
+ if self.style_runs() {
+ stages::project_for_markdown(pages)
+ } else {
+ stages::project(pages)
+ }
+ }
+
pub fn parse_from_pages(&self, pages: Vec<Page>, outline: Vec<OutlineTarget>) -> ParseResult {
let total_pages = pages.len().min(u32::MAX as usize) as u32;
- let mut parsed_pages = stages::project(pages);
+ let mut parsed_pages = self.project(pages);
let full_text = if let Some(md) = apply_layout(self, &mut parsed_pages, &outline) {
md
@@ -1078,7 +1096,7 @@ impl LiteParse {
let all_blocks = if page_filtered { None } else { all_blocks };
let markdown_out = self.config.output_format == crate::config::OutputFormat::Markdown;
- let mut parsed_pages = stages::project(pages);
+ let mut parsed_pages = self.project(pages);
for ((page, blocks), stats) in parsed_pages.iter_mut().zip(&page_blocks).zip(page_stats) {
if markdown_out {
page.markdown = stages::render_blocks(blocks);
diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rs
index a557348..e64283e 100644
--- a/crates/liteparse/src/projection.rs
+++ b/crates/liteparse/src/projection.rs
@@ -447,6 +447,7 @@ fn form_lines(
median_width: f32,
median_height: f32,
page_width: f32,
+ style_runs: bool,
) -> Vec<Vec<ProjectedTextItem>> {
// Y-tolerance for sorting: items within this threshold are considered same line
let y_sort_tolerance: f32 = (median_height * 0.5).max(5.0);
@@ -846,10 +847,11 @@ fn form_lines(
.chars()
.next()
.is_some_and(|c| c.is_alphabetic());
- // An emphasis change (bold / italic / struck / monospace) keeps the items
- // apart: a merged item carries one font, so the second run would render in
- // the first one's style.
- let style_change = !same_emphasis(&prev.item, &item.item);
+ // For markdown, an emphasis change (bold / italic / struck / monospace) keeps
+ // the items apart: a merged item carries one font, so the second run would render
+ // in the first one's style. Plain text has no emphasis, and its grid places
+ // columns from the merged items, so it keeps merging across style changes.
+ let style_change = style_runs && !same_emphasis(&prev.item, &item.item);
if y_compatible
&& !both_are_numbers
&& !alpha_alpha_gap
@@ -1777,6 +1779,7 @@ fn detect_and_render_flowing_lines(
fn project_to_grid(
page: &Page,
mut projection_boxes: Vec<ProjectedTextItem>,
+ style_runs: bool,
) -> (Vec<ProjectedTextItem>, String) {
if projection_boxes.is_empty() {
return (Vec::new(), String::new());
@@ -1823,6 +1826,7 @@ fn project_to_grid(
median_width,
median_height,
page.page_width,
+ style_runs,
);
if lines.is_empty() {
return (Vec::new(), String::new());
@@ -2902,6 +2906,16 @@ fn clean_rendered_text(text: &str) -> String {
}
pub fn project_pages_to_grid(pages: Vec<Page>) -> Vec<ParsedPage> {
+ project_pages(pages, false)
+}
+
+/// [`project_pages_to_grid`] for markdown output: items whose emphasis differs stay apart, so
+/// each run renders in its own style.
+pub fn project_pages_for_markdown(pages: Vec<Page>) -> Vec<ParsedPage> {
+ project_pages(pages, true)
+}
+
+fn project_pages(pages: Vec<Page>, style_runs: bool) -> Vec<ParsedPage> {
pages
.into_iter()
.map(|page| {
@@ -2927,7 +2941,7 @@ pub fn project_pages_to_grid(pages: Vec<Page>) -> Vec<ParsedPage> {
})
.collect();
- let (projected_items, text) = project_to_grid(&page, projection_boxes);
+ let (projected_items, text) = project_to_grid(&page, projection_boxes, style_runs);
// Detect figure regions from the page's vector graphics before
// XY-cut runs so the layout recursion can treat them as obstacles
// (partition the page around figures rather than slicing through).
@@ -5365,7 +5379,7 @@ mod tests {
projected_item("", 30.0, 0.0, 20.0),
];
- let (_, text) = project_to_grid(&page, projection_boxes);
+ let (_, text) = project_to_grid(&page, projection_boxes, false);
assert!(text.is_empty());
}
@@ -5408,7 +5422,7 @@ mod tests {
item_at(".7", 316.478, 149.76, 4.66, 8.42),
item_at("2.3", 370.9, 149.76, 10.6, 8.42),
];
- let lines = form_lines(&mut items, 10.0, 8.42, 612.0);
+ let lines = form_lines(&mut items, 10.0, 8.42, 612.0, true);
assert_eq!(lines.len(), 1);
assert!(lines[0].iter().any(|i| i.item.text == "7.7"));
assert!(lines[0].iter().all(|i| !i.is_margin_line_number));
@@ -5423,7 +5437,7 @@ mod tests {
item_at("12", 310.0, 100.0, 8.0, 10.0),
item_at("right column text", 330.0, 100.0, 230.0, 10.0),
];
- let lines = form_lines(&mut items, 10.0, 10.0, 612.0);
+ let lines = form_lines(&mut items, 10.0, 10.0, 612.0, true);
let flagged: Vec<_> = lines
.iter()
.flatten()
diff --git a/crates/liteparse/src/stages.rs b/crates/liteparse/src/stages.rs
index f74eb07..63b70c8 100644
--- a/crates/liteparse/src/stages.rs
+++ b/crates/liteparse/src/stages.rs
@@ -375,6 +375,11 @@ pub fn project(pages: Vec<Page>) -> Vec<ParsedPage> {
projection::project_pages_to_grid(pages)
}
+/// [`project`] for markdown output: emphasis runs within a line stay separate items.
+pub fn project_for_markdown(pages: Vec<Page>) -> Vec<ParsedPage> {
+ projection::project_pages_for_markdown(pages)
+}
+
// ── Markdown ───────────────────────────────────────────────────────────
/// Whole-document signals the block classifier needs: computed once over