A fast, helpful, and open-source document parser
Keep plain text and JSON output as on main: emphasis and super/subscript splitting only for markdown and layout blocks
Logan Markewich55b50d88cb0b
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex 2ce87eb..4fb1e44 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -362,6 +362,7 @@ fn extract_single_page(glyph_resolver,output_options.emit_word_boxes,output_options.extract_text_metadata,+ output_options.split_scripts,&consumed,)?;items.extend(emitted.into_iter().map(math_item));@@ -506,6 +507,9 @@ pub struct ExtractionOutputOptions {pub extract_form_fields: bool,pub extract_structure_tree: bool,pub emit_word_boxes: bool,+ /// Split items at a superscript / subscript run (a size step with a baseline shift), so+ /// markdown can mark it. Plain text keeps such runs in their word.+ pub split_scripts: bool,}fn non_empty(value: &Option<String>) -> Option<String> {@@ -1586,6 +1590,7 @@ fn extract_page_text_items(glyph_resolver: Option<&dyn crate::GlyphResolver>,emit_word_boxes: bool,extract_text_metadata: bool,+ split_scripts: bool,math_chars: &std::collections::HashSet<i32>,) -> Result<Vec<TextItem>, LiteParseError> {let char_count = text_page.char_count();@@ -1883,23 +1888,24 @@ fn extract_page_text_items(// text it follows. A declared size change alone is no script: producers draw the same// visible text with different size / matrix pairs ("290-300." after a reference's// "Nat. Energy 2018, 3,"), and splitting those let projection misplace the piece.- let size_break = seg.font_height.is_some_and(|sh| {- let m = obj_meta.meta_for(&ch, cv.text_object());- let fh = if m.font_size > 0.0 {- m.scale_y.map(|sy| m.font_size * sy)- } else {- None- };- let shifted = (vp_strict.bottom - seg.last_char_bottom).abs()- > 0.15- * (seg.vp_bottom - seg.vp_top)- .abs()- .max(vp_strict.bottom - vp_strict.top);- shifted- && fh.is_some_and(|fh| {- sh > 0.0 && fh > 0.0 && (fh < sh * 0.78 || sh < fh * 0.78)- })- });+ let size_break = split_scripts+ && seg.font_height.is_some_and(|sh| {+ let m = obj_meta.meta_for(&ch, cv.text_object());+ let fh = if m.font_size > 0.0 {+ m.scale_y.map(|sy| m.font_size * sy)+ } else {+ None+ };+ let shifted = (vp_strict.bottom - seg.last_char_bottom).abs()+ > 0.15+ * (seg.vp_bottom - seg.vp_top)+ .abs()+ .max(vp_strict.bottom - vp_strict.top);+ shifted+ && fh.is_some_and(|fh| {+ sh > 0.0 && fh > 0.0 && (fh < sh * 0.78 || sh < fh * 0.78)+ })+ });if !y_overlap || line_changed || gap >= MAX_INLINE_GAP || dot_leader_break || size_break{seg.flush(&mut items);diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex 6cf7a9f..129e8fd 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -375,6 +375,7 @@ impl LiteParse {extract_annotations: self.config.extract_annotations,extract_form_fields: self.config.extract_form_fields,extract_structure_tree: self.config.extract_structure_tree,+ split_scripts: self.style_runs(),},}}@@ -901,7 +902,7 @@ impl LiteParse {.collect();// Grid projection- let mut parsed_pages = stages::project(pages);+ let mut parsed_pages = self.project(pages);// Attach per-page complexity signals, including the layout signals// that need the projected page (same as `is_complex()` reports).@@ -984,9 +985,26 @@ impl LiteParse {/// configured output formatter, so it touches neither PDFium nor OCR and/// is fully synchronous. Used when an external extractor (e.g. with its/// own font-recovery pipeline) owns text extraction.+ /// Markdown output and layout blocks keep emphasis and super/subscript runs apart; plain+ /// text and JSON keep such runs inside their word, as the text grid was always built.+ /// Blocks follow markdown so `extract_blocks` reports the same blocks under every format.+ fn style_runs(&self) -> bool {+ self.config.output_format == crate::config::OutputFormat::Markdown+ || self.config.extract_blocks+ }++ /// Grid projection (see [`Self::style_runs`]).+ fn project(&self, pages: Vec<Page>) -> Vec<ParsedPage> {+ if self.style_runs() {+ stages::project_for_markdown(pages)+ } else {+ stages::project(pages)+ }+ }+pub fn parse_from_pages(&self, pages: Vec<Page>, outline: Vec<OutlineTarget>) -> ParseResult {let total_pages = pages.len().min(u32::MAX as usize) as u32;- let mut parsed_pages = stages::project(pages);+ let mut parsed_pages = self.project(pages);let full_text = if let Some(md) = apply_layout(self, &mut parsed_pages, &outline) {md@@ -1078,7 +1096,7 @@ impl LiteParse {let all_blocks = if page_filtered { None } else { all_blocks };let markdown_out = self.config.output_format == crate::config::OutputFormat::Markdown;- let mut parsed_pages = stages::project(pages);+ let mut parsed_pages = self.project(pages);for ((page, blocks), stats) in parsed_pages.iter_mut().zip(&page_blocks).zip(page_stats) {if markdown_out {page.markdown = stages::render_blocks(blocks);diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rsindex a557348..e64283e 100644--- a/crates/liteparse/src/projection.rs+++ b/crates/liteparse/src/projection.rs@@ -447,6 +447,7 @@ fn form_lines(median_width: f32,median_height: f32,page_width: f32,+ style_runs: bool,) -> Vec<Vec<ProjectedTextItem>> {// Y-tolerance for sorting: items within this threshold are considered same linelet y_sort_tolerance: f32 = (median_height * 0.5).max(5.0);@@ -846,10 +847,11 @@ fn form_lines(.chars().next().is_some_and(|c| c.is_alphabetic());- // An emphasis change (bold / italic / struck / monospace) keeps the items- // apart: a merged item carries one font, so the second run would render in- // the first one's style.- let style_change = !same_emphasis(&prev.item, &item.item);+ // For markdown, an emphasis change (bold / italic / struck / monospace) keeps+ // the items apart: a merged item carries one font, so the second run would render+ // in the first one's style. Plain text has no emphasis, and its grid places+ // columns from the merged items, so it keeps merging across style changes.+ let style_change = style_runs && !same_emphasis(&prev.item, &item.item);if y_compatible&& !both_are_numbers&& !alpha_alpha_gap@@ -1777,6 +1779,7 @@ fn detect_and_render_flowing_lines(fn project_to_grid(page: &Page,mut projection_boxes: Vec<ProjectedTextItem>,+ style_runs: bool,) -> (Vec<ProjectedTextItem>, String) {if projection_boxes.is_empty() {return (Vec::new(), String::new());@@ -1823,6 +1826,7 @@ fn project_to_grid(median_width,median_height,page.page_width,+ style_runs,);if lines.is_empty() {return (Vec::new(), String::new());@@ -2902,6 +2906,16 @@ fn clean_rendered_text(text: &str) -> String {}pub fn project_pages_to_grid(pages: Vec<Page>) -> Vec<ParsedPage> {+ project_pages(pages, false)+}++/// [`project_pages_to_grid`] for markdown output: items whose emphasis differs stay apart, so+/// each run renders in its own style.+pub fn project_pages_for_markdown(pages: Vec<Page>) -> Vec<ParsedPage> {+ project_pages(pages, true)+}++fn project_pages(pages: Vec<Page>, style_runs: bool) -> Vec<ParsedPage> {pages.into_iter().map(|page| {@@ -2927,7 +2941,7 @@ pub fn project_pages_to_grid(pages: Vec<Page>) -> Vec<ParsedPage> {}).collect();- let (projected_items, text) = project_to_grid(&page, projection_boxes);+ let (projected_items, text) = project_to_grid(&page, projection_boxes, style_runs);// Detect figure regions from the page's vector graphics before// XY-cut runs so the layout recursion can treat them as obstacles// (partition the page around figures rather than slicing through).@@ -5365,7 +5379,7 @@ mod tests {projected_item("", 30.0, 0.0, 20.0),];- let (_, text) = project_to_grid(&page, projection_boxes);+ let (_, text) = project_to_grid(&page, projection_boxes, false);assert!(text.is_empty());}@@ -5408,7 +5422,7 @@ mod tests {item_at(".7", 316.478, 149.76, 4.66, 8.42),item_at("2.3", 370.9, 149.76, 10.6, 8.42),];- let lines = form_lines(&mut items, 10.0, 8.42, 612.0);+ let lines = form_lines(&mut items, 10.0, 8.42, 612.0, true);assert_eq!(lines.len(), 1);assert!(lines[0].iter().any(|i| i.item.text == "7.7"));assert!(lines[0].iter().all(|i| !i.is_margin_line_number));@@ -5423,7 +5437,7 @@ mod tests {item_at("12", 310.0, 100.0, 8.0, 10.0),item_at("right column text", 330.0, 100.0, 230.0, 10.0),];- let lines = form_lines(&mut items, 10.0, 10.0, 612.0);+ let lines = form_lines(&mut items, 10.0, 10.0, 612.0, true);let flagged: Vec<_> = lines.iter().flatten()diff --git a/crates/liteparse/src/stages.rs b/crates/liteparse/src/stages.rsindex f74eb07..63b70c8 100644--- a/crates/liteparse/src/stages.rs+++ b/crates/liteparse/src/stages.rs@@ -375,6 +375,11 @@ pub fn project(pages: Vec<Page>) -> Vec<ParsedPage> {projection::project_pages_to_grid(pages)}+/// [`project`] for markdown output: emphasis runs within a line stay separate items.+pub fn project_for_markdown(pages: Vec<Page>) -> Vec<ParsedPage> {+ projection::project_pages_for_markdown(pages)+}+// ── Markdown ───────────────────────────────────────────────────────────/// Whole-document signals the block classifier needs: computed once over