A fast, helpful, and open-source document parser
Merge origin/main into logan/native-blocks-api
Logan Markewichdfcbaba919fa
diff --cc Cargo.lockindex 2887c22,a64fcc2..5abe237--- a/Cargo.lock+++ b/Cargo.lock@@@ -2016,7 -2016,7 +2016,7 @@@ checksum = "92daf443525c4cce67b150400bc[[package]]name = "liteparse"- version = "2.11.0"-version = "2.14.1"++version = "2.14.2"dependencies = ["blake3","clap",@@@ -2048,7 -2048,7 +2048,7 @@@[[package]]name = "liteparse-napi"- version = "2.11.0"-version = "2.14.1"++version = "2.14.2"dependencies = ["image","liteparse",@@@ -2064,7 -2064,7 +2064,7 @@@[[package]]name = "liteparse-pdfium"- version = "1.5.0"-version = "1.8.0"++version = "1.9.0"dependencies = ["blake3","liteparse-pdfium-sys",@@@ -2072,7 -2072,7 +2072,7 @@@[[package]]name = "liteparse-pdfium-sys"- version = "1.5.0"-version = "1.8.0"++version = "1.9.0"dependencies = ["bindgen","flate2",@@@ -2083,7 -2083,7 +2083,7 @@@[[package]]name = "liteparse-python"- version = "2.11.0"-version = "2.14.1"++version = "2.14.2"dependencies = ["anyhow","clap",@@@ -2099,7 -2099,7 +2099,7 @@@[[package]]name = "liteparse-wasm"- version = "2.11.0"-version = "2.14.1"++version = "2.14.2"dependencies = ["console_error_panic_hook","js-sys",diff --cc crates/liteparse/src/layout.rsindex 0000000,0bf9fde..569a2bbmode 000000,100644..100644--- a/crates/liteparse/src/layout.rs+++ b/crates/liteparse/src/layout.rs@@@ -1,0 -1,176 +1,211 @@@+ //! Public, serializable view of the markdown classifier's block decomposition.+ //!+ //! The classifier's internal [`Block`](crate::markdown_layout::Block) is a Rust+ //! enum with per-variant payloads. That shape can't cross the napi or PyO3+ //! boundary — neither supports enums carrying data — so the public type is a+ //! single flat struct discriminated by `kind`, with the variant-specific fields+ //! as `Option`s. This mirrors how the crate already exposes+ //! `StructureAttributeValue` to the bindings, and keeps one block shape across+ //! Rust, JSON, Node, Python, and WASM instead of three divergent ones.+ //!+ //! Every field that doesn't apply to a block's `kind` is `None` and is skipped+ //! during serialization, so a heading serializes as `{kind, text, level, bbox}`+ //! rather than a wall of nulls.++ use serde::Serialize;+-use crate::markdown_layout::{Block, Cell, PositionedBlock};++use crate::markdown_layout::{Block, Cell, PositionedBlock, SpanCell};+ use crate::types::Rect;++ /// One table cell: its rendered text and the region it occupied.+ ///+ /// `bbox` is `None` for cells with no ink behind them — padding inserted to+ /// square off a ragged grid, or halves of a merged run split at an estimated+ /// position rather than an observed boundary.+ #[derive(Debug, Clone, Serialize)]+ pub struct LayoutCell {+ pub text: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub bbox: Option<Rect>,++ /// Merge spans, present only on `merged_table` cells (and only when > 1).++ /// A cell absorbed by a neighbour's span is absent from its row entirely,++ /// matching HTML's occupancy model.++ #[serde(skip_serializing_if = "Option::is_none")]++ pub colspan: Option<u16>,++ #[serde(skip_serializing_if = "Option::is_none")]++ pub rowspan: Option<u16>,+ }++ impl From<&Cell> for LayoutCell {+ fn from(c: &Cell) -> Self {+ LayoutCell {+ text: c.text.clone(),+ bbox: c.bbox.clone(),++ colspan: None,++ rowspan: None,++ }++ }++}++++impl From<&SpanCell> for LayoutCell {++ fn from(c: &SpanCell) -> Self {++ LayoutCell {++ text: c.text.clone(),++ bbox: None,++ colspan: (c.colspan > 1).then_some(c.colspan),++ rowspan: (c.rowspan > 1).then_some(c.rowspan),+ }+ }+ }++ /// A classified block plus where it sits on the page.+ ///+ /// `kind` discriminates the block; see each field for which kinds populate it.+ /// Blocks appear in reading order, matching the order the markdown renderer+ /// emits them, so the Nth block here is the Nth block of that page's markdown.+ #[derive(Debug, Clone, Serialize)]+ pub struct LayoutBlock {+ /// One of `heading`, `paragraph`, `list_item`, `code`, `table`,- /// `grid_fallback`, `rule`, `figure`.++ /// `merged_table`, `grid_fallback`, `rule`, `figure`.+ pub kind: &'static str,+ /// Rendered text for the text-bearing kinds (`heading`, `paragraph`,+ /// `list_item`). Table text lives in `header`/`rows`; code and grid text in+ /// `lines`.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub text: Option<String>,+ /// Heading level (1–6), or list nesting depth for `list_item`.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub level: Option<u8>,+ /// Whether the block's text is uniformly bold / italic. `paragraph` and+ /// `list_item` only; omitted when false.+ #[serde(skip_serializing_if = "std::ops::Not::not")]+ pub bold: bool,+ #[serde(skip_serializing_if = "std::ops::Not::not")]+ pub italic: bool,+ /// `list_item`: whether the list is ordered, and the original marker as it+ /// appeared on the page (`138.`, `iii)`, `•`).+ #[serde(skip_serializing_if = "Option::is_none")]+ pub ordered: Option<bool>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub marker: Option<String>,+ /// Verbatim source lines for `code` and `grid_fallback`.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub lines: Option<Vec<String>>,+ /// Best-effort language hint for `code`.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub lang: Option<String>,+ /// `table`: the header row, when one was detected.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub header: Option<Vec<LayoutCell>>,- /// `table`: the body rows.++ /// `table`: the body rows. `merged_table`: all rows (header rows lead,++ /// counted by `header_rows`); rows are ragged — cells covered by a++ /// neighbour's span are absent.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub rows: Option<Vec<Vec<LayoutCell>>>,++ /// `merged_table`: how many leading rows of `rows` are header rows.++ #[serde(skip_serializing_if = "Option::is_none")]++ pub header_rows: Option<usize>,+ /// `figure`: the image's page-scoped id and encoded format, matching the+ /// `img_{id}.{format}` target the markdown renderer emits.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub id: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub format: Option<String>,+ /// Region of the page this block occupies, in the same top-left, 72-DPI+ /// viewport space as `text_items`. The union of every source line that fed+ /// the block, so a wrapped heading or multi-line paragraph reports its full+ /// band. `None` when the block has no page geometry behind it.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub bbox: Option<Rect>,+ }++ impl LayoutBlock {+ /// Base value with every variant-specific field cleared.+ fn of(kind: &'static str, bbox: Option<Rect>) -> Self {+ LayoutBlock {+ kind,+ text: None,+ level: None,+ bold: false,+ italic: false,+ ordered: None,+ marker: None,+ lines: None,+ lang: None,+ header: None,+ rows: None,++ header_rows: None,+ id: None,+ format: None,+ bbox,+ }+ }+ }++ fn cells(row: &[Cell]) -> Vec<LayoutCell> {+ row.iter().map(LayoutCell::from).collect()+ }++ impl From<&PositionedBlock> for LayoutBlock {+ fn from(pb: &PositionedBlock) -> Self {+ let bbox = pb.bbox.clone();+ match &pb.block {+ Block::Heading { level, text } => LayoutBlock {+ text: Some(text.clone()),+ level: Some(*level),+ ..LayoutBlock::of("heading", bbox)+ },+ Block::Paragraph { text, bold, italic } => LayoutBlock {+ text: Some(text.clone()),+ bold: *bold,+ italic: *italic,+ ..LayoutBlock::of("paragraph", bbox)+ },+ Block::ListItem {+ ordered,+ marker,+ level,+ text,+ bold,+ italic,+ } => LayoutBlock {+ text: Some(text.clone()),+ level: Some(*level),+ ordered: Some(*ordered),+ marker: Some(marker.clone()),+ bold: *bold,+ italic: *italic,+ ..LayoutBlock::of("list_item", bbox)+ },+ Block::CodeBlock { lines, lang } => LayoutBlock {+ lines: Some(lines.clone()),+ lang: lang.clone(),+ ..LayoutBlock::of("code", bbox)+ },+ Block::Table { header, rows } => LayoutBlock {+ header: header.as_ref().map(|h| cells(h)),+ rows: Some(rows.iter().map(|r| cells(r)).collect()),+ ..LayoutBlock::of("table", bbox)+ },++ Block::MergedTable { rows, header_rows } => LayoutBlock {++ header_rows: Some(*header_rows),++ rows: Some(++ rows.iter()++ .map(|r| r.iter().map(LayoutCell::from).collect())++ .collect(),++ ),++ ..LayoutBlock::of("merged_table", bbox)++ },+ Block::GridFallback { lines } => LayoutBlock {+ lines: Some(lines.clone()),+ ..LayoutBlock::of("grid_fallback", bbox)+ },+ Block::HorizontalRule => LayoutBlock::of("rule", bbox),+ Block::Figure { id, format } => LayoutBlock {+ id: Some(id.clone()),+ format: Some(format.clone()),+ ..LayoutBlock::of("figure", bbox)+ },+ }+ }+ }diff --cc crates/liteparse/src/markdown_layout/blocks.rsindex 3d184f5,ece6f05..d0c155e--- a/crates/liteparse/src/markdown_layout/blocks.rs+++ b/crates/liteparse/src/markdown_layout/blocks.rs@@@ -1,7 -1,82 +1,120 @@@use super::inline::escape_inline;use super::paragraphs::{ParaAccum, is_soft_hyphen_break};use super::tables::escape_table_cell;+ use crate::types::Rect;++ /// One table cell: rendered text plus, when the cell came from real page+ /// content, the region it occupied. `bbox` is `None` for cells that exist only+ /// to square off a ragged grid (padding inserted when rows disagree on column+ /// count) — those occupy no ink on the page, and reporting a rect for them+ /// would invent geometry the classifier never saw.+ #[derive(Debug, Clone, Default)]+ pub struct Cell {+ pub text: String,+ pub bbox: Option<Rect>,+ }++ impl Cell {+ /// Cell carrying text and the region it was read from.+ pub fn located(text: impl Into<String>, bbox: Rect) -> Self {+ Cell {+ text: text.into(),+ bbox: Some(bbox),+ }+ }++ /// Borrow the cell's text. Lets call sites that only care about content+ /// read a `Cell` about as tersely as the `String` it replaced.+ pub fn as_str(&self) -> &str {+ &self.text+ }+ }++ // Text-only construction (`"a".into()`, `s.to_string().into()`)+ impl From<&str> for Cell {+ fn from(text: &str) -> Self {+ Cell {+ text: text.to_string(),+ bbox: None,+ }+ }+ }++ impl From<String> for Cell {+ fn from(text: String) -> Self {+ Cell { text, bbox: None }+ }+ }++ // Cells compare by content alone. A cell's box is derived metadata describing+ // where the text was read from, not part of its identity — two cells holding+ // the same text are the same table content whether or not either one knows its+ // coordinates. This is also what lets assertions compare a detected grid+ // against a plain literal of expected strings.+ impl PartialEq for Cell {+ fn eq(&self, other: &Self) -> bool {+ self.text == other.text+ }+ }++ impl Eq for Cell {}++ impl PartialEq<str> for Cell {+ fn eq(&self, other: &str) -> bool {+ self.text == other+ }+ }++ impl PartialEq<&str> for Cell {+ fn eq(&self, other: &&str) -> bool {+ self.text == *other+ }+ }++ impl PartialEq<String> for Cell {+ fn eq(&self, other: &String) -> bool {+ self.text == *other+ }+ }+++/// One cell of a [`Block::MergedTable`]. `colspan`/`rowspan` are 1 for an++/// ordinary cell; the cells they cover are absent from `rows` rather than++/// present-and-empty, matching HTML's occupancy model. Distinct from [`Cell`]:++/// that type carries the page geometry a cell was *read from*, this one the++/// merge structure a producer *declared*.++#[derive(Debug, Clone, PartialEq, Eq)]++pub struct SpanCell {++ pub text: String,++ pub colspan: u16,++ pub rowspan: u16,++}++++impl SpanCell {++ /// A plain 1x1 cell.++ pub fn new(text: impl Into<String>) -> Self {++ Self {++ text: text.into(),++ colspan: 1,++ rowspan: 1,++ }++ }++++ /// A cell spanning `colspan` columns and `rowspan` rows. Zero is coerced to++ /// 1 — a zero span is meaningless in HTML and `rowspan="0"` means "to the++ /// end of the section", which is not what any producer here intends.++ pub fn spanning(text: impl Into<String>, colspan: u16, rowspan: u16) -> Self {++ Self {++ text: text.into(),++ colspan: colspan.max(1),++ rowspan: rowspan.max(1),++ }++ }++++ fn is_plain(&self) -> bool {++ self.colspan <= 1 && self.rowspan <= 1++ }++}+/// Coarse block representation: the output of page classification, consumed by/// `render_blocks` to produce the final markdown string.#[derive(Debug, Clone)]@@@ -36,23 -111,9 +149,23 @@@ pub enum Block/// when the first row didn't qualify (e.g. wasn't bold and the table mode/// can't otherwise distinguish it).Table {- header: Option<Vec<String>>,- rows: Vec<Vec<String>>,+ header: Option<Vec<Cell>>,+ rows: Vec<Vec<Cell>>,},+ /// Table whose cells carry explicit spans. Emitted by producers that know+ /// the merge structure outright (e.g. a structural reader with access to+ /// `gridSpan` / `vMerge`) rather than inferring it from coordinates.+ ///+ /// `rows` is *not* rectangular: a cell absorbed by a neighbour's+ /// `colspan`/`rowspan` is omitted entirely, exactly as in HTML. `header_rows`+ /// is the count of leading rows that are header rows (`<th>`); 0 means none.+ ///+ /// Renders as a GFM pipe table when it degenerates to a plain grid, and as+ /// HTML `<table>` otherwise — pipe tables cannot express spans at all.+ MergedTable {- rows: Vec<Vec<Cell>>,++ rows: Vec<Vec<SpanCell>>,+ header_rows: usize,+ },/// Tabular-looking region we couldn't classify confidently — rendered/// verbatim inside a fenced block to preserve visual structure for the/// downstream LLM. Strictly better than emitting a mangled pipe table.@@@ -146,127 -171,90 +223,181 @@@ fn wrap_emphasis(text: &str, bold: bool}}+ /// A classified block plus the region of the page it was read from.+ ///+ /// `bbox` is the union of the boxes of every source line that fed the block, so+ /// a wrapped heading or a multi-line paragraph reports the whole band it+ /// occupies. It is `None` only for blocks with no page geometry behind them —+ /// synthesized content, or sources that never carried coordinates.+ #[derive(Debug, Clone)]+ pub struct PositionedBlock {+ pub block: Block,+ pub bbox: Option<Rect>,+ }++ impl PositionedBlock {+ pub fn new(block: Block, bbox: Option<Rect>) -> Self {+ PositionedBlock { block, bbox }+ }++ /// A block with no known geometry.+ pub fn unlocated(block: Block) -> Self {+ PositionedBlock { block, bbox: None }+ }++ /// Grow this block's box to also cover `other`.+ fn absorb(&mut self, other: &Option<Rect>) {+ if let Some(r) = other {+ Rect::extend(&mut self.bbox, r);+ }+ }+ }++ /// Heal words hyphenated across a soft line wrap that the classifier split into+ /// two *separate* paragraph blocks: `…they dis-` ‖ `lodged…`. When a plain+ /// paragraph ends with `<letter>-` and the next is a plain paragraph starting+ /// lowercase, fuse them — dropping the hyphen and joining with no separator —+ /// and union their boxes, since the result spans both source regions.+ ///+ /// This runs as a pass over the block list rather than inside the renderer so+ /// that the blocks callers receive are the same ones that produced the+ /// markdown. The lowercase and plain-text gates keep real compounds (`well-`+ /// then a capitalized `Known`) and emphasised/heading/table starts intact.+ pub fn splice_soft_hyphens(blocks: Vec<PositionedBlock>) -> Vec<PositionedBlock> {+ let mut out: Vec<PositionedBlock> = Vec::with_capacity(blocks.len());+ for pb in blocks {+ // Both sides must be unemphasised: an emphasised predecessor renders+ // with a trailing `**`/`*`, so its final character was never the+ // hyphen this splice keys off.+ let joinable = match (out.last().map(|p| &p.block), &pb.block) {+ (+ Some(Block::Paragraph {+ text: prev,+ bold: false,+ italic: false,+ }),+ Block::Paragraph {+ text,+ bold: false,+ italic: false,+ },+ ) => is_soft_hyphen_break(prev, text).then(|| text.clone()),+ _ => None,+ };+ if let Some(tail) = joinable {+ let prev = out.last_mut().expect("gate matched on a previous block");+ if let Block::Paragraph { text, .. } = &mut prev.block {+ while text.ends_with(|c: char| c.is_whitespace()) {+ text.pop();+ }+ text.pop(); // the soft hyphen+ text.push_str(&tail);+ }+ prev.absorb(&pb.bbox);+ continue;+ }+ out.push(pb);+ }+ out+ }++/// Render a GFM pipe table. `head` is the header row (already chosen by the+/// caller); `body` the remaining rows. No-ops when there are no columns.- fn render_pipe_table(head: Option<&[String]>, body: &[Vec<String>], out: &mut String) {++fn render_pipe_table(head: Option<&[Cell]>, body: &[Vec<Cell>], out: &mut String) {+ let column_count = head.map(|h| h.len()).unwrap_or(0);+ if column_count == 0 {+ return;+ }+ out.push_str("| ");+ for (i, cell) in head.unwrap().iter().enumerate() {+ if i > 0 {+ out.push_str(" | ");+ }- out.push_str(&escape_table_cell(cell));++ out.push_str(&escape_table_cell(&cell.text));+ }+ out.push_str(" |\n");+ out.push('|');+ for _ in 0..column_count {+ out.push_str("---|");+ }+ for row in body {+ out.push_str("\n| ");+ for (i, cell) in row.iter().enumerate() {+ if i > 0 {+ out.push_str(" | ");+ }- out.push_str(&escape_table_cell(cell));++ out.push_str(&escape_table_cell(&cell.text));+ }+ out.push_str(" |");+ }+}++/// Escape a cell for HTML `<td>` content. Markdown specials are deliberately+/// left alone — inside an HTML block they are literal text, and escaping them+/// would surface backslashes in the rendered output.+fn escape_html_cell(s: &str) -> String {+ let mut out = String::with_capacity(s.len());+ for c in s.chars() {+ match c {+ '&' => out.push_str("&"),+ '<' => out.push_str("<"),+ '>' => out.push_str(">"),+ '\n' => out.push(' '),+ _ => out.push(c),+ }+ }+ out+}++/// True when a merged table degenerates to a plain rectangular grid, in which+/// case the pipe rendering is both valid and nicer to read.- fn is_plain_grid(rows: &[Vec<Cell>], header_rows: usize) -> bool {++fn is_plain_grid(rows: &[Vec<SpanCell>], header_rows: usize) -> bool {+ // GFM has exactly one header row, or none.+ if header_rows > 1 {+ return false;+ }- if !rows.iter().all(|r| r.iter().all(Cell::is_plain)) {++ if !rows.iter().all(|r| r.iter().all(SpanCell::is_plain)) {+ return false;+ }+ // Ragged rows can only have come from spans we've just ruled out, but check+ // anyway: a pipe table with uneven rows renders as a broken table.+ let width = rows.first().map(|r| r.len()).unwrap_or(0);+ rows.iter().all(|r| r.len() == width)+}++/// Render a merged table as HTML. Cells covered by a neighbour's span are+/// simply absent from `rows`, so this is a direct transcription.- fn render_html_table(rows: &[Vec<Cell>], header_rows: usize, out: &mut String) {++fn render_html_table(rows: &[Vec<SpanCell>], header_rows: usize, out: &mut String) {+ out.push_str("<table>");+ for (r, row) in rows.iter().enumerate() {+ out.push_str("\n<tr>");+ let tag = if r < header_rows { "th" } else { "td" };+ for cell in row {+ out.push_str("\n<");+ out.push_str(tag);+ if cell.colspan > 1 {+ out.push_str(&format!(" colspan=\"{}\"", cell.colspan));+ }+ if cell.rowspan > 1 {+ out.push_str(&format!(" rowspan=\"{}\"", cell.rowspan));+ }+ out.push('>');+ out.push_str(&escape_html_cell(&cell.text));+ out.push_str("</");+ out.push_str(tag);+ out.push('>');+ }+ out.push_str("\n</tr>");+ }+ out.push_str("\n</table>");+}+/// Render a list of blocks to a markdown string.- pub fn render_blocks(blocks: &[Block]) -> String {+ pub fn render_blocks(blocks: &[PositionedBlock]) -> String {let mut out = String::new();- for (i, block) in blocks.iter().enumerate() {+ for (i, positioned) in blocks.iter().enumerate() {+ let block = &positioned.block;if i > 0 {- // A word hyphenated across a soft line wrap sometimes lands in two- // *separate* paragraph blocks (the classifier mis-split mid-word):- // `…they dis-` ‖ `lodged…`. When the previous block ends with- // `<letter>-` and this block is a plain paragraph beginning with a- // lowercase letter, splice them: drop the hyphen and join with no- // separator, healing both the word and the spurious break. The- // lowercase + plain-text gates keep real compounds (`well-` then a- // capitalized `Known`) and emphasised/heading/table starts intact.- if let (- Block::Paragraph { .. },- Block::Paragraph {- text,- bold: false,- italic: false,- },- ) = (&blocks[i - 1], block)- && is_soft_hyphen_break(&out, text)- {- while out.ends_with(|c: char| c.is_whitespace()) {- out.pop();- }- out.pop(); // the soft hyphen- out.push_str(text);- continue;- }// Consecutive list items render as a tight list (single newline).// Everything else gets a blank line between blocks.let tight = matches!(block, Block::ListItem { .. })@@@ -322,30 -310,31 +453,31 @@@None => (None, rows.as_slice()),},};- let column_count = head.map(|h| h.len()).unwrap_or(0);- if column_count == 0 {+ render_pipe_table(head, body, &mut out);+ }+ Block::MergedTable { rows, header_rows } => {+ if rows.is_empty() {continue;}- out.push_str("| ");- for (i, cell) in head.unwrap().iter().enumerate() {- if i > 0 {- out.push_str(" | ");- }- out.push_str(&escape_table_cell(&cell.text));- }- out.push_str(" |\n");- out.push('|');- for _ in 0..column_count {- out.push_str("---|");- }- for row in body {- out.push_str("\n| ");- for (i, cell) in row.iter().enumerate() {- if i > 0 {- out.push_str(" | ");- }- out.push_str(&escape_table_cell(&cell.text));- }- out.push_str(" |");+ if is_plain_grid(rows, *header_rows) {+ // Degenerate case — no merges, so pipes are expressive- // enough. Reuse the exact same rendering the PDF path gets,- // including the promote-first-row-when-headerless rule.- let plain: Vec<Vec<String>> = rows++ // enough. Reuse the exact same rendering `Block::Table`++ // gets, including the promote-first-row-when-headerless++ // rule.++ let plain: Vec<Vec<Cell>> = rows+ .iter()- .map(|r| r.iter().map(|c| c.text.clone()).collect())++ .map(|r| r.iter().map(|c| Cell::from(c.text.clone())).collect())+ .collect();+ // `header_rows` is 0 or 1 here (>1 forces HTML). Either way+ // the first row leads: as the real header, or promoted into+ // one because GFM has no headerless table.+ let (head, body) = match plain.split_first() {+ Some((first, rest)) => (Some(first.as_slice()), rest),+ None => (None, plain.as_slice()),+ };+ render_pipe_table(head, body, &mut out);+ } else {+ render_html_table(rows, *header_rows, &mut out);}}Block::GridFallback { lines } => {@@@ -518,101 -515,42 +658,132 @@@ mod testsbold: false,italic: false,};+ let spliced = |a: &str, b: &str| {+ splice_soft_hyphens(vec![+ PositionedBlock::unlocated(p(a)),+ PositionedBlock::unlocated(p(b)),+ ])+ };+ let rendered = |a: &str, b: &str| render_blocks(&spliced(a, b));// Mid-word hyphen split into two paragraphs heals into one.- let s = render_blocks(&[p("they dis-"), p("lodged the part")]);- assert_eq!(s, "they dislodged the part");+ assert_eq!(+ rendered("they dis-", "lodged the part"),+ "they dislodged the part"+ );// Capitalized continuation is a real compound break — left intact.- let s = render_blocks(&[p("the well-"), p("Known fact")]);- assert_eq!(s, "the well-\n\nKnown fact");+ assert_eq!(+ rendered("the well-", "Known fact"),+ "the well-\n\nKnown fact"+ );// Trailing dash not preceded by a letter doesn't splice.- let s = render_blocks(&[p("a -"), p("dash line")]);- assert_eq!(s, "a -\n\ndash line");+ assert_eq!(rendered("a -", "dash line"), "a -\n\ndash line");+ // A spliced block covers both source regions; an unspliced pair stays+ // as two blocks with their own boxes.+ let r = |y: f32| Rect {+ x: 10.0,+ y,+ width: 100.0,+ height: 12.0,+ };+ let joined = splice_soft_hyphens(vec![+ PositionedBlock::new(p("they dis-"), Some(r(50.0))),+ PositionedBlock::new(p("lodged the part"), Some(r(70.0))),+ ]);+ assert_eq!(joined.len(), 1);+ let bbox = joined[0].bbox.clone().expect("merged block keeps geometry");+ assert_eq!((bbox.y, bbox.height), (50.0, 32.0));}+ #[test]+ fn merged_table_without_spans_renders_as_pipes() {+ // A merge-aware producer that happens to emit a plain grid must be+ // byte-identical to the PDF path's pipe table — otherwise the same+ // table looks different depending on which pipeline produced it.+ let plain = Block::Table {+ header: Some(vec!["a".into(), "b".into()]),+ rows: vec![vec!["1".into(), "2".into()]],+ };+ let merged = Block::MergedTable {+ header_rows: 1,+ rows: vec![- vec![Cell::new("a"), Cell::new("b")],- vec![Cell::new("1"), Cell::new("2")],++ vec![SpanCell::new("a"), SpanCell::new("b")],++ vec![SpanCell::new("1"), SpanCell::new("2")],+ ],+ };- assert_eq!(render_blocks(&[plain]), render_blocks(&[merged]));++ assert_eq!(render(vec![plain]), render(vec![merged]));+ }++ #[test]+ fn merged_table_with_spans_renders_as_html() {+ // Cells covered by a span are absent, not empty — row 1 has one cell.- let s = render_blocks(&[Block::MergedTable {++ let s = render(vec![Block::MergedTable {+ header_rows: 1,+ rows: vec![- vec![Cell::spanning("wide", 2, 1)],- vec![Cell::spanning("tall", 1, 2), Cell::new("x")],- vec![Cell::new("y")],++ vec![SpanCell::spanning("wide", 2, 1)],++ vec![SpanCell::spanning("tall", 1, 2), SpanCell::new("x")],++ vec![SpanCell::new("y")],+ ],+ }]);+ assert_eq!(+ s,+ "<table>\n<tr>\n<th colspan=\"2\">wide</th>\n</tr>\n\+ <tr>\n<td rowspan=\"2\">tall</td>\n<td>x</td>\n</tr>\n\+ <tr>\n<td>y</td>\n</tr>\n</table>"+ );+ }++ #[test]+ fn merged_table_escapes_html_not_markdown() {+ // Inside an HTML block, markdown specials are literal; `<` is not.- let s = render_blocks(&[Block::MergedTable {++ let s = render(vec![Block::MergedTable {+ header_rows: 0,- rows: vec![vec![Cell::new("a|b *c* <d> &e"), Cell::spanning("f", 2, 1)]],++ rows: vec![vec![++ SpanCell::new("a|b *c* <d> &e"),++ SpanCell::spanning("f", 2, 1),++ ]],+ }]);+ assert!(s.contains("<td>a|b *c* <d> &e</td>"), "{s}");+ }++ #[test]+ fn merged_table_multiple_header_rows_force_html() {+ // Two header rows have no pipe-table representation even with no spans.- let s = render_blocks(&[Block::MergedTable {++ let s = render(vec![Block::MergedTable {+ header_rows: 2,+ rows: vec![- vec![Cell::new("a"), Cell::new("b")],- vec![Cell::new("c"), Cell::new("d")],- vec![Cell::new("1"), Cell::new("2")],++ vec![SpanCell::new("a"), SpanCell::new("b")],++ vec![SpanCell::new("c"), SpanCell::new("d")],++ vec![SpanCell::new("1"), SpanCell::new("2")],+ ],+ }]);+ assert!(s.starts_with("<table>"), "{s}");+ assert_eq!(s.matches("<th>").count(), 4);+ assert_eq!(s.matches("<td>").count(), 2);+ }++ #[test]+ fn merged_table_ragged_rows_force_html() {+ // Uneven row widths with no declared spans would render as a broken+ // pipe table; HTML at least preserves the cells.- let s = render_blocks(&[Block::MergedTable {++ let s = render(vec![Block::MergedTable {+ header_rows: 1,- rows: vec![vec![Cell::new("a"), Cell::new("b")], vec![Cell::new("1")]],++ rows: vec![++ vec![SpanCell::new("a"), SpanCell::new("b")],++ vec![SpanCell::new("1")],++ ],+ }]);+ assert!(s.starts_with("<table>"), "{s}");+ }++ #[test]+ fn zero_span_is_coerced_to_one() {+ // `rowspan="0"` means "to the end of the section" in HTML — never what+ // a producer means here, so it must not survive into the output.- let c = Cell::spanning("x", 0, 0);++ let c = SpanCell::spanning("x", 0, 0);+ assert_eq!((c.colspan, c.rowspan), (1, 1));+ assert!(c.is_plain());+ }+#[test]fn render_table_without_header_promotes_first_row() {let blocks = vecs — a producer without per-block geometry wraps each++ /// with `PositionedBlock::unlocated`. When `extract_blocks` is on, the++ /// supplied blocks (and their boxes) also populate each page's `blocks`,++ /// mirroring the classifier path.+ ///+ /// Doc-level markdown renders from `all_blocks` when given, NOT by+ /// joining the page renders: `render_blocks` is context-sensitive across+ /// block boundaries (tight lists join with `\n`, soft-hyphen paragraph+ /// splices join with nothing), so a page-boundary join would perturb+ /// whitespace wherever a page break falls inside a list. The per-page+ /// markdown is the paged *view* of the same blocks. `target_pages` /+ /// `max_pages` filtering drops `all_blocks` and falls back to joining+ /// the surviving pages.+ pub fn parse_from_blocks(+ &self,+ pages: Vec<Page>,- page_blocks: Vec<Vec<crate::markdown_layout::Block>>,- all_blocks: Option<Vec<crate::markdown_layout::Block>>,++ page_blocks: Vec<Vec<crate::markdown_layout::PositionedBlock>>,++ all_blocks: Option<Vec<crate::markdown_layout::PositionedBlock>>,+ outline: Vec<OutlineTarget>,+ images: Vec<crate::types::ExtractedImage>,+ page_stats: Vec<Option<crate::ocr_merge::PageComplexityStats>>,+ ) -> Result<ParseResult, LiteParseError> {++ // Reported before page filtering, matching every other path.++ let total_pages = pages.len().min(u32::MAX as usize) as u32;+ let (pages, page_blocks, page_stats, page_filtered) =+ self.select_block_pages(pages, page_blocks, page_stats)?;+ let all_blocks = if page_filtered { None } else { all_blocks };++ let markdown_out = self.config.output_format == crate::config::OutputFormat::Markdown;+ let mut parsed_pages = projection::project_pages_to_grid(pages);+ for ((page, blocks), stats) in parsed_pages.iter_mut().zip(&page_blocks).zip(page_stats) {+ if markdown_out {+ page.markdown = crate::markdown_layout::render_blocks(blocks);+ }++ if self.config.extract_blocks {++ page.blocks = Some(++ blocks++ .iter()++ .map(crate::layout::LayoutBlock::from)++ .collect(),++ );++ }+ page.complexity = stats;+ if !self.config.extract_content_bounds {+ page.content_bounds = None;+ }+ }++ let full_text = if markdown_out {+ match all_blocks {+ Some(blocks) => crate::markdown_layout::render_blocks(&blocks),+ None => parsed_pages+ .iter()+ .map(|p| p.markdown.as_str())+ .collect::<Vec<_>>()+ .join("\n\n-----\n\n"),+ }+ } else {+ parsed_pages+ .iter()+ .map(|p| p.text.as_str())+ .collect::<Vec<_>>()+ .join("\n\n")+ };++ Ok(ParseResult {++ total_pages,+ pages: parsed_pages,++ page_errors: Vec::new(),+ text: full_text,+ outline,+ images,++ screenshots: Vec::new(),+ image_error_count: 0,+ form_type: None,+ creator: None,+ producer: None,+ doc_meta: None,+ // Office containers carry no XFA packets; `Some([])` keeps the+ // JSON shape identical to the conversion path, which also finds+ // none in a LibreOffice-produced PDF.+ xfa_packets: self.config.extract_xfa_packets.then(Vec::new),+ })+ }+/// Generate screenshots of document pages as PNG bytes.////// Non-PDF files are automatically converted to PDF first (requires@@@ -970,75 -1197,6 +1334,75 @@@ mod testsassert!(item.trailing_space_generated);}+ /// `parse_from_blocks`: markdown comes from the block model, page text+ /// from grid projection over the text items, and the doc-level markdown+ /// renders from `all_blocks` rather than a page join.+ #[test]+ fn parse_from_blocks_renders_blocks_and_projects_text() {- use crate::markdown_layout::Block;++ use crate::markdown_layout::{Block, PositionedBlock};+- let blocks = vec![Block::Heading {++ let blocks = vec![PositionedBlock::unlocated(Block::Heading {+ level: 1,+ text: "hello".into(),- }];++ })];+ let result = LiteParse::new(LiteParseConfig {+ output_format: crate::config::OutputFormat::Markdown,+ ..Default::default()+ })+ .parse_from_blocks(+ vec![page_with_text_metadata()],+ vec![blocks.clone()],+ Some(blocks),+ vec![],+ vec![],+ vec![None],+ )+ .expect("no page filter configured");+ assert_eq!(result.pages.len(), 1);+ assert_eq!(result.pages[0].markdown, "# hello");+ assert_eq!(result.text, "# hello");+ // Projection, not a reading-order join: the projected page text+ // carries the item's text content.+ assert!(result.pages[0].text.contains("hello"));+ }++ /// Page filtering drops `all_blocks` (a doc-level render would leak+ /// filtered pages' content) and falls back to joining survivors.+ #[test]+ fn parse_from_blocks_page_filter_drops_all_blocks() {- use crate::markdown_layout::Block;++ use crate::markdown_layout::{Block, PositionedBlock};++ let mut p2 = page_with_text_metadata();+ p2.page_number = 2;+ p2.text_items[0].text = "world".into();- let b1 = vec![Block::Heading {++ let b1 = vec![PositionedBlock::unlocated(Block::Heading {+ level: 1,+ text: "hello".into(),- }];- let b2 = vec![Block::Heading {++ })];++ let b2 = vec![PositionedBlock::unlocated(Block::Heading {+ level: 1,+ text: "world".into(),- }];++ })];+ let all = b1.iter().chain(&b2).cloned().collect::<Vec<_>>();+ let result = LiteParse::new(LiteParseConfig {+ output_format: crate::config::OutputFormat::Markdown,+ target_pages: Some("2".into()),+ ..Default::default()+ })+ .parse_from_blocks(+ vec![page_with_text_metadata(), p2],+ vec![b1, b2],+ Some(all),+ vec![],+ vec![],+ vec![None, None],+ )+ .expect("valid page filter");+ assert_eq!(result.pages.len(), 1);+ assert_eq!(result.pages[0].page_number, 2);+ assert_eq!(result.text, "# world");+ }+#[test]fn image_extraction_is_opt_in_but_embed_mode_implies_it() {let default = LiteParseConfig::default();