Skip to content

A fast, helpful, and open-source document parser

Merge origin/main into logan/native-blocks-api

Logan Markewichdfcbaba919fa
diff --cc Cargo.lock
index 2887c22,a64fcc2..5abe237
--- a/Cargo.lock
+++ b/Cargo.lock
@@@ -2016,7 -2016,7 +2016,7 @@@ checksum = "92daf443525c4cce67b150400bc
[[package]]
name = "liteparse"
- version = "2.11.0"
-version = "2.14.1"
++version = "2.14.2"
dependencies = [
"blake3",
"clap",
@@@ -2048,7 -2048,7 +2048,7 @@@
[[package]]
name = "liteparse-napi"
- version = "2.11.0"
-version = "2.14.1"
++version = "2.14.2"
dependencies = [
"image",
"liteparse",
@@@ -2064,7 -2064,7 +2064,7 @@@
[[package]]
name = "liteparse-pdfium"
- version = "1.5.0"
-version = "1.8.0"
++version = "1.9.0"
dependencies = [
"blake3",
"liteparse-pdfium-sys",
@@@ -2072,7 -2072,7 +2072,7 @@@
[[package]]
name = "liteparse-pdfium-sys"
- version = "1.5.0"
-version = "1.8.0"
++version = "1.9.0"
dependencies = [
"bindgen",
"flate2",
@@@ -2083,7 -2083,7 +2083,7 @@@
[[package]]
name = "liteparse-python"
- version = "2.11.0"
-version = "2.14.1"
++version = "2.14.2"
dependencies = [
"anyhow",
"clap",
@@@ -2099,7 -2099,7 +2099,7 @@@
[[package]]
name = "liteparse-wasm"
- version = "2.11.0"
-version = "2.14.1"
++version = "2.14.2"
dependencies = [
"console_error_panic_hook",
"js-sys",
diff --cc crates/liteparse/src/layout.rs
index 0000000,0bf9fde..569a2bb
mode 000000,100644..100644
--- a/crates/liteparse/src/layout.rs
+++ b/crates/liteparse/src/layout.rs
@@@ -1,0 -1,176 +1,211 @@@
+ //! Public, serializable view of the markdown classifier's block decomposition.
+ //!
+ //! The classifier's internal [`Block`](crate::markdown_layout::Block) is a Rust
+ //! enum with per-variant payloads. That shape can't cross the napi or PyO3
+ //! boundary — neither supports enums carrying data — so the public type is a
+ //! single flat struct discriminated by `kind`, with the variant-specific fields
+ //! as `Option`s. This mirrors how the crate already exposes
+ //! `StructureAttributeValue` to the bindings, and keeps one block shape across
+ //! Rust, JSON, Node, Python, and WASM instead of three divergent ones.
+ //!
+ //! Every field that doesn't apply to a block's `kind` is `None` and is skipped
+ //! during serialization, so a heading serializes as `{kind, text, level, bbox}`
+ //! rather than a wall of nulls.
+
+ use serde::Serialize;
+
-use crate::markdown_layout::{Block, Cell, PositionedBlock};
++use crate::markdown_layout::{Block, Cell, PositionedBlock, SpanCell};
+ use crate::types::Rect;
+
+ /// One table cell: its rendered text and the region it occupied.
+ ///
+ /// `bbox` is `None` for cells with no ink behind them — padding inserted to
+ /// square off a ragged grid, or halves of a merged run split at an estimated
+ /// position rather than an observed boundary.
+ #[derive(Debug, Clone, Serialize)]
+ pub struct LayoutCell {
+ pub text: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub bbox: Option<Rect>,
++ /// Merge spans, present only on `merged_table` cells (and only when > 1).
++ /// A cell absorbed by a neighbour's span is absent from its row entirely,
++ /// matching HTML's occupancy model.
++ #[serde(skip_serializing_if = "Option::is_none")]
++ pub colspan: Option<u16>,
++ #[serde(skip_serializing_if = "Option::is_none")]
++ pub rowspan: Option<u16>,
+ }
+
+ impl From<&Cell> for LayoutCell {
+ fn from(c: &Cell) -> Self {
+ LayoutCell {
+ text: c.text.clone(),
+ bbox: c.bbox.clone(),
++ colspan: None,
++ rowspan: None,
++ }
++ }
++}
++
++impl From<&SpanCell> for LayoutCell {
++ fn from(c: &SpanCell) -> Self {
++ LayoutCell {
++ text: c.text.clone(),
++ bbox: None,
++ colspan: (c.colspan > 1).then_some(c.colspan),
++ rowspan: (c.rowspan > 1).then_some(c.rowspan),
+ }
+ }
+ }
+
+ /// A classified block plus where it sits on the page.
+ ///
+ /// `kind` discriminates the block; see each field for which kinds populate it.
+ /// Blocks appear in reading order, matching the order the markdown renderer
+ /// emits them, so the Nth block here is the Nth block of that page's markdown.
+ #[derive(Debug, Clone, Serialize)]
+ pub struct LayoutBlock {
+ /// One of `heading`, `paragraph`, `list_item`, `code`, `table`,
- /// `grid_fallback`, `rule`, `figure`.
++ /// `merged_table`, `grid_fallback`, `rule`, `figure`.
+ pub kind: &'static str,
+ /// Rendered text for the text-bearing kinds (`heading`, `paragraph`,
+ /// `list_item`). Table text lives in `header`/`rows`; code and grid text in
+ /// `lines`.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub text: Option<String>,
+ /// Heading level (1–6), or list nesting depth for `list_item`.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub level: Option<u8>,
+ /// Whether the block's text is uniformly bold / italic. `paragraph` and
+ /// `list_item` only; omitted when false.
+ #[serde(skip_serializing_if = "std::ops::Not::not")]
+ pub bold: bool,
+ #[serde(skip_serializing_if = "std::ops::Not::not")]
+ pub italic: bool,
+ /// `list_item`: whether the list is ordered, and the original marker as it
+ /// appeared on the page (`138.`, `iii)`, `•`).
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub ordered: Option<bool>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub marker: Option<String>,
+ /// Verbatim source lines for `code` and `grid_fallback`.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub lines: Option<Vec<String>>,
+ /// Best-effort language hint for `code`.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub lang: Option<String>,
+ /// `table`: the header row, when one was detected.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub header: Option<Vec<LayoutCell>>,
- /// `table`: the body rows.
++ /// `table`: the body rows. `merged_table`: all rows (header rows lead,
++ /// counted by `header_rows`); rows are ragged — cells covered by a
++ /// neighbour's span are absent.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub rows: Option<Vec<Vec<LayoutCell>>>,
++ /// `merged_table`: how many leading rows of `rows` are header rows.
++ #[serde(skip_serializing_if = "Option::is_none")]
++ pub header_rows: Option<usize>,
+ /// `figure`: the image's page-scoped id and encoded format, matching the
+ /// `img_{id}.{format}` target the markdown renderer emits.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub id: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub format: Option<String>,
+ /// Region of the page this block occupies, in the same top-left, 72-DPI
+ /// viewport space as `text_items`. The union of every source line that fed
+ /// the block, so a wrapped heading or multi-line paragraph reports its full
+ /// band. `None` when the block has no page geometry behind it.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub bbox: Option<Rect>,
+ }
+
+ impl LayoutBlock {
+ /// Base value with every variant-specific field cleared.
+ fn of(kind: &'static str, bbox: Option<Rect>) -> Self {
+ LayoutBlock {
+ kind,
+ text: None,
+ level: None,
+ bold: false,
+ italic: false,
+ ordered: None,
+ marker: None,
+ lines: None,
+ lang: None,
+ header: None,
+ rows: None,
++ header_rows: None,
+ id: None,
+ format: None,
+ bbox,
+ }
+ }
+ }
+
+ fn cells(row: &[Cell]) -> Vec<LayoutCell> {
+ row.iter().map(LayoutCell::from).collect()
+ }
+
+ impl From<&PositionedBlock> for LayoutBlock {
+ fn from(pb: &PositionedBlock) -> Self {
+ let bbox = pb.bbox.clone();
+ match &pb.block {
+ Block::Heading { level, text } => LayoutBlock {
+ text: Some(text.clone()),
+ level: Some(*level),
+ ..LayoutBlock::of("heading", bbox)
+ },
+ Block::Paragraph { text, bold, italic } => LayoutBlock {
+ text: Some(text.clone()),
+ bold: *bold,
+ italic: *italic,
+ ..LayoutBlock::of("paragraph", bbox)
+ },
+ Block::ListItem {
+ ordered,
+ marker,
+ level,
+ text,
+ bold,
+ italic,
+ } => LayoutBlock {
+ text: Some(text.clone()),
+ level: Some(*level),
+ ordered: Some(*ordered),
+ marker: Some(marker.clone()),
+ bold: *bold,
+ italic: *italic,
+ ..LayoutBlock::of("list_item", bbox)
+ },
+ Block::CodeBlock { lines, lang } => LayoutBlock {
+ lines: Some(lines.clone()),
+ lang: lang.clone(),
+ ..LayoutBlock::of("code", bbox)
+ },
+ Block::Table { header, rows } => LayoutBlock {
+ header: header.as_ref().map(|h| cells(h)),
+ rows: Some(rows.iter().map(|r| cells(r)).collect()),
+ ..LayoutBlock::of("table", bbox)
+ },
++ Block::MergedTable { rows, header_rows } => LayoutBlock {
++ header_rows: Some(*header_rows),
++ rows: Some(
++ rows.iter()
++ .map(|r| r.iter().map(LayoutCell::from).collect())
++ .collect(),
++ ),
++ ..LayoutBlock::of("merged_table", bbox)
++ },
+ Block::GridFallback { lines } => LayoutBlock {
+ lines: Some(lines.clone()),
+ ..LayoutBlock::of("grid_fallback", bbox)
+ },
+ Block::HorizontalRule => LayoutBlock::of("rule", bbox),
+ Block::Figure { id, format } => LayoutBlock {
+ id: Some(id.clone()),
+ format: Some(format.clone()),
+ ..LayoutBlock::of("figure", bbox)
+ },
+ }
+ }
+ }
diff --cc crates/liteparse/src/markdown_layout/blocks.rs
index 3d184f5,ece6f05..d0c155e
--- a/crates/liteparse/src/markdown_layout/blocks.rs
+++ b/crates/liteparse/src/markdown_layout/blocks.rs
@@@ -1,7 -1,82 +1,120 @@@
use super::inline::escape_inline;
use super::paragraphs::{ParaAccum, is_soft_hyphen_break};
use super::tables::escape_table_cell;
+ use crate::types::Rect;
+
+ /// One table cell: rendered text plus, when the cell came from real page
+ /// content, the region it occupied. `bbox` is `None` for cells that exist only
+ /// to square off a ragged grid (padding inserted when rows disagree on column
+ /// count) — those occupy no ink on the page, and reporting a rect for them
+ /// would invent geometry the classifier never saw.
+ #[derive(Debug, Clone, Default)]
+ pub struct Cell {
+ pub text: String,
+ pub bbox: Option<Rect>,
+ }
+
+ impl Cell {
+ /// Cell carrying text and the region it was read from.
+ pub fn located(text: impl Into<String>, bbox: Rect) -> Self {
+ Cell {
+ text: text.into(),
+ bbox: Some(bbox),
+ }
+ }
+
+ /// Borrow the cell's text. Lets call sites that only care about content
+ /// read a `Cell` about as tersely as the `String` it replaced.
+ pub fn as_str(&self) -> &str {
+ &self.text
+ }
+ }
+
+ // Text-only construction (`"a".into()`, `s.to_string().into()`)
+ impl From<&str> for Cell {
+ fn from(text: &str) -> Self {
+ Cell {
+ text: text.to_string(),
+ bbox: None,
+ }
+ }
+ }
+
+ impl From<String> for Cell {
+ fn from(text: String) -> Self {
+ Cell { text, bbox: None }
+ }
+ }
+
+ // Cells compare by content alone. A cell's box is derived metadata describing
+ // where the text was read from, not part of its identity — two cells holding
+ // the same text are the same table content whether or not either one knows its
+ // coordinates. This is also what lets assertions compare a detected grid
+ // against a plain literal of expected strings.
+ impl PartialEq for Cell {
+ fn eq(&self, other: &Self) -> bool {
+ self.text == other.text
+ }
+ }
+
+ impl Eq for Cell {}
+
+ impl PartialEq<str> for Cell {
+ fn eq(&self, other: &str) -> bool {
+ self.text == other
+ }
+ }
+
+ impl PartialEq<&str> for Cell {
+ fn eq(&self, other: &&str) -> bool {
+ self.text == *other
+ }
+ }
+
+ impl PartialEq<String> for Cell {
+ fn eq(&self, other: &String) -> bool {
+ self.text == *other
+ }
+ }
+
++/// One cell of a [`Block::MergedTable`]. `colspan`/`rowspan` are 1 for an
++/// ordinary cell; the cells they cover are absent from `rows` rather than
++/// present-and-empty, matching HTML's occupancy model. Distinct from [`Cell`]:
++/// that type carries the page geometry a cell was *read from*, this one the
++/// merge structure a producer *declared*.
++#[derive(Debug, Clone, PartialEq, Eq)]
++pub struct SpanCell {
++ pub text: String,
++ pub colspan: u16,
++ pub rowspan: u16,
++}
++
++impl SpanCell {
++ /// A plain 1x1 cell.
++ pub fn new(text: impl Into<String>) -> Self {
++ Self {
++ text: text.into(),
++ colspan: 1,
++ rowspan: 1,
++ }
++ }
++
++ /// A cell spanning `colspan` columns and `rowspan` rows. Zero is coerced to
++ /// 1 — a zero span is meaningless in HTML and `rowspan="0"` means "to the
++ /// end of the section", which is not what any producer here intends.
++ pub fn spanning(text: impl Into<String>, colspan: u16, rowspan: u16) -> Self {
++ Self {
++ text: text.into(),
++ colspan: colspan.max(1),
++ rowspan: rowspan.max(1),
++ }
++ }
++
++ fn is_plain(&self) -> bool {
++ self.colspan <= 1 && self.rowspan <= 1
++ }
++}
+
/// Coarse block representation: the output of page classification, consumed by
/// `render_blocks` to produce the final markdown string.
#[derive(Debug, Clone)]
@@@ -36,23 -111,9 +149,23 @@@ pub enum Block
/// when the first row didn't qualify (e.g. wasn't bold and the table mode
/// can't otherwise distinguish it).
Table {
- header: Option<Vec<String>>,
- rows: Vec<Vec<String>>,
+ header: Option<Vec<Cell>>,
+ rows: Vec<Vec<Cell>>,
},
+ /// Table whose cells carry explicit spans. Emitted by producers that know
+ /// the merge structure outright (e.g. a structural reader with access to
+ /// `gridSpan` / `vMerge`) rather than inferring it from coordinates.
+ ///
+ /// `rows` is *not* rectangular: a cell absorbed by a neighbour's
+ /// `colspan`/`rowspan` is omitted entirely, exactly as in HTML. `header_rows`
+ /// is the count of leading rows that are header rows (`<th>`); 0 means none.
+ ///
+ /// Renders as a GFM pipe table when it degenerates to a plain grid, and as
+ /// HTML `<table>` otherwise — pipe tables cannot express spans at all.
+ MergedTable {
- rows: Vec<Vec<Cell>>,
++ rows: Vec<Vec<SpanCell>>,
+ header_rows: usize,
+ },
/// Tabular-looking region we couldn't classify confidently — rendered
/// verbatim inside a fenced block to preserve visual structure for the
/// downstream LLM. Strictly better than emitting a mangled pipe table.
@@@ -146,127 -171,90 +223,181 @@@ fn wrap_emphasis(text: &str, bold: bool
}
}
+ /// A classified block plus the region of the page it was read from.
+ ///
+ /// `bbox` is the union of the boxes of every source line that fed the block, so
+ /// a wrapped heading or a multi-line paragraph reports the whole band it
+ /// occupies. It is `None` only for blocks with no page geometry behind them —
+ /// synthesized content, or sources that never carried coordinates.
+ #[derive(Debug, Clone)]
+ pub struct PositionedBlock {
+ pub block: Block,
+ pub bbox: Option<Rect>,
+ }
+
+ impl PositionedBlock {
+ pub fn new(block: Block, bbox: Option<Rect>) -> Self {
+ PositionedBlock { block, bbox }
+ }
+
+ /// A block with no known geometry.
+ pub fn unlocated(block: Block) -> Self {
+ PositionedBlock { block, bbox: None }
+ }
+
+ /// Grow this block's box to also cover `other`.
+ fn absorb(&mut self, other: &Option<Rect>) {
+ if let Some(r) = other {
+ Rect::extend(&mut self.bbox, r);
+ }
+ }
+ }
+
+ /// Heal words hyphenated across a soft line wrap that the classifier split into
+ /// two *separate* paragraph blocks: `…they dis-` ‖ `lodged…`. When a plain
+ /// paragraph ends with `<letter>-` and the next is a plain paragraph starting
+ /// lowercase, fuse them — dropping the hyphen and joining with no separator —
+ /// and union their boxes, since the result spans both source regions.
+ ///
+ /// This runs as a pass over the block list rather than inside the renderer so
+ /// that the blocks callers receive are the same ones that produced the
+ /// markdown. The lowercase and plain-text gates keep real compounds (`well-`
+ /// then a capitalized `Known`) and emphasised/heading/table starts intact.
+ pub fn splice_soft_hyphens(blocks: Vec<PositionedBlock>) -> Vec<PositionedBlock> {
+ let mut out: Vec<PositionedBlock> = Vec::with_capacity(blocks.len());
+ for pb in blocks {
+ // Both sides must be unemphasised: an emphasised predecessor renders
+ // with a trailing `**`/`*`, so its final character was never the
+ // hyphen this splice keys off.
+ let joinable = match (out.last().map(|p| &p.block), &pb.block) {
+ (
+ Some(Block::Paragraph {
+ text: prev,
+ bold: false,
+ italic: false,
+ }),
+ Block::Paragraph {
+ text,
+ bold: false,
+ italic: false,
+ },
+ ) => is_soft_hyphen_break(prev, text).then(|| text.clone()),
+ _ => None,
+ };
+ if let Some(tail) = joinable {
+ let prev = out.last_mut().expect("gate matched on a previous block");
+ if let Block::Paragraph { text, .. } = &mut prev.block {
+ while text.ends_with(|c: char| c.is_whitespace()) {
+ text.pop();
+ }
+ text.pop(); // the soft hyphen
+ text.push_str(&tail);
+ }
+ prev.absorb(&pb.bbox);
+ continue;
+ }
+ out.push(pb);
+ }
+ out
+ }
+
+/// Render a GFM pipe table. `head` is the header row (already chosen by the
+/// caller); `body` the remaining rows. No-ops when there are no columns.
- fn render_pipe_table(head: Option<&[String]>, body: &[Vec<String>], out: &mut String) {
++fn render_pipe_table(head: Option<&[Cell]>, body: &[Vec<Cell>], out: &mut String) {
+ let column_count = head.map(|h| h.len()).unwrap_or(0);
+ if column_count == 0 {
+ return;
+ }
+ out.push_str("| ");
+ for (i, cell) in head.unwrap().iter().enumerate() {
+ if i > 0 {
+ out.push_str(" | ");
+ }
- out.push_str(&escape_table_cell(cell));
++ out.push_str(&escape_table_cell(&cell.text));
+ }
+ out.push_str(" |\n");
+ out.push('|');
+ for _ in 0..column_count {
+ out.push_str("---|");
+ }
+ for row in body {
+ out.push_str("\n| ");
+ for (i, cell) in row.iter().enumerate() {
+ if i > 0 {
+ out.push_str(" | ");
+ }
- out.push_str(&escape_table_cell(cell));
++ out.push_str(&escape_table_cell(&cell.text));
+ }
+ out.push_str(" |");
+ }
+}
+
+/// Escape a cell for HTML `<td>` content. Markdown specials are deliberately
+/// left alone — inside an HTML block they are literal text, and escaping them
+/// would surface backslashes in the rendered output.
+fn escape_html_cell(s: &str) -> String {
+ let mut out = String::with_capacity(s.len());
+ for c in s.chars() {
+ match c {
+ '&' => out.push_str("&amp;"),
+ '<' => out.push_str("&lt;"),
+ '>' => out.push_str("&gt;"),
+ '\n' => out.push(' '),
+ _ => out.push(c),
+ }
+ }
+ out
+}
+
+/// True when a merged table degenerates to a plain rectangular grid, in which
+/// case the pipe rendering is both valid and nicer to read.
- fn is_plain_grid(rows: &[Vec<Cell>], header_rows: usize) -> bool {
++fn is_plain_grid(rows: &[Vec<SpanCell>], header_rows: usize) -> bool {
+ // GFM has exactly one header row, or none.
+ if header_rows > 1 {
+ return false;
+ }
- if !rows.iter().all(|r| r.iter().all(Cell::is_plain)) {
++ if !rows.iter().all(|r| r.iter().all(SpanCell::is_plain)) {
+ return false;
+ }
+ // Ragged rows can only have come from spans we've just ruled out, but check
+ // anyway: a pipe table with uneven rows renders as a broken table.
+ let width = rows.first().map(|r| r.len()).unwrap_or(0);
+ rows.iter().all(|r| r.len() == width)
+}
+
+/// Render a merged table as HTML. Cells covered by a neighbour's span are
+/// simply absent from `rows`, so this is a direct transcription.
- fn render_html_table(rows: &[Vec<Cell>], header_rows: usize, out: &mut String) {
++fn render_html_table(rows: &[Vec<SpanCell>], header_rows: usize, out: &mut String) {
+ out.push_str("<table>");
+ for (r, row) in rows.iter().enumerate() {
+ out.push_str("\n<tr>");
+ let tag = if r < header_rows { "th" } else { "td" };
+ for cell in row {
+ out.push_str("\n<");
+ out.push_str(tag);
+ if cell.colspan > 1 {
+ out.push_str(&format!(" colspan=\"{}\"", cell.colspan));
+ }
+ if cell.rowspan > 1 {
+ out.push_str(&format!(" rowspan=\"{}\"", cell.rowspan));
+ }
+ out.push('>');
+ out.push_str(&escape_html_cell(&cell.text));
+ out.push_str("</");
+ out.push_str(tag);
+ out.push('>');
+ }
+ out.push_str("\n</tr>");
+ }
+ out.push_str("\n</table>");
+}
+
/// Render a list of blocks to a markdown string.
- pub fn render_blocks(blocks: &[Block]) -> String {
+ pub fn render_blocks(blocks: &[PositionedBlock]) -> String {
let mut out = String::new();
- for (i, block) in blocks.iter().enumerate() {
+ for (i, positioned) in blocks.iter().enumerate() {
+ let block = &positioned.block;
if i > 0 {
- // A word hyphenated across a soft line wrap sometimes lands in two
- // *separate* paragraph blocks (the classifier mis-split mid-word):
- // `…they dis-` ‖ `lodged…`. When the previous block ends with
- // `<letter>-` and this block is a plain paragraph beginning with a
- // lowercase letter, splice them: drop the hyphen and join with no
- // separator, healing both the word and the spurious break. The
- // lowercase + plain-text gates keep real compounds (`well-` then a
- // capitalized `Known`) and emphasised/heading/table starts intact.
- if let (
- Block::Paragraph { .. },
- Block::Paragraph {
- text,
- bold: false,
- italic: false,
- },
- ) = (&blocks[i - 1], block)
- && is_soft_hyphen_break(&out, text)
- {
- while out.ends_with(|c: char| c.is_whitespace()) {
- out.pop();
- }
- out.pop(); // the soft hyphen
- out.push_str(text);
- continue;
- }
// Consecutive list items render as a tight list (single newline).
// Everything else gets a blank line between blocks.
let tight = matches!(block, Block::ListItem { .. })
@@@ -322,30 -310,31 +453,31 @@@
None => (None, rows.as_slice()),
},
};
- let column_count = head.map(|h| h.len()).unwrap_or(0);
- if column_count == 0 {
+ render_pipe_table(head, body, &mut out);
+ }
+ Block::MergedTable { rows, header_rows } => {
+ if rows.is_empty() {
continue;
}
- out.push_str("| ");
- for (i, cell) in head.unwrap().iter().enumerate() {
- if i > 0 {
- out.push_str(" | ");
- }
- out.push_str(&escape_table_cell(&cell.text));
- }
- out.push_str(" |\n");
- out.push('|');
- for _ in 0..column_count {
- out.push_str("---|");
- }
- for row in body {
- out.push_str("\n| ");
- for (i, cell) in row.iter().enumerate() {
- if i > 0 {
- out.push_str(" | ");
- }
- out.push_str(&escape_table_cell(&cell.text));
- }
- out.push_str(" |");
+ if is_plain_grid(rows, *header_rows) {
+ // Degenerate case — no merges, so pipes are expressive
- // enough. Reuse the exact same rendering the PDF path gets,
- // including the promote-first-row-when-headerless rule.
- let plain: Vec<Vec<String>> = rows
++ // enough. Reuse the exact same rendering `Block::Table`
++ // gets, including the promote-first-row-when-headerless
++ // rule.
++ let plain: Vec<Vec<Cell>> = rows
+ .iter()
- .map(|r| r.iter().map(|c| c.text.clone()).collect())
++ .map(|r| r.iter().map(|c| Cell::from(c.text.clone())).collect())
+ .collect();
+ // `header_rows` is 0 or 1 here (>1 forces HTML). Either way
+ // the first row leads: as the real header, or promoted into
+ // one because GFM has no headerless table.
+ let (head, body) = match plain.split_first() {
+ Some((first, rest)) => (Some(first.as_slice()), rest),
+ None => (None, plain.as_slice()),
+ };
+ render_pipe_table(head, body, &mut out);
+ } else {
+ render_html_table(rows, *header_rows, &mut out);
}
}
Block::GridFallback { lines } => {
@@@ -518,101 -515,42 +658,132 @@@ mod tests
bold: false,
italic: false,
};
+ let spliced = |a: &str, b: &str| {
+ splice_soft_hyphens(vec![
+ PositionedBlock::unlocated(p(a)),
+ PositionedBlock::unlocated(p(b)),
+ ])
+ };
+ let rendered = |a: &str, b: &str| render_blocks(&spliced(a, b));
// Mid-word hyphen split into two paragraphs heals into one.
- let s = render_blocks(&[p("they dis-"), p("lodged the part")]);
- assert_eq!(s, "they dislodged the part");
+ assert_eq!(
+ rendered("they dis-", "lodged the part"),
+ "they dislodged the part"
+ );
// Capitalized continuation is a real compound break — left intact.
- let s = render_blocks(&[p("the well-"), p("Known fact")]);
- assert_eq!(s, "the well-\n\nKnown fact");
+ assert_eq!(
+ rendered("the well-", "Known fact"),
+ "the well-\n\nKnown fact"
+ );
// Trailing dash not preceded by a letter doesn't splice.
- let s = render_blocks(&[p("a -"), p("dash line")]);
- assert_eq!(s, "a -\n\ndash line");
+ assert_eq!(rendered("a -", "dash line"), "a -\n\ndash line");
+ // A spliced block covers both source regions; an unspliced pair stays
+ // as two blocks with their own boxes.
+ let r = |y: f32| Rect {
+ x: 10.0,
+ y,
+ width: 100.0,
+ height: 12.0,
+ };
+ let joined = splice_soft_hyphens(vec![
+ PositionedBlock::new(p("they dis-"), Some(r(50.0))),
+ PositionedBlock::new(p("lodged the part"), Some(r(70.0))),
+ ]);
+ assert_eq!(joined.len(), 1);
+ let bbox = joined[0].bbox.clone().expect("merged block keeps geometry");
+ assert_eq!((bbox.y, bbox.height), (50.0, 32.0));
}
+ #[test]
+ fn merged_table_without_spans_renders_as_pipes() {
+ // A merge-aware producer that happens to emit a plain grid must be
+ // byte-identical to the PDF path's pipe table — otherwise the same
+ // table looks different depending on which pipeline produced it.
+ let plain = Block::Table {
+ header: Some(vec!["a".into(), "b".into()]),
+ rows: vec![vec!["1".into(), "2".into()]],
+ };
+ let merged = Block::MergedTable {
+ header_rows: 1,
+ rows: vec![
- vec![Cell::new("a"), Cell::new("b")],
- vec![Cell::new("1"), Cell::new("2")],
++ vec![SpanCell::new("a"), SpanCell::new("b")],
++ vec![SpanCell::new("1"), SpanCell::new("2")],
+ ],
+ };
- assert_eq!(render_blocks(&[plain]), render_blocks(&[merged]));
++ assert_eq!(render(vec![plain]), render(vec![merged]));
+ }
+
+ #[test]
+ fn merged_table_with_spans_renders_as_html() {
+ // Cells covered by a span are absent, not empty — row 1 has one cell.
- let s = render_blocks(&[Block::MergedTable {
++ let s = render(vec![Block::MergedTable {
+ header_rows: 1,
+ rows: vec![
- vec![Cell::spanning("wide", 2, 1)],
- vec![Cell::spanning("tall", 1, 2), Cell::new("x")],
- vec![Cell::new("y")],
++ vec![SpanCell::spanning("wide", 2, 1)],
++ vec![SpanCell::spanning("tall", 1, 2), SpanCell::new("x")],
++ vec![SpanCell::new("y")],
+ ],
+ }]);
+ assert_eq!(
+ s,
+ "<table>\n<tr>\n<th colspan=\"2\">wide</th>\n</tr>\n\
+ <tr>\n<td rowspan=\"2\">tall</td>\n<td>x</td>\n</tr>\n\
+ <tr>\n<td>y</td>\n</tr>\n</table>"
+ );
+ }
+
+ #[test]
+ fn merged_table_escapes_html_not_markdown() {
+ // Inside an HTML block, markdown specials are literal; `<` is not.
- let s = render_blocks(&[Block::MergedTable {
++ let s = render(vec![Block::MergedTable {
+ header_rows: 0,
- rows: vec![vec![Cell::new("a|b *c* <d> &e"), Cell::spanning("f", 2, 1)]],
++ rows: vec![vec![
++ SpanCell::new("a|b *c* <d> &e"),
++ SpanCell::spanning("f", 2, 1),
++ ]],
+ }]);
+ assert!(s.contains("<td>a|b *c* &lt;d&gt; &amp;e</td>"), "{s}");
+ }
+
+ #[test]
+ fn merged_table_multiple_header_rows_force_html() {
+ // Two header rows have no pipe-table representation even with no spans.
- let s = render_blocks(&[Block::MergedTable {
++ let s = render(vec![Block::MergedTable {
+ header_rows: 2,
+ rows: vec![
- vec![Cell::new("a"), Cell::new("b")],
- vec![Cell::new("c"), Cell::new("d")],
- vec![Cell::new("1"), Cell::new("2")],
++ vec![SpanCell::new("a"), SpanCell::new("b")],
++ vec![SpanCell::new("c"), SpanCell::new("d")],
++ vec![SpanCell::new("1"), SpanCell::new("2")],
+ ],
+ }]);
+ assert!(s.starts_with("<table>"), "{s}");
+ assert_eq!(s.matches("<th>").count(), 4);
+ assert_eq!(s.matches("<td>").count(), 2);
+ }
+
+ #[test]
+ fn merged_table_ragged_rows_force_html() {
+ // Uneven row widths with no declared spans would render as a broken
+ // pipe table; HTML at least preserves the cells.
- let s = render_blocks(&[Block::MergedTable {
++ let s = render(vec![Block::MergedTable {
+ header_rows: 1,
- rows: vec![vec![Cell::new("a"), Cell::new("b")], vec![Cell::new("1")]],
++ rows: vec![
++ vec![SpanCell::new("a"), SpanCell::new("b")],
++ vec![SpanCell::new("1")],
++ ],
+ }]);
+ assert!(s.starts_with("<table>"), "{s}");
+ }
+
+ #[test]
+ fn zero_span_is_coerced_to_one() {
+ // `rowspan="0"` means "to the end of the section" in HTML — never what
+ // a producer means here, so it must not survive into the output.
- let c = Cell::spanning("x", 0, 0);
++ let c = SpanCell::spanning("x", 0, 0);
+ assert_eq!((c.colspan, c.rowspan), (1, 1));
+ assert!(c.is_plain());
+ }
+
#[test]
fn render_table_without_header_promotes_first_row() {
let blocks = vec![Block::Table {
diff --cc crates/liteparse/src/markdown_layout/mod.rs
index a02ede1,2865366..6c9534c
--- a/crates/liteparse/src/markdown_layout/mod.rs
+++ b/crates/liteparse/src/markdown_layout/mod.rs
@@@ -18,10 -18,9 +18,10 @@@ mod paragraphs
mod repetition;
mod tables;
- pub use blocks::{Block, Cell, render_blocks};
-pub use blocks::{Block, Cell, PositionedBlock, render_blocks, splice_soft_hyphens};
++pub use blocks::{Block, Cell, PositionedBlock, SpanCell, render_blocks, splice_soft_hyphens};
pub use classify::classify_page_with_filters;
pub use headings::{build_heading_map, compute_body_size};
+pub use inline::{apply_link, escape_inline};
pub use repetition::{compute_header_footer_set, detect_single_page_chrome};
pub use tables::detect_table_rects;
pub(crate) use tables::{count_text_table_runs, validated_ruled_table_rects};
diff --cc crates/liteparse/src/parser.rs
index 8b68006,d163d40..9b33177
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@@ -716,125 -950,6 +950,143 @@@ impl LiteParse
}
}
+ /// Page selection for `parse_from_blocks`: `target_pages` filters by
+ /// 1-based page number, then `max_pages` truncates. Returns the aligned
+ /// `(pages, page_blocks, page_stats)` triple plus whether any filtering
+ /// occurred — the caller drops the doc-level `all_blocks` when it did.
+ fn select_block_pages(
+ &self,
+ pages: Vec<Page>,
- page_blocks: Vec<Vec<crate::markdown_layout::Block>>,
++ page_blocks: Vec<Vec<crate::markdown_layout::PositionedBlock>>,
+ complexity: Vec<Option<crate::ocr_merge::PageComplexityStats>>,
+ ) -> Result<
+ (
+ Vec<Page>,
- Vec<Vec<crate::markdown_layout::Block>>,
++ Vec<Vec<crate::markdown_layout::PositionedBlock>>,
+ Vec<Option<crate::ocr_merge::PageComplexityStats>>,
+ bool,
+ ),
+ LiteParseError,
+ > {
- let mut selected: Vec<((Page, Vec<crate::markdown_layout::Block>), _)> =
++ let mut selected: Vec<((Page, Vec<crate::markdown_layout::PositionedBlock>), _)> =
+ pages.into_iter().zip(page_blocks).zip(complexity).collect();
+ let mut page_filtered = false;
+ if let Some(targets) = self.resolve_target_pages()? {
+ let keep: std::collections::HashSet<usize> =
+ targets.iter().map(|p| *p as usize).collect();
+ selected.retain(|((p, _), _)| keep.contains(&p.page_number));
+ page_filtered = true;
+ }
+ if selected.len() > self.config.max_pages {
+ selected.truncate(self.config.max_pages);
+ page_filtered = true;
+ }
+ let mut pages = Vec::with_capacity(selected.len());
+ let mut page_blocks = Vec::with_capacity(selected.len());
+ let mut page_stats = Vec::with_capacity(selected.len());
+ for ((p, b), s) in selected {
+ pages.push(p);
+ page_blocks.push(b);
+ page_stats.push(s);
+ }
+ Ok((pages, page_blocks, page_stats, page_filtered))
+ }
+
+ /// `parse_from_pages`'s structural sibling: the caller owns block-level
+ /// structure (headings, lists, tables, merged cells) alongside the text
+ /// items, so per-page markdown renders from the block model with
+ /// `render_blocks` while page `.text` still comes from grid projection
+ /// over the supplied text items — the same spatial text layout every
+ /// other path produces. Like `parse_from_pages` this never touches
+ /// PDFium or OCR and is fully synchronous; text items arrive in viewport
+ /// space (top-left origin, 72 DPI). `page_blocks` must align 1:1 with
+ /// `pages`; `page_stats` carries per-page complexity (or `None`s).
++ /// Blocks arrive as [`PositionedBlock`](crate::markdown_layout::
++ /// PositionedBlock)s — a producer without per-block geometry wraps each
++ /// with `PositionedBlock::unlocated`. When `extract_blocks` is on, the
++ /// supplied blocks (and their boxes) also populate each page's `blocks`,
++ /// mirroring the classifier path.
+ ///
+ /// Doc-level markdown renders from `all_blocks` when given, NOT by
+ /// joining the page renders: `render_blocks` is context-sensitive across
+ /// block boundaries (tight lists join with `\n`, soft-hyphen paragraph
+ /// splices join with nothing), so a page-boundary join would perturb
+ /// whitespace wherever a page break falls inside a list. The per-page
+ /// markdown is the paged *view* of the same blocks. `target_pages` /
+ /// `max_pages` filtering drops `all_blocks` and falls back to joining
+ /// the surviving pages.
+ pub fn parse_from_blocks(
+ &self,
+ pages: Vec<Page>,
- page_blocks: Vec<Vec<crate::markdown_layout::Block>>,
- all_blocks: Option<Vec<crate::markdown_layout::Block>>,
++ page_blocks: Vec<Vec<crate::markdown_layout::PositionedBlock>>,
++ all_blocks: Option<Vec<crate::markdown_layout::PositionedBlock>>,
+ outline: Vec<OutlineTarget>,
+ images: Vec<crate::types::ExtractedImage>,
+ page_stats: Vec<Option<crate::ocr_merge::PageComplexityStats>>,
+ ) -> Result<ParseResult, LiteParseError> {
++ // Reported before page filtering, matching every other path.
++ let total_pages = pages.len().min(u32::MAX as usize) as u32;
+ let (pages, page_blocks, page_stats, page_filtered) =
+ self.select_block_pages(pages, page_blocks, page_stats)?;
+ let all_blocks = if page_filtered { None } else { all_blocks };
+
+ let markdown_out = self.config.output_format == crate::config::OutputFormat::Markdown;
+ let mut parsed_pages = projection::project_pages_to_grid(pages);
+ for ((page, blocks), stats) in parsed_pages.iter_mut().zip(&page_blocks).zip(page_stats) {
+ if markdown_out {
+ page.markdown = crate::markdown_layout::render_blocks(blocks);
+ }
++ if self.config.extract_blocks {
++ page.blocks = Some(
++ blocks
++ .iter()
++ .map(crate::layout::LayoutBlock::from)
++ .collect(),
++ );
++ }
+ page.complexity = stats;
+ if !self.config.extract_content_bounds {
+ page.content_bounds = None;
+ }
+ }
+
+ let full_text = if markdown_out {
+ match all_blocks {
+ Some(blocks) => crate::markdown_layout::render_blocks(&blocks),
+ None => parsed_pages
+ .iter()
+ .map(|p| p.markdown.as_str())
+ .collect::<Vec<_>>()
+ .join("\n\n-----\n\n"),
+ }
+ } else {
+ parsed_pages
+ .iter()
+ .map(|p| p.text.as_str())
+ .collect::<Vec<_>>()
+ .join("\n\n")
+ };
+
+ Ok(ParseResult {
++ total_pages,
+ pages: parsed_pages,
++ page_errors: Vec::new(),
+ text: full_text,
+ outline,
+ images,
++ screenshots: Vec::new(),
+ image_error_count: 0,
+ form_type: None,
+ creator: None,
+ producer: None,
+ doc_meta: None,
+ // Office containers carry no XFA packets; `Some([])` keeps the
+ // JSON shape identical to the conversion path, which also finds
+ // none in a LibreOffice-produced PDF.
+ xfa_packets: self.config.extract_xfa_packets.then(Vec::new),
+ })
+ }
+
/// Generate screenshots of document pages as PNG bytes.
///
/// Non-PDF files are automatically converted to PDF first (requires
@@@ -970,75 -1197,6 +1334,75 @@@ mod tests
assert!(item.trailing_space_generated);
}
+ /// `parse_from_blocks`: markdown comes from the block model, page text
+ /// from grid projection over the text items, and the doc-level markdown
+ /// renders from `all_blocks` rather than a page join.
+ #[test]
+ fn parse_from_blocks_renders_blocks_and_projects_text() {
- use crate::markdown_layout::Block;
++ use crate::markdown_layout::{Block, PositionedBlock};
+
- let blocks = vec![Block::Heading {
++ let blocks = vec![PositionedBlock::unlocated(Block::Heading {
+ level: 1,
+ text: "hello".into(),
- }];
++ })];
+ let result = LiteParse::new(LiteParseConfig {
+ output_format: crate::config::OutputFormat::Markdown,
+ ..Default::default()
+ })
+ .parse_from_blocks(
+ vec![page_with_text_metadata()],
+ vec![blocks.clone()],
+ Some(blocks),
+ vec![],
+ vec![],
+ vec![None],
+ )
+ .expect("no page filter configured");
+ assert_eq!(result.pages.len(), 1);
+ assert_eq!(result.pages[0].markdown, "# hello");
+ assert_eq!(result.text, "# hello");
+ // Projection, not a reading-order join: the projected page text
+ // carries the item's text content.
+ assert!(result.pages[0].text.contains("hello"));
+ }
+
+ /// Page filtering drops `all_blocks` (a doc-level render would leak
+ /// filtered pages' content) and falls back to joining survivors.
+ #[test]
+ fn parse_from_blocks_page_filter_drops_all_blocks() {
- use crate::markdown_layout::Block;
++ use crate::markdown_layout::{Block, PositionedBlock};
+
+ let mut p2 = page_with_text_metadata();
+ p2.page_number = 2;
+ p2.text_items[0].text = "world".into();
- let b1 = vec![Block::Heading {
++ let b1 = vec![PositionedBlock::unlocated(Block::Heading {
+ level: 1,
+ text: "hello".into(),
- }];
- let b2 = vec![Block::Heading {
++ })];
++ let b2 = vec![PositionedBlock::unlocated(Block::Heading {
+ level: 1,
+ text: "world".into(),
- }];
++ })];
+ let all = b1.iter().chain(&b2).cloned().collect::<Vec<_>>();
+ let result = LiteParse::new(LiteParseConfig {
+ output_format: crate::config::OutputFormat::Markdown,
+ target_pages: Some("2".into()),
+ ..Default::default()
+ })
+ .parse_from_blocks(
+ vec![page_with_text_metadata(), p2],
+ vec![b1, b2],
+ Some(all),
+ vec![],
+ vec![],
+ vec![None, None],
+ )
+ .expect("valid page filter");
+ assert_eq!(result.pages.len(), 1);
+ assert_eq!(result.pages[0].page_number, 2);
+ assert_eq!(result.text, "# world");
+ }
+
#[test]
fn image_extraction_is_opt_in_but_embed_mode_implies_it() {
let default = LiteParseConfig::default();