A fast, helpful, and open-source document parser
wip
Logan Markewich6ca990eb34a9
diff --git a/crates/liteparse-docx/ATTRIBUTION.md b/crates/liteparse-docx/ATTRIBUTION.mdindex 92a8bf4..4eb1c11 100644--- a/crates/liteparse-docx/ATTRIBUTION.md+++ b/crates/liteparse-docx/ATTRIBUTION.md@@ -66,6 +66,20 @@ render/mod.rs sections: estimate_cursor_y, resolve_and_layout,7. Test-only constructor edits: `FontRegistry::new(skia_safe::FontMgr::new())`→ `FontRegistry::new()` across layout test modules.8. Unknown-element tolerance — see below.+9. **Block→page instrumentation (liteparse-only, no upstream equivalent).**+ `LayoutedPage.block_starts` records, per page, the flattened body-block+ indices (over the concatenation of every section's `blocks`) whose first+ content command landed on that page — this feeds liteparse's per-page+ markdown split. Supporting plumbing: `BuiltSection.source_indices`+ (`layout/build/mod.rs`), `SectionStart.block_sources` ++ `PageLayoutState.pending_block_start`/`note_block_start` with flush sites+ at the four content-append points (`layout/section/layout.rs`), and the+ `body_base` offset accumulation in `render/mod.rs::layout_document`.+ Deliberately a side-channel field rather than a `DrawCommand` variant — the+ command enum is matched exhaustively across the crate by design, and a page+ field inherits checkpoint/replay and `Continuous`-continuation semantics+ for free. Verified behaviour-preserving: `layout_probe` command censuses+ over the 48-doc corpus + long documents are byte-identical before/after.## Layout parity (measured 2026-08-13, macOS host)diff --git a/crates/liteparse-docx/src/render/layout/build/mod.rs b/crates/liteparse-docx/src/render/layout/build/mod.rsindex e25471c..7c43679 100644--- a/crates/liteparse-docx/src/render/layout/build/mod.rs+++ b/crates/liteparse-docx/src/render/layout/build/mod.rs@@ -101,6 +101,12 @@ pub struct BuildState {/// Built section output — layout blocks.pub struct BuiltSection {pub blocks: Vec<LayoutBlock>,+ /// liteparse instrumentation: `blocks[i]` was built from+ /// `section.blocks[source_indices[i]]`. Section-local indices — the+ /// document-level flattening offset is applied by `layout_document`.+ /// Source blocks that build to nothing (section breaks, drop-cap+ /// paragraphs merged into their successor) have no entry.+ pub source_indices: Vec<usize>,}/// Build layout blocks for one section by recursing into its block tree.@@ -111,21 +117,25 @@ pub fn build_section_blocks(state: &mut BuildState,) -> BuiltSection {let mut pending_dropcap: Option<crate::render::layout::paragraph::DropCapInfo> = None;- let blocks: Vec<LayoutBlock> = section- .blocks- .iter()- .filter_map(|block| {- build_block(- block,- config.content_width(),- ctx,- state,- &mut pending_dropcap,- )- })- .collect();-- BuiltSection { blocks }+ let mut blocks = Vec::new();+ let mut source_indices = Vec::new();+ for (idx, block) in section.blocks.iter().enumerate() {+ if let Some(built) = build_block(+ block,+ config.content_width(),+ ctx,+ state,+ &mut pending_dropcap,+ ) {+ blocks.push(built);+ source_indices.push(idx);+ }+ }++ BuiltSection {+ blocks,+ source_indices,+ }}/// §17.11.2: build the document's endnote content, rendered once at the end ofdiff --git a/crates/liteparse-docx/src/render/layout/draw_command.rs b/crates/liteparse-docx/src/render/layout/draw_command.rsindex 8919a58..4656860 100644--- a/crates/liteparse-docx/src/render/layout/draw_command.rs+++ b/crates/liteparse-docx/src/render/layout/draw_command.rs@@ -362,6 +362,15 @@ impl DrawCommand {pub struct LayoutedPage {pub commands: Vec<DrawCommand>,pub page_size: PtSize,+ /// liteparse instrumentation (no upstream equivalent): flattened body-block+ /// indices — into the concatenation of every section's `blocks`, section+ /// breaks included — whose first content command landed on this page.+ /// Deliberately a side-channel field rather than a `DrawCommand` variant:+ /// the command enum is matched exhaustively across the crate by design, and+ /// a field rides along with the page through checkpoint/replay and+ /// `Continuous`-section continuation for free. Blocks that emit no content+ /// (empty paragraphs) appear on no page; consumers fill forward.+ pub block_starts: Vec<usize>,}impl LayoutedPage {@@ -369,6 +378,7 @@ impl LayoutedPage {Self {commands: Vec::new(),page_size,+ block_starts: Vec::new(),}}}diff --git a/crates/liteparse-docx/src/render/layout/section/layout.rs b/crates/liteparse-docx/src/render/layout/section/layout.rsindex 7cd0b14..6db6b94 100644--- a/crates/liteparse-docx/src/render/layout/section/layout.rs+++ b/crates/liteparse-docx/src/render/layout/section/layout.rs@@ -100,6 +100,11 @@ struct PageLayoutState<'doc> {/// §17.3.3.1: an inline page break at the end of a paragraph defers the/// page break to the start of the next block.pending_page_break: bool,+ /// liteparse instrumentation: the current block's flattened source index,+ /// waiting to be recorded on the page that receives the block's first+ /// content command (`note_block_start`). Re-assigned at the top of the+ /// block loop — including on replay — so it needs no checkpoint entry.+ pending_block_start: Option<usize>,}impl<'doc> PageLayoutState<'doc> {@@ -135,6 +140,18 @@ impl<'doc> PageLayoutState<'doc> {page_start_block: 0,prev_table_style_id: None,pending_page_break: false,+ pending_block_start: None,+ }+ }++ /// Record the pending block's source index on the page currently being+ /// assembled. Called immediately before a block's first content commands+ /// are appended; `take()` makes it record once per block, on the page+ /// where content actually landed (a block that emits nothing records+ /// nowhere, and consumers fill forward).+ fn note_block_start(&mut self) {+ if let Some(idx) = self.pending_block_start.take() {+ self.current_page.block_starts.push(idx);}}@@ -1103,6 +1120,9 @@ fn emit_split_paragraph<'doc>();}let segment_x = col_x(state.current_col);+ if !segment.commands.is_empty() {+ state.note_block_start();+ }for mut cmd in segment.commands {cmd.shift_y(state.cursor_y);cmd.shift_x(segment_x);@@ -1228,6 +1248,10 @@ pub(crate) struct SectionStart<'a> {/// §17.10.6: logical number of the section's first page, with/// `w:pgNumType/@start` applied. Drives §20.4.3.1 float mirroring.pub logical_page_base: usize,+ /// liteparse instrumentation: flattened document-level source index for+ /// each entry of `blocks`, parallel to it (see `BuiltSection`). Callers+ /// that don't need block→page provenance pass `&[]`.+ pub block_sources: &'a [usize],}/// Lay out a sequence of blocks into pages.@@ -1255,6 +1279,7 @@ pub fn layout_section(// A section laid out on its own starts at page 1 — the §17.10.6// renumbering only exists across a document's section list.logical_page_base: 1,+ block_sources: &[],},)}@@ -1273,6 +1298,7 @@ pub(crate) fn layout_section_with_clearance(continuation,clearance,logical_page_base,+ block_sources,} = start;let content_width = config.content_width();let num_cols = config.num_columns();@@ -1308,6 +1334,11 @@ pub(crate) fn layout_section_with_clearance('blocks: while block_idx < blocks.len() {let block = &blocks[block_idx];+ // liteparse instrumentation: arm the block→page marker for this block.+ // Re-assigned on every iteration — including `continue 'blocks`+ // replays, whose checkpoint restore has already rolled back any+ // marker the replayed page recorded.+ state.pending_block_start = block_sources.get(block_idx).copied();// §17.3.3.1: a deferred inline page break from the previous block// forces this block onto a new page.if state.pending_page_break {@@ -1773,6 +1804,9 @@ pub(crate) fn layout_section_with_clearance(col_x(state.current_col).raw(),para.size.height.raw());+ if !para.commands.is_empty() {+ state.note_block_start();+ }for mut cmd in para.commands {cmd.shift_y(state.cursor_y);cmd.shift_x(col_x(state.current_col));@@ -2031,6 +2065,9 @@ pub(crate) fn layout_section_with_clearance(};let slice_height = slice.size.height;+ if !slice.commands.is_empty() {+ state.note_block_start();+ }for mut cmd in slice.commands {cmd.shift_y(y_start);cmd.shift_x(table_x);@@ -2112,6 +2149,9 @@ pub(crate) fn layout_section_with_clearance(// Continuation slice — start a new page.state.push_new_page(block_idx, &ctx);}+ if !slice.commands.is_empty() {+ state.note_block_start();+ }for mut cmd in slice.commands {cmd.shift_y(state.cursor_y);cmd.shift_x(table_x);diff --git a/crates/liteparse-docx/src/render/layout/section/mod.rs b/crates/liteparse-docx/src/render/layout/section/mod.rsindex 39403dd..fbfe581 100644--- a/crates/liteparse-docx/src/render/layout/section/mod.rs+++ b/crates/liteparse-docx/src/render/layout/section/mod.rs@@ -313,6 +313,7 @@ mod tests {continuation: None,clearance: &clearance,logical_page_base: 2,+ block_sources: &[],},);@@ -862,6 +863,7 @@ mod tests {continuation: None,clearance: &clearance,logical_page_base: 1,+ block_sources: &[],},);@@ -936,6 +938,7 @@ mod tests {continuation: None,clearance: &clearance,logical_page_base: 1,+ block_sources: &[],},);@@ -1011,6 +1014,7 @@ mod tests {continuation: None,clearance: &clearance,logical_page_base: 1,+ block_sources: &[],},);@@ -1741,6 +1745,7 @@ mod tests {continuation: None,clearance: &clearance,logical_page_base: 1,+ block_sources: &[],},);@@ -1813,6 +1818,7 @@ mod tests {continuation: None,clearance: &clearance,logical_page_base: 1,+ block_sources: &[],},);let separator_y = pages[0]@@ -3382,4 +3388,129 @@ mod tests {"all words land in one column or the other");}++ // ── block_starts instrumentation (liteparse) ────────────────────────────+ // The block→page markers feed liteparse's per-page markdown split; the+ // contract is "a block's index appears on exactly the page that received+ // its first content command, and blocks that emit nothing appear nowhere".++ fn layout_with_sources(+ blocks: &[LayoutBlock],+ sources: &[usize],+ ) -> Vec<crate::render::layout::draw_command::LayoutedPage> {+ let config = small_config();+ let clearance = HeaderFooterClearance::uniform(&config);+ super::layout::layout_section_with_clearance(+ blocks,+ &config,+ None,+ Pt::ZERO,+ Pt::new(14.0),+ super::layout::SectionStart {+ continuation: None,+ clearance: &clearance,+ logical_page_base: 1,+ block_sources: sources,+ },+ )+ }++ #[test]+ fn block_starts_record_the_page_where_each_block_lands() {+ // Global (flattened) indices are opaque to layout — pass arbitrary+ // non-contiguous ones and expect them back verbatim.+ let blocks = vec![+ styled_para_block("p1", false, false),+ styled_para_block("p2", false, true),+ styled_para_block("p3", false, true),+ ];+ let pages = layout_with_sources(&blocks, &[10, 20, 30]);+ assert_eq!(pages.len(), 3, "page_break_before splits them");+ assert_eq!(pages[0].block_starts, vec![10]);+ assert_eq!(pages[1].block_starts, vec![20]);+ assert_eq!(pages[2].block_starts, vec![30]);+ }++ #[test]+ fn a_split_paragraph_records_only_its_start_page() {+ // 12 full-width lines at 14pt in an 80pt body: the paragraph must+ // break across pages, and the marker belongs to the page holding the+ // first segment only.+ let fragments: Vec<Fragment> = (0..12)+ .map(|i| text_frag(&format!("line{i}"), 170.0, 14.0))+ .collect();+ let blocks = vec![LayoutBlock::Paragraph {+ fragments,+ style: ParagraphStyle::default(),+ page_break_before: false,+ footnotes: vec![],+ floating_images: vec![],+ floating_shapes: vec![],+ }];+ let pages = layout_with_sources(&blocks, &[5]);+ assert!(+ pages.len() >= 2,+ "the paragraph spans pages: {}",+ pages.len()+ );+ assert_eq!(pages[0].block_starts, vec![5]);+ for (i, page) in pages.iter().enumerate().skip(1) {+ assert!(+ page.block_starts.is_empty(),+ "continuation page {i} must not re-record the block"+ );+ }+ }++ #[test]+ fn a_block_that_emits_nothing_records_nowhere() {+ let blocks = vec![+ styled_para_block("p1", false, false),+ LayoutBlock::Paragraph {+ fragments: vec![],+ style: ParagraphStyle::default(),+ page_break_before: false,+ footnotes: vec![],+ floating_images: vec![],+ floating_shapes: vec![],+ },+ styled_para_block("p3", false, false),+ ];+ let pages = layout_with_sources(&blocks, &[0, 1, 2]);+ let all: Vec<usize> = pages.iter().flat_map(|p| p.block_starts.clone()).collect();+ assert!(+ all.contains(&0) && all.contains(&2),+ "content blocks recorded: {all:?}"+ );+ assert!(+ !all.contains(&1),+ "the empty paragraph must not be recorded: {all:?}"+ );+ }++ #[test]+ fn a_paginated_table_records_only_its_first_slice_page() {+ let rows: Vec<(String, bool)> = (0..12).map(|i| (format!("row{i}"), false)).collect();+ let rows_ref: Vec<(&str, bool)> = rows.iter().map(|(t, h)| (t.as_str(), *h)).collect();+ let blocks = vec![one_column_table(&rows_ref)];+ let pages = layout_with_sources(&blocks, &[7]);+ assert!(pages.len() >= 2, "the table paginates: {}", pages.len());+ assert_eq!(pages[0].block_starts, vec![7]);+ for page in &pages[1..] {+ assert!(+ page.block_starts.is_empty(),+ "continuation slices must not re-record"+ );+ }+ }++ #[test]+ fn without_sources_no_markers_are_recorded() {+ let blocks = vec![+ styled_para_block("p1", false, false),+ styled_para_block("p2", false, true),+ ];+ let pages = layout_with_sources(&blocks, &[]);+ assert!(pages.iter().all(|p| p.block_starts.is_empty()));+ }}diff --git a/crates/liteparse-docx/src/render/mod.rs b/crates/liteparse-docx/src/render/mod.rsindex 02ee39b..939a781 100644--- a/crates/liteparse-docx/src/render/mod.rs+++ b/crates/liteparse-docx/src/render/mod.rs@@ -142,6 +142,11 @@ pub fn layout_document(let mut pending_continuation: Option<layout::section::ContinuationState> = None;let even_and_odd = resolved.even_and_odd_headers;+ // liteparse instrumentation: running offset turning each section's local+ // block indices into indices over the flattened concatenation of every+ // section's `blocks` — the same flattening the structure emitter walks.+ let mut body_base: usize = 0;+// Phase 1: layout all sections to determine total page count.for (section_idx, section) in resolved.sections.iter().enumerate() {let config = PageConfig::from_section(§ion.properties);@@ -161,6 +166,9 @@ pub fn layout_document();let built = build_section_blocks(section, &config, &ctx, &mut state);+ let block_sources: Vec<usize> =+ built.source_indices.iter().map(|i| i + body_base).collect();+ body_base += section.blocks.len();let measure_fn = |text: &str,font: &layout::fragment::FontProps|-> (dimension::Pt, layout::fragment::TextMetrics) {@@ -186,6 +194,7 @@ pub fn layout_document(continuation,clearance: &clearance,logical_page_base,+ block_sources: &block_sources,},);diff --git a/crates/liteparse-napi/Cargo.toml b/crates/liteparse-napi/Cargo.tomlindex 668fd1e..88e779e 100644--- a/crates/liteparse-napi/Cargo.toml+++ b/crates/liteparse-napi/Cargo.toml@@ -10,8 +10,9 @@ description = "Node.js native bindings for LiteParse"crate-type = ["cdylib"][features]-default = ["tesseract"]+default = ["tesseract", "docx-native"]tesseract = ["liteparse/tesseract"]+docx-native = ["liteparse/docx-native"][dependencies]liteparse = { package = "liteparse", version = "2.11.0", path = "../liteparse", default-features = false }diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rsindex a2ee1ae..c44dd4d 100644--- a/crates/liteparse-napi/src/types.rs+++ b/crates/liteparse-napi/src/types.rs@@ -59,6 +59,10 @@ pub struct JsLiteParseConfig {/// Keep running headers/footers in markdown output instead of stripping/// repeated page-band lines and page chrome (default false).pub keep_headers_footers: Option<bool>,+ /// Parse DOCX natively instead of converting via LibreOffice (default+ /// true in builds with the docx-native feature; other formats always use+ /// the conversion path). Set false to force the LibreOffice path.+ pub office_native: Option<bool>,/// Extract all PDF annotations as page-scoped structured data.pub extract_annotations: Option<bool>,/// Extract AcroForm widget fields and values.@@ -186,6 +190,9 @@ impl JsLiteParseConfig {if let Some(v) = self.keep_headers_footers {cfg.keep_headers_footers = v;}+ if let Some(v) = self.office_native {+ cfg.office_native = v;+ }if let Some(v) = self.extract_annotations {cfg.extract_annotations = v;}@@ -274,6 +281,7 @@ impl JsLiteParseConfig {image_output_dir: cfg.image_output_dir.clone(),extract_links: Some(cfg.extract_links),keep_headers_footers: Some(cfg.keep_headers_footers),+ office_native: Some(cfg.office_native),extract_annotations: Some(cfg.extract_annotations),extract_form_fields: Some(cfg.extract_form_fields),extract_structure_tree: Some(cfg.extract_structure_tree),diff --git a/crates/liteparse-python/Cargo.toml b/crates/liteparse-python/Cargo.tomlindex a1ffe4b..8e62cff 100644--- a/crates/liteparse-python/Cargo.toml+++ b/crates/liteparse-python/Cargo.toml@@ -11,8 +11,9 @@ name = "_liteparse"crate-type = ["cdylib"][features]-default = ["tesseract"]+default = ["tesseract", "docx-native"]tesseract = ["liteparse/tesseract"]+docx-native = ["liteparse/docx-native"][dependencies]liteparse = { package = "liteparse", version = "2.11.0", path = "../liteparse", default-features = false }diff --git a/crates/liteparse-python/src/cli.rs b/crates/liteparse-python/src/cli.rsindex 2a7c128..9cc4989 100644--- a/crates/liteparse-python/src/cli.rs+++ b/crates/liteparse-python/src/cli.rs@@ -37,6 +37,10 @@ struct ParseCommand {format: String,#[arg(long)]no_ocr: bool,+ /// Force the LibreOffice conversion path for DOCX instead of the native+ /// parser.+ #[arg(long)]+ no_office_native: bool,#[arg(long, default_value = "eng")]ocr_language: String,#[arg(long, default_value = None)]@@ -149,6 +153,10 @@ struct BatchParseCommand {format: String,#[arg(long)]no_ocr: bool,+ /// Force the LibreOffice conversion path for DOCX instead of the native+ /// parser.+ #[arg(long)]+ no_office_native: bool,#[arg(long, default_value = "eng")]ocr_language: String,#[arg(long, default_value = None)]@@ -266,6 +274,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {let mut config = LiteParseConfig {ocr_language: cmd.ocr_language,ocr_enabled: !cmd.no_ocr,+ office_native: !cmd.no_office_native,tessdata_path: cmd.tessdata_path,max_pages: cmd.max_pages,target_pages: cmd.target_pages,@@ -364,6 +373,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {let mut config = LiteParseConfig {ocr_language: cmd.ocr_language,ocr_enabled: !cmd.no_ocr,+ office_native: !cmd.no_office_native,tessdata_path: cmd.tessdata_path,max_pages: cmd.max_pages,target_pages: None,diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rsindex 2859a2c..17d8a4e 100644--- a/crates/liteparse-python/src/lib.rs+++ b/crates/liteparse-python/src/lib.rs@@ -1062,6 +1062,8 @@ struct PyLiteParseConfig {#[pyo3(get)]keep_headers_footers: bool,#[pyo3(get)]+ office_native: bool,+ #[pyo3(get)]extract_annotations: bool,#[pyo3(get)]extract_form_fields: bool,@@ -1140,6 +1142,7 @@ impl PyLiteParseConfig {},extract_links: cfg.extract_links,keep_headers_footers: cfg.keep_headers_footers,+ office_native: cfg.office_native,extract_annotations: cfg.extract_annotations,extract_form_fields: cfg.extract_form_fields,extract_structure_tree: cfg.extract_structure_tree,@@ -1199,6 +1202,7 @@ impl LiteParse {image_output_dir = None,extract_links = None,keep_headers_footers = None,+ office_native = None,extract_annotations = None,extract_form_fields = None,extract_structure_tree = None,@@ -1235,6 +1239,7 @@ impl LiteParse {image_output_dir: Option<String>,extract_links: Option<bool>,keep_headers_footers: Option<bool>,+ office_native: Option<bool>,extract_annotations: Option<bool>,extract_form_fields: Option<bool>,extract_structure_tree: Option<bool>,@@ -1315,6 +1320,9 @@ impl LiteParse {if let Some(v) = keep_headers_footers {cfg.keep_headers_footers = v;}+ if let Some(v) = office_native {+ cfg.office_native = v;+ }if let Some(v) = extract_annotations {cfg.extract_annotations = v;}diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rsindex afc7fda..a74d906 100644--- a/crates/liteparse-wasm/src/lib.rs+++ b/crates/liteparse-wasm/src/lib.rs@@ -59,6 +59,10 @@ pub struct LiteParseConfig {/// Keep running headers/footers in markdown output instead of stripping/// repeated page-band lines and page chrome. Default false.keep_headers_footers: Option<bool>,+ /// Parse DOCX natively instead of converting via LibreOffice. Accepted+ /// and echoed for config round-trip parity; the native path is not+ /// compiled into the wasm build yet, so it has no effect here.+ office_native: Option<bool>,extract_annotations: Option<bool>,extract_form_fields: Option<bool>,extract_structure_tree: Option<bool>,@@ -167,6 +171,9 @@ impl LiteParseConfig {if let Some(v) = self.keep_headers_footers {cfg.keep_headers_footers = v;}+ if let Some(v) = self.office_native {+ cfg.office_native = v;+ }if let Some(v) = self.extract_annotations {cfg.extract_annotations = v;}@@ -257,6 +264,7 @@ impl LiteParseConfig {extract_images: Some(cfg.extract_images),extract_links: Some(cfg.extract_links),keep_headers_footers: Some(cfg.keep_headers_footers),+ office_native: Some(cfg.office_native),extract_annotations: Some(cfg.extract_annotations),extract_form_fields: Some(cfg.extract_form_fields),extract_structure_tree: Some(cfg.extract_structure_tree),diff --git a/crates/liteparse/Cargo.toml b/crates/liteparse/Cargo.tomlindex dd37a9a..d7c831b 100644--- a/crates/liteparse/Cargo.toml+++ b/crates/liteparse/Cargo.toml@@ -19,7 +19,7 @@ name = "docx_blocks"required-features = ["docx-native"][features]-default = ["tesseract"]+default = ["tesseract", "docx-native"]oar-ocr = ["dep:oar-ocr"]oar-ocr-auto-download = ["oar-ocr", "oar-ocr/auto-download"]oar-ocr-cuda = ["oar-ocr", "oar-ocr/cuda"]diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rsindex a33db5e..b74033b 100644--- a/crates/liteparse/src/config.rs+++ b/crates/liteparse/src/config.rs@@ -53,6 +53,17 @@ pub struct LiteParseConfig {/// downstream than risk losing content. Only affects Markdown output.#[serde(default)]pub keep_headers_footers: bool,+ /// Parse DOCX natively (vendored layout engine) instead of converting via+ /// LibreOffice. Default `true`; effective only when built with the+ /// `docx-native` cargo feature and the input is a `.docx` — every other+ /// format always takes the conversion path. Set `false` to force the+ /// LibreOffice path (A/B comparison). The native path never runs OCR+ /// (text is born digital; a text-sparse DOCX falls back to conversion so+ /// scanned images still OCR), produces no screenshots, and falls back to+ /// conversion for config it cannot honor (image/annotation/form/structure+ /// extraction, crop, complexity).+ #[serde(default = "default_true")]+ pub office_native: bool,/// Extract all PDF annotations into each parsed page. Default `false`./// This is independent of `extract_links`, which only controls Markdown/// link reconstruction.@@ -224,6 +235,7 @@ impl Default for LiteParseConfig {image_output_dir: None,extract_links: true,keep_headers_footers: false,+ office_native: true,extract_annotations: false,extract_form_fields: false,extract_structure_tree: false,@@ -245,6 +257,10 @@ impl Default for LiteParseConfig {}/// Returns the default number of OCR workers: CPU cores - 1, minimum 1.+fn default_true() -> bool {+ true+}+fn default_num_workers() -> usize {std::thread::available_parallelism().map(|n| n.get().saturating_sub(1).max(1))diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rsindex 48cb191..5fd2181 100644--- a/crates/liteparse/src/main.rs+++ b/crates/liteparse/src/main.rs@@ -54,6 +54,12 @@ struct ParseCommand {#[arg(long)]no_ocr: bool,+ /// Force the LibreOffice conversion path for DOCX instead of the native+ /// parser (builds with the docx-native feature parse DOCX natively by+ /// default)+ #[arg(long)]+ no_office_native: bool,+/// OCR language (Tesseract format, e.g. "eng", "fra", "deu")#[arg(long, default_value = "eng")]ocr_language: String,@@ -205,6 +211,12 @@ struct BatchParseCommand {#[arg(long)]no_ocr: bool,+ /// Force the LibreOffice conversion path for DOCX instead of the native+ /// parser (builds with the docx-native feature parse DOCX natively by+ /// default)+ #[arg(long)]+ no_office_native: bool,+/// OCR language (Tesseract format, e.g. "eng", "fra", "deu")#[arg(long, default_value = "eng")]ocr_language: String,@@ -385,6 +397,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {let mut config = LiteParseConfig {ocr_language: cmd.ocr_language,ocr_enabled: !cmd.no_ocr,+ office_native: !cmd.no_office_native,tessdata_path: cmd.tessdata_path,max_pages: cmd.max_pages,target_pages: cmd.target_pages,@@ -488,6 +501,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {let mut config = LiteParseConfig {ocr_language: cmd.ocr_language,ocr_enabled: !cmd.no_ocr,+ office_native: !cmd.no_office_native,tessdata_path: cmd.tessdata_path,max_pages: cmd.max_pages,target_pages: None,diff --git a/crates/liteparse/src/office/docx.rs b/crates/liteparse/src/office/docx.rsindex 175a85b..3b23a5a 100644--- a/crates/liteparse/src/office/docx.rs+++ b/crates/liteparse/src/office/docx.rs@@ -35,15 +35,41 @@ pub fn docx_to_blocks(data: &[u8]) -> Result<Vec<Block>, LiteParseError> {/// Emit blocks for an already-resolved document.pub fn emit(doc: &ResolvedDocument) -> Vec<Block> {- let body: Vec<DocxBlock> = doc- .sections- .iter()- .flat_map(|s| s.blocks.iter().cloned())- .collect();+ emit_with_sources(doc).into_iter().map(|(b, _)| b).collect()+}++/// Where an emitted [`Block`] came from, for the layout block→page join.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum BlockSource {+ /// Index into the flattened concatenation of every section's blocks+ /// (section breaks included) — the same flattening whose indices the+ /// layout engine records in `LayoutedPage::block_starts`.+ Body(usize),+ /// Footnote/endnote bodies, and the `HorizontalRule` that precedes them.+ Note,+}+/// [`emit`], with each block tagged by the source body block that produced it.+///+/// One source block maps to zero or one emitted blocks today (empty paragraphs+/// and section breaks emit nothing; nothing merges across source blocks), but+/// the tagging tolerates zero-or-more either way.+pub fn emit_with_sources(doc: &ResolvedDocument) -> Vec<(Block, BlockSource)> {let mut em = Emitter::new(doc);- let mut out = Vec::new();- em.blocks(&body, &mut out);+ let mut out: Vec<(Block, BlockSource)> = Vec::new();+ // One Emitter across all blocks: list counters must persist so an ordered+ // list interrupted by prose continues its count.+ let mut scratch = Vec::new();+ let mut flat_idx = 0usize;+ for section in &doc.sections {+ for block in §ion.blocks {+ em.blocks(std::slice::from_ref(block), &mut scratch);+ for b in scratch.drain(..) {+ out.push((b, BlockSource::Body(flat_idx)));+ }+ flat_idx += 1;+ }+ }// Note bodies are appended after the document, behind a rule. Emit *all* of// them, not just those whose reference we walked past: notes are frequently@@ -60,8 +86,8 @@ pub fn emit(doc: &ResolvedDocument) -> Vec<Block> {}}if !notes.is_empty() {- out.push(Block::HorizontalRule);- out.extend(notes);+ out.push((Block::HorizontalRule, BlockSource::Note));+ out.extend(notes.into_iter().map(|b| (b, BlockSource::Note)));}out}@@ -570,4 +596,59 @@ mod tests {);assert_eq!(text, "**ab**!");}++ /// Provenance tagging must not change what is emitted; body indices must+ /// be non-decreasing (the block→page join walks them with a forward+ /// cursor); and note bodies must form a `Note`-tagged tail behind the+ /// rule, so the page split can send them to the last page.+ #[test]+ fn emit_with_sources_is_emit_plus_tags() {+ let data = std::fs::read(concat!(+ env!("CARGO_MANIFEST_DIR"),+ "/../../docx_files/legal/courts_3rd_circuit_ch4.docx"+ ))+ .expect("corpus fixture");+ let parsed = liteparse_docx::docx::parse(&data).expect("fixture parses");+ let doc = liteparse_docx::render::resolve::resolve(parsed);++ let tagged = emit_with_sources(&doc);+ let plain = emit(&doc);+ assert_eq!(+ format!("{plain:?}"),+ format!("{:?}", tagged.iter().map(|(b, _)| b).collect::<Vec<_>>()),+ "tagging must be a pure annotation"+ );++ let body: Vec<usize> = tagged+ .iter()+ .filter_map(|(_, s)| match s {+ BlockSource::Body(i) => Some(*i),+ BlockSource::Note => None,+ })+ .collect();+ assert!(!body.is_empty());+ assert!(+ body.windows(2).all(|w| w[0] <= w[1]),+ "body indices are non-decreasing"+ );++ let first_note = tagged+ .iter()+ .position(|(_, s)| *s == BlockSource::Note)+ .expect("fixture has footnotes");+ assert!(+ matches!(tagged[first_note].0, Block::HorizontalRule),+ "notes start with the separating rule"+ );+ assert!(+ tagged[first_note..]+ .iter()+ .all(|(_, s)| *s == BlockSource::Note),+ "notes are a contiguous tail"+ );+ assert!(+ tagged[first_note..].len() > 1,+ "note bodies follow the rule"+ );+ }}diff --git a/crates/liteparse/src/office/docx_layout.rs b/crates/liteparse/src/office/docx_layout.rsnew file mode 100644index 0000000..7140cdf--- /dev/null+++ b/crates/liteparse/src/office/docx_layout.rs@@ -0,0 +1,471 @@+//! `LayoutedPage` draw commands → [`Page`]/[`TextItem`] geometry.+//!+//! The stage-2 tap (see `NATIVE_OFFICE_PLAN.md`): the vendored layout engine+//! ends at per-page [`DrawCommand`] streams in Pt with a top-left origin —+//! already liteparse viewport space, no unit conversion and no y-flip. What a+//! `Text` command does *not* carry is a bounding box: `position` is the+//! baseline origin and there is no width field. Both come from re-measuring+//! with the same [`TextMeasurer`]/[`FontRegistry`] the layout ran with, which+//! reproduces the engine's own advances bit-for-bit.+//!+//! Facts that are geometric inferences on the PDF path arrive here as data:+//! `LinkAnnotation` → [`TextItem::link`], `Outline` marks →+//! [`OutlineTarget`]s. Two deliberate v1 gaps, both documented in the plan:+//! `TextItem::strike` stays `false` (the vendored layout never draws+//! strikethrough lines — markdown strike comes from the block emitter's+//! cascade, which reads the source instead of geometry), and `Image`/`Path`+//! commands feed nothing (no Figure blocks exist on the native markdown path+//! yet; `ParseResult.images` stays empty).++use std::collections::HashMap;++use liteparse_docx::render::fonts::FontRegistry;+use liteparse_docx::render::layout::draw_command::{DrawCommand, LayoutedPage, OutlineMark};+use liteparse_docx::render::layout::fragment::FontProps;+use liteparse_docx::render::layout::measurer::TextMeasurer;++use crate::types::{OutlineTarget, Page, Rect, TextItem, WordBox};++/// Everything the native pipeline taps out of a laid-out document.+pub struct NativeLayout {+ /// One [`Page`] per physical page, in order, with real geometry.+ pub pages: Vec<Page>,+ /// Outline entries in reading order, `page_index` zero-based, `y_pdf` in+ /// PDF user space (bottom-left origin) per the [`OutlineTarget`] contract.+ pub outline: Vec<OutlineTarget>,+ /// Flattened body-block index → zero-based physical page where the+ /// block's first content landed (min page on the rare duplicate).+ pub block_pages: HashMap<usize, usize>,+}++/// Horizontal/vertical fraction of a link rectangle an item must cover for+/// the link to attach. Link rects are per-word, so a run-level item normally+/// covers its words' rects fully; the threshold only guards against grazing+/// overlaps from neighbouring lines.+const LINK_MIN_COVER_FRACTION: f32 = 0.5;++/// Convert laid-out pages into liteparse [`Page`]s + outline + block→page map.+///+/// `registry` must be the same registry `layout_document` ran with — the+/// re-measure only reproduces the engine's line breaks against identical font+/// resolution. `emit_word_boxes` mirrors `LiteParseConfig::emit_word_boxes`:+/// when false, `TextItem::words` stays empty and the per-word prefix measures+/// are skipped entirely.+pub fn layout_to_pages(+ layouted: &[LayoutedPage],+ registry: &FontRegistry,+ emit_word_boxes: bool,+) -> NativeLayout {+ let measurer = TextMeasurer::new(registry);+ let mut pages = Vec::with_capacity(layouted.len());+ let mut outline: Vec<OutlineTarget> = Vec::new();+ let mut block_pages: HashMap<usize, usize> = HashMap::new();++ for (page_idx, lp) in layouted.iter().enumerate() {+ let page_height = lp.page_size.height.raw();+ let mut text_items: Vec<TextItem> = Vec::new();+ let mut links: Vec<(Rect, String)> = Vec::new();+ // Indices into `outline` whose heading bracket is still open and has+ // no y yet — resolved by the first Text command inside the bracket.+ let mut open_headings: Vec<usize> = Vec::new();++ for cmd in &lp.commands {+ match cmd {+ DrawCommand::Text {+ position,+ text,+ font_family,+ char_spacing,+ font_size,+ bold,+ italic,+ color,+ text_scale,+ } => {+ let props = FontProps {+ family: font_family.clone(),+ size: *font_size,+ bold: *bold,+ italic: *italic,+ underline: false,+ char_spacing: *char_spacing,+ text_scale: *text_scale,+ underline_position: liteparse_docx::render::dimension::Pt::ZERO,+ underline_thickness: liteparse_docx::render::dimension::Pt::ZERO,+ };+ let (advance, metrics) = measurer.measure(text, &props);+ let ascent = metrics.ascent.raw();+ let descent = metrics.descent.raw();+ let x = position.x.raw();+ let top = position.y.raw() - ascent;+ let width = advance.raw();+ let height = ascent + descent;++ let words = if emit_word_boxes {+ word_boxes(text, &props, &measurer, x, top, height)+ } else {+ Vec::new()+ };++ let item = TextItem {+ text: text.to_string(),+ x,+ y: top,+ width,+ height,+ rotation: 0.0,+ font_name: Some(font_family.to_string()),+ font_size: Some(font_size.raw()),+ // The PDF path's font_height is font_size × text-matrix+ // y-scale; the native path has no CTM, so scale is 1.+ font_height: Some(font_size.raw()),+ font_ascent: Some(ascent),+ // PDFium's convention: descent is negative below the+ // baseline. `TextMetrics.descent` is positive-down.+ font_descent: Some(-descent),+ font_weight: Some(if *bold { 700 } else { 400 }),+ text_width: Some(width),+ fill_color: Some(format!(+ "ff{:02x}{:02x}{:02x}",+ color.r, color.g, color.b+ )),+ words,+ ..TextItem::default()+ };++ for &oi in &open_headings {+ if outline[oi].y_pdf.is_none() {+ outline[oi].y_pdf = Some(page_height - top);+ }+ }++ text_items.push(item);+ }+ DrawCommand::EmojiCluster {+ rect, text, size, ..+ } => {+ text_items.push(TextItem {+ text: text.clone(),+ x: rect.origin.x.raw(),+ y: rect.origin.y.raw(),+ width: rect.size.width.raw(),+ height: rect.size.height.raw(),+ font_size: Some(size.raw()),+ ..TextItem::default()+ });+ }+ DrawCommand::LinkAnnotation { rect, url } => {+ links.push((+ Rect {+ x: rect.origin.x.raw(),+ y: rect.origin.y.raw(),+ width: rect.size.width.raw(),+ height: rect.size.height.raw(),+ },+ url.to_string(),+ ));+ }+ DrawCommand::Outline(mark) => match mark {+ OutlineMark::Begin(h) => {+ outline.push(OutlineTarget {+ level: h.level.value(),+ title: h.title.to_string(),+ page_index: page_idx as i32,+ y_pdf: None,+ });+ open_headings.push(outline.len() - 1);+ }+ OutlineMark::End => {+ open_headings.pop();+ }+ },+ // Underline/strike: the vendored layout emits underlines but+ // never strikethrough; TextItem has no underline field and+ // strike detection over border/separator lines would only+ // false-positive. Images/paths/rects: deferred (module docs).+ DrawCommand::Underline { .. }+ | DrawCommand::Line { .. }+ | DrawCommand::Rect { .. }+ | DrawCommand::Image { .. }+ | DrawCommand::Path { .. }+ | DrawCommand::InternalLink { .. }+ | DrawCommand::NamedDestination { .. } => {}+ }+ }++ assign_links(&mut text_items, &links);++ for &b in &lp.block_starts {+ block_pages+ .entry(b)+ .and_modify(|p| *p = (*p).min(page_idx))+ .or_insert(page_idx);+ }++ let content_bounds = union_bounds(&text_items);+ pages.push(Page {+ page_number: page_idx + 1,+ page_width: lp.page_size.width.raw(),+ page_height,+ content_bounds,+ text_items,+ graphics: Vec::new(),+ vector_graphics: None,+ struct_nodes: Vec::new(),+ image_refs: Vec::new(),+ annotations: None,+ form_fields: None,+ structure_tree: None,+ });+ }++ NativeLayout {+ pages,+ outline,+ block_pages,+ }+}++/// Per-word boxes via prefix re-measure. The measurer's advance arithmetic is+/// linear in the string (cmap advances + per-char spacing, no shaping), so a+/// prefix measure is exact — `width(a+b) == width(a) + width(b)`.+fn word_boxes(+ text: &str,+ props: &FontProps,+ measurer: &TextMeasurer<'_>,+ item_x: f32,+ item_y: f32,+ item_height: f32,+) -> Vec<WordBox> {+ let mut words = Vec::new();+ let mut search_from = 0usize;+ for word in text.split_whitespace() {+ // Locate this word's byte range (split_whitespace loses offsets).+ let rel = match text[search_from..].find(word) {+ Some(r) => r,+ None => return Vec::new(),+ };+ let start = search_from + rel;+ let end = start + word.len();+ search_from = end;+ let x0 = measurer.measure(&text[..start], props).0.raw();+ let x1 = measurer.measure(&text[..end], props).0.raw();+ words.push(WordBox {+ text: word.to_string(),+ x: item_x + x0,+ y: item_y,+ width: x1 - x0,+ height: item_height,+ });+ }+ words+}++/// Attach link URLs to the items covering each link rectangle. Rects come one+/// per word from the layout engine; an item takes the first rect it covers.+fn assign_links(items: &mut [TextItem], links: &[(Rect, String)]) {+ if links.is_empty() {+ return;+ }+ for item in items.iter_mut() {+ if item.link.is_some() {+ continue;+ }+ for (r, url) in links {+ let ox = (item.x + item.width).min(r.x + r.width) - item.x.max(r.x);+ let oy = (item.y + item.height).min(r.y + r.height) - item.y.max(r.y);+ if ox >= r.width * LINK_MIN_COVER_FRACTION && oy >= r.height * LINK_MIN_COVER_FRACTION {+ item.link = Some(url.clone());+ break;+ }+ }+ }+}++fn union_bounds(items: &[TextItem]) -> Option<Rect> {+ let mut it = items.iter();+ let first = it.next()?;+ let (mut x0, mut y0) = (first.x, first.y);+ let (mut x1, mut y1) = (first.x + first.width, first.y + first.height);+ for i in it {+ x0 = x0.min(i.x);+ y0 = y0.min(i.y);+ x1 = x1.max(i.x + i.width);+ y1 = y1.max(i.y + i.height);+ }+ Some(Rect {+ x: x0,+ y: y0,+ width: x1 - x0,+ height: y1 - y0,+ })+}++#[cfg(test)]+mod tests {+ use super::*;+ use liteparse_docx::render::dimension::Pt;+ use liteparse_docx::render::geometry::{PtOffset, PtRect, PtSize};+ use liteparse_docx::render::layout::draw_command::OutlineHeading;+ use liteparse_docx::render::resolve::color::RgbColor;+ use std::rc::Rc;++ fn registry() -> FontRegistry {+ FontRegistry::new()+ }++ fn text_cmd(x: f32, y: f32, text: &str, size: f32) -> DrawCommand {+ DrawCommand::Text {+ position: PtOffset::new(Pt::new(x), Pt::new(y)),+ text: Rc::from(text),+ font_family: Rc::from("Arial"),+ char_spacing: Pt::ZERO,+ font_size: Pt::new(size),+ bold: false,+ italic: false,+ color: RgbColor::BLACK,+ text_scale: 1.0,+ }+ }++ fn page(commands: Vec<DrawCommand>) -> LayoutedPage {+ LayoutedPage {+ commands,+ page_size: PtSize::new(Pt::new(612.0), Pt::new(792.0)),+ block_starts: Vec::new(),+ }+ }++ #[test]+ fn text_bbox_hangs_from_the_baseline_by_ascent() {+ let reg = registry();+ let measurer = TextMeasurer::new(®);+ let props = FontProps {+ family: Rc::from("Arial"),+ size: Pt::new(12.0),+ bold: false,+ italic: false,+ underline: false,+ char_spacing: Pt::ZERO,+ text_scale: 1.0,+ underline_position: Pt::ZERO,+ underline_thickness: Pt::ZERO,+ };+ let (advance, metrics) = measurer.measure("Hello", &props);++ let out = layout_to_pages(+ &[page(vec![text_cmd(72.0, 100.0, "Hello", 12.0)])],+ ®,+ true,+ );+ let item = &out.pages[0].text_items[0];+ assert_eq!(item.x, 72.0);+ assert_eq!(item.y, 100.0 - metrics.ascent.raw());+ assert_eq!(item.width, advance.raw());+ assert_eq!(item.height, metrics.ascent.raw() + metrics.descent.raw());+ assert_eq!(item.font_descent, Some(-metrics.descent.raw()));+ assert_eq!(out.pages[0].page_number, 1);+ // Word boxes partition the advance: two words, gap between them.+ let out2 = layout_to_pages(+ &[page(vec![text_cmd(0.0, 100.0, "ab cd", 12.0)])],+ ®,+ true,+ );+ let words = &out2.pages[0].text_items[0].words;+ assert_eq!(words.len(), 2);+ assert_eq!(words[0].x, 0.0);+ assert!(+ words[1].x > words[0].x + words[0].width,+ "gap for the space"+ );+ }++ #[test]+ fn links_attach_to_covered_items_only() {+ let reg = registry();+ let m = TextMeasurer::new(®);+ let props = FontProps {+ family: Rc::from("Arial"),+ size: Pt::new(12.0),+ bold: false,+ italic: false,+ underline: false,+ char_spacing: Pt::ZERO,+ text_scale: 1.0,+ underline_position: Pt::ZERO,+ underline_thickness: Pt::ZERO,+ };+ let (w, metrics) = m.measure("click", &props);+ let link_rect = PtRect::from_xywh(+ Pt::new(72.0),+ Pt::new(100.0 - metrics.ascent.raw()),+ w,+ Pt::new(metrics.ascent.raw() + metrics.descent.raw()),+ );+ let out = layout_to_pages(+ &[page(vec![+ text_cmd(72.0, 100.0, "click", 12.0),+ text_cmd(72.0, 300.0, "plain", 12.0),+ DrawCommand::LinkAnnotation {+ rect: link_rect,+ url: Rc::from("https://example.com"),+ },+ ])],+ ®,+ false,+ );+ let items = &out.pages[0].text_items;+ assert_eq!(items[0].link.as_deref(), Some("https://example.com"));+ assert_eq!(items[1].link, None);+ }++ #[test]+ fn outline_marks_become_targets_with_pdf_space_y() {+ let reg = registry();+ let m = TextMeasurer::new(®);+ let props = FontProps {+ family: Rc::from("Arial"),+ size: Pt::new(12.0),+ bold: false,+ italic: false,+ underline: false,+ char_spacing: Pt::ZERO,+ text_scale: 1.0,+ underline_position: Pt::ZERO,+ underline_thickness: Pt::ZERO,+ };+ let ascent = m.measure("Title", &props).1.ascent.raw();+ let out = layout_to_pages(+ &[page(vec![+ DrawCommand::Outline(OutlineMark::Begin(OutlineHeading {+ node_id: 1,+ level: liteparse_docx::model::OutlineLevel::new(2),+ title: Rc::from("Title"),+ })),+ text_cmd(72.0, 100.0, "Title", 12.0),+ DrawCommand::Outline(OutlineMark::End),+ ])],+ ®,+ false,+ );+ assert_eq!(out.outline.len(), 1);+ let t = &out.outline[0];+ assert_eq!((t.level, t.page_index), (2, 0));+ assert_eq!(t.title, "Title");+ // Bottom-left PDF space: page_height − viewport top of the heading.+ assert_eq!(t.y_pdf, Some(792.0 - (100.0 - ascent)));+ }++ #[test]+ fn block_starts_become_a_min_page_map() {+ let reg = registry();+ let mut p0 = page(vec![]);+ p0.block_starts = vec![3, 4];+ let mut p1 = page(vec![]);+ p1.block_starts = vec![4, 5];+ let out = layout_to_pages(&[p0, p1], ®, false);+ assert_eq!(out.block_pages[&3], 0);+ assert_eq!(out.block_pages[&4], 0, "duplicate takes the earliest page");+ assert_eq!(out.block_pages[&5], 1);+ }+}diff --git a/crates/liteparse/src/office/mod.rs b/crates/liteparse/src/office/mod.rsindex 1db846b..bf99d1d 100644--- a/crates/liteparse/src/office/mod.rs+++ b/crates/liteparse/src/office/mod.rs@@ -12,3 +12,27 @@#[cfg(feature = "docx-native")]pub mod docx;+#[cfg(feature = "docx-native")]+pub mod docx_layout;++/// The input's bytes when it is a `.docx` the native path should try, `None`+/// otherwise. Path inputs are matched on extension only (`.doc`/`.docm`/+/// `.dotx` stay on the conversion path); byte inputs by container sniffing.+/// A path that fails to read returns `None` so the conversion path surfaces+/// its usual error for missing/unreadable files.+#[cfg(all(feature = "docx-native", not(target_arch = "wasm32")))]+pub(crate) fn docx_bytes(input: &crate::types::PdfInput) -> Option<std::borrow::Cow<'_, [u8]>> {+ use crate::types::PdfInput;+ match input {+ PdfInput::Path(p) => {+ let ext = std::path::Path::new(p).extension()?;+ if !ext.eq_ignore_ascii_case("docx") {+ return None;+ }+ std::fs::read(p).ok().map(std::borrow::Cow::Owned)+ }+ PdfInput::Bytes(b) => (crate::conversion::guess_extension_from_data(b).as_deref()+ == Some("docx"))+ .then_some(std::borrow::Cow::Borrowed(b.as_slice())),+ }+}diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex 5f21718..b9425c2 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -362,6 +362,41 @@ impl LiteParse {self.validate_output_config()?;+ // Native DOCX path: parse + resolve + layout in-process, no+ // LibreOffice, no PDFium, no OCR. Fail-open: any error (or panic — the+ // layout engine is young) logs and falls through to the conversion+ // path, as does a config asking for something the native path cannot+ // honor and a text-sparse document when OCR is on.+ #[cfg(all(feature = "docx-native", not(target_arch = "wasm32")))]+ if self.config.office_native {+ if let Some(reason) = self.native_docx_ineligible_reason() {+ if crate::office::docx_bytes(&input).is_some() {+ log(&format!(+ "[liteparse] office_native: conversion path ({reason})"+ ));+ }+ } else if let Some(bytes) = crate::office::docx_bytes(&input) {+ match self.try_parse_docx_native(&bytes) {+ Ok(Some(result)) => {+ let total =+ web_time::Instant::now().duration_since(t0).as_secs_f64() * 1000.0;+ log(&format!("[liteparse] native docx total: {:.1}ms", total));+ return Ok(result);+ }+ Ok(None) => {+ log(+ "[liteparse] office_native: text-sparse docx, falling back to conversion path for OCR",+ );+ }+ Err(e) => {+ log(&format!(+ "[liteparse] office_native failed ({e}); falling back to conversion path"+ ));+ }+ }+ }+ }+#[cfg(not(target_arch = "wasm32"))]let (validated_input, _guard) =conversion::resolve_pdf_input(input, self.config.password.as_deref(), false).await?;@@ -716,6 +751,212 @@ impl LiteParse {}}+ /// Config the native DOCX path cannot honor — these force the conversion+ /// path so the caller still gets what they asked for. `doc_meta` is not+ /// in the list: it is `None` for converted inputs already, and the native+ /// path keeps that rule.+ #[cfg(all(feature = "docx-native", not(target_arch = "wasm32")))]+ fn native_docx_ineligible_reason(&self) -> Option<&'static str> {+ let c = &self.config;+ if c.effective_extract_images() {+ Some("extract_images")+ } else if c.extract_annotations {+ Some("extract_annotations")+ } else if c.extract_form_fields {+ Some("extract_form_fields")+ } else if c.extract_structure_tree {+ Some("extract_structure_tree")+ } else if c.extract_xfa_packets {+ Some("extract_xfa_packets")+ } else if c.extract_vector_graphics {+ Some("extract_vector_graphics")+ } else if c.include_complexity {+ Some("include_complexity")+ } else if c.crop_box.is_some() {+ Some("crop_box")+ } else if c.skip_diagonal_text {+ Some("skip_diagonal_text")+ } else {+ None+ }+ }++ /// Run the native DOCX pipeline. `Ok(None)` means "eligible fallback":+ /// the document is text-sparse and OCR is enabled, so the conversion path+ /// (which can OCR embedded images) serves the caller better. Panics from+ /// the vendored layout engine are caught and mapped to `Err` so the+ /// caller degrades to LibreOffice instead of aborting.+ #[cfg(all(feature = "docx-native", not(target_arch = "wasm32")))]+ fn try_parse_docx_native(&self, data: &[u8]) -> Result<Option<ParseResult>, LiteParseError> {+ std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {+ self.parse_docx_native_inner(data)+ }))+ .unwrap_or_else(|_| {+ Err(LiteParseError::Conversion(+ "native docx layout panicked".to_string(),+ ))+ })+ }++ /// Total text length below which a DOCX counts as text-sparse (likely a+ /// scanned document pasted into Word) when OCR is enabled.+ #[cfg(all(feature = "docx-native", not(target_arch = "wasm32")))]+ const NATIVE_TEXT_SPARSE_CHARS: usize = 32;++ #[cfg(all(feature = "docx-native", not(target_arch = "wasm32")))]+ fn parse_docx_native_inner(&self, data: &[u8]) -> Result<Option<ParseResult>, LiteParseError> {+ use crate::office::{docx, docx_layout};++ let parsed = liteparse_docx::docx::parse(data)+ .map_err(|e| LiteParseError::Conversion(format!("docx parse failed: {e}")))?;+ let resolved = liteparse_docx::render::resolve::resolve(parsed);+ // Build the registry directly rather than via `resolve_and_layout`,+ // whose `.expect` panics on a font-less host; here that is a clean+ // fallback to the conversion path instead.+ let registry = liteparse_docx::render::fonts::FontRegistry::build(+ &resolved.embedded_fonts,+ &resolved.font_families,+ )+ .map_err(|e| LiteParseError::Conversion(format!("font registry: {e}")))?;+ let layouted = liteparse_docx::render::layout_document(&resolved, ®istry);+ let native =+ docx_layout::layout_to_pages(&layouted, ®istry, self.config.emit_word_boxes);++ if self.config.ocr_enabled {+ let total_chars: usize = native+ .pages+ .iter()+ .flat_map(|p| &p.text_items)+ .map(|i| i.text.trim().len())+ .sum();+ if total_chars < Self::NATIVE_TEXT_SPARSE_CHARS {+ return Ok(None);+ }+ }++ let tagged = docx::emit_with_sources(&resolved);+ let n_pages = native.pages.len();+ let page_blocks = split_blocks_by_page(&tagged, &native.block_pages, n_pages);+ let all_blocks: Vec<crate::markdown_layout::Block> =+ tagged.into_iter().map(|(b, _)| b).collect();++ // Page selection, mirroring the PDF path: `target_pages` filters by+ // 1-based page number, then `max_pages` truncates. The outline stays+ // whole-document, as it does on the PDF path. A page selection also+ // narrows the doc-level text to the surviving pages' blocks.+ let mut selected: Vec<(Page, Vec<crate::markdown_layout::Block>)> =+ native.pages.into_iter().zip(page_blocks).collect();+ let mut page_filtered = false;+ if let Some(targets) = self.resolve_target_pages()? {+ let keep: std::collections::HashSet<usize> =+ targets.iter().map(|p| *p as usize).collect();+ selected.retain(|(p, _)| keep.contains(&p.page_number));+ page_filtered = true;+ }+ if selected.len() > self.config.max_pages {+ selected.truncate(self.config.max_pages);+ page_filtered = true;+ }+ let (pages, page_blocks): (Vec<_>, Vec<_>) = selected.into_iter().unzip();++ Ok(Some(self.parse_from_native_blocks(+ pages,+ page_blocks,+ (!page_filtered).then_some(all_blocks),+ native.outline,+ )))+ }++ /// `parse_from_pages`'s native sibling: no projection — the block model+ /// arrives from the DOCX source, so markdown is `render_blocks` per page+ /// and page text is a straight reading-order join of the text items.+ ///+ /// Doc-level markdown renders from `all_blocks` when given, NOT by+ /// joining the page renders: `render_blocks` is context-sensitive across+ /// block boundaries (tight lists join with `\n`, soft-hyphen paragraph+ /// splices join with nothing), so a page-boundary join would perturb+ /// whitespace wherever a page break falls inside a list. Rendering the+ /// whole sequence keeps `ParseResult.text` byte-identical to the pure+ /// structure path; the per-page markdown is the paged *view* of the same+ /// blocks. `None` (page-filtered parse) falls back to joining the+ /// surviving pages.+ #[cfg(all(feature = "docx-native", not(target_arch = "wasm32")))]+ fn parse_from_native_blocks(+ &self,+ pages: Vec<Page>,+ page_blocks: Vec<Vec<crate::markdown_layout::Block>>,+ all_blocks: Option<Vec<crate::markdown_layout::Block>>,+ outline: Vec<OutlineTarget>,+ ) -> ParseResult {+ let markdown_out = self.config.output_format == crate::config::OutputFormat::Markdown;+ let parsed_pages: Vec<ParsedPage> = pages+ .into_iter()+ .zip(page_blocks)+ .map(|(p, blocks)| {+ let text = native_page_text(&p.text_items);+ let markdown = if markdown_out {+ crate::markdown_layout::render_blocks(&blocks)+ } else {+ String::new()+ };+ ParsedPage {+ page_number: p.page_number,+ page_width: p.page_width,+ page_height: p.page_height,+ content_bounds: self+ .config+ .extract_content_bounds+ .then_some(p.content_bounds)+ .flatten(),+ text,+ markdown,+ text_items: p.text_items,+ projected_lines: Vec::new(),+ regions: crate::types::Region::default(),+ graphics: Vec::new(),+ vector_graphics: None,+ figures: Vec::new(),+ struct_nodes: Vec::new(),+ image_refs: Vec::new(),+ complexity: None,+ annotations: None,+ form_fields: None,+ structure_tree: None,+ }+ })+ .collect();++ let full_text = if markdown_out {+ match all_blocks {+ Some(blocks) => crate::markdown_layout::render_blocks(&blocks),+ None => parsed_pages+ .iter()+ .map(|p| p.markdown.as_str())+ .collect::<Vec<_>>()+ .join("\n\n-----\n\n"),+ }+ } else {+ parsed_pages+ .iter()+ .map(|p| p.text.as_str())+ .collect::<Vec<_>>()+ .join("\n\n")+ };++ ParseResult {+ pages: parsed_pages,+ text: full_text,+ outline,+ images: Vec::new(),+ image_error_count: 0,+ form_type: None,+ creator: None,+ producer: None,+ doc_meta: None,+ xfa_packets: None,+ }+ }+/// Generate screenshots of document pages as PNG bytes.////// Non-PDF files are automatically converted to PDF first (requires@@ -780,6 +1021,74 @@ impl LiteParse {}}+/// Distribute the emitted blocks across pages using the layout's block→page+/// map. A forward cursor plus `max` keeps the per-page lists a partition of+/// the input in its original order — concatenating them reproduces the+/// doc-level markdown exactly — while filling forward for blocks the layout+/// recorded nowhere (empty paragraphs) and clamping any out-of-order page+/// assignment a float relocation might produce. `Note` blocks land on the+/// last page: doc-level output must keep notes at the very end, and many+/// emitted notes (referenced only from headers or textboxes) have no+/// physical body page at all.+#[cfg(all(feature = "docx-native", not(target_arch = "wasm32")))]+fn split_blocks_by_page(+ tagged: &[(+ crate::markdown_layout::Block,+ crate::office::docx::BlockSource,+ )],+ block_pages: &std::collections::HashMap<usize, usize>,+ n_pages: usize,+) -> Vec<Vec<crate::markdown_layout::Block>> {+ use crate::office::docx::BlockSource;+ let n_pages = n_pages.max(1);+ let mut out: Vec<Vec<crate::markdown_layout::Block>> =+ (0..n_pages).map(|_| Vec::new()).collect();+ let mut current = 0usize;+ for (block, source) in tagged {+ let page = match source {+ BlockSource::Body(i) => block_pages+ .get(i)+ .copied()+ .map_or(current, |p| current.max(p)),+ BlockSource::Note => n_pages - 1,+ };+ let page = page.min(n_pages - 1);+ current = page;+ out[page].push(block.clone());+ }+ out+}++/// Reading-order plain text from native text items: emission order is the+/// engine's layout order, so lines are rebuilt by baseline proximity and+/// word gaps rather than by re-sorting (which would interleave columns).+#[cfg(all(feature = "docx-native", not(target_arch = "wasm32")))]+fn native_page_text(items: &[crate::types::TextItem]) -> String {+ /// Fraction of the font size an x-gap must exceed to count as a missing+ /// inter-word space (word commands carry no trailing spaces). Adjacent+ /// same-word run splits sit at ~0 gap; a real space is ~0.25em.+ const WORD_GAP_EM: f32 = 0.15;+ let mut out = String::new();+ let mut prev: Option<&crate::types::TextItem> = None;+ for item in items.iter().filter(|i| !i.text.is_empty()) {+ if let Some(p) = prev {+ let font_size = p.font_size.unwrap_or(12.0).max(1.0);+ let baseline = item.y + item.font_ascent.unwrap_or(item.height);+ let prev_baseline = p.y + p.font_ascent.unwrap_or(p.height);+ if (baseline - prev_baseline).abs() < 0.5 * font_size {+ if item.x - (p.x + p.width) > WORD_GAP_EM * font_size {+ out.push(' ');+ }+ } else {+ out.push('\n');+ }+ }+ out.push_str(&item.text);+ prev = Some(item);+ }+ out+}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rsindex 71e40dc..3f71d7d 100644--- a/crates/liteparse/tests/integration_test.rs+++ b/crates/liteparse/tests/integration_test.rs@@ -286,3 +286,56 @@ async fn test_annotation_text_complexity_reason() {page.reasons);}++/// Native DOCX path end-to-end: real geometry per page, links as data, and+/// doc-level markdown byte-identical to the pure structure path+/// (`docx_to_blocks` → `render_blocks`) — the invariant that pins the+/// docx_rules_eval corpus score.+#[cfg(feature = "docx-native")]+#[tokio::test]+#[serial]+async fn test_parse_docx_native_integration() {+ let path = "../../docx_files/legal/uk_parl_media_bill_ia.docx";+ let lit = LiteParse::new(LiteParseConfig {+ output_format: liteparse::config::OutputFormat::Markdown,+ quiet: true,+ ..Default::default()+ });+ let parsed = lit.parse(path).await.expect("native parse succeeds");++ let data = std::fs::read(path).expect("fixture readable");+ let blocks = liteparse::office::docx::docx_to_blocks(&data).expect("blocks");+ let expected = liteparse::markdown_layout::render_blocks(&blocks);+ assert_eq!(+ parsed.text, expected,+ "native pipeline markdown must match the structure path byte-for-byte"+ );++ assert!(+ parsed.pages.len() > 1,+ "multi-page doc: {}",+ parsed.pages.len()+ );+ assert!(+ parsed.pages.iter().all(|p| !p.text_items.is_empty()),+ "every page carries real text geometry"+ );+ let linked = parsed+ .pages+ .iter()+ .flat_map(|p| &p.text_items)+ .filter(|i| i.link.is_some())+ .count();+ assert!(linked > 0, "hyperlinks arrive as TextItem.link");+ assert!(+ !parsed.outline.is_empty(),+ "outline extracted from headings"+ );+ assert!(+ parsed+ .pages+ .iter()+ .all(|p| p.page_width > 0.0 && p.page_height > 0.0),+ "real page dimensions"+ );+}diff --git a/packages/node/src/cli.ts b/packages/node/src/cli.tsindex 3b2a7e8..91a4aa3 100644--- a/packages/node/src/cli.ts+++ b/packages/node/src/cli.ts@@ -87,6 +87,10 @@ programcollectHeader,).option("--no-ocr", "Disable OCR")+ .option(+ "--no-office-native",+ "Force the LibreOffice conversion path for DOCX instead of the native parser",+ ).option("--ocr-language <lang>", "OCR language (default: eng)").option("--max-pages <n>", "Max pages to parse", parseInt).option(@@ -132,6 +136,7 @@ programif (opts.extractImages) config.extractImages = true;if (opts.links === false) config.extractLinks = false;if (opts.keepHeadersFooters) config.keepHeadersFooters = true;+ if (opts.officeNative === false) config.officeNative = false;if (opts.extractAnnotations) config.extractAnnotations = true;if (opts.extractFormFields) config.extractFormFields = true;if (opts.extractStructureTree) config.extractStructureTree = true;@@ -317,6 +322,10 @@ program.argument("<output-dir>", "Output directory").option("--format <format>", 'Output format: json|text|markdown (default: "text")').option("--no-ocr", "Disable OCR")+ .option(+ "--no-office-native",+ "Force the LibreOffice conversion path for DOCX instead of the native parser",+ ).option("--ocr-language <lang>", "OCR language (default: eng)").option("--ocr-server-url <url>", "HTTP OCR server URL").option(@@ -362,6 +371,7 @@ programconst format = (opts.format as string) ?? "text";config.outputFormat = format as "json" | "text" | "markdown";if (opts.ocr === false) config.ocrEnabled = false;+ if (opts.officeNative === false) config.officeNative = false;if (opts.ocrLanguage) config.ocrLanguage = opts.ocrLanguage as string;if (opts.ocrServerUrl)config.ocrServerUrl = opts.ocrServerUrl as string;diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.tsindex cd49388..4861c03 100644--- a/packages/node/src/lib.ts+++ b/packages/node/src/lib.ts@@ -40,6 +40,8 @@ export interface LiteParseConfig {extractLinks: boolean;/** Keep running headers/footers in markdown output instead of stripping repeated page-band lines and page chrome (default: false). */keepHeadersFooters: boolean;+ /** Parse DOCX natively instead of converting via LibreOffice (default: true; other formats always use the conversion path). */+ officeNative: boolean;/** Extract all PDF annotations into each parsed page (default: false). */extractAnnotations: boolean;/** Extract AcroForm widget fields and values (default: false). */@@ -538,6 +540,7 @@ export class LiteParse {imageOutputDir: userConfig.imageOutputDir,extractLinks: userConfig.extractLinks,keepHeadersFooters: userConfig.keepHeadersFooters,+ officeNative: userConfig.officeNative,extractAnnotations: userConfig.extractAnnotations,extractFormFields: userConfig.extractFormFields,extractStructureTree: userConfig.extractStructureTree,@@ -579,6 +582,7 @@ export class LiteParse {imageOutputDir: resolved.imageOutputDir ?? undefined,extractLinks: resolved.extractLinks ?? true,keepHeadersFooters: resolved.keepHeadersFooters ?? false,+ officeNative: resolved.officeNative ?? true,extractAnnotations: resolved.extractAnnotations ?? false,extractFormFields: resolved.extractFormFields ?? false,extractStructureTree: resolved.extractStructureTree ?? false,diff --git a/packages/node/src/native.ts b/packages/node/src/native.tsindex 874a1bf..0cfd66f 100644--- a/packages/node/src/native.ts+++ b/packages/node/src/native.ts@@ -34,6 +34,7 @@ export interface LiteParseNativeConfig {imageOutputDir?: string;extractLinks?: boolean;keepHeadersFooters?: boolean;+ officeNative?: boolean;extractAnnotations?: boolean;extractFormFields?: boolean;extractStructureTree?: boolean;diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.pyindex a2b27b4..5c2829c 100644--- a/packages/python/liteparse/parser.py+++ b/packages/python/liteparse/parser.py@@ -395,6 +395,7 @@ class LiteParse:image_output_dir: Optional[Union[str, Path]] = None,extract_links: Optional[bool] = None,keep_headers_footers: Optional[bool] = None,+ office_native: Optional[bool] = None,extract_annotations: Optional[bool] = None,extract_form_fields: Optional[bool] = None,extract_structure_tree: Optional[bool] = None,@@ -435,6 +436,9 @@ class LiteParse:keep_headers_footers: Keep running headers/footers in markdownoutput instead of stripping repeated page-band lines and pagechrome (default: False).+ office_native: Parse DOCX natively instead of converting via+ LibreOffice (default: True; other formats always use the+ conversion path). Set False to force the LibreOffice path.image_output_dir: Directory where extracted embedded images arewritten. Requires ``extract_images=True`` and returns filenames/paths in each ``ExtractedImage``.@@ -519,6 +523,8 @@ class LiteParse:kwargs["extract_links"] = extract_linksif keep_headers_footers is not None:kwargs["keep_headers_footers"] = keep_headers_footers+ if office_native is not None:+ kwargs["office_native"] = office_nativeif extract_annotations is not None:kwargs["extract_annotations"] = extract_annotationsif extract_form_fields is not None:@@ -699,6 +705,7 @@ class LiteParse:image_output_dir=cfg.image_output_dir,extract_links=cfg.extract_links,keep_headers_footers=cfg.keep_headers_footers,+ office_native=cfg.office_native,extract_annotations=cfg.extract_annotations,extract_form_fields=cfg.extract_form_fields,extract_structure_tree=cfg.extract_structure_tree,diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.pyindex d22da18..7fa9517 100644--- a/packages/python/liteparse/types.py+++ b/packages/python/liteparse/types.py@@ -406,6 +406,9 @@ class LiteParseConfig:#: Keep running headers/footers in markdown output instead of stripping#: repeated page-band lines and page chrome.keep_headers_footers: bool = False+ #: Parse DOCX natively instead of converting via LibreOffice (other+ #: formats always use the conversion path).+ office_native: bool = Trueextract_images: bool = Falseextract_vector_graphics: bool = Falseextract_xfa_packets: bool = False