A fast, helpful, and open-source document parser
Merge branch 'main' into pierre/parse-screenshots
Logan Markewich2d02d4aa1dcf
diff --cc crates/liteparse-napi/src/types.rsindex 764557a,34c2ee6..88e2bb7--- a/crates/liteparse-napi/src/types.rs+++ b/crates/liteparse-napi/src/types.rs@@@ -32,9 -32,9 +32,12 @@@ pub struct JsLiteParseConfigpub max_pages: Option<u32>,/// Specific pages to parse (e.g., "1-5,10,15-20").pub target_pages: Option<String>,+ /// Render parsed pages to PNG and return them in `ParseResult.screenshots`.+ /// Default false; PNG payloads can be large.+ pub extract_screenshots: Option<bool>,+ /// Continue after page-level extraction failures and return them in+ /// `ParseResult.pageErrors`. Default false.+ pub continue_on_page_error: Option<bool>,/// DPI for rendering pages (used for OCR and screenshots).pub dpi: Option<f64>,/// Output format: "json", "text", or "markdown".@@@ -148,9 -148,9 +151,12 @@@ impl JsLiteParseConfigif let Some(v) = self.target_pages {cfg.target_pages = Some(v);}+ if let Some(v) = self.extract_screenshots {+ cfg.extract_screenshots = v;+ }+ if let Some(v) = self.continue_on_page_error {+ cfg.continue_on_page_error = v;+ }if let Some(v) = self.dpi {cfg.dpi = v as f32;}@@@ -261,7 -261,7 +267,8 @@@tessdata_path: cfg.tessdata_path.clone(),max_pages: Some(cfg.max_pages as u32),target_pages: cfg.target_pages.clone(),+ extract_screenshots: Some(cfg.extract_screenshots),+ continue_on_page_error: Some(cfg.continue_on_page_error),dpi: Some(cfg.dpi as f64),output_format: Some(match cfg.output_format {OutputFormat::Json => "json".to_string(),@@@ -888,10 -888,12 +895,13 @@@ impl JsParsedPage#[napi(object)]#[derive(Clone)]pub struct JsParseResult {+ /// Total source-document pages before target/max-page filtering.+ pub total_pages: u32,pub pages: Vec<JsParsedPage>,+ pub page_errors: Vec<JsPageError>,pub text: String,pub images: Vec<JsExtractedImage>,+ pub screenshots: Vec<JsScreenshotResult>,pub image_error_count: u32,pub form_type: Option<i32>,/// The document's `/Info` `Creator` entry, when present.diff --cc crates/liteparse-python/src/lib.rsindex 9d3fee7,912aae4..c465df7--- a/crates/liteparse-python/src/lib.rs+++ b/crates/liteparse-python/src/lib.rs@@@ -603,10 -605,10 +605,12 @@@ struct PyParseResult#[pyo3(get)]images: Vec<PyExtractedImage>,#[pyo3(get)]+ screenshots: Vec<PyScreenshotResult>,+ #[pyo3(get)]image_error_count: u32,#[pyo3(get)]+ page_errors: Vec<PyPageError>,+ #[pyo3(get)]form_type: Option<i32>,#[pyo3(get)]creator: Option<String>,@@@ -731,12 -743,15 +745,20 @@@ impl PyParseResult.into_iter().map(PyExtractedImage::from_rust).collect(),+ screenshots: result+ .screenshots+ .into_iter()+ .map(PyScreenshotResult::from_rust)+ .collect(),image_error_count: result.image_error_count,+ page_errors: result+ .page_errors+ .into_iter()+ .map(|error| PyPageError {+ page_num: error.page_number,+ message: error.message,+ })+ .collect(),form_type: result.form_type,creator: result.creator,producer: result.producer,@@@ -1075,8 -1066,8 +1097,10 @@@ struct PyLiteParseConfig#[pyo3(get)]target_pages: Option<String>,#[pyo3(get)]+ extract_screenshots: bool,+ #[pyo3(get)]+ continue_on_page_error: bool,+ #[pyo3(get)]dpi: f32,#[pyo3(get)]output_format: String,@@@ -1156,7 -1147,7 +1180,8 @@@ impl PyLiteParseConfigtessdata_path: cfg.tessdata_path.clone(),max_pages: cfg.max_pages,target_pages: cfg.target_pages.clone(),+ extract_screenshots: cfg.extract_screenshots,+ continue_on_page_error: cfg.continue_on_page_error,dpi: cfg.dpi,output_format: match cfg.output_format {OutputFormat::Json => "json".to_string(),@@@ -1222,7 -1283,7 +1317,8 @@@ impl LiteParsetessdata_path = None,max_pages = None,target_pages = None,+ extract_screenshots = None,+ continue_on_page_error = None,dpi = None,output_format = None,preserve_very_small_text = None,@@@ -1259,7 -1320,7 +1355,8 @@@tessdata_path: Option<String>,max_pages: Option<usize>,target_pages: Option<String>,+ extract_screenshots: Option<bool>,+ continue_on_page_error: Option<bool>,dpi: Option<f32>,output_format: Option<String>,preserve_very_small_text: Option<bool>,@@@ -1310,9 -1371,9 +1407,12 @@@if let Some(v) = target_pages {cfg.target_pages = Some(v);}+ if let Some(v) = extract_screenshots {+ cfg.extract_screenshots = v;+ }+ if let Some(v) = continue_on_page_error {+ cfg.continue_on_page_error = v;+ }if let Some(v) = dpi {cfg.dpi = v;}diff --cc crates/liteparse/src/config.rsindex 1c91bca,9713240..a4bca65--- a/crates/liteparse/src/config.rs+++ b/crates/liteparse/src/config.rs@@@ -19,10 -19,12 +19,16 @@@ pub struct LiteParseConfigpub max_pages: usize,/// Specific pages to parse (e.g., "1-5,10,15-20"). None means all pages.pub target_pages: Option<String>,+ /// Render parsed pages to PNG and return them in `ParseResult.screenshots`.+ /// Default `false`; PNG payloads can be large.+ #[serde(default)]+ pub extract_screenshots: bool,+ /// Continue parsing after a page-level PDFium extraction failure and+ /// report it in `ParseResult.page_errors`. Default `false` preserves the+ /// fail-fast behavior. Document-open and document-level failures remain+ /// fatal regardless of this setting.+ #[serde(default)]+ pub continue_on_page_error: bool,/// DPI for rendering pages (used for OCR and screenshots).pub dpi: f32,/// Output format.@@@ -217,7 -219,7 +223,8 @@@ impl Default for LiteParseConfigtessdata_path: None,max_pages: 1000,target_pages: None,+ extract_screenshots: false,+ continue_on_page_error: false,dpi: 150.0,output_format: OutputFormat::Json,preserve_very_small_text: false,@@@ -358,7 -367,7 +372,8 @@@ mod tests// OCR defaults on only when a built-in engine is compiled in.assert_eq!(c.ocr_enabled, cfg!(feature = "tesseract"));assert_eq!(c.max_pages, 1000);+ assert!(!c.extract_screenshots);+ assert!(!c.continue_on_page_error);assert_eq!(c.dpi, 150.0);assert_eq!(c.output_format, OutputFormat::Json);assert!(!c.preserve_very_small_text);diff --cc crates/liteparse/src/parser.rsindex 31b7501,481c45a..ceda9df--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@@ -544,13 -629,12 +633,13 @@@ impl LiteParse// run document actions and paint computed field appearances that// have no appearance stream to flatten.//-- // Plain OCR rendering does not need it — flattening promotes the-- // widget appearances into page content, so the raster is the same-- // either way. Complexity likewise runs on the flattened document-- // by design (see `is_complex`).- let needs_pristine_document =- flattened_form_widgets && self.config.ocr_enabled && self.config.render_form_fields;++ // Plain rendering (OCR rasters, screenshots) does not need it —++ // flattening promotes the widget appearances into page content,++ // so the raster is the same either way. Complexity likewise runs++ // on the flattened document by design (see `is_complex`).+ let needs_pristine_document = flattened_form_widgets+ && (self.config.ocr_enabled || self.config.extract_screenshots)+ && self.config.render_form_fields;let pristine_document = needs_pristine_document.then(|| extract::load_document_from_input(&lib, document_input, password)).transpose()?;@@@ -593,34 -690,6 +695,35 @@@} else {Vec::new()};+ #[cfg(not(target_arch = "wasm32"))]+ let screenshots = if self.config.extract_screenshots {+ let page_numbers = pages+ .iter()+ .map(|page| page.page_number as u32)+ .collect::<Vec<_>>();+ render::render_document_pages(+ analysis_document,+ Some(&page_numbers),+ self.config.dpi,+ self.config.detect_screenshot_rects,+ self.config.render_form_fields,++ self.config.continue_on_page_error,+ )?+ .into_iter()+ .map(|page| ScreenshotResult {+ page_num: page.page_num,+ width: page.width,+ height: page.height,+ image_bytes: page.png_bytes,+ is_solid_fill: page.is_solid_fill,+ rects: page.rects,+ })+ .collect()+ } else {+ Vec::new()+ };+ #[cfg(target_arch = "wasm32")]+ let screenshots = Vec::new();// `lib` is dropped here, releasing the PDFium lock.(pages,diff --cc crates/liteparse/src/render.rsindex 4a7ee70,d8c961f..43cb98d--- a/crates/liteparse/src/render.rs+++ b/crates/liteparse/src/render.rs@@@ -46,6 -46,6 +46,7 @@@ pub fn render_pages_to_pngdpi,detect_rects,render_form_fields,++ false,)}@@@ -55,6 -55,6 +56,7 @@@ pub(crate) fn render_document_pagesdpi: f32,detect_rects: bool,render_form_fields: bool,++ continue_on_page_error: bool,) -> Result<Vec<RenderedPage>, LiteParseError> {let page_count = document.page_count() as u32;let pages: Vec<u32> = match page_numbers {@@@ -71,43 -71,43 +73,55 @@@let mut results = Vec::with_capacity(pages.len());for page_num in pages {-- if page_num < 1 || page_num > page_count {-- return Err(LiteParseError::Other(format!(-- "page {page_num} out of range (document has {page_count} pages)"-- )));-- }++ let page_render = (|| -> Result<RenderedPage, LiteParseError> {++ if page_num < 1 || page_num > page_count {++ return Err(LiteParseError::Other(format!(++ "page {page_num} out of range (document has {page_count} pages)"++ )));++ }++++ let page = document.page((page_num - 1) as i32)?;++ let bitmap = page.render_with_form(dpi, form.as_ref())?;++ let width = bitmap.width() as u32;++ let height = bitmap.height() as u32;++ let rgba = bitmap.to_rgba();-- let page = document.page((page_num - 1) as i32)?;-- let bitmap = page.render_with_form(dpi, form.as_ref())?;-- let width = bitmap.width() as u32;-- let height = bitmap.height() as u32;-- let rgba = bitmap.to_rgba();++ let is_solid_fill = is_solid_fill_rgba(&rgba, width as usize, height as usize);++ // A solid-fill page has no structure to find; skip the scan (this is++ // also the extract binary's cheap blank-page short-circuit).++ let rects = if detect_rects && !is_solid_fill {++ find_solid_rects_rgba(++ &rgba,++ width as usize,++ height as usize,++ page.width(),++ page.height(),++ )++ } else {++ Vec::new()++ };-- let is_solid_fill = is_solid_fill_rgba(&rgba, width as usize, height as usize);-- // A solid-fill page has no structure to find; skip the scan (this is-- // also the extract binary's cheap blank-page short-circuit).-- let rects = if detect_rects && !is_solid_fill {-- find_solid_rects_rgba(-- &rgba,-- width as usize,-- height as usize,-- page.width(),-- page.height(),-- )-- } else {-- Vec::new()-- };++ let png_bytes = encode_png(&rgba, width, height)?;-- let png_bytes = encode_png(&rgba, width, height)?;++ Ok(RenderedPage {++ page_num,++ width,++ height,++ png_bytes,++ is_solid_fill,++ rects,++ })++ })();-- results.push(RenderedPage {-- page_num,-- width,-- height,-- png_bytes,-- is_solid_fill,-- rects,-- });++ match page_render {++ Ok(rendered) => results.push(rendered),++ // The page's text is already extracted; a tolerant parse keeps++ // it and just forgoes this page's screenshot.++ Err(error) if continue_on_page_error => eprintln!(++ "[render] page {page_num} failed: {error} — skipping its screenshot (continue_on_page_error)"++ ),++ Err(error) => return Err(error),++ }}Ok(results)diff --cc packages/node/README.mdindex e9c3b73,e1931d7..52c9b8c--- a/packages/node/README.md+++ b/packages/node/README.md@@@ -55,7 -77,7 +77,8 @@@ const parser = new LiteParse(tessdataPath: undefined, // Path to tessdata directory (optional)maxPages: 1000, // Max pages to parsetargetPages: '1-5,10', // Specific pages (optional)+ extractScreenshots: false, // Return parsed pages as PNG buffers+ continueOnPageError: false, // Skip broken pages and return pageErrorsdpi: 150, // Rendering DPIoutputFormat: 'json', // "json" | "text" | "markdown"imageMode: 'placeholder', // Markdown image handling: "placeholder" | "off" | "embed"diff --cc packages/node/native.d.tsindex 6a37ce9,e4bb544..c7eb8d2--- a/packages/node/native.d.ts+++ b/packages/node/native.d.ts@@@ -21,11 -21,11 +21,16 @@@ export interface JsLiteParseConfigmaxPages?: number/** Specific pages to parse (e.g., "1-5,10,15-20"). */targetPages?: string+ /**+ * Render parsed pages to PNG and return them in `ParseResult.screenshots`.+ * Default false; PNG payloads can be large.+ */+ extractScreenshots?: boolean+ /**+ * Continue after page-level extraction failures and return them in+ * `ParseResult.pageErrors`. Default false.+ */+ continueOnPageError?: boolean/** DPI for rendering pages (used for OCR and screenshots). */dpi?: number/** Output format: "json", "text", or "markdown". */@@@ -316,10 -330,12 +335,13 @@@ export interface JsFormFieldselectedOptions: Array<string>}export interface JsParseResult {+ /** Total source-document pages before target/max-page filtering. */+ totalPages: numberpages: Array<JsParsedPage>+ pageErrors: Array<JsPageError>text: stringimages: Array<JsExtractedImage>+ screenshots: Array<JsScreenshotResult>imageErrorCount: numberformType?: number/** The document's `/Info` `Creator` entry, when present. */diff --cc packages/node/src/lib.tsindex 4df0fe5,cbd75e9..21df191--- a/packages/node/src/lib.ts+++ b/packages/node/src/lib.ts@@@ -29,8 -28,8 +29,10 @@@ export interface LiteParseConfigtessdataPath?: string;maxPages: number;targetPages?: string;+ /** Render parsed pages to PNG and return them in `ParseResult.screenshots`. */+ extractScreenshots: boolean;+ /** Continue after page-level extraction failures and collect `pageErrors`. */+ continueOnPageError: boolean;dpi: number;outputFormat: OutputFormat;/** How to surface raster images in markdown output (default: "placeholder"). */@@@ -536,7 -552,7 +557,8 @@@ export class LiteParsetessdataPath: userConfig.tessdataPath,maxPages: userConfig.maxPages,targetPages: userConfig.targetPages,+ extractScreenshots: userConfig.extractScreenshots,+ continueOnPageError: userConfig.continueOnPageError,dpi: userConfig.dpi,outputFormat: userConfig.outputFormat,imageMode: userConfig.imageMode,@@@ -578,7 -594,7 +600,8 @@@tessdataPath: resolved.tessdataPath ?? undefined,maxPages: resolved.maxPages ?? 1000,targetPages: resolved.targetPages ?? undefined,+ extractScreenshots: resolved.extractScreenshots ?? false,+ continueOnPageError: resolved.continueOnPageError ?? false,dpi: resolved.dpi ?? 150,outputFormat: (resolved.outputFormat as OutputFormat) ?? "json",imageMode: (resolved.imageMode as ImageMode) ?? "placeholder",@@@ -721,6 -771,22 +778,23 @@@ function toComplexity(s: NativePageComp};}+ function toParseResult(result: NativeParseResult): ParseResult {+ return {+ totalPages: result.totalPages,+ pages: result.pages.map(toPage),+ pageErrors: result.pageErrors ?? [],+ text: result.text,+ images: (result.images ?? []).map(toImage),++ screenshots: (result.screenshots ?? []).map(toScreenshot),+ imageErrorCount: result.imageErrorCount ?? 0,+ formType: result.formType,+ creator: result.creator,+ producer: result.producer,+ docMeta: result.docMeta,+ xfaPackets: result.xfaPackets,+ };+ }+function toPage(p: NativeParsedPage): ParsedPage {return {pageNum: p.pageNum,diff --cc packages/node/src/native.tsindex 00c0015,0868b91..4991dfb--- a/packages/node/src/native.ts+++ b/packages/node/src/native.ts@@@ -27,7 -27,7 +27,8 @@@ export interface LiteParseNativeConfigtessdataPath?: string;maxPages?: number;targetPages?: string;+ extractScreenshots?: boolean;+ continueOnPageError?: boolean;dpi?: number;outputFormat?: string;imageMode?: string;@@@ -235,10 -235,11 +236,12 @@@ export interface NativeExtractedImage}export interface NativeParseResult {+ totalPages: number;pages: NativeParsedPage[];+ pageErrors: Array<{ pageNum: number; message: string }>;text: string;images: NativeExtractedImage[];+ screenshots: NativeScreenshotResult[];imageErrorCount: number;formType?: number;creator?: string;diff --cc packages/python/README.mdindex bef05ae,788424d..0517bd7--- a/packages/python/README.md+++ b/packages/python/README.md@@@ -54,7 -55,7 +55,8 @@@ parser = LiteParsetessdata_path=None, # Path to tessdata directory (optional)max_pages=1000, # Max pages to parsetarget_pages="1-5,10", # Specific pages (optional)+ extract_screenshots=False, # Return parsed pages as PNG bytes+ continue_on_page_error=False, # Skip broken pages and return page_errorsdpi=150, # Rendering DPIoutput_format="json", # "json" | "text" | "markdown"image_mode="placeholder", # Markdown image handling: "placeholder" | "off" | "embed"diff --cc packages/python/liteparse/parser.pyindex 62eed93,487260d..a3a2aaf--- a/packages/python/liteparse/parser.py+++ b/packages/python/liteparse/parser.py@@@ -339,29 -341,13 +341,34 @@@ def _convert_native_result(native_resulreturn ParseResult(pages=pages,text=native_result.text,+ total_pages=getattr(native_result, "total_pages", len(pages)),images=images,+ screenshots=[+ ScreenshotResult(+ page_num=screenshot.page_num,+ width=screenshot.width,+ height=screenshot.height,+ image_bytes=screenshot.image_bytes,+ is_solid_fill=getattr(screenshot, "is_solid_fill", False),+ rects=[+ ScreenshotRect(+ x=rect.x,+ y=rect.y,+ width=rect.width,+ height=rect.height,+ color=rect.color,+ is_line=rect.is_line,+ )+ for rect in getattr(screenshot, "rects", [])+ ],+ )+ for screenshot in getattr(native_result, "screenshots", [])+ ],image_error_count=getattr(native_result, "image_error_count", 0),+ page_errors=[+ PageError(page_num=error.page_num, message=error.message)+ for error in getattr(native_result, "page_errors", [])+ ],form_type=getattr(native_result, "form_type", None),creator=getattr(native_result, "creator", None),producer=getattr(native_result, "producer", None),@@@ -405,7 -391,7 +412,8 @@@ class LiteParsetessdata_path: Optional[str] = None,max_pages: Optional[int] = None,target_pages: Optional[str] = None,+ extract_screenshots: Optional[bool] = None,+ continue_on_page_error: Optional[bool] = None,dpi: Optional[float] = None,output_format: Optional[str] = None,preserve_very_small_text: Optional[bool] = None,@@@ -446,8 -432,9 +454,11 @@@tessdata_path: Path to tessdata directory for Tesseractmax_pages: Maximum number of pages to parsetarget_pages: Specific pages to parse (e.g., "1-5,10,15-20")+ extract_screenshots: Render parsed pages to PNG and return them in+ ``ParseResult.screenshots``. Default False; PNG payloads can be large.+ continue_on_page_error: Skip page-level PDF extraction failures and+ return them in ``ParseResult.page_errors``. Document-level+ failures remain fatal. Default False.dpi: DPI for rendering (affects OCR quality)output_format: Output format: "json", "text", or "markdown" (default: "json")preserve_very_small_text: Whether to preserve very small text@@@ -521,8 -508,8 +532,10 @@@kwargs["max_pages"] = max_pagesif target_pages is not None:kwargs["target_pages"] = target_pages+ if extract_screenshots is not None:+ kwargs["extract_screenshots"] = extract_screenshots+ if continue_on_page_error is not None:+ kwargs["continue_on_page_error"] = continue_on_page_errorif dpi is not None:kwargs["dpi"] = dpiif output_format is not None:@@@ -715,7 -774,7 +800,8 @@@tessdata_path=cfg.tessdata_path,max_pages=cfg.max_pages,target_pages=cfg.target_pages,+ extract_screenshots=cfg.extract_screenshots,+ continue_on_page_error=cfg.continue_on_page_error,dpi=cfg.dpi,output_format=cfg.output_format,preserve_very_small_text=cfg.preserve_very_small_text,diff --cc packages/python/liteparse/types.pyindex cef6a26,18b8db3..9705fa9--- a/packages/python/liteparse/types.py+++ b/packages/python/liteparse/types.py@@@ -248,9 -255,11 +255,12 @@@ class ParseResult"""Result of parsing a document."""pages: List[ParsedPage]text: str+ #: Total source-document pages before target/max-page filtering.+ total_pages: int = 0images: List[ExtractedImage] = field(default_factory=list)+ screenshots: List["ScreenshotResult"] = field(default_factory=list)image_error_count: int = 0+ page_errors: List[PageError] = field(default_factory=list)#: PDFium form type, present only when ``extract_form_fields=True``.form_type: Optional[int] = None#: The document's ``/Info`` ``Creator`` entry, when present.@@@ -383,7 -405,7 +406,8 @@@ class LiteParseConfigtessdata_path: Optional[str]max_pages: inttarget_pages: Optional[str]+ extract_screenshots: bool+ continue_on_page_error: booldpi: floatoutput_format: strpreserve_very_small_text: bool