A fast, helpful, and open-source document parser
review: clarify batch API surface, add deterministic session cleanup
Logan Markewich7d3f93a31352
diff --git a/crates/liteparse-napi/src/lib.rs b/crates/liteparse-napi/src/lib.rsindex ffcf0af..c7e7ba3 100644--- a/crates/liteparse-napi/src/lib.rs+++ b/crates/liteparse-napi/src/lib.rs@@ -48,14 +48,16 @@ impl LiteParse {Ok(JsParseResult::from_rust(&result, &self.config))}- /// Open a document for bounded-memory batch parsing.+ /// Open a document for bounded-memory batch parsing. Internal plumbing+ /// for the JS wrapper's `parseBatches()` — prefer that; it also closes+ /// the session for you.////// Converts a non-PDF source once, then yields `batchSize` pages at a time/// via `nextBatch()` (default 25). Cross-page passes (repeated/// header/footer removal, image deduplication) see only the pages in their/// own batch, so output can differ from a whole-document `parse()`.#[napi]- pub async fn open(+ pub async fn open_batch_session(&self,input: Either<String, Buffer>,batch_size: Option<u32>,@@ -72,13 +74,13 @@ impl LiteParse {let session = self.inner- .open(pdf_input, batch_size)+ .open_batch_session(pdf_input, batch_size).await.map_err(|e| Error::from_reason(e.to_string()))?;Ok(ParseSession {total_pages: session.total_pages(),- inner: std::sync::Arc::new(tokio::sync::Mutex::new(session)),+ inner: std::sync::Arc::new(tokio::sync::Mutex::new(Some(session))),config: self.config.clone(),})}@@ -169,17 +171,20 @@ impl LiteParse {}}-/// A document opened once and parsed in bounded page batches.+/// A document opened once and parsed in bounded page batches. Internal+/// plumbing for the JS wrapper's `parseBatches()` — prefer that.///-/// Created by `LiteParse.open()`. The converted-PDF temporary file for a-/// non-PDF source lives as long as the session, so conversion is paid once no-/// matter how many batches are consumed.+/// Created by `LiteParse.openBatchSession()`. The converted-PDF temporary+/// file for a non-PDF source lives as long as the session, so conversion is+/// paid once no matter how many batches are consumed. Call `close()` when+/// abandoning the session early — otherwise that temp file waits for GC.#[napi]pub struct ParseSession {/// The core session is `&mut` per batch, but napi hands out `&self`, so/// the mutation is serialized here. Concurrent `nextBatch()` calls queue/// rather than interleave, which also keeps batch order well-defined.- inner: std::sync::Arc<tokio::sync::Mutex<liteparse::ParseSession>>,+ /// `None` after `close()`.+ inner: std::sync::Arc<tokio::sync::Mutex<Option<liteparse::ParseSession>>>,config: liteparse::config::LiteParseConfig,total_pages: u32,}@@ -193,12 +198,14 @@ impl ParseSession {}/// Parse and return the next batch, or `null` once every page within- /// `maxPages` has been yielded.+ /// `maxPages` has been yielded. Rejects if the session is closed.#[napi]pub async fn next_batch(&self) -> Result<Option<JsParseBatch>> {let inner = self.inner.clone();let mut session = inner.lock().await;let batch = session+ .as_mut()+ .ok_or_else(|| Error::from_reason("session is closed"))?.next_batch().await.map_err(|e| Error::from_reason(e.to_string()))?;@@ -209,6 +216,20 @@ impl ParseSession {result: JsParseResult::from_rust(&batch.result, &self.config),}))}++ /// Release the session's resources now — most importantly the converted+ /// temporary PDF for a non-PDF source, which otherwise lives until the+ /// JS object is garbage collected. Idempotent; `nextBatch()` rejects+ /// afterwards.+ #[napi]+ pub async fn close(&self) -> Result<()> {+ let inner = self.inner.clone();+ let mut session = inner.lock().await;+ // Dropping the core session drops the conversion guard, which+ // removes the temp file.+ session.take();+ Ok(())+ }}/// Search text items for phrase matches, returning merged items with combined bounding boxes.diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rsindex 2be21e6..c9c16b2 100644--- a/crates/liteparse-python/src/lib.rs+++ b/crates/liteparse-python/src/lib.rs@@ -1172,8 +1172,11 @@ impl PyLiteParseConfig {// Batch parsing// ----------------------------------------------------------------------------/// One batch of pages from a `ParseSession`.-#[pyclass(frozen, name = "ParseBatch")]+/// One batch of pages from a `_ParseSession`. Internal plumbing for the+/// wrapper's `parse_batches()`, which converts it into the public+/// `liteparse.types.ParseBatch` dataclass — the underscore name keeps the+/// two from colliding.+#[pyclass(frozen, name = "_ParseBatch", skip_from_py_object)]#[derive(Clone)]struct PyParseBatch {/// First source page in this batch (1-indexed).@@ -1187,13 +1190,14 @@ struct PyParseBatch {result: PyParseResult,}-/// A document opened once and parsed in bounded page batches.+/// A document opened once and parsed in bounded page batches. Internal+/// plumbing for the wrapper's `parse_batches()` — prefer that.////// Iterate it directly to consume every batch:///-/// for batch in parser.open("large.pdf", batch_size=20):+/// for batch in parser.open_batch_session("large.pdf", batch_size=20):/// handle(batch.result.pages)-#[pyclass(name = "ParseSession", unsendable)]+#[pyclass(name = "_ParseSession", unsendable)]struct PyParseSession {inner: liteparse::ParseSession,runtime: std::sync::Arc<tokio::runtime::Runtime>,@@ -1464,13 +1468,14 @@ impl LiteParse {}/// Open a document from a file path for bounded-memory batch parsing.+ /// Internal plumbing for the wrapper's `parse_batches()` — prefer that.///- /// Converts a non-PDF source once and returns a `ParseSession` yielding+ /// Converts a non-PDF source once and returns a `_ParseSession` yielding/// `batch_size` pages at a time. Cross-page passes (repeated header/footer/// removal, image deduplication) see only the pages in their own batch, so/// output can differ from a whole-document `parse()`.#[pyo3(signature = (input, batch_size = None))]- fn open(+ fn open_batch_session(&self,py: Python<'_>,input: String,@@ -1480,8 +1485,9 @@ impl LiteParse {}/// Open a document from raw bytes for bounded-memory batch parsing.+ /// Internal plumbing for the wrapper's `parse_batches()` — prefer that.#[pyo3(signature = (data, batch_size = None))]- fn open_bytes(+ fn open_batch_session_bytes(&self,py: Python<'_>,data: Vec<u8>,@@ -1583,8 +1589,8 @@ impl LiteParse {}impl LiteParse {- /// Shared body of `open` / `open_bytes`. Not a `#[pymethods]` entry, so it- /// stays off the Python surface.+ /// Shared body of `open_batch_session` / `open_batch_session_bytes`. Not+ /// a `#[pymethods]` entry, so it stays off the Python surface.fn open_session(&self,py: Python<'_>,@@ -1593,7 +1599,10 @@ impl LiteParse {) -> PyResult<PyParseSession> {let batch_size = batch_size.unwrap_or(liteparse::DEFAULT_PAGE_BATCH_SIZE);let session = py- .detach(|| self.runtime.block_on(self.inner.open(input, batch_size)))+ .detach(|| {+ self.runtime+ .block_on(self.inner.open_batch_session(input, batch_size))+ }).map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;Ok(PyParseSession {diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rsindex cce034f..57176cc 100644--- a/crates/liteparse-wasm/src/lib.rs+++ b/crates/liteparse-wasm/src/lib.rs@@ -1110,7 +1110,8 @@ impl LiteParse {/// Cross-page passes (repeated header/footer removal, image deduplication)/// see only the pages in their own batch, so output can differ from a/// whole-document `parse()`.- pub async fn open(+ #[wasm_bindgen(js_name = openBatchSession)]+ pub async fn open_batch_session(&self,data: Vec<u8>,batch_size: Option<usize>,@@ -1118,7 +1119,7 @@ impl LiteParse {let batch_size = batch_size.unwrap_or(liteparse::DEFAULT_PAGE_BATCH_SIZE);let session = self.inner- .open(PdfInput::Bytes(data), batch_size)+ .open_batch_session(PdfInput::Bytes(data), batch_size).await.map_err(|e| JsError::new(&format!("open failed: {}", e)))?;@@ -1144,8 +1145,9 @@ pub struct ParseBatch {/// A document opened once and parsed in bounded page batches.///-/// Created by `LiteParse.open()`. Call `nextBatch()` until it returns-/// `undefined`.+/// Created by `LiteParse.openBatchSession()`. Call `nextBatch()` until it+/// returns `undefined`, then `free()` the session to release its wasm-side+/// memory promptly (wasm-bindgen objects are not garbage collected).#[wasm_bindgen]pub struct ParseSession {inner: liteparse::ParseSession,@@ -1162,6 +1164,9 @@ impl ParseSession {/// Parse and return the next batch, or `undefined` once every page within/// `maxPages` has been yielded.+ ///+ /// Batches are parsed one at a time: await each call before making the+ /// next (a concurrent call throws wasm-bindgen's recursive-borrow error).#[wasm_bindgen(js_name = nextBatch)]pub async fn next_batch(&mut self) -> Result<Option<ParseBatch>, JsError> {let batch = selfdiff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rsindex 972a5e1..64ddcf2 100644--- a/crates/liteparse/src/config.rs+++ b/crates/liteparse/src/config.rs@@ -259,7 +259,8 @@ fn default_num_workers() -> usize {/// input.const MAX_TARGET_PAGES: u64 = 100_000;-/// Pages per batch when a caller of [`crate::parser::LiteParse::open`] does+/// Pages per batch when a caller of+/// [`crate::parser::LiteParse::open_batch_session`] does/// not pick a size. Small enough to keep the materialized result bounded;/// large enough that the per-batch document reopen stays modest (it costs/// roughly 13% at this size on a 457-page document, ~4% at 50).diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex 7e20a4e..e72e085 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -895,7 +895,7 @@ impl LiteParse {////// Returns an error if `target_pages` is configured — an explicit page/// selection and generated batch ranges would be ambiguous together.- pub async fn open(+ pub async fn open_batch_session(&self,input: PdfInput,batch_size: usize,@@ -944,9 +944,9 @@ impl LiteParse {/// A document opened once and parsed in bounded page batches.///-/// Created by [`LiteParse::open`]. The converted-PDF temporary file (for-/// non-PDF sources) lives as long as the session, so conversion is paid once-/// no matter how many batches are consumed. The PDFium document itself is+/// Created by [`LiteParse::open_batch_session`]. The converted-PDF temporary+/// file (for non-PDF sources) lives as long as the session, so conversion is+/// paid once no matter how many batches are consumed. The PDFium document is/// reopened per batch, which measures at a few percent of a batch's extraction/// cost and keeps the process-global PDFium lock free between batches.pub struct ParseSession {@@ -1208,4 +1208,51 @@ mod tests {assert_eq!(pages[0].markdown, "intro\n\n\n\noutro");assert_eq!(full_text, pages[0].markdown);}++ /// A non-PDF source is converted exactly once, when the session opens:+ /// the converted temporary PDF outlives every batch and is only removed+ /// when the session is dropped. Lives here rather than in the integration+ /// tests because it asserts against [`ResolvedInput`] internals.+ #[tokio::test]+ #[serial_test::serial]+ async fn test_batch_session_owns_converted_source_for_its_lifetime() {+ if std::env::var("SKIP_INTEGRATION_TESTS").as_deref() == Ok("yes") {+ return;+ }+ let parser = LiteParse::new(LiteParseConfig {+ ocr_enabled: false,+ quiet: true,+ ..LiteParseConfig::default()+ });+ let mut session = parser+ .open_batch_session(+ PdfInput::Path("../../integration_tests_data/sample3.doc".to_string()),+ 1,+ )+ .await+ .expect("should convert and open a .doc");++ assert!(session.input.is_converted());+ let converted_path = match &session.input.input {+ PdfInput::Path(p) => p.clone(),+ PdfInput::Bytes(_) => panic!("a converted .doc should resolve to a temp file path"),+ };++ let mut pages = 0;+ while let Some(batch) = session.next_batch().await.expect("batch should parse") {+ pages += batch.result.pages.len();+ assert!(+ std::path::Path::new(&converted_path).exists(),+ "converted temp PDF should outlive every batch — a missing file \+ would mean it was re-resolved or cleaned up per batch"+ );+ }+ assert_eq!(pages, 2);++ drop(session);+ assert!(+ !std::path::Path::new(&converted_path).exists(),+ "dropping the session should clean up the converted temp PDF"+ );+ }}diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rsindex f0eff78..8eb2796 100644--- a/crates/liteparse/tests/integration_test.rs+++ b/crates/liteparse/tests/integration_test.rs@@ -132,7 +132,7 @@ async fn test_batch_parse_matches_whole_document() {.expect("whole-document parse should succeed");let mut session = LiteParse::new(config())- .open(PdfInput::Path(path.to_string()), 2)+ .open_batch_session(PdfInput::Path(path.to_string()), 2).await.expect("should open a session");assert_eq!(session.total_pages(), whole.total_pages);@@ -169,7 +169,7 @@ async fn test_batch_parse_respects_max_pages() {max_pages: 2,..LiteParseConfig::default()})- .open(+ .open_batch_session(PdfInput::Path("../../integration_tests_data/filled_acroform.pdf".to_string()),100,)@@ -201,7 +201,7 @@ async fn test_batch_parse_rejects_target_pages() {target_pages: Some("1-2".into()),..LiteParseConfig::default()})- .open(+ .open_batch_session(PdfInput::Path("../../integration_tests_data/filled_acroform.pdf".to_string()),25,)@@ -212,38 +212,6 @@ async fn test_batch_parse_rejects_target_pages() {);}-/// A non-PDF source is converted once when the session opens, not once per-/// batch — the whole reason batching lives in core rather than in a binding.-#[tokio::test]-#[serial]-async fn test_batch_parse_converts_office_source_once() {- let env_var = std::env::var("SKIP_INTEGRATION_TESTS");- if let Ok(v) = env_var- && v == "yes"- {- return;- }- let mut session = LiteParse::new(LiteParseConfig {- ocr_enabled: false,- quiet: true,- ..LiteParseConfig::default()- })- .open(- PdfInput::Path("../../integration_tests_data/sample3.doc".to_string()),- 1,- )- .await- .expect("should convert and open a .doc");-- // The converted temporary PDF is owned by the session, so every batch- // after the first resolves against a file that no longer needs LibreOffice.- let mut pages = 0;- while let Some(batch) = session.next_batch().await.expect("batch should parse") {- pages += batch.result.pages.len();- }- assert_eq!(pages, 2);-}-#[tokio::test]#[serial]async fn test_parse_bytes_office_integration() {diff --git a/packages/node/native.d.ts b/packages/node/native.d.tsindex dd0e4e3..6689e54 100644--- a/packages/node/native.d.ts+++ b/packages/node/native.d.ts@@ -482,14 +482,16 @@ export declare class LiteParse {/** Parse a document. Accepts a file path (string) or raw PDF bytes (Buffer). */parse(input: string | Buffer): Promise<JsParseResult>/**- * Open a document for bounded-memory batch parsing.+ * Open a document for bounded-memory batch parsing. Internal plumbing+ * for the JS wrapper's `parseBatches()` — prefer that; it also closes+ * the session for you.** Converts a non-PDF source once, then yields `batchSize` pages at a time* via `nextBatch()` (default 25). Cross-page passes (repeated* header/footer removal, image deduplication) see only the pages in their* own batch, so output can differ from a whole-document `parse()`.*/- open(input: string | Buffer, batchSize?: number | undefined | null): Promise<ParseSession>+ openBatchSession(input: string | Buffer, batchSize?: number | undefined | null): Promise<ParseSession>/*** Parse from pre-extracted pages, skipping PDFium text extraction.*@@ -518,18 +520,27 @@ export declare class LiteParse {get config(): JsLiteParseConfig}/**- * A document opened once and parsed in bounded page batches.+ * A document opened once and parsed in bounded page batches. Internal+ * plumbing for the JS wrapper's `parseBatches()` — prefer that.*- * Created by `LiteParse.open()`. The converted-PDF temporary file for a- * non-PDF source lives as long as the session, so conversion is paid once no- * matter how many batches are consumed.+ * Created by `LiteParse.openBatchSession()`. The converted-PDF temporary+ * file for a non-PDF source lives as long as the session, so conversion is+ * paid once no matter how many batches are consumed. Call `close()` when+ * abandoning the session early — otherwise that temp file waits for GC.*/export declare class ParseSession {/** Total pages in the source document, before `maxPages` or batching. */get totalPages(): number/*** Parse and return the next batch, or `null` once every page within- * `maxPages` has been yielded.+ * `maxPages` has been yielded. Rejects if the session is closed.*/nextBatch(): Promise<JsParseBatch | null>+ /**+ * Release the session's resources now — most importantly the converted+ * temporary PDF for a non-PDF source, which otherwise lives until the+ * JS object is garbage collected. Idempotent; `nextBatch()` rejects+ * afterwards.+ */+ close(): Promise<void>}diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.tsindex 33bd2a3..a01dbf6 100644--- a/packages/node/src/lib.ts+++ b/packages/node/src/lib.ts@@ -633,35 +633,49 @@ export class LiteParse {* Each yielded result is independent and becomes collectible once the caller* advances the iterator, so a consumer that does not retain batches never* holds more than one batch of pages in memory. A non-PDF source is- * converted once when the iterator starts, not once per batch.+ * converted once when the iterator starts, not once per batch; its temporary+ * file is released when iteration ends — including an early `break` or+ * `throw`, which run the generator's cleanup.** Cross-page passes see only the pages in their own batch, so repeated* header/footer removal and image deduplication are batch-local and the* output can differ from `parse()`. Prefer `parse()` unless the size of the* materialized result is the problem.*- * Throws if the parser is configured with `targetPages` — an explicit page- * selection and generated batch ranges are ambiguous together.+ * As with any async generator, work starts on the first `next()` call, so+ * errors (an unreadable file, or a parser configured with `targetPages` —+ * ambiguous with generated batch ranges) surface on the first iteration+ * rather than when `parseBatches()` itself is called.*/async *parseBatches(input: LiteParseInput,options: ParseBatchOptions = {},): AsyncGenerator<ParseBatch> {const nativeInput = typeof input === "string" ? input : Buffer.from(input);- const session = await this._native.open(nativeInput, options.batchSize);- const totalPages = session.totalPages;+ const session = await this._native.openBatchSession(+ nativeInput,+ options.batchSize,+ );+ try {+ const totalPages = session.totalPages;- for (;;) {- const batch = await session.nextBatch();- if (batch == null) {- return;+ for (;;) {+ const batch = await session.nextBatch();+ if (batch == null) {+ return;+ }+ yield {+ startPage: batch.startPage,+ endPage: batch.endPage,+ totalPages,+ result: toParseResult(batch.result),+ };}- yield {- startPage: batch.startPage,- endPage: batch.endPage,- totalPages,- result: toParseResult(batch.result),- };+ } finally {+ // Frees the session's converted-PDF temp file now instead of at GC —+ // this runs on normal exhaustion and when the consumer abandons the+ // loop early.+ await session.close();}}diff --git a/packages/node/src/native.ts b/packages/node/src/native.tsindex ea5f602..72f46af 100644--- a/packages/node/src/native.ts+++ b/packages/node/src/native.ts@@ -324,12 +324,13 @@ export interface NativeParseBatch {export interface NativeParseSession {nextBatch(): Promise<NativeParseBatch | null>;+ close(): Promise<void>;readonly totalPages: number;}export interface LiteParseNative {parse(input: string | Buffer): Promise<NativeParseResult>;- open(+ openBatchSession(input: string | Buffer,batchSize?: number,): Promise<NativeParseSession>;diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.pyindex 43b4982..dc645f4 100644--- a/packages/python/liteparse/parser.py+++ b/packages/python/liteparse/parser.py@@ -616,21 +616,34 @@ class LiteParse:Raises:ParseError: If parsing fails, or if the parser was constructedwith ``target_pages`` (ambiguous with generated batch ranges).+ Open errors are raised here, when ``parse_batches`` is called;+ per-batch parse errors are raised from the iterator.FileNotFoundError: If the file doesn't exist."""+ # Validate and open eagerly — this is not a generator function, so a+ # missing file or a target_pages conflict raises here rather than on+ # the first iteration of the returned iterator.try:if isinstance(file_data, bytes):- session = self._native.open_bytes(file_data, batch_size)+ session = self._native.open_batch_session_bytes(+ file_data, batch_size+ )else:file_path = Path(file_data)if not file_path.exists():raise FileNotFoundError(f"File not found: {file_path}")- session = self._native.open(str(file_path.absolute()), batch_size)+ session = self._native.open_batch_session(+ str(file_path.absolute()), batch_size+ )except FileNotFoundError:raiseexcept Exception as e:raise ParseError(str(e)) from e+ return self._iter_batches(session)++ @staticmethod+ def _iter_batches(session: Any) -> Iterator[ParseBatch]:total_pages = session.total_pageswhile True:try: