Skip to content

A fast, helpful, and open-source document parser

review: clarify batch API surface, add deterministic session cleanup

Logan Markewich7d3f93a31352
diff --git a/crates/liteparse-napi/src/lib.rs b/crates/liteparse-napi/src/lib.rs
index ffcf0af..c7e7ba3 100644
--- a/crates/liteparse-napi/src/lib.rs
+++ b/crates/liteparse-napi/src/lib.rs
@@ -48,14 +48,16 @@ impl LiteParse {
Ok(JsParseResult::from_rust(&result, &self.config))
}
- /// Open a document for bounded-memory batch parsing.
+ /// Open a document for bounded-memory batch parsing. Internal plumbing
+ /// for the JS wrapper's `parseBatches()` — prefer that; it also closes
+ /// the session for you.
///
/// Converts a non-PDF source once, then yields `batchSize` pages at a time
/// via `nextBatch()` (default 25). Cross-page passes (repeated
/// header/footer removal, image deduplication) see only the pages in their
/// own batch, so output can differ from a whole-document `parse()`.
#[napi]
- pub async fn open(
+ pub async fn open_batch_session(
&self,
input: Either<String, Buffer>,
batch_size: Option<u32>,
@@ -72,13 +74,13 @@ impl LiteParse {
let session = self
.inner
- .open(pdf_input, batch_size)
+ .open_batch_session(pdf_input, batch_size)
.await
.map_err(|e| Error::from_reason(e.to_string()))?;
Ok(ParseSession {
total_pages: session.total_pages(),
- inner: std::sync::Arc::new(tokio::sync::Mutex::new(session)),
+ inner: std::sync::Arc::new(tokio::sync::Mutex::new(Some(session))),
config: self.config.clone(),
})
}
@@ -169,17 +171,20 @@ impl LiteParse {
}
}
-/// A document opened once and parsed in bounded page batches.
+/// A document opened once and parsed in bounded page batches. Internal
+/// plumbing for the JS wrapper's `parseBatches()` — prefer that.
///
-/// Created by `LiteParse.open()`. The converted-PDF temporary file for a
-/// non-PDF source lives as long as the session, so conversion is paid once no
-/// matter how many batches are consumed.
+/// Created by `LiteParse.openBatchSession()`. The converted-PDF temporary
+/// file for a non-PDF source lives as long as the session, so conversion is
+/// paid once no matter how many batches are consumed. Call `close()` when
+/// abandoning the session early — otherwise that temp file waits for GC.
#[napi]
pub struct ParseSession {
/// The core session is `&mut` per batch, but napi hands out `&self`, so
/// the mutation is serialized here. Concurrent `nextBatch()` calls queue
/// rather than interleave, which also keeps batch order well-defined.
- inner: std::sync::Arc<tokio::sync::Mutex<liteparse::ParseSession>>,
+ /// `None` after `close()`.
+ inner: std::sync::Arc<tokio::sync::Mutex<Option<liteparse::ParseSession>>>,
config: liteparse::config::LiteParseConfig,
total_pages: u32,
}
@@ -193,12 +198,14 @@ impl ParseSession {
}
/// Parse and return the next batch, or `null` once every page within
- /// `maxPages` has been yielded.
+ /// `maxPages` has been yielded. Rejects if the session is closed.
#[napi]
pub async fn next_batch(&self) -> Result<Option<JsParseBatch>> {
let inner = self.inner.clone();
let mut session = inner.lock().await;
let batch = session
+ .as_mut()
+ .ok_or_else(|| Error::from_reason("session is closed"))?
.next_batch()
.await
.map_err(|e| Error::from_reason(e.to_string()))?;
@@ -209,6 +216,20 @@ impl ParseSession {
result: JsParseResult::from_rust(&batch.result, &self.config),
}))
}
+
+ /// Release the session's resources now — most importantly the converted
+ /// temporary PDF for a non-PDF source, which otherwise lives until the
+ /// JS object is garbage collected. Idempotent; `nextBatch()` rejects
+ /// afterwards.
+ #[napi]
+ pub async fn close(&self) -> Result<()> {
+ let inner = self.inner.clone();
+ let mut session = inner.lock().await;
+ // Dropping the core session drops the conversion guard, which
+ // removes the temp file.
+ session.take();
+ Ok(())
+ }
}
/// Search text items for phrase matches, returning merged items with combined bounding boxes.
diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rs
index 2be21e6..c9c16b2 100644
--- a/crates/liteparse-python/src/lib.rs
+++ b/crates/liteparse-python/src/lib.rs
@@ -1172,8 +1172,11 @@ impl PyLiteParseConfig {
// Batch parsing
// ---------------------------------------------------------------------------
-/// One batch of pages from a `ParseSession`.
-#[pyclass(frozen, name = "ParseBatch")]
+/// One batch of pages from a `_ParseSession`. Internal plumbing for the
+/// wrapper's `parse_batches()`, which converts it into the public
+/// `liteparse.types.ParseBatch` dataclass — the underscore name keeps the
+/// two from colliding.
+#[pyclass(frozen, name = "_ParseBatch", skip_from_py_object)]
#[derive(Clone)]
struct PyParseBatch {
/// First source page in this batch (1-indexed).
@@ -1187,13 +1190,14 @@ struct PyParseBatch {
result: PyParseResult,
}
-/// A document opened once and parsed in bounded page batches.
+/// A document opened once and parsed in bounded page batches. Internal
+/// plumbing for the wrapper's `parse_batches()` — prefer that.
///
/// Iterate it directly to consume every batch:
///
-/// for batch in parser.open("large.pdf", batch_size=20):
+/// for batch in parser.open_batch_session("large.pdf", batch_size=20):
/// handle(batch.result.pages)
-#[pyclass(name = "ParseSession", unsendable)]
+#[pyclass(name = "_ParseSession", unsendable)]
struct PyParseSession {
inner: liteparse::ParseSession,
runtime: std::sync::Arc<tokio::runtime::Runtime>,
@@ -1464,13 +1468,14 @@ impl LiteParse {
}
/// Open a document from a file path for bounded-memory batch parsing.
+ /// Internal plumbing for the wrapper's `parse_batches()` — prefer that.
///
- /// Converts a non-PDF source once and returns a `ParseSession` yielding
+ /// Converts a non-PDF source once and returns a `_ParseSession` yielding
/// `batch_size` pages at a time. Cross-page passes (repeated header/footer
/// removal, image deduplication) see only the pages in their own batch, so
/// output can differ from a whole-document `parse()`.
#[pyo3(signature = (input, batch_size = None))]
- fn open(
+ fn open_batch_session(
&self,
py: Python<'_>,
input: String,
@@ -1480,8 +1485,9 @@ impl LiteParse {
}
/// Open a document from raw bytes for bounded-memory batch parsing.
+ /// Internal plumbing for the wrapper's `parse_batches()` — prefer that.
#[pyo3(signature = (data, batch_size = None))]
- fn open_bytes(
+ fn open_batch_session_bytes(
&self,
py: Python<'_>,
data: Vec<u8>,
@@ -1583,8 +1589,8 @@ impl LiteParse {
}
impl LiteParse {
- /// Shared body of `open` / `open_bytes`. Not a `#[pymethods]` entry, so it
- /// stays off the Python surface.
+ /// Shared body of `open_batch_session` / `open_batch_session_bytes`. Not
+ /// a `#[pymethods]` entry, so it stays off the Python surface.
fn open_session(
&self,
py: Python<'_>,
@@ -1593,7 +1599,10 @@ impl LiteParse {
) -> PyResult<PyParseSession> {
let batch_size = batch_size.unwrap_or(liteparse::DEFAULT_PAGE_BATCH_SIZE);
let session = py
- .detach(|| self.runtime.block_on(self.inner.open(input, batch_size)))
+ .detach(|| {
+ self.runtime
+ .block_on(self.inner.open_batch_session(input, batch_size))
+ })
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
Ok(PyParseSession {
diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rs
index cce034f..57176cc 100644
--- a/crates/liteparse-wasm/src/lib.rs
+++ b/crates/liteparse-wasm/src/lib.rs
@@ -1110,7 +1110,8 @@ impl LiteParse {
/// Cross-page passes (repeated header/footer removal, image deduplication)
/// see only the pages in their own batch, so output can differ from a
/// whole-document `parse()`.
- pub async fn open(
+ #[wasm_bindgen(js_name = openBatchSession)]
+ pub async fn open_batch_session(
&self,
data: Vec<u8>,
batch_size: Option<usize>,
@@ -1118,7 +1119,7 @@ impl LiteParse {
let batch_size = batch_size.unwrap_or(liteparse::DEFAULT_PAGE_BATCH_SIZE);
let session = self
.inner
- .open(PdfInput::Bytes(data), batch_size)
+ .open_batch_session(PdfInput::Bytes(data), batch_size)
.await
.map_err(|e| JsError::new(&format!("open failed: {}", e)))?;
@@ -1144,8 +1145,9 @@ pub struct ParseBatch {
/// A document opened once and parsed in bounded page batches.
///
-/// Created by `LiteParse.open()`. Call `nextBatch()` until it returns
-/// `undefined`.
+/// Created by `LiteParse.openBatchSession()`. Call `nextBatch()` until it
+/// returns `undefined`, then `free()` the session to release its wasm-side
+/// memory promptly (wasm-bindgen objects are not garbage collected).
#[wasm_bindgen]
pub struct ParseSession {
inner: liteparse::ParseSession,
@@ -1162,6 +1164,9 @@ impl ParseSession {
/// Parse and return the next batch, or `undefined` once every page within
/// `maxPages` has been yielded.
+ ///
+ /// Batches are parsed one at a time: await each call before making the
+ /// next (a concurrent call throws wasm-bindgen's recursive-borrow error).
#[wasm_bindgen(js_name = nextBatch)]
pub async fn next_batch(&mut self) -> Result<Option<ParseBatch>, JsError> {
let batch = self
diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rs
index 972a5e1..64ddcf2 100644
--- a/crates/liteparse/src/config.rs
+++ b/crates/liteparse/src/config.rs
@@ -259,7 +259,8 @@ fn default_num_workers() -> usize {
/// input.
const MAX_TARGET_PAGES: u64 = 100_000;
-/// Pages per batch when a caller of [`crate::parser::LiteParse::open`] does
+/// Pages per batch when a caller of
+/// [`crate::parser::LiteParse::open_batch_session`] does
/// not pick a size. Small enough to keep the materialized result bounded;
/// large enough that the per-batch document reopen stays modest (it costs
/// roughly 13% at this size on a 457-page document, ~4% at 50).
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index 7e20a4e..e72e085 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -895,7 +895,7 @@ impl LiteParse {
///
/// Returns an error if `target_pages` is configured — an explicit page
/// selection and generated batch ranges would be ambiguous together.
- pub async fn open(
+ pub async fn open_batch_session(
&self,
input: PdfInput,
batch_size: usize,
@@ -944,9 +944,9 @@ impl LiteParse {
/// A document opened once and parsed in bounded page batches.
///
-/// Created by [`LiteParse::open`]. The converted-PDF temporary file (for
-/// non-PDF sources) lives as long as the session, so conversion is paid once
-/// no matter how many batches are consumed. The PDFium document itself is
+/// Created by [`LiteParse::open_batch_session`]. The converted-PDF temporary
+/// file (for non-PDF sources) lives as long as the session, so conversion is
+/// paid once no matter how many batches are consumed. The PDFium document is
/// reopened per batch, which measures at a few percent of a batch's extraction
/// cost and keeps the process-global PDFium lock free between batches.
pub struct ParseSession {
@@ -1208,4 +1208,51 @@ mod tests {
assert_eq!(pages[0].markdown, "intro\n\n![](img_p1_1.jpg)\n\noutro");
assert_eq!(full_text, pages[0].markdown);
}
+
+ /// A non-PDF source is converted exactly once, when the session opens:
+ /// the converted temporary PDF outlives every batch and is only removed
+ /// when the session is dropped. Lives here rather than in the integration
+ /// tests because it asserts against [`ResolvedInput`] internals.
+ #[tokio::test]
+ #[serial_test::serial]
+ async fn test_batch_session_owns_converted_source_for_its_lifetime() {
+ if std::env::var("SKIP_INTEGRATION_TESTS").as_deref() == Ok("yes") {
+ return;
+ }
+ let parser = LiteParse::new(LiteParseConfig {
+ ocr_enabled: false,
+ quiet: true,
+ ..LiteParseConfig::default()
+ });
+ let mut session = parser
+ .open_batch_session(
+ PdfInput::Path("../../integration_tests_data/sample3.doc".to_string()),
+ 1,
+ )
+ .await
+ .expect("should convert and open a .doc");
+
+ assert!(session.input.is_converted());
+ let converted_path = match &session.input.input {
+ PdfInput::Path(p) => p.clone(),
+ PdfInput::Bytes(_) => panic!("a converted .doc should resolve to a temp file path"),
+ };
+
+ let mut pages = 0;
+ while let Some(batch) = session.next_batch().await.expect("batch should parse") {
+ pages += batch.result.pages.len();
+ assert!(
+ std::path::Path::new(&converted_path).exists(),
+ "converted temp PDF should outlive every batch — a missing file \
+ would mean it was re-resolved or cleaned up per batch"
+ );
+ }
+ assert_eq!(pages, 2);
+
+ drop(session);
+ assert!(
+ !std::path::Path::new(&converted_path).exists(),
+ "dropping the session should clean up the converted temp PDF"
+ );
+ }
}
diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rs
index f0eff78..8eb2796 100644
--- a/crates/liteparse/tests/integration_test.rs
+++ b/crates/liteparse/tests/integration_test.rs
@@ -132,7 +132,7 @@ async fn test_batch_parse_matches_whole_document() {
.expect("whole-document parse should succeed");
let mut session = LiteParse::new(config())
- .open(PdfInput::Path(path.to_string()), 2)
+ .open_batch_session(PdfInput::Path(path.to_string()), 2)
.await
.expect("should open a session");
assert_eq!(session.total_pages(), whole.total_pages);
@@ -169,7 +169,7 @@ async fn test_batch_parse_respects_max_pages() {
max_pages: 2,
..LiteParseConfig::default()
})
- .open(
+ .open_batch_session(
PdfInput::Path("../../integration_tests_data/filled_acroform.pdf".to_string()),
100,
)
@@ -201,7 +201,7 @@ async fn test_batch_parse_rejects_target_pages() {
target_pages: Some("1-2".into()),
..LiteParseConfig::default()
})
- .open(
+ .open_batch_session(
PdfInput::Path("../../integration_tests_data/filled_acroform.pdf".to_string()),
25,
)
@@ -212,38 +212,6 @@ async fn test_batch_parse_rejects_target_pages() {
);
}
-/// A non-PDF source is converted once when the session opens, not once per
-/// batch — the whole reason batching lives in core rather than in a binding.
-#[tokio::test]
-#[serial]
-async fn test_batch_parse_converts_office_source_once() {
- let env_var = std::env::var("SKIP_INTEGRATION_TESTS");
- if let Ok(v) = env_var
- && v == "yes"
- {
- return;
- }
- let mut session = LiteParse::new(LiteParseConfig {
- ocr_enabled: false,
- quiet: true,
- ..LiteParseConfig::default()
- })
- .open(
- PdfInput::Path("../../integration_tests_data/sample3.doc".to_string()),
- 1,
- )
- .await
- .expect("should convert and open a .doc");
-
- // The converted temporary PDF is owned by the session, so every batch
- // after the first resolves against a file that no longer needs LibreOffice.
- let mut pages = 0;
- while let Some(batch) = session.next_batch().await.expect("batch should parse") {
- pages += batch.result.pages.len();
- }
- assert_eq!(pages, 2);
-}
-
#[tokio::test]
#[serial]
async fn test_parse_bytes_office_integration() {
diff --git a/packages/node/native.d.ts b/packages/node/native.d.ts
index dd0e4e3..6689e54 100644
--- a/packages/node/native.d.ts
+++ b/packages/node/native.d.ts
@@ -482,14 +482,16 @@ export declare class LiteParse {
/** Parse a document. Accepts a file path (string) or raw PDF bytes (Buffer). */
parse(input: string | Buffer): Promise<JsParseResult>
/**
- * Open a document for bounded-memory batch parsing.
+ * Open a document for bounded-memory batch parsing. Internal plumbing
+ * for the JS wrapper's `parseBatches()` — prefer that; it also closes
+ * the session for you.
*
* Converts a non-PDF source once, then yields `batchSize` pages at a time
* via `nextBatch()` (default 25). Cross-page passes (repeated
* header/footer removal, image deduplication) see only the pages in their
* own batch, so output can differ from a whole-document `parse()`.
*/
- open(input: string | Buffer, batchSize?: number | undefined | null): Promise<ParseSession>
+ openBatchSession(input: string | Buffer, batchSize?: number | undefined | null): Promise<ParseSession>
/**
* Parse from pre-extracted pages, skipping PDFium text extraction.
*
@@ -518,18 +520,27 @@ export declare class LiteParse {
get config(): JsLiteParseConfig
}
/**
- * A document opened once and parsed in bounded page batches.
+ * A document opened once and parsed in bounded page batches. Internal
+ * plumbing for the JS wrapper's `parseBatches()` — prefer that.
*
- * Created by `LiteParse.open()`. The converted-PDF temporary file for a
- * non-PDF source lives as long as the session, so conversion is paid once no
- * matter how many batches are consumed.
+ * Created by `LiteParse.openBatchSession()`. The converted-PDF temporary
+ * file for a non-PDF source lives as long as the session, so conversion is
+ * paid once no matter how many batches are consumed. Call `close()` when
+ * abandoning the session early — otherwise that temp file waits for GC.
*/
export declare class ParseSession {
/** Total pages in the source document, before `maxPages` or batching. */
get totalPages(): number
/**
* Parse and return the next batch, or `null` once every page within
- * `maxPages` has been yielded.
+ * `maxPages` has been yielded. Rejects if the session is closed.
*/
nextBatch(): Promise<JsParseBatch | null>
+ /**
+ * Release the session's resources now — most importantly the converted
+ * temporary PDF for a non-PDF source, which otherwise lives until the
+ * JS object is garbage collected. Idempotent; `nextBatch()` rejects
+ * afterwards.
+ */
+ close(): Promise<void>
}
diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts
index 33bd2a3..a01dbf6 100644
--- a/packages/node/src/lib.ts
+++ b/packages/node/src/lib.ts
@@ -633,35 +633,49 @@ export class LiteParse {
* Each yielded result is independent and becomes collectible once the caller
* advances the iterator, so a consumer that does not retain batches never
* holds more than one batch of pages in memory. A non-PDF source is
- * converted once when the iterator starts, not once per batch.
+ * converted once when the iterator starts, not once per batch; its temporary
+ * file is released when iteration ends — including an early `break` or
+ * `throw`, which run the generator's cleanup.
*
* Cross-page passes see only the pages in their own batch, so repeated
* header/footer removal and image deduplication are batch-local and the
* output can differ from `parse()`. Prefer `parse()` unless the size of the
* materialized result is the problem.
*
- * Throws if the parser is configured with `targetPages` — an explicit page
- * selection and generated batch ranges are ambiguous together.
+ * As with any async generator, work starts on the first `next()` call, so
+ * errors (an unreadable file, or a parser configured with `targetPages` —
+ * ambiguous with generated batch ranges) surface on the first iteration
+ * rather than when `parseBatches()` itself is called.
*/
async *parseBatches(
input: LiteParseInput,
options: ParseBatchOptions = {},
): AsyncGenerator<ParseBatch> {
const nativeInput = typeof input === "string" ? input : Buffer.from(input);
- const session = await this._native.open(nativeInput, options.batchSize);
- const totalPages = session.totalPages;
+ const session = await this._native.openBatchSession(
+ nativeInput,
+ options.batchSize,
+ );
+ try {
+ const totalPages = session.totalPages;
- for (;;) {
- const batch = await session.nextBatch();
- if (batch == null) {
- return;
+ for (;;) {
+ const batch = await session.nextBatch();
+ if (batch == null) {
+ return;
+ }
+ yield {
+ startPage: batch.startPage,
+ endPage: batch.endPage,
+ totalPages,
+ result: toParseResult(batch.result),
+ };
}
- yield {
- startPage: batch.startPage,
- endPage: batch.endPage,
- totalPages,
- result: toParseResult(batch.result),
- };
+ } finally {
+ // Frees the session's converted-PDF temp file now instead of at GC —
+ // this runs on normal exhaustion and when the consumer abandons the
+ // loop early.
+ await session.close();
}
}
diff --git a/packages/node/src/native.ts b/packages/node/src/native.ts
index ea5f602..72f46af 100644
--- a/packages/node/src/native.ts
+++ b/packages/node/src/native.ts
@@ -324,12 +324,13 @@ export interface NativeParseBatch {
export interface NativeParseSession {
nextBatch(): Promise<NativeParseBatch | null>;
+ close(): Promise<void>;
readonly totalPages: number;
}
export interface LiteParseNative {
parse(input: string | Buffer): Promise<NativeParseResult>;
- open(
+ openBatchSession(
input: string | Buffer,
batchSize?: number,
): Promise<NativeParseSession>;
diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py
index 43b4982..dc645f4 100644
--- a/packages/python/liteparse/parser.py
+++ b/packages/python/liteparse/parser.py
@@ -616,21 +616,34 @@ class LiteParse:
Raises:
ParseError: If parsing fails, or if the parser was constructed
with ``target_pages`` (ambiguous with generated batch ranges).
+ Open errors are raised here, when ``parse_batches`` is called;
+ per-batch parse errors are raised from the iterator.
FileNotFoundError: If the file doesn't exist.
"""
+ # Validate and open eagerly — this is not a generator function, so a
+ # missing file or a target_pages conflict raises here rather than on
+ # the first iteration of the returned iterator.
try:
if isinstance(file_data, bytes):
- session = self._native.open_bytes(file_data, batch_size)
+ session = self._native.open_batch_session_bytes(
+ file_data, batch_size
+ )
else:
file_path = Path(file_data)
if not file_path.exists():
raise FileNotFoundError(f"File not found: {file_path}")
- session = self._native.open(str(file_path.absolute()), batch_size)
+ session = self._native.open_batch_session(
+ str(file_path.absolute()), batch_size
+ )
except FileNotFoundError:
raise
except Exception as e:
raise ParseError(str(e)) from e
+ return self._iter_batches(session)
+
+ @staticmethod
+ def _iter_batches(session: Any) -> Iterator[ParseBatch]:
total_pages = session.total_pages
while True:
try: