Skip to content

A fast, helpful, and open-source document parser

feat: worker pools

Logan Markewichaac108a46fbc
diff --git a/README.md b/README.md
index 4de9580..aaaf091 100644
--- a/README.md
+++ b/README.md
@@ -43,6 +43,7 @@ hard stuff so your models see clean, structured data and markdown.
- **Markdown Output**: Structured Markdown with headings, tables, lists, images, and links — great for feeding LLMs and RAG pipelines
- **Bounding Boxes**: Precise text positioning information
- **Multi-language**: Use from Rust, Node.js/TypeScript, Python, or the browser (WASM)
+- **Worker Pool Mode** (Python & Node.js): Parse in persistent worker processes for true parallelism (PDFium otherwise serializes concurrent parses) and hard per-parse timeouts — rogue documents are killed, identified by name, and never stall the pipeline
- **Multi-platform**: Linux, macOS (Intel/ARM), Windows
```mermaid
diff --git a/packages/node/README.md b/packages/node/README.md
index 52c9b8c..b20700c 100644
--- a/packages/node/README.md
+++ b/packages/node/README.md
@@ -150,6 +150,47 @@ const result = await parser.parse(pdfBytes);
console.log(result.text);
```
+## Worker Pool and Hard Timeouts
+
+PDFium is not thread-safe, so all parses inside one process serialize on a
+process-global lock — even `Promise.all` over multiple `parse()` calls runs
+them one at a time. And a parse stuck inside a single PDFium call cannot be
+interrupted from within the process. For high-throughput services, run parses
+in a pool of persistent worker processes instead:
+
+```typescript
+import { LiteParse, ParseTimeoutError } from '@llamaindex/liteparse';
+
+const parser = new LiteParse({ poolSize: 4, parseTimeoutMs: 15_000 });
+await parser.warmUp(); // optional: pre-initialize workers (~45ms total)
+
+try {
+ const result = await parser.parse('document.pdf');
+} catch (e) {
+ if (e instanceof ParseTimeoutError) {
+ console.warn(`killed rogue document: ${e.source} (deadline ${e.timeoutMs}ms)`);
+ }
+}
+
+parser.close(); // frees workers immediately; an idle pool never blocks exit
+```
+
+- **Real parallelism**: each worker has its own PDFium instance, so `poolSize`
+ parses run genuinely concurrently — `Promise.all` becomes actually parallel
+ with no caller changes.
+- **Hard kill-switch**: `parseTimeoutMs` is enforced by SIGKILLing the worker
+ process (and forking a replacement), so a timed-out parse is guaranteed
+ dead — no wedged thread holding a lock. The error names the document, so
+ rogue files identify themselves in your logs.
+- **Low overhead**: workers are forked once and reused. Per-parse overhead is
+ IPC serialization only (sub-millisecond for small documents).
+
+`parseTimeoutMs` requires `poolSize` by design: an in-process deadline cannot
+actually stop a parse, and LiteParse won't offer a timeout it can't enforce.
+(`worker_threads` wouldn't help either — workers share the process and its
+PDFium lock, and `terminate()` cannot preempt native code.) Only `parse()` is
+routed through the pool; other methods run in-process.
+
## Screenshots
Generate PNG screenshots of document pages:
diff --git a/packages/node/package.json b/packages/node/package.json
index 7a28dc6..ded5bcf 100644
--- a/packages/node/package.json
+++ b/packages/node/package.json
@@ -43,6 +43,7 @@
"build:pdfium": "bash scripts/copy-pdfium.sh .",
"build:ts": "tsc",
"build": "npm run build:rs && npm run build:pdfium && npm run build:ts",
+ "test": "npm run build:ts && node --test tests/",
"prepublishOnly": "npm run build"
},
"dependencies": {
diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts
index 8ef08de..e48ad97 100644
--- a/packages/node/src/lib.ts
+++ b/packages/node/src/lib.ts
@@ -11,6 +11,9 @@ import {
type NativePageComplexityStats,
type NativeScreenshotResult,
} from "./native.js";
+import { WorkerPool, ParseTimeoutError } from "./pool.js";
+
+export { ParseTimeoutError };
// ---------------------------------------------------------------------------
// Public types — match the existing TypeScript API
@@ -20,6 +23,32 @@ export type LiteParseInput = string | Buffer | Uint8Array;
export type OutputFormat = "json" | "text" | "markdown";
export type ImageMode = "off" | "placeholder" | "embed";
+/** Options for pool mode: parsing in persistent, killable worker processes. */
+export interface PoolOptions {
+ /**
+ * Route `parse()` through a pool of this many persistent worker processes
+ * instead of parsing in-process. Each worker has its own PDFium instance,
+ * so pooled parses run genuinely in parallel — in-process parses (including
+ * concurrent `parse()` promises) serialize on a process-global PDFium lock
+ * — and a stuck document can be killed without leaking a wedged thread.
+ * Workers are forked once and reused; per-parse overhead is IPC
+ * serialization only. Only `parse()` is routed through the pool; other
+ * methods still run in-process. Call `close()` when done (an idle pool
+ * never keeps the event loop alive, but explicit shutdown frees workers
+ * immediately).
+ */
+ poolSize?: number;
+ /**
+ * Hard per-parse deadline in milliseconds. Requires `poolSize`: a deadline
+ * is only honest when it can actually stop the work, and an in-process
+ * parse stuck inside a single PDFium FFI call cannot be interrupted — the
+ * pool enforces the deadline by SIGKILLing the worker process. On expiry
+ * the parse rejects with {@link ParseTimeoutError} (naming the document)
+ * and a fresh worker replaces the killed one.
+ */
+ parseTimeoutMs?: number;
+}
+
export interface LiteParseConfig {
ocrLanguage: string;
ocrEnabled: boolean;
@@ -606,8 +635,9 @@ export interface LayoutComplexityStats {
export class LiteParse {
private _native: LiteParseNative;
private _config: LiteParseConfig;
+ private _pool: WorkerPool | null = null;
- constructor(userConfig: Partial<LiteParseConfig> = {}) {
+ constructor(userConfig: Partial<LiteParseConfig> & PoolOptions = {}) {
const nativeConfig: LiteParseNativeConfig = {
ocrLanguage: userConfig.ocrLanguage,
ocrEnabled: userConfig.ocrEnabled,
@@ -650,6 +680,26 @@ export class LiteParse {
this._native = new native.LiteParse(nativeConfig);
+ if (
+ userConfig.parseTimeoutMs !== undefined &&
+ userConfig.poolSize === undefined
+ ) {
+ throw new Error(
+ "parseTimeoutMs requires poolSize: an in-process parse stuck inside " +
+ "a PDFium call cannot be interrupted, so a timeout without process " +
+ "isolation would be a promise LiteParse cannot keep. Set poolSize " +
+ "(e.g. poolSize: 1) to enforce the deadline by killing the worker " +
+ "process.",
+ );
+ }
+ if (userConfig.poolSize !== undefined) {
+ this._pool = new WorkerPool(
+ nativeConfig as unknown as Record<string, unknown>,
+ userConfig.poolSize,
+ userConfig.parseTimeoutMs,
+ );
+ }
+
// Read back the resolved config from the native side
const resolved = this._native.config;
this._config = {
@@ -697,10 +747,39 @@ export class LiteParse {
// Convert Uint8Array to Buffer for the native side
const nativeInput =
typeof input === "string" ? input : Buffer.from(input);
+ if (this._pool !== null) {
+ const source =
+ typeof nativeInput === "string"
+ ? nativeInput
+ : `<${nativeInput.byteLength} bytes>`;
+ return this._pool.parse(nativeInput, source);
+ }
const result: NativeParseResult = await this._native.parse(nativeInput);
return toParseResult(result);
}
+ /**
+ * Resolves once every pool worker is initialized. No-op without `poolSize`.
+ *
+ * Optional: the first parse on each worker waits for its init anyway. Call
+ * this before latency-sensitive traffic to avoid paying worker startup on
+ * the first request.
+ */
+ async warmUp(): Promise<void> {
+ if (this._pool !== null) await this._pool.warmUp();
+ }
+
+ /**
+ * Shut down pool workers, if pool mode is enabled. Idempotent.
+ *
+ * Without `poolSize` this is a no-op. An idle pool never keeps the event
+ * loop alive and workers exit when the parent does, so forgetting to call
+ * this leaks nothing past process exit.
+ */
+ close(): void {
+ if (this._pool !== null) this._pool.close();
+ }
+
/**
* Parse a document in bounded-memory page batches of `batchSize` pages.
*
@@ -839,7 +918,8 @@ function toComplexity(s: NativePageComplexityStats): PageComplexityStats {
};
}
-function toParseResult(result: NativeParseResult): ParseResult {
+/** @internal Exported for pool-worker.ts only; not public API. */
+export function toParseResult(result: NativeParseResult): ParseResult {
return {
totalPages: result.totalPages,
pages: result.pages.map(toPage),
diff --git a/packages/node/src/pool-worker.ts b/packages/node/src/pool-worker.ts
new file mode 100644
index 0000000..bae1cd5
--- /dev/null
+++ b/packages/node/src/pool-worker.ts
@@ -0,0 +1,66 @@
+// Entry point for pool worker processes, forked by pool.ts.
+//
+// Each worker holds its own native LiteParse instance — and therefore its own
+// PDFium instance and PDFium lock — so pooled parses run genuinely in
+// parallel and a stuck parse can be killed by killing this process. The
+// parent speaks over the fork() IPC channel (advanced serialization), one
+// request in flight at a time.
+
+import { native, type LiteParseNative } from "./native.js";
+import { toParseResult } from "./lib.js";
+
+// A worker must never outlive its pool: when the parent exits (or kills the
+// IPC channel), shut down instead of lingering as an orphan.
+process.on("disconnect", () => process.exit(0));
+
+/** IPC deserialization yields plain Uint8Arrays; the native API wants Buffers. */
+function asBuffer(data: Buffer | Uint8Array): Buffer {
+ return Buffer.isBuffer(data)
+ ? data
+ : Buffer.from(data.buffer, data.byteOffset, data.byteLength);
+}
+
+type WorkerMessage =
+ | { type: "init"; config: Record<string, unknown> }
+ | { type: "parse"; payload: string | Uint8Array }
+ | { type: "stop" };
+
+let parser: LiteParseNative | null = null;
+
+process.on("message", async (msg: WorkerMessage) => {
+ switch (msg.type) {
+ case "init": {
+ try {
+ parser = new native.LiteParse(msg.config);
+ } catch (e) {
+ process.send!({
+ type: "initError",
+ message: e instanceof Error ? e.message : String(e),
+ });
+ process.exit(1);
+ }
+ process.send!({ type: "ready" });
+ return;
+ }
+ case "parse": {
+ if (parser === null) {
+ process.send!({ type: "err", message: "worker not initialized" });
+ return;
+ }
+ try {
+ const input =
+ typeof msg.payload === "string" ? msg.payload : asBuffer(msg.payload);
+ const result = await parser.parse(input);
+ process.send!({ type: "ok", result: toParseResult(result) });
+ } catch (e) {
+ process.send!({
+ type: "err",
+ message: e instanceof Error ? e.message : String(e),
+ });
+ }
+ return;
+ }
+ case "stop":
+ process.exit(0);
+ }
+});
diff --git a/packages/node/src/pool.ts b/packages/node/src/pool.ts
new file mode 100644
index 0000000..5b7fa84
--- /dev/null
+++ b/packages/node/src/pool.ts
@@ -0,0 +1,305 @@
+// Process-isolated worker pool for LiteParse.
+//
+// PDFium work happens behind a process-global lock, and a single PDFium FFI
+// call cannot be interrupted from inside the process — worker_threads would
+// not help (same process, same lock, and terminate() cannot preempt native
+// code). The only enforceable timeout is a process boundary: parses run in
+// forked worker processes (see pool-worker.ts), and a worker that blows its
+// deadline is SIGKILLed and replaced. Because each worker has its own PDFium,
+// pooled parses also run genuinely in parallel instead of queueing on the
+// lock.
+//
+// Not public API; use `new LiteParse({ poolSize, parseTimeoutMs })`.
+
+import { fork, type ChildProcess } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import type { ParseResult } from "./lib.js";
+
+/** A pooled parse exceeded `parseTimeoutMs` and its worker was killed.
+ *
+ * Only thrown in pool mode, where the deadline is enforced by killing the
+ * worker process — the timed-out parse is guaranteed dead, not still running
+ * in the background. `source` names the document (file path, or `<N bytes>`
+ * for byte inputs); log it to identify the documents that stall your
+ * pipeline.
+ */
+export class ParseTimeoutError extends Error {
+ readonly source: string;
+ readonly timeoutMs: number;
+
+ constructor(message: string, source: string, timeoutMs: number) {
+ super(message);
+ this.name = "ParseTimeoutError";
+ this.source = source;
+ this.timeoutMs = timeoutMs;
+ }
+}
+
+class WorkerTimeout extends Error {}
+class WorkerCrashed extends Error {}
+
+type WorkerResponse =
+ | { type: "ready" }
+ | { type: "initError"; message: string }
+ | { type: "ok"; result: ParseResult }
+ | { type: "err"; message: string };
+
+const WORKER_PATH = fileURLToPath(new URL("./pool-worker.js", import.meta.url));
+
+/** IPC deserialization yields plain Uint8Arrays where the API promises
+ * Buffers; rewrap in place (no copy). */
+function reviveBuffers(result: ParseResult): ParseResult {
+ for (const image of result.images ?? []) {
+ if (image.bytes && !Buffer.isBuffer(image.bytes)) {
+ const b = image.bytes as Uint8Array;
+ image.bytes = Buffer.from(b.buffer, b.byteOffset, b.byteLength);
+ }
+ }
+ for (const shot of result.screenshots ?? []) {
+ if (shot.imageBuffer && !Buffer.isBuffer(shot.imageBuffer)) {
+ const b = shot.imageBuffer as Uint8Array;
+ shot.imageBuffer = Buffer.from(b.buffer, b.byteOffset, b.byteLength);
+ }
+ }
+ return result;
+}
+
+class WorkerHandle {
+ private child: ChildProcess;
+ private readyPromise: Promise<void>;
+ private pending: {
+ resolve: (r: ParseResult) => void;
+ reject: (e: Error) => void;
+ } | null = null;
+ private dead = false;
+
+ constructor(config: Record<string, unknown>) {
+ this.child = fork(WORKER_PATH, [], {
+ serialization: "advanced",
+ // stdout/stderr inherited: parse logs and crash traces stay visible.
+ stdio: ["ignore", "inherit", "inherit", "ipc"],
+ });
+
+ let readyResolve!: () => void;
+ let readyReject!: (e: Error) => void;
+ this.readyPromise = new Promise<void>((resolve, reject) => {
+ readyResolve = resolve;
+ readyReject = reject;
+ });
+ // ready() may never be awaited if the worker is retired first.
+ this.readyPromise.catch(() => {});
+
+ this.child.on("message", (msg: WorkerResponse) => {
+ if (msg.type === "ready") {
+ readyResolve();
+ if (this.pending === null) this.idle();
+ } else if (msg.type === "initError") {
+ readyReject(new Error(msg.message));
+ } else if (this.pending) {
+ const { resolve, reject } = this.pending;
+ this.pending = null;
+ this.idle();
+ if (msg.type === "ok") resolve(reviveBuffers(msg.result));
+ else reject(new WorkerCrashed(msg.message));
+ }
+ });
+ const onGone = (cause: string) => {
+ this.dead = true;
+ readyReject(new WorkerCrashed(cause));
+ if (this.pending) {
+ const { reject } = this.pending;
+ this.pending = null;
+ reject(new WorkerCrashed(cause));
+ }
+ };
+ this.child.on("error", (e) => onGone(e.message));
+ this.child.on("exit", (code, signal) =>
+ onGone(`worker exited (code=${code}, signal=${signal})`),
+ );
+
+ // Stay ref'd until the init handshake arrives — a caller awaiting
+ // warmUp()/ready() must keep the event loop alive. idle() runs on
+ // "ready"; after that, an idle worker never holds the loop open.
+ this.child.send({ type: "init", config });
+ }
+
+ /** Resolves when the worker's native parser is constructed. Init time
+ * never counts toward the parse deadline — the deadline is a promise about
+ * parsing, not about process startup. */
+ ready(): Promise<void> {
+ return this.readyPromise;
+ }
+
+ /** An idle pool must not hold the parent's event loop open. */
+ private idle(): void {
+ this.child.unref();
+ this.child.channel?.unref();
+ }
+
+ request(
+ payload: string | Buffer,
+ timeoutMs: number | undefined,
+ ): Promise<ParseResult> {
+ if (this.dead) {
+ return Promise.reject(new WorkerCrashed("worker already exited"));
+ }
+ this.child.ref();
+ this.child.channel?.ref();
+ return new Promise<ParseResult>((resolve, reject) => {
+ let timer: NodeJS.Timeout | undefined;
+ const settle =
+ (fn: (v: never) => void) =>
+ (value: never): void => {
+ if (timer !== undefined) clearTimeout(timer);
+ fn(value);
+ };
+ this.pending = {
+ resolve: settle(resolve) as (r: ParseResult) => void,
+ reject: settle(reject) as (e: Error) => void,
+ };
+ if (timeoutMs !== undefined) {
+ timer = setTimeout(() => {
+ if (this.pending) {
+ const { reject: rejectPending } = this.pending;
+ this.pending = null;
+ rejectPending(new WorkerTimeout());
+ }
+ }, timeoutMs);
+ }
+ this.child.send({ type: "parse", payload });
+ });
+ }
+
+ kill(): void {
+ this.dead = true;
+ this.child.kill("SIGKILL");
+ }
+
+ /** Graceful shutdown; escalates to SIGKILL if the worker doesn't exit. */
+ stop(): void {
+ if (this.dead) return;
+ this.dead = true;
+ try {
+ this.child.send({ type: "stop" });
+ } catch {
+ // channel already closed
+ }
+ const escalate = setTimeout(() => this.child.kill("SIGKILL"), 5000);
+ escalate.unref();
+ this.child.once("exit", () => clearTimeout(escalate));
+ this.idle();
+ }
+}
+
+export class WorkerPool {
+ private config: Record<string, unknown>;
+ private timeoutMs: number | undefined;
+ private workers = new Set<WorkerHandle>();
+ private idle: WorkerHandle[] = [];
+ private waiters: Array<{
+ resolve: (w: WorkerHandle) => void;
+ reject: (e: Error) => void;
+ }> = [];
+ private closed = false;
+
+ constructor(
+ config: Record<string, unknown>,
+ poolSize: number,
+ parseTimeoutMs?: number,
+ ) {
+ if (!Number.isInteger(poolSize) || poolSize < 1) {
+ throw new Error("poolSize must be an integer >= 1");
+ }
+ if (parseTimeoutMs !== undefined && !(parseTimeoutMs > 0)) {
+ throw new Error("parseTimeoutMs must be > 0");
+ }
+ this.config = config;
+ this.timeoutMs = parseTimeoutMs;
+ // Spawn eagerly: children load the addon and construct their native
+ // parsers concurrently while the caller goes on with its own startup.
+ for (let i = 0; i < poolSize; i++) {
+ this.spawnWorker();
+ }
+ }
+
+ private spawnWorker(): void {
+ const worker = new WorkerHandle(this.config);
+ this.workers.add(worker);
+ this.release(worker);
+ }
+
+ private acquire(): Promise<WorkerHandle> {
+ const worker = this.idle.pop();
+ if (worker !== undefined) return Promise.resolve(worker);
+ return new Promise((resolve, reject) =>
+ this.waiters.push({ resolve, reject }),
+ );
+ }
+
+ private release(worker: WorkerHandle): void {
+ if (this.closed) {
+ this.workers.delete(worker);
+ worker.stop();
+ return;
+ }
+ const waiter = this.waiters.shift();
+ if (waiter !== undefined) waiter.resolve(worker);
+ else this.idle.push(worker);
+ }
+
+ private retire(worker: WorkerHandle): void {
+ worker.kill();
+ this.workers.delete(worker);
+ if (!this.closed) this.spawnWorker();
+ }
+
+ /** Run one parse on an idle worker.
+ *
+ * Waits for a free worker first; `parseTimeoutMs` bounds the parse itself,
+ * not the wait. */
+ async parse(payload: string | Buffer, source: string): Promise<ParseResult> {
+ if (this.closed) throw new Error("parser pool is closed");
+ const worker = await this.acquire();
+ try {
+ await worker.ready();
+ const result = await worker.request(payload, this.timeoutMs);
+ this.release(worker);
+ return result;
+ } catch (e) {
+ this.retire(worker);
+ if (e instanceof WorkerTimeout) {
+ throw new ParseTimeoutError(
+ `parse of ${source} exceeded ${this.timeoutMs}ms; the worker process was killed`,
+ source,
+ this.timeoutMs!,
+ );
+ }
+ if (e instanceof WorkerCrashed) {
+ throw new Error(
+ `liteparse worker process died while parsing ${source}: ${e.message}`,
+ );
+ }
+ throw e;
+ }
+ }
+
+ /** Resolves when every worker is initialized. Optional — the first parse
+ * per worker waits for init anyway. */
+ async warmUp(): Promise<void> {
+ await Promise.all([...this.workers].map((w) => w.ready()));
+ }
+
+ /** Shut down all workers. Idempotent. Busy workers are stopped as their
+ * in-flight parses finish. */
+ close(): void {
+ if (this.closed) return;
+ this.closed = true;
+ for (const waiter of this.waiters.splice(0)) {
+ waiter.reject(new Error("parser pool is closed"));
+ }
+ for (const worker of this.idle.splice(0)) {
+ this.workers.delete(worker);
+ worker.stop();
+ }
+ }
+}
diff --git a/packages/node/tests/pool.test.mjs b/packages/node/tests/pool.test.mjs
new file mode 100644
index 0000000..ca767cc
--- /dev/null
+++ b/packages/node/tests/pool.test.mjs
@@ -0,0 +1,167 @@
+// E2E tests for pool mode (process-isolated workers with hard timeouts).
+// Run with: npm test (node --test tests/)
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { deflateSync } from "node:zlib";
+import { fileURLToPath } from "node:url";
+import { writeFileSync, mkdtempSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { LiteParse, ParseTimeoutError } from "../dist/lib.js";
+
+const SAMPLE_PDF = fileURLToPath(
+ new URL("../../../integration_tests_data/sample.pdf", import.meta.url),
+);
+const CFG = { quiet: true, ocrEnabled: false, outputFormat: "text" };
+
+/**
+ * One page with a dense grid of tiny text objects. Parse time is superlinear
+ * in the object count (PDFium's text-page assembly), which makes a small file
+ * arbitrarily slow to parse — the same shape as the rogue documents that
+ * stall prod pipelines, used here to exercise the timeout path.
+ */
+function gridPdf(nObjects) {
+ const cols = Math.max(1, Math.floor(Math.sqrt(nObjects)));
+ const parts = [];
+ for (let i = 0; i < nObjects; i++) {
+ const x = (20 + (i % cols) * 2.5).toFixed(1);
+ const y = (770 - Math.floor(i / cols) * 2.5).toFixed(1);
+ parts.push(`BT /F1 2 Tf ${x} ${y} Td (x) Tj ET`);
+ }
+ const stream = deflateSync(Buffer.from(parts.join("\n")));
+ const width = Math.floor(40 + cols * 2.5);
+ const height = Math.floor(40 + (nObjects / cols) * 2.5);
+ const objects = [
+ Buffer.from("<< /Type /Catalog /Pages 2 0 R >>"),
+ Buffer.from("<< /Type /Pages /Kids [3 0 R] /Count 1 >>"),
+ Buffer.from(
+ `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 ${width} ${height}] ` +
+ "/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",
+ ),
+ Buffer.from("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"),
+ Buffer.concat([
+ Buffer.from(`<< /Length ${stream.length} /Filter /FlateDecode >>\nstream\n`),
+ stream,
+ Buffer.from("\nendstream"),
+ ]),
+ ];
+ const chunks = [Buffer.from("%PDF-1.4\n")];
+ const offsets = [];
+ let position = chunks[0].length;
+ objects.forEach((body, index) => {
+ offsets.push(position);
+ const chunk = Buffer.concat([
+ Buffer.from(`${index + 1} 0 obj\n`),
+ body,
+ Buffer.from("\nendobj\n"),
+ ]);
+ chunks.push(chunk);
+ position += chunk.length;
+ });
+ const xref = [`xref`, `0 ${objects.length + 1}`, `0000000000 65535 f `]
+ .concat(offsets.map((off) => `${String(off).padStart(10, "0")} 00000 n `))
+ .join("\n");
+ chunks.push(
+ Buffer.from(
+ `${xref}\ntrailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\n` +
+ `startxref\n${position}\n%%EOF\n`,
+ ),
+ );
+ return Buffer.concat(chunks);
+}
+
+test("pool matches in-process results", { skip: !existsSync(SAMPLE_PDF) }, async () => {
+ const inproc = new LiteParse(CFG);
+ const expected = await inproc.parse(SAMPLE_PDF);
+ const pooled = new LiteParse({ ...CFG, poolSize: 2 });
+ try {
+ const fromPath = await pooled.parse(SAMPLE_PDF);
+ const { readFileSync } = await import("node:fs");
+ const fromBytes = await pooled.parse(readFileSync(SAMPLE_PDF));
+ assert.equal(fromPath.text, expected.text);
+ assert.equal(fromBytes.text, expected.text);
+ assert.equal(fromPath.pages.length, expected.pages.length);
+ } finally {
+ pooled.close();
+ }
+});
+
+test("parseTimeoutMs requires poolSize", () => {
+ assert.throws(() => new LiteParse({ parseTimeoutMs: 5000 }), /poolSize/);
+});
+
+test("timeout kills worker and pool recovers", { skip: !existsSync(SAMPLE_PDF) }, async () => {
+ const slowDoc = gridPdf(64000); // parses in seconds, not before the deadline
+ const pooled = new LiteParse({ ...CFG, poolSize: 1, parseTimeoutMs: 500 });
+ try {
+ await pooled.warmUp();
+ await assert.rejects(pooled.parse(slowDoc), (e) => {
+ assert.ok(e instanceof ParseTimeoutError);
+ assert.equal(e.timeoutMs, 500);
+ assert.match(e.source, /bytes/);
+ return true;
+ });
+ // The replacement worker must serve subsequent parses.
+ const result = await pooled.parse(SAMPLE_PDF);
+ assert.ok(result.text.length > 0);
+ } finally {
+ pooled.close();
+ }
+});
+
+test("timeout names the file", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "liteparse-pool-"));
+ const slowPath = join(dir, "rogue.pdf");
+ writeFileSync(slowPath, gridPdf(64000));
+ const pooled = new LiteParse({ ...CFG, poolSize: 1, parseTimeoutMs: 500 });
+ try {
+ await assert.rejects(pooled.parse(slowPath), (e) => {
+ assert.ok(e instanceof ParseTimeoutError);
+ assert.equal(e.source, slowPath);
+ return true;
+ });
+ } finally {
+ pooled.close();
+ }
+});
+
+test("parse errors propagate from the worker", async () => {
+ const pooled = new LiteParse({ ...CFG, poolSize: 1 });
+ try {
+ await assert.rejects(
+ pooled.parse(Buffer.from("%PDF-1.4 not actually a pdf")),
+ (e) => !(e instanceof ParseTimeoutError),
+ );
+ // Worker survives a failed parse and serves the next one.
+ if (existsSync(SAMPLE_PDF)) {
+ const result = await pooled.parse(SAMPLE_PDF);
+ assert.ok(result.text.length > 0);
+ }
+ } finally {
+ pooled.close();
+ }
+});
+
+test("close() shuts down the pool", { skip: !existsSync(SAMPLE_PDF) }, async () => {
+ const pooled = new LiteParse({ ...CFG, poolSize: 2 });
+ await pooled.parse(SAMPLE_PDF);
+ pooled.close();
+ await assert.rejects(pooled.parse(SAMPLE_PDF), /closed/);
+ pooled.close(); // idempotent
+});
+
+test("pooled parses run concurrently", { skip: !existsSync(SAMPLE_PDF) }, async () => {
+ const pooled = new LiteParse({ ...CFG, poolSize: 4 });
+ try {
+ await pooled.warmUp();
+ const results = await Promise.all(
+ Array.from({ length: 8 }, () => pooled.parse(SAMPLE_PDF)),
+ );
+ assert.equal(results.length, 8);
+ for (const result of results) assert.ok(result.text.length > 0);
+ } finally {
+ pooled.close();
+ }
+});
diff --git a/packages/python/README.md b/packages/python/README.md
index 0517bd7..c01a489 100644
--- a/packages/python/README.md
+++ b/packages/python/README.md
@@ -114,6 +114,40 @@ with open("document.pdf", "rb") as f:
print(result.text)
```
+## Worker Pool and Hard Timeouts
+
+PDFium is not thread-safe, so in-process parses serialize on a process-global
+lock — and a parse stuck inside a single PDFium call cannot be interrupted
+from within the process. For high-throughput services, run parses in a pool
+of persistent worker processes instead:
+
+```python
+from liteparse import LiteParse, ParseTimeoutError
+
+parser = LiteParse(pool_size=4, parse_timeout=15)
+parser.warm_up() # optional: pre-initialize workers (~60ms total)
+
+try:
+ result = parser.parse("document.pdf")
+except ParseTimeoutError as e:
+ print(f"killed rogue document: {e.source} (deadline {e.timeout}s)")
+
+parser.close() # or use `with LiteParse(...) as parser:`
+```
+
+- **Real parallelism**: each worker has its own PDFium instance, so `pool_size`
+ parses run genuinely concurrently instead of queueing on the global lock.
+- **Hard kill-switch**: `parse_timeout` is enforced by killing the worker
+ process (and spawning a replacement), so a timed-out parse is guaranteed
+ dead — no wedged thread holding a lock. The exception names the document,
+ so rogue files identify themselves in your logs.
+- **Low overhead**: workers are spawned once and reused. Per-parse overhead is
+ IPC serialization only (sub-millisecond for small documents).
+
+`parse_timeout` requires `pool_size` by design: an in-process deadline cannot
+actually stop a parse, and LiteParse won't offer a timeout it can't enforce.
+Only `parse()` is routed through the pool; other methods run in-process.
+
## Screenshots
Generate PNG screenshots of document pages:
diff --git a/packages/python/liteparse/__init__.py b/packages/python/liteparse/__init__.py
index 53b9c7b..00bafdb 100644
--- a/packages/python/liteparse/__init__.py
+++ b/packages/python/liteparse/__init__.py
@@ -25,6 +25,7 @@ from .types import (
ScreenshotRect,
ScreenshotResult,
ParseError,
+ ParseTimeoutError,
VectorGraphics,
VectorLine,
VectorShape,
@@ -59,6 +60,7 @@ __all__ = [
"ExtractedImage",
"ImageRect",
"ParseError",
+ "ParseTimeoutError",
"search_items",
"VectorGraphics",
"VectorLine",
diff --git a/packages/python/liteparse/_pool.py b/packages/python/liteparse/_pool.py
new file mode 100644
index 0000000..b7b83bc
--- /dev/null
+++ b/packages/python/liteparse/_pool.py
@@ -0,0 +1,324 @@
+"""Process-isolated worker pool for LiteParse.
+
+PDFium work happens behind a process-global lock, and a single PDFium FFI
+call cannot be interrupted from inside the process — a thread stuck in one
+holds the lock until the call returns, no matter what deadline the caller
+wanted. The only enforceable timeout is a process boundary: run parses in
+dedicated worker processes and kill a worker outright when it blows its
+deadline.
+
+This module implements that boundary:
+
+- Each worker is a subprocess running ``python -m liteparse._pool`` with its
+ own native LiteParse instance (and therefore its own PDFium instance and
+ its own PDFium lock). Workers are persistent — spawn and import cost is
+ paid once per worker, not per parse.
+- Requests and responses are pickled frames over the worker's stdin/stdout.
+ The worker's real stdout is reserved for the protocol; fd 1 is redirected
+ to stderr immediately so stray prints (native progress logs, user hooks)
+ cannot corrupt it.
+- A parse that exceeds ``parse_timeout`` gets its worker SIGKILLed and a
+ replacement spawned; the caller receives :class:`ParseTimeoutError` naming
+ the document. Because each worker has its own PDFium, one stuck document
+ never blocks the others (no cross-worker lock convoy).
+
+The pool is not public API; use ``LiteParse(pool_size=..., parse_timeout=...)``.
+"""
+
+import os
+import pickle
+import queue
+import struct
+import subprocess
+import sys
+import threading
+from typing import Any, BinaryIO, Dict, List, Optional, Tuple, Union
+
+from .types import ParseError, ParseTimeoutError
+
+_FRAME_HEADER = struct.Struct("!I")
+
+# Sentinel posted by a worker's reader thread when its stdout hits EOF (the
+# process exited or was killed).
+_DEAD = object()
+
+# How long to allow a worker to spawn, import liteparse, and construct its
+# native parser. Deliberately generous: init time is excluded from
+# parse_timeout, and a machine slow enough to trip this has bigger problems.
+_INIT_TIMEOUT_S = 120.0
+
+
+class _WorkerTimeout(Exception):
+ """Internal: the worker did not respond within the parse deadline."""
+
+
+class _WorkerCrashed(Exception):
+ """Internal: the worker process exited before responding."""
+
+
+class _WorkerInitFailed(Exception):
+ """Internal: the worker's native parser could not be constructed."""
+
+
+def _write_frame(stream: BinaryIO, obj: Any) -> None:
+ data = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
+ stream.write(_FRAME_HEADER.pack(len(data)))
+ stream.write(data)
+ stream.flush()
+
+
+def _read_frame(stream: BinaryIO) -> Optional[Any]:
+ """Read one frame, or None on EOF/truncation."""
+ header = b""
+ while len(header) < _FRAME_HEADER.size:
+ chunk = stream.read(_FRAME_HEADER.size - len(header))
+ if not chunk:
+ return None
+ header += chunk
+ (length,) = _FRAME_HEADER.unpack(header)
+ data = b""
+ while len(data) < length:
+ chunk = stream.read(length - len(data))
+ if not chunk:
+ return None
+ data += chunk
+ return pickle.loads(data)
+
+
+def _worker_main() -> int:
+ """Entry point of a worker subprocess (``python -m liteparse._pool``)."""
+ # Reserve the real stdout for protocol frames and point fd 1 at stderr so
+ # that nothing else in the process can write into the frame stream.
+ out = os.fdopen(os.dup(1), "wb")
+ os.dup2(2, 1)
+ stdin = sys.stdin.buffer
+
+ config = _read_frame(stdin)
+ if config is None:
+ return 1
+ try:
+ from liteparse._liteparse import LiteParse as _NativeLiteParse
+
+ native = _NativeLiteParse(**config)
+ except Exception as e: # noqa: BLE001 - forwarded to the parent verbatim
+ _write_frame(out, ("init_error", f"{type(e).__name__}: {e}"))
+ return 1
+ _write_frame(out, ("ready", None))
+
+ from liteparse.parser import _convert_native_result
+
+ while True:
+ msg = _read_frame(stdin)
+ if msg is None or msg[0] == "stop":
+ return 0
+ _, payload = msg
+ try:
+ if isinstance(payload, bytes):
+ native_result = native.parse_bytes(payload)
+ else:
+ native_result = native.parse(payload)
+ _write_frame(out, ("ok", _convert_native_result(native_result)))
+ except Exception as e: # noqa: BLE001 - forwarded to the parent verbatim
+ _write_frame(out, ("err", f"{type(e).__name__}: {e}"))
+
+
+class _Worker:
+ """One persistent worker subprocess plus its reader thread."""
+
+ def __init__(self, config: Dict[str, Any]):
+ self._proc = subprocess.Popen(
+ [sys.executable, "-m", "liteparse._pool"],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=None, # inherit: parse logs and tracebacks stay visible
+ )
+ self._responses: "queue.Queue[Any]" = queue.Queue()
+ self._ready = False
+ reader = threading.Thread(
+ target=self._read_loop, daemon=True, name="liteparse-pool-reader"
+ )
+ reader.start()
+ _write_frame(self._proc.stdin, config)
+
+ def _read_loop(self) -> None:
+ while True:
+ frame = _read_frame(self._proc.stdout)
+ if frame is None:
+ self._responses.put(_DEAD)
+ return
+ self._responses.put(frame)
+
+ def _ensure_ready(self) -> None:
+ """Consume the init handshake. Init time never counts toward the
+ parse deadline — the deadline is a promise about parsing, not about
+ interpreter startup."""
+ if self._ready:
+ return
+ try:
+ frame = self._responses.get(timeout=_INIT_TIMEOUT_S)
+ except queue.Empty:
+ raise _WorkerCrashed("worker failed to initialize in time") from None
+ if frame is _DEAD:
+ raise _WorkerCrashed("worker exited during initialization")
+ kind, data = frame
+ if kind == "init_error":
+ # The worker process has already exited; the caller must retire
+ # this worker rather than reuse it.
+ raise _WorkerInitFailed(data)
+ self._ready = True
+
+ def request(
+ self, payload: Union[str, bytes], timeout: Optional[float]
+ ) -> Tuple[str, Any]:
+ self._ensure_ready()
+ try:
+ _write_frame(self._proc.stdin, ("parse", payload))
+ except (BrokenPipeError, OSError) as e:
+ raise _WorkerCrashed(str(e)) from None
+ try:
+ frame = self._responses.get(timeout=timeout)
+ except queue.Empty:
+ raise _WorkerTimeout() from None
+ if frame is _DEAD:
+ raise _WorkerCrashed("worker exited mid-parse")
+ return frame
+
+ def kill(self) -> None:
+ self._proc.kill()
+ try:
+ self._proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ pass
+ self._close_pipes()
+
+ def stop(self) -> None:
+ """Graceful shutdown; falls back to kill if the worker doesn't exit."""
+ try:
+ _write_frame(self._proc.stdin, ("stop", None))
+ self._proc.stdin.close()
+ except (BrokenPipeError, OSError):
+ pass
+ try:
+ self._proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ self._proc.kill()
+ self._close_pipes()
+
+ def _close_pipes(self) -> None:
+ for stream in (self._proc.stdin, self._proc.stdout):
+ try:
+ if stream:
+ stream.close()
+ except OSError:
+ pass
+
+
+class WorkerPool:
+ """Fixed-size pool of parse worker processes with a hard kill deadline."""
+
+ def __init__(
+ self,
+ config: Dict[str, Any],
+ pool_size: int,
+ parse_timeout: Optional[float] = None,
+ ):
+ if pool_size < 1:
+ raise ValueError("pool_size must be >= 1")
+ if parse_timeout is not None and parse_timeout <= 0:
+ raise ValueError("parse_timeout must be > 0 seconds")
+ self._config = dict(config)
+ self._timeout = parse_timeout
+ self._idle: "queue.Queue[_Worker]" = queue.Queue()
+ self._lock = threading.Lock()
+ self._workers: List[_Worker] = []
+ self._closed = False
+ # Spawn eagerly: children import and construct their native parsers
+ # concurrently while the caller goes on with its own startup.
+ for _ in range(pool_size):
+ self._spawn_worker()
+
+ def _spawn_worker(self) -> None:
+ worker = _Worker(self._config)
+ with self._lock:
+ self._workers.append(worker)
+ self._idle.put(worker)
+
+ def _retire_worker(self, worker: _Worker) -> None:
+ worker.kill()
+ with self._lock:
+ if worker in self._workers:
+ self._workers.remove(worker)
+
+ def parse(self, payload: Union[str, bytes], source: str) -> Any:
+ """Run one parse on an idle worker.
+
+ Blocks until a worker is free; ``parse_timeout`` bounds the parse
+ itself, not the wait for a free worker.
+ """
+ if self._closed:
+ raise ParseError("parser pool is closed")
+ worker = self._idle.get()
+ replace = False
+ try:
+ status, data = worker.request(payload, self._timeout)
+ except _WorkerTimeout:
+ replace = True
+ timeout = self._timeout
+ raise ParseTimeoutError(
+ f"parse of {source} exceeded {timeout}s; "
+ "the worker process was killed",
+ source=source,
+ timeout=timeout,
+ ) from None
+ except _WorkerCrashed as e:
+ replace = True
+ raise ParseError(
+ f"liteparse worker process died while parsing {source}: {e}"
+ ) from None
+ except _WorkerInitFailed as e:
+ replace = True
+ raise ParseError(f"liteparse worker failed to initialize: {e}") from None
+ finally:
+ if replace:
+ self._retire_worker(worker)
+ if not self._closed:
+ self._spawn_worker()
+ elif self._closed:
+ worker.stop()
+ else:
+ self._idle.put(worker)
+ if status == "ok":
+ return data
+ raise ParseError(data)
+
+ def warm_up(self) -> None:
+ """Block until every worker has finished initializing.
+
+ Optional — the first parse per worker waits for init anyway. Useful
+ before latency-sensitive traffic or benchmarks.
+ """
+ workers = list(self._workers)
+ for worker in workers:
+ worker._ensure_ready()
+
+ def close(self) -> None:
+ """Shut down all workers. Idempotent."""
+ if self._closed:
+ return
+ self._closed = True
+ # Stop the workers we can grab; busy workers are stopped by parse()'s
+ # finally-block when they come back (see the _closed check there).
+ with self._lock:
+ workers = list(self._workers)
+ self._workers.clear()
+ while True:
+ try:
+ self._idle.get_nowait()
+ except queue.Empty:
+ break
+ for worker in workers:
+ worker.stop()
+
+
+if __name__ == "__main__":
+ sys.exit(_worker_main())
diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py
index f48d566..b34d68f 100644
--- a/packages/python/liteparse/parser.py
+++ b/packages/python/liteparse/parser.py
@@ -489,6 +489,8 @@ class LiteParse:
skip_diagonal_text: Optional[bool] = None,
include_complexity: Optional[bool] = None,
extract_vector_graphics: Optional[bool] = None,
+ pool_size: Optional[int] = None,
+ parse_timeout: Optional[float] = None,
):
"""
Initialize LiteParse parser.
@@ -568,6 +570,23 @@ class LiteParse:
pass.
extract_vector_graphics: Expose page-scoped vector shapes and
merged horizontal/vertical line segments. Default False.
+ pool_size: Route :meth:`parse` through a pool of this many
+ persistent worker processes instead of parsing in-process.
+ Each worker has its own PDFium instance, so pooled parses run
+ genuinely in parallel (in-process parses serialize on a
+ process-global PDFium lock) and a stuck document can be
+ killed without leaking a wedged thread. Workers are spawned
+ once and reused; per-parse overhead is IPC serialization
+ only. Other methods (:meth:`parse_batches`,
+ :meth:`is_complex`, :meth:`screenshot`) still run in-process.
+ Call :meth:`close` (or use ``with``) to shut workers down.
+ parse_timeout: Hard per-parse deadline in seconds. Requires
+ ``pool_size``: a deadline is only honest when it can actually
+ stop the work, and an in-process parse stuck inside a single
+ PDFium FFI call cannot be interrupted — the pool enforces the
+ deadline by killing the worker process. On expiry the parse
+ raises :class:`ParseTimeoutError` (naming the document) and a
+ fresh worker replaces the killed one.
"""
kwargs = {}
if ocr_enabled is not None:
@@ -647,6 +666,52 @@ class LiteParse:
self._native = _NativeLiteParse(**kwargs)
+ if parse_timeout is not None and pool_size is None:
+ raise ValueError(
+ "parse_timeout requires pool_size: an in-process parse stuck "
+ "inside a PDFium call cannot be interrupted, so a timeout "
+ "without process isolation would be a promise LiteParse "
+ "cannot keep. Set pool_size (e.g. pool_size=1) to enforce "
+ "the deadline by killing the worker process."
+ )
+ self._pool = None
+ if pool_size is not None:
+ from ._pool import WorkerPool
+
+ self._pool = WorkerPool(kwargs, pool_size, parse_timeout)
+
+ def close(self) -> None:
+ """Shut down pool workers, if pool mode is enabled. Idempotent.
+
+ Without ``pool_size`` this is a no-op. Workers also exit on their own
+ when the parent process does, so forgetting to call this leaks
+ nothing past interpreter exit.
+ """
+ if self._pool is not None:
+ self._pool.close()
+
+ def warm_up(self) -> None:
+ """Block until all pool workers are initialized (no-op without pool).
+
+ Optional: the first parse on each worker waits for its init anyway.
+ Call this before latency-sensitive traffic to avoid paying worker
+ startup on the first request.
+ """
+ if self._pool is not None:
+ self._pool.warm_up()
+
+ def __enter__(self) -> "LiteParse":
+ return self
+
+ def __exit__(self, *exc_info: Any) -> None:
+ self.close()
+
+ def __del__(self) -> None:
+ try:
+ self.close()
+ except Exception:
+ pass
+
def parse(
self,
file_data: Union[str, Path, bytes],
@@ -662,19 +727,29 @@ class LiteParse:
Raises:
ParseError: If parsing fails.
+ ParseTimeoutError: In pool mode, if the parse exceeded
+ ``parse_timeout`` (the worker process is killed and replaced).
FileNotFoundError: If the file doesn't exist.
"""
+ if isinstance(file_data, bytes):
+ payload: Union[str, bytes] = file_data
+ source = f"<{len(file_data)} bytes>"
+ else:
+ file_path = Path(file_data)
+ if not file_path.exists():
+ raise FileNotFoundError(f"File not found: {file_path}")
+ payload = str(file_path.absolute())
+ source = payload
+
+ if self._pool is not None:
+ return self._pool.parse(payload, source)
+
try:
- if isinstance(file_data, bytes):
- native_result = self._native.parse_bytes(file_data)
+ if isinstance(payload, bytes):
+ native_result = self._native.parse_bytes(payload)
else:
- file_path = Path(file_data)
- if not file_path.exists():
- raise FileNotFoundError(f"File not found: {file_path}")
- native_result = self._native.parse(str(file_path.absolute()))
+ native_result = self._native.parse(payload)
return _convert_native_result(native_result)
- except FileNotFoundError:
- raise
except Exception as e:
raise ParseError(str(e)) from e
diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.py
index c796c75..ba60687 100644
--- a/packages/python/liteparse/types.py
+++ b/packages/python/liteparse/types.py
@@ -493,3 +493,23 @@ class LiteParseConfig:
class ParseError(Exception):
"""Exception raised when parsing fails."""
pass
+
+
+class ParseTimeoutError(ParseError):
+ """A pooled parse exceeded ``parse_timeout`` and its worker was killed.
+
+ Only raised in pool mode (``LiteParse(pool_size=...)``), where the
+ deadline is enforced by killing the worker process — the timed-out parse
+ is guaranteed dead, not still running in the background.
+
+ Attributes:
+ source: The document that timed out — the file path, or
+ ``"<N bytes>"`` for byte inputs. Log this to identify the
+ documents that stall your pipeline.
+ timeout: The deadline that was exceeded, in seconds.
+ """
+
+ def __init__(self, message: str, *, source: str, timeout: float):
+ super().__init__(message)
+ self.source = source
+ self.timeout = timeout
diff --git a/packages/python/tests/test_pool_e2e.py b/packages/python/tests/test_pool_e2e.py
new file mode 100644
index 0000000..82d955a
--- /dev/null
+++ b/packages/python/tests/test_pool_e2e.py
@@ -0,0 +1,119 @@
+"""E2E tests for pool mode (process-isolated workers with hard timeouts)."""
+
+import math
+import zlib
+from pathlib import Path
+
+import pytest
+
+from liteparse import LiteParse, ParseError, ParseTimeoutError
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+SAMPLE_PDF = REPO_ROOT / "integration_tests_data" / "sample.pdf"
+
+CFG = dict(quiet=True, ocr_enabled=False, output_format="text")
+
+
+def _grid_pdf(n_objects: int) -> bytes:
+ """One page with a dense grid of tiny text objects.
+
+ Parse time is superlinear in the object count (PDFium's text-page
+ assembly), which makes a small file arbitrarily slow to parse — the same
+ shape as the rogue documents that stall prod pipelines, used here to
+ exercise the timeout path.
+ """
+ cols = int(math.sqrt(n_objects)) or 1
+ parts = [
+ f"BT /F1 2 Tf {20 + (i % cols) * 2.5:.1f} "
+ f"{770 - (i // cols) * 2.5:.1f} Td (x) Tj ET"
+ for i in range(n_objects)
+ ]
+ stream = zlib.compress("\n".join(parts).encode())
+ width = int(40 + cols * 2.5)
+ height = int(40 + (n_objects // cols) * 2.5)
+ objects = [
+ b"<< /Type /Catalog /Pages 2 0 R >>",
+ b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+ b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 %d %d] "
+ b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>"
+ % (width, height),
+ b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+ b"<< /Length %d /Filter /FlateDecode >>\nstream\n%s\nendstream"
+ % (len(stream), stream),
+ ]
+ out = bytearray(b"%PDF-1.4\n")
+ offsets = []
+ for i, body in enumerate(objects, start=1):
+ offsets.append(len(out))
+ out += b"%d 0 obj\n" % i + body + b"\nendobj\n"
+ xref_pos = len(out)
+ out += b"xref\n0 %d\n" % (len(objects) + 1)
+ out += b"0000000000 65535 f \n"
+ for off in offsets:
+ out += b"%010d 00000 n \n" % off
+ out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (
+ len(objects) + 1,
+ xref_pos,
+ )
+ return bytes(out)
+
+
+def sample_pdf() -> Path:
+ if not SAMPLE_PDF.exists():
+ pytest.skip(f"Test document not found: {SAMPLE_PDF}")
+ return SAMPLE_PDF
+
+
+def test_pool_matches_in_process(sample_pdf):
+ expected = LiteParse(**CFG).parse(sample_pdf)
+ with LiteParse(**CFG, pool_size=2) as pooled:
+ from_path = pooled.parse(sample_pdf)
+ from_bytes = pooled.parse(sample_pdf.read_bytes())
+ assert from_path.text == expected.text
+ assert from_bytes.text == expected.text
+ assert len(from_path.pages) == len(expected.pages)
+
+
+def test_parse_timeout_requires_pool():
+ with pytest.raises(ValueError, match="pool_size"):
+ LiteParse(parse_timeout=5)
+
+
+def test_timeout_kills_worker_and_pool_recovers(sample_pdf):
+ slow_doc = _grid_pdf(64000) # parses in seconds, not before the deadline
+ with LiteParse(**CFG, pool_size=1, parse_timeout=0.5) as pooled:
+ pooled.warm_up()
+ with pytest.raises(ParseTimeoutError) as excinfo:
+ pooled.parse(slow_doc)
+ assert excinfo.value.timeout == 0.5
+ assert "bytes" in excinfo.value.source
+ # The replacement worker must serve subsequent parses.
+ result = pooled.parse(sample_pdf)
+ assert result.text
+
+
+def test_timeout_names_the_file(tmp_path):
+ slow_path = tmp_path / "rogue.pdf"
+ slow_path.write_bytes(_grid_pdf(64000))
+ with LiteParse(**CFG, pool_size=1, parse_timeout=0.5) as pooled:
+ with pytest.raises(ParseTimeoutError) as excinfo:
+ pooled.parse(slow_path)
+ assert excinfo.value.source == str(slow_path.absolute())
+
+
+def test_parse_error_propagates_from_worker():
+ with LiteParse(**CFG, pool_size=1) as pooled:
+ with pytest.raises(ParseError):
+ pooled.parse(b"%PDF-1.4 not actually a pdf")
+
+
+def test_close_shuts_down_workers(sample_pdf):
+ pooled = LiteParse(**CFG, pool_size=2)
+ pooled.parse(sample_pdf)
+ pooled.close()
+ workers = list(pooled._pool._workers)
+ assert workers == []
+ with pytest.raises(ParseError, match="closed"):
+ pooled.parse(sample_pdf)
+ pooled.close() # idempotent