Skip to content

Automate browser based workflows with AI

fix(SKY-10960): OCR scanned PDFs page-by-page in file_url_parser (#6722)

Cindy Li48f8e7bb0cba
diff --git a/skyvern/constants.py b/skyvern/constants.py
index 104d91e70..f405f3fac 100644
--- a/skyvern/constants.py
+++ b/skyvern/constants.py
@@ -108,6 +108,12 @@ class ScrapeType(StrEnum):
SCRAPE_TYPE_ORDER = [ScrapeType.NORMAL, ScrapeType.NORMAL, ScrapeType.RELOAD]
DEFAULT_MAX_TOKENS = 100000
MAX_FILE_PARSE_INPUT_TOKENS = 256_000
+# Safety ceiling on how many pages of a scanned/image-only PDF are OCR'd via the vision-LLM
+# fallback. Each page is a separate LLM call, so this bounds cost; MAX_FILE_PARSE_INPUT_TOKENS
+# still bounds the concatenated text handed to downstream extraction.
+MAX_PDF_OCR_PAGES = 50
+# Max concurrent per-page OCR calls when transcribing a scanned PDF.
+PDF_OCR_PAGE_CONCURRENCY = 5
MAX_IMAGE_MESSAGES = 10
SCROLL_AMOUNT_MULTIPLIER = 100
EXTRACT_ACTION_SCROLL_AMOUNT = 500 # pixels per scroll action from extract-action prompt
diff --git a/skyvern/forge/sdk/utils/pdf_parser.py b/skyvern/forge/sdk/utils/pdf_parser.py
index 7b02c6b38..7363e6f10 100644
--- a/skyvern/forge/sdk/utils/pdf_parser.py
+++ b/skyvern/forge/sdk/utils/pdf_parser.py
@@ -13,7 +13,7 @@ import pdfplumber
import structlog
from pypdf import PdfReader
-from skyvern.constants import MAX_FILE_PARSE_INPUT_TOKENS
+from skyvern.constants import MAX_FILE_PARSE_INPUT_TOKENS, MAX_PDF_OCR_PAGES
from skyvern.exceptions import PDFParsingError
from skyvern.forge.sdk.utils.sanitization import sanitize_postgres_text
from skyvern.utils.token_counter import count_tokens
@@ -216,20 +216,21 @@ def validate_pdf_file(
def render_pdf_pages_as_images(
file_path: str,
file_identifier: str | None = None,
- max_pages: int = 10,
+ max_pages: int | None = MAX_PDF_OCR_PAGES,
resolution: int = 150,
) -> list[bytes]:
"""Render PDF pages as PNG images using pdfplumber.
- Returns a list of PNG byte buffers, one per page (up to max_pages).
- Intended for vision-LLM fallback when text extraction returns nothing
- (e.g. scanned / image-based PDFs).
+ Returns a list of PNG byte buffers, one per page. Defaults to a bounded page
+ cap; pass ``max_pages=None`` to deliberately render every page. Intended for
+ vision-LLM fallback when text extraction returns nothing (e.g. scanned /
+ image-based PDFs).
"""
identifier = file_identifier or file_path
page_images: list[bytes] = []
with pdfplumber.open(file_path) as pdf:
- pages_to_render = min(len(pdf.pages), max_pages)
+ pages_to_render = len(pdf.pages) if max_pages is None else min(len(pdf.pages), max_pages)
if pages_to_render < len(pdf.pages):
LOG.warning(
"PDF has more pages than max_pages for image rendering, truncating",
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index fa2ad2253..2a265ae22 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -43,7 +43,9 @@ from skyvern.constants import (
AZURE_BLOB_STORAGE_MAX_UPLOAD_FILE_COUNT,
GET_DOWNLOADED_FILES_TIMEOUT,
MAX_FILE_PARSE_INPUT_TOKENS,
+ MAX_PDF_OCR_PAGES,
MAX_UPLOAD_FILE_COUNT,
+ PDF_OCR_PAGE_CONCURRENCY,
SAVE_DOWNLOADED_FILES_TIMEOUT,
)
from skyvern.exceptions import (
@@ -5627,31 +5629,27 @@ class FileParserBlock(Block):
if extracted_text.strip():
return extracted_text
- # Scanned / image-based PDF — render pages as images and use vision LLM
+ # Scanned / image-based PDF — render pages as images and OCR each page in
+ # its own vision-LLM call. A single call covering every page collapses a
+ # multi-page document down to the first page or two.
LOG.info(
"PDF text extraction returned empty content, falling back to vision LLM OCR",
file_url=self.file_url,
)
try:
- page_images = render_pdf_pages_as_images(file_path, file_identifier=self.file_url)
+ page_images = await asyncio.to_thread(
+ render_pdf_pages_as_images,
+ file_path,
+ file_identifier=self.file_url,
+ max_pages=MAX_PDF_OCR_PAGES,
+ )
if not page_images:
return extracted_text
-
- llm_prompt = prompt_engine.load_prompt("extract-text-from-image")
- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- self.override_llm_key, default=app.LLM_API_HANDLER
- )
- # OCR transcription intentionally skips system_prompt
- # It still applies to the downstream extract-information-from-file-text call.
- llm_response = await llm_api_handler(
- prompt=llm_prompt,
- prompt_name="extract-text-from-image",
- screenshots=page_images,
- force_dict=True,
+ return await self._ocr_pdf_pages(
+ page_images,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
)
- return llm_response.get("extracted_text", "")
except Exception:
LOG.exception(
"Failed to extract text from PDF via vision LLM fallback",
@@ -5659,6 +5657,80 @@ class FileParserBlock(Block):
)
raise
+ async def _ocr_pdf_pages(
+ self,
+ page_images: list[bytes],
+ workflow_run_block_id: str | None = None,
+ organization_id: str | None = None,
+ ) -> str:
+ """OCR each rendered PDF page in its own vision-LLM call and concatenate.
+
+ Per-page transcription avoids the single-call collapse where a multi-page
+ document is summarized down to its first page(s). Pages are transcribed with
+ bounded concurrency, reassembled in page order with page markers, and
+ truncated at a page boundary once MAX_FILE_PARSE_INPUT_TOKENS is reached.
+ """
+ llm_prompt = prompt_engine.load_prompt("extract-text-from-image")
+ llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
+ self.override_llm_key, default=app.LLM_API_HANDLER
+ )
+ semaphore = asyncio.Semaphore(PDF_OCR_PAGE_CONCURRENCY)
+
+ async def _ocr_page(page_image: bytes) -> str:
+ async with semaphore:
+ # OCR transcription intentionally skips system_prompt; it still applies
+ # to the downstream extract-information-from-file-text call.
+ llm_response = await llm_api_handler(
+ prompt=llm_prompt,
+ prompt_name="extract-text-from-image",
+ screenshots=[page_image],
+ force_dict=True,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ return llm_response.get("extracted_text", "") or ""
+
+ page_results = await asyncio.gather(
+ *(_ocr_page(page_image) for page_image in page_images),
+ return_exceptions=True,
+ )
+
+ # A total OCR outage must fail the block, not record an empty success — match the
+ # prior single-call path, which propagated OCR errors. Partial failures are skipped below.
+ errors = [r for r in page_results if isinstance(r, BaseException)]
+ if errors and len(errors) == len(page_results):
+ raise errors[0]
+
+ page_chunks: list[str] = []
+ current_tokens = 0
+ for page_number, result in enumerate(page_results, start=1):
+ if isinstance(result, BaseException):
+ LOG.warning(
+ "Failed to OCR a PDF page via vision LLM, skipping it",
+ file_url=self.file_url,
+ page_number=page_number,
+ error=str(result),
+ )
+ continue
+ page_text = result.strip()
+ if not page_text:
+ continue
+ chunk = f"--- Page {page_number} ---\n{page_text}"
+ chunk_tokens = count_tokens(chunk)
+ if current_tokens + chunk_tokens > MAX_FILE_PARSE_INPUT_TOKENS:
+ LOG.warning(
+ "PDF OCR text exceeds token limit, truncating at page boundary",
+ file_url=self.file_url,
+ pages_included=page_number - 1,
+ total_pages=len(page_results),
+ max_tokens=MAX_FILE_PARSE_INPUT_TOKENS,
+ )
+ break
+ current_tokens += chunk_tokens
+ page_chunks.append(chunk)
+
+ return "\n\n".join(page_chunks)
+
async def _parse_image_file(
self,
file_path: str,
diff --git a/tests/unit/test_block_llm_cost_kwargs_wiring.py b/tests/unit/test_block_llm_cost_kwargs_wiring.py
index e406edefe..a396794b2 100644
--- a/tests/unit/test_block_llm_cost_kwargs_wiring.py
+++ b/tests/unit/test_block_llm_cost_kwargs_wiring.py
@@ -18,7 +18,7 @@ BLOCK_PY = (
BLOCK_ATTRIBUTED_METHODS = {
"_generate_workflow_run_block_description", # BaseBlock description gen
"send_prompt", # TextPromptBlock
- "_parse_pdf_file", # FileParserBlock PDF vision
+ "_ocr_pdf_pages", # FileParserBlock per-page PDF vision OCR
"_parse_image_file", # FileParserBlock image OCR
"_extract_with_ai", # FileParserBlock schema extract
"execute", # PDFParserBlock (deprecated) execute
diff --git a/tests/unit/workflow/test_file_parser_block.py b/tests/unit/workflow/test_file_parser_block.py
index 7a7db3522..ee36a7dad 100644
--- a/tests/unit/workflow/test_file_parser_block.py
+++ b/tests/unit/workflow/test_file_parser_block.py
@@ -7,9 +7,11 @@ token truncation, and error handling for DOCX files.
from __future__ import annotations
+import asyncio
import json
from datetime import datetime, timezone
from pathlib import Path
+from typing import Any
from unittest.mock import AsyncMock, MagicMock
import docx
@@ -380,3 +382,159 @@ class TestExtractWithAiSerialization:
_, kwargs = mock_load.call_args
assert kwargs["extracted_text_content"] == "Hello\nWorld"
+
+
+class TestOcrPdfPages:
+ """Tests for the per-page vision-LLM OCR path for scanned PDFs (SKY-10960)."""
+
+ @staticmethod
+ def _patch_handler(mp: pytest.MonkeyPatch, handler: AsyncMock) -> None:
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.LLMAPIHandlerFactory.get_override_llm_api_handler",
+ lambda *a, **kw: handler,
+ )
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt",
+ MagicMock(return_value="prompt"),
+ )
+
+ async def test_each_page_transcribed_and_concatenated_in_order(self) -> None:
+ """One LLM call per page; every page is concatenated in page order with markers."""
+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)
+ page_text = {
+ b"img-1": "Cover sheet",
+ b"img-2": "Demographics",
+ b"img-3": "Provider: JORDAN SAMPLE MD",
+ }
+
+ async def fake_handler(**kwargs: Any) -> dict[str, str]:
+ (image,) = kwargs["screenshots"]
+ return {"extracted_text": page_text[image]}
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_handler(mp, handler)
+
+ result = await block._ocr_pdf_pages([b"img-1", b"img-2", b"img-3"])
+
+ # One call per page — the multi-page document is never sent as a single collapsed call.
+ assert handler.await_count == 3
+ for call in handler.await_args_list:
+ assert len(call.kwargs["screenshots"]) == 1
+ # Late-page content survives the single-call collapse.
+ assert "Provider: JORDAN SAMPLE MD" in result
+ assert "Cover sheet" in result and "Demographics" in result
+ assert result.index("--- Page 1 ---") < result.index("--- Page 2 ---") < result.index("--- Page 3 ---")
+
+ async def test_order_preserved_when_a_later_page_resolves_first(self) -> None:
+ """Output stays in page order even if an earlier page's call finishes last."""
+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)
+
+ async def fake_handler(**kwargs: Any) -> dict[str, str]:
+ (image,) = kwargs["screenshots"]
+ if image == b"slow-1":
+ await asyncio.sleep(0.05)
+ return {"extracted_text": "first page body"}
+ return {"extracted_text": "second page body"}
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_handler(mp, handler)
+
+ result = await block._ocr_pdf_pages([b"slow-1", b"fast-2"])
+
+ assert result.index("first page body") < result.index("second page body")
+
+ async def test_failed_page_is_skipped_not_fatal(self) -> None:
+ """A per-page OCR failure is logged and skipped; the other pages still extract."""
+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)
+
+ async def fake_handler(**kwargs: Any) -> dict[str, str]:
+ (image,) = kwargs["screenshots"]
+ if image == b"bad":
+ raise RuntimeError("vision model timeout")
+ return {"extracted_text": f"text for {image.decode()}"}
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_handler(mp, handler)
+
+ result = await block._ocr_pdf_pages([b"good-1", b"bad", b"good-3"])
+
+ assert handler.await_count == 3
+ assert "text for good-1" in result and "text for good-3" in result
+ assert "--- Page 2 ---" not in result
+
+ async def test_all_pages_failed_raises(self) -> None:
+ """A total OCR outage (every page errors) propagates instead of returning empty text."""
+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)
+
+ async def fake_handler(**kwargs: Any) -> dict[str, str]:
+ raise RuntimeError("vision model outage")
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_handler(mp, handler)
+
+ with pytest.raises(RuntimeError, match="vision model outage"):
+ await block._ocr_pdf_pages([b"p1", b"p2", b"p3"])
+
+ async def test_empty_pages_contribute_nothing(self) -> None:
+ """Pages that OCR to empty text add neither a marker nor content."""
+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)
+
+ async def fake_handler(**kwargs: Any) -> dict[str, str]:
+ (image,) = kwargs["screenshots"]
+ return {"extracted_text": "" if image == b"blank" else "real content"}
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_handler(mp, handler)
+
+ result = await block._ocr_pdf_pages([b"blank", b"page-2"])
+
+ assert "--- Page 1 ---" not in result
+ assert "--- Page 2 ---" in result and "real content" in result
+
+ async def test_truncates_at_page_boundary_on_token_limit(self) -> None:
+ """Concatenation stops at a page boundary once the token budget is exceeded."""
+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)
+
+ async def fake_handler(**kwargs: Any) -> dict[str, str]:
+ (image,) = kwargs["screenshots"]
+ return {"extracted_text": f"content {image.decode()}"}
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_handler(mp, handler)
+ # Each page chunk counts as 10 tokens; a 15-token budget admits only the first page.
+ mp.setattr("skyvern.forge.sdk.workflow.models.block.count_tokens", lambda text: 10)
+ mp.setattr("skyvern.forge.sdk.workflow.models.block.MAX_FILE_PARSE_INPUT_TOKENS", 15)
+
+ result = await block._ocr_pdf_pages([b"page-1", b"page-2", b"page-3"])
+
+ assert "--- Page 1 ---" in result
+ assert "--- Page 2 ---" not in result and "--- Page 3 ---" not in result
+
+ async def test_parse_pdf_file_routes_empty_text_to_per_page_ocr(self) -> None:
+ """A scanned PDF (no extractable text layer) is routed through per-page OCR."""
+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)
+
+ async def fake_handler(**kwargs: Any) -> dict[str, str]:
+ (image,) = kwargs["screenshots"]
+ return {"extracted_text": f"page {image.decode()}"}
+
+ with pytest.MonkeyPatch.context() as mp:
+ handler = AsyncMock(side_effect=fake_handler)
+ self._patch_handler(mp, handler)
+ mp.setattr("skyvern.forge.sdk.workflow.models.block.extract_pdf_file", lambda *a, **kw: "")
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.render_pdf_pages_as_images",
+ lambda *a, **kw: [b"A", b"B"],
+ )
+
+ result = await block._parse_pdf_file("/tmp/scan.pdf")
+
+ assert handler.await_count == 2
+ assert "page A" in result and "page B" in result