Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-10960): OCR scanned PDFs page-by-page in file_url_parser (#6722)
Cindy Li48f8e7bb0cba
diff --git a/skyvern/constants.py b/skyvern/constants.pyindex 104d91e70..f405f3fac 100644--- a/skyvern/constants.py+++ b/skyvern/constants.py@@ -108,6 +108,12 @@ class ScrapeType(StrEnum):SCRAPE_TYPE_ORDER = [ScrapeType.NORMAL, ScrapeType.NORMAL, ScrapeType.RELOAD]DEFAULT_MAX_TOKENS = 100000MAX_FILE_PARSE_INPUT_TOKENS = 256_000+# Safety ceiling on how many pages of a scanned/image-only PDF are OCR'd via the vision-LLM+# fallback. Each page is a separate LLM call, so this bounds cost; MAX_FILE_PARSE_INPUT_TOKENS+# still bounds the concatenated text handed to downstream extraction.+MAX_PDF_OCR_PAGES = 50+# Max concurrent per-page OCR calls when transcribing a scanned PDF.+PDF_OCR_PAGE_CONCURRENCY = 5MAX_IMAGE_MESSAGES = 10SCROLL_AMOUNT_MULTIPLIER = 100EXTRACT_ACTION_SCROLL_AMOUNT = 500 # pixels per scroll action from extract-action promptdiff --git a/skyvern/forge/sdk/utils/pdf_parser.py b/skyvern/forge/sdk/utils/pdf_parser.pyindex 7b02c6b38..7363e6f10 100644--- a/skyvern/forge/sdk/utils/pdf_parser.py+++ b/skyvern/forge/sdk/utils/pdf_parser.py@@ -13,7 +13,7 @@ import pdfplumberimport structlogfrom pypdf import PdfReader-from skyvern.constants import MAX_FILE_PARSE_INPUT_TOKENS+from skyvern.constants import MAX_FILE_PARSE_INPUT_TOKENS, MAX_PDF_OCR_PAGESfrom skyvern.exceptions import PDFParsingErrorfrom skyvern.forge.sdk.utils.sanitization import sanitize_postgres_textfrom skyvern.utils.token_counter import count_tokens@@ -216,20 +216,21 @@ def validate_pdf_file(def render_pdf_pages_as_images(file_path: str,file_identifier: str | None = None,- max_pages: int = 10,+ max_pages: int | None = MAX_PDF_OCR_PAGES,resolution: int = 150,) -> list[bytes]:"""Render PDF pages as PNG images using pdfplumber.- Returns a list of PNG byte buffers, one per page (up to max_pages).- Intended for vision-LLM fallback when text extraction returns nothing- (e.g. scanned / image-based PDFs).+ Returns a list of PNG byte buffers, one per page. Defaults to a bounded page+ cap; pass ``max_pages=None`` to deliberately render every page. Intended for+ vision-LLM fallback when text extraction returns nothing (e.g. scanned /+ image-based PDFs)."""identifier = file_identifier or file_pathpage_images: list[bytes] = []with pdfplumber.open(file_path) as pdf:- pages_to_render = min(len(pdf.pages), max_pages)+ pages_to_render = len(pdf.pages) if max_pages is None else min(len(pdf.pages), max_pages)if pages_to_render < len(pdf.pages):LOG.warning("PDF has more pages than max_pages for image rendering, truncating",diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex fa2ad2253..2a265ae22 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -43,7 +43,9 @@ from skyvern.constants import (AZURE_BLOB_STORAGE_MAX_UPLOAD_FILE_COUNT,GET_DOWNLOADED_FILES_TIMEOUT,MAX_FILE_PARSE_INPUT_TOKENS,+ MAX_PDF_OCR_PAGES,MAX_UPLOAD_FILE_COUNT,+ PDF_OCR_PAGE_CONCURRENCY,SAVE_DOWNLOADED_FILES_TIMEOUT,)from skyvern.exceptions import (@@ -5627,31 +5629,27 @@ class FileParserBlock(Block):if extracted_text.strip():return extracted_text- # Scanned / image-based PDF — render pages as images and use vision LLM+ # Scanned / image-based PDF — render pages as images and OCR each page in+ # its own vision-LLM call. A single call covering every page collapses a+ # multi-page document down to the first page or two.LOG.info("PDF text extraction returned empty content, falling back to vision LLM OCR",file_url=self.file_url,)try:- page_images = render_pdf_pages_as_images(file_path, file_identifier=self.file_url)+ page_images = await asyncio.to_thread(+ render_pdf_pages_as_images,+ file_path,+ file_identifier=self.file_url,+ max_pages=MAX_PDF_OCR_PAGES,+ )if not page_images:return extracted_text-- llm_prompt = prompt_engine.load_prompt("extract-text-from-image")- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- self.override_llm_key, default=app.LLM_API_HANDLER- )- # OCR transcription intentionally skips system_prompt- # It still applies to the downstream extract-information-from-file-text call.- llm_response = await llm_api_handler(- prompt=llm_prompt,- prompt_name="extract-text-from-image",- screenshots=page_images,- force_dict=True,+ return await self._ocr_pdf_pages(+ page_images,workflow_run_block_id=workflow_run_block_id,organization_id=organization_id,)- return llm_response.get("extracted_text", "")except Exception:LOG.exception("Failed to extract text from PDF via vision LLM fallback",@@ -5659,6 +5657,80 @@ class FileParserBlock(Block):)raise+ async def _ocr_pdf_pages(+ self,+ page_images: list[bytes],+ workflow_run_block_id: str | None = None,+ organization_id: str | None = None,+ ) -> str:+ """OCR each rendered PDF page in its own vision-LLM call and concatenate.++ Per-page transcription avoids the single-call collapse where a multi-page+ document is summarized down to its first page(s). Pages are transcribed with+ bounded concurrency, reassembled in page order with page markers, and+ truncated at a page boundary once MAX_FILE_PARSE_INPUT_TOKENS is reached.+ """+ llm_prompt = prompt_engine.load_prompt("extract-text-from-image")+ llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(+ self.override_llm_key, default=app.LLM_API_HANDLER+ )+ semaphore = asyncio.Semaphore(PDF_OCR_PAGE_CONCURRENCY)++ async def _ocr_page(page_image: bytes) -> str:+ async with semaphore:+ # OCR transcription intentionally skips system_prompt; it still applies+ # to the downstream extract-information-from-file-text call.+ llm_response = await llm_api_handler(+ prompt=llm_prompt,+ prompt_name="extract-text-from-image",+ screenshots=[page_image],+ force_dict=True,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )+ return llm_response.get("extracted_text", "") or ""++ page_results = await asyncio.gather(+ *(_ocr_page(page_image) for page_image in page_images),+ return_exceptions=True,+ )++ # A total OCR outage must fail the block, not record an empty success — match the+ # prior single-call path, which propagated OCR errors. Partial failures are skipped below.+ errors = [r for r in page_results if isinstance(r, BaseException)]+ if errors and len(errors) == len(page_results):+ raise errors[0]++ page_chunks: list[str] = []+ current_tokens = 0+ for page_number, result in enumerate(page_results, start=1):+ if isinstance(result, BaseException):+ LOG.warning(+ "Failed to OCR a PDF page via vision LLM, skipping it",+ file_url=self.file_url,+ page_number=page_number,+ error=str(result),+ )+ continue+ page_text = result.strip()+ if not page_text:+ continue+ chunk = f"--- Page {page_number} ---\n{page_text}"+ chunk_tokens = count_tokens(chunk)+ if current_tokens + chunk_tokens > MAX_FILE_PARSE_INPUT_TOKENS:+ LOG.warning(+ "PDF OCR text exceeds token limit, truncating at page boundary",+ file_url=self.file_url,+ pages_included=page_number - 1,+ total_pages=len(page_results),+ max_tokens=MAX_FILE_PARSE_INPUT_TOKENS,+ )+ break+ current_tokens += chunk_tokens+ page_chunks.append(chunk)++ return "\n\n".join(page_chunks)+async def _parse_image_file(self,file_path: str,diff --git a/tests/unit/test_block_llm_cost_kwargs_wiring.py b/tests/unit/test_block_llm_cost_kwargs_wiring.pyindex e406edefe..a396794b2 100644--- a/tests/unit/test_block_llm_cost_kwargs_wiring.py+++ b/tests/unit/test_block_llm_cost_kwargs_wiring.py@@ -18,7 +18,7 @@ BLOCK_PY = (BLOCK_ATTRIBUTED_METHODS = {"_generate_workflow_run_block_description", # BaseBlock description gen"send_prompt", # TextPromptBlock- "_parse_pdf_file", # FileParserBlock PDF vision+ "_ocr_pdf_pages", # FileParserBlock per-page PDF vision OCR"_parse_image_file", # FileParserBlock image OCR"_extract_with_ai", # FileParserBlock schema extract"execute", # PDFParserBlock (deprecated) executediff --git a/tests/unit/workflow/test_file_parser_block.py b/tests/unit/workflow/test_file_parser_block.pyindex 7a7db3522..ee36a7dad 100644--- a/tests/unit/workflow/test_file_parser_block.py+++ b/tests/unit/workflow/test_file_parser_block.py@@ -7,9 +7,11 @@ token truncation, and error handling for DOCX files.from __future__ import annotations+import asyncioimport jsonfrom datetime import datetime, timezonefrom pathlib import Path+from typing import Anyfrom unittest.mock import AsyncMock, MagicMockimport docx@@ -380,3 +382,159 @@ class TestExtractWithAiSerialization:_, kwargs = mock_load.call_argsassert kwargs["extracted_text_content"] == "Hello\nWorld"+++class TestOcrPdfPages:+ """Tests for the per-page vision-LLM OCR path for scanned PDFs (SKY-10960)."""++ @staticmethod+ def _patch_handler(mp: pytest.MonkeyPatch, handler: AsyncMock) -> None:+ mp.setattr(+ "skyvern.forge.sdk.workflow.models.block.LLMAPIHandlerFactory.get_override_llm_api_handler",+ lambda *a, **kw: handler,+ )+ mp.setattr(+ "skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt",+ MagicMock(return_value="prompt"),+ )++ async def test_each_page_transcribed_and_concatenated_in_order(self) -> None:+ """One LLM call per page; every page is concatenated in page order with markers."""+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)+ page_text = {+ b"img-1": "Cover sheet",+ b"img-2": "Demographics",+ b"img-3": "Provider: JORDAN SAMPLE MD",+ }++ async def fake_handler(**kwargs: Any) -> dict[str, str]:+ (image,) = kwargs["screenshots"]+ return {"extracted_text": page_text[image]}++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_handler(mp, handler)++ result = await block._ocr_pdf_pages([b"img-1", b"img-2", b"img-3"])++ # One call per page — the multi-page document is never sent as a single collapsed call.+ assert handler.await_count == 3+ for call in handler.await_args_list:+ assert len(call.kwargs["screenshots"]) == 1+ # Late-page content survives the single-call collapse.+ assert "Provider: JORDAN SAMPLE MD" in result+ assert "Cover sheet" in result and "Demographics" in result+ assert result.index("--- Page 1 ---") < result.index("--- Page 2 ---") < result.index("--- Page 3 ---")++ async def test_order_preserved_when_a_later_page_resolves_first(self) -> None:+ """Output stays in page order even if an earlier page's call finishes last."""+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)++ async def fake_handler(**kwargs: Any) -> dict[str, str]:+ (image,) = kwargs["screenshots"]+ if image == b"slow-1":+ await asyncio.sleep(0.05)+ return {"extracted_text": "first page body"}+ return {"extracted_text": "second page body"}++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_handler(mp, handler)++ result = await block._ocr_pdf_pages([b"slow-1", b"fast-2"])++ assert result.index("first page body") < result.index("second page body")++ async def test_failed_page_is_skipped_not_fatal(self) -> None:+ """A per-page OCR failure is logged and skipped; the other pages still extract."""+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)++ async def fake_handler(**kwargs: Any) -> dict[str, str]:+ (image,) = kwargs["screenshots"]+ if image == b"bad":+ raise RuntimeError("vision model timeout")+ return {"extracted_text": f"text for {image.decode()}"}++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_handler(mp, handler)++ result = await block._ocr_pdf_pages([b"good-1", b"bad", b"good-3"])++ assert handler.await_count == 3+ assert "text for good-1" in result and "text for good-3" in result+ assert "--- Page 2 ---" not in result++ async def test_all_pages_failed_raises(self) -> None:+ """A total OCR outage (every page errors) propagates instead of returning empty text."""+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)++ async def fake_handler(**kwargs: Any) -> dict[str, str]:+ raise RuntimeError("vision model outage")++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_handler(mp, handler)++ with pytest.raises(RuntimeError, match="vision model outage"):+ await block._ocr_pdf_pages([b"p1", b"p2", b"p3"])++ async def test_empty_pages_contribute_nothing(self) -> None:+ """Pages that OCR to empty text add neither a marker nor content."""+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)++ async def fake_handler(**kwargs: Any) -> dict[str, str]:+ (image,) = kwargs["screenshots"]+ return {"extracted_text": "" if image == b"blank" else "real content"}++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_handler(mp, handler)++ result = await block._ocr_pdf_pages([b"blank", b"page-2"])++ assert "--- Page 1 ---" not in result+ assert "--- Page 2 ---" in result and "real content" in result++ async def test_truncates_at_page_boundary_on_token_limit(self) -> None:+ """Concatenation stops at a page boundary once the token budget is exceeded."""+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)++ async def fake_handler(**kwargs: Any) -> dict[str, str]:+ (image,) = kwargs["screenshots"]+ return {"extracted_text": f"content {image.decode()}"}++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_handler(mp, handler)+ # Each page chunk counts as 10 tokens; a 15-token budget admits only the first page.+ mp.setattr("skyvern.forge.sdk.workflow.models.block.count_tokens", lambda text: 10)+ mp.setattr("skyvern.forge.sdk.workflow.models.block.MAX_FILE_PARSE_INPUT_TOKENS", 15)++ result = await block._ocr_pdf_pages([b"page-1", b"page-2", b"page-3"])++ assert "--- Page 1 ---" in result+ assert "--- Page 2 ---" not in result and "--- Page 3 ---" not in result++ async def test_parse_pdf_file_routes_empty_text_to_per_page_ocr(self) -> None:+ """A scanned PDF (no extractable text layer) is routed through per-page OCR."""+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)++ async def fake_handler(**kwargs: Any) -> dict[str, str]:+ (image,) = kwargs["screenshots"]+ return {"extracted_text": f"page {image.decode()}"}++ with pytest.MonkeyPatch.context() as mp:+ handler = AsyncMock(side_effect=fake_handler)+ self._patch_handler(mp, handler)+ mp.setattr("skyvern.forge.sdk.workflow.models.block.extract_pdf_file", lambda *a, **kw: "")+ mp.setattr(+ "skyvern.forge.sdk.workflow.models.block.render_pdf_pages_as_images",+ lambda *a, **kw: [b"A", b"B"],+ )++ result = await block._parse_pdf_file("/tmp/scan.pdf")++ assert handler.await_count == 2+ assert "page A" in result and "page B" in result