Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-10960): improve file_url_parser OCR fidelity + make its model PostHog-configurable (#6867)
Cindy Li89ffbccbdba0
diff --git a/skyvern/forge/prompts/skyvern/extract-text-from-image.j2 b/skyvern/forge/prompts/skyvern/extract-text-from-image.j2index a577d1e9a..fb60a3336 100644--- a/skyvern/forge/prompts/skyvern/extract-text-from-image.j2+++ b/skyvern/forge/prompts/skyvern/extract-text-from-image.j2@@ -8,6 +8,9 @@ Reply in JSON format with the following keys:}TEXT EXTRACTION GUIDELINES:+- Transcribe ONLY characters that are actually present and legible in the image.+- NEVER infer, guess, complete, summarize, translate, or invent text. Do not add words, names, dates, headings, form fields, boilerplate, or any content that is not physically printed on the page.+- If a word or region is unclear, transcribe what you can and mark the rest [illegible]. Do not substitute a plausible word.- Preserve reading order (top to bottom, left to right)- For tables: format as rows separated by newlines, columns separated by " | "- For multi-column layouts: extract each column separately, separated by blank linesdiff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex f368833e3..f180f7e8b 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -5965,6 +5965,21 @@ class FileParserBlock(Block):)raise+ async def _resolve_file_parser_handler(+ self, prompt_type: str, distinct_id: str | None, organization_id: str | None+ ) -> LLMAPIHandler:+ """Resolve the default handler for a file-parser prompt type.++ Honors the LLM_CONFIG_BY_PROMPT_TYPE PostHog flag (keyed by prompt type) so the+ OCR and extraction models can be set without a deploy; falls back to the primary+ handler. A block-level override_llm_key still takes precedence at the call site.+ """+ if distinct_id:+ posthog_handler = await get_llm_handler_for_prompt_type(prompt_type, distinct_id, organization_id)+ if posthog_handler:+ return posthog_handler+ return app.LLM_API_HANDLER+async def _ocr_pdf_pages(self,page_images: list[bytes],@@ -5979,8 +5994,11 @@ class FileParserBlock(Block):truncated at a page boundary once MAX_FILE_PARSE_INPUT_TOKENS is reached."""llm_prompt = prompt_engine.load_prompt("extract-text-from-image")+ default_handler = await self._resolve_file_parser_handler(+ "extract-text-from-image", workflow_run_block_id, organization_id+ )llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- self.override_llm_key, default=app.LLM_API_HANDLER+ self.override_llm_key, default=default_handler)semaphore = asyncio.Semaphore(PDF_OCR_PAGE_CONCURRENCY)@@ -6051,8 +6069,11 @@ class FileParserBlock(Block):image_bytes = f.read()llm_prompt = prompt_engine.load_prompt("extract-text-from-image")+ default_handler = await self._resolve_file_parser_handler(+ "extract-text-from-image", workflow_run_block_id, organization_id+ )llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- self.override_llm_key, default=app.LLM_API_HANDLER+ self.override_llm_key, default=default_handler)# OCR transcription intentionally skips system_prompt — see# _parse_pdf_file_with_vision_ocr for rationale.@@ -6169,7 +6190,10 @@ class FileParserBlock(Block):)llm_key = self.override_llm_key- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(llm_key, default=app.LLM_API_HANDLER)+ default_handler = await self._resolve_file_parser_handler(+ "extract-information-from-file-text", workflow_run_block_id, organization_id+ )+ llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(llm_key, default=default_handler)llm_response = await llm_api_handler(prompt=llm_prompt,