Skip to content

Automate browser based workflows with AI

Keep a fetched OTP code until an action types it and enforce the expected type on the webhook parse (SKY-18274) (#8898)

Aaron Perezc3e9a23ffdba
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index 8db64e69f..8f634dd18 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -16865,6 +16865,12 @@
}
],
"title": "Worksheet"
+ },
+ "force_ocr": {
+ "type": "boolean",
+ "title": "Force Ocr",
+ "description": "PDF only: ignore the PDF's embedded text layer and read the pages with vision OCR instead. Use it when a PDF's text layer is unreadable in a way Skyvern does not detect on its own. OCR reads at most the first 50 pages, so a longer PDF loses the rest; leave this off unless the text layer is unusable.",
+ "default": false
}
},
"type": "object",
@@ -16957,6 +16963,12 @@
}
],
"title": "Worksheet"
+ },
+ "force_ocr": {
+ "type": "boolean",
+ "title": "Force Ocr",
+ "description": "PDF only: ignore the PDF's embedded text layer and read the pages with vision OCR instead. Use it when a PDF's text layer is unreadable in a way Skyvern does not detect on its own. OCR reads at most the first 50 pages, so a longer PDF loses the rest; leave this off unless the text layer is unusable.",
+ "default": false
}
},
"type": "object",
diff --git a/skyvern-frontend/src/routes/workflows/editor/fileParserWorksheet.test.ts b/skyvern-frontend/src/routes/workflows/editor/fileParserWorksheet.test.ts
index 375b78866..88633a1e4 100644
--- a/skyvern-frontend/src/routes/workflows/editor/fileParserWorksheet.test.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/fileParserWorksheet.test.ts
@@ -37,6 +37,7 @@ const FILE_PARSER_BLOCK: FileURLParserBlock = {
file_type: "excel",
json_schema: null,
worksheet: "Query Log",
+ force_ocr: false,
};
describe("File Parser worksheet round-trip", () => {
@@ -67,4 +68,21 @@ describe("File Parser worksheet round-trip", () => {
expect((block as FileUrlParserBlockYAML).worksheet).toBe("Query Log");
});
+
+ test("force OCR survives block -> node -> block", () => {
+ const node = convertToNode(
+ { id: "fp2" },
+ { ...FILE_PARSER_BLOCK, file_type: "pdf", force_ocr: true },
+ true,
+ );
+ expect(isFileParserNode(node)).toBe(true);
+ if (!isFileParserNode(node)) {
+ return;
+ }
+ expect(node.data.forceOcr).toBe(true);
+
+ const [block] = getWorkflowBlocks([node], []);
+
+ expect((block as FileUrlParserBlockYAML).force_ocr).toBe(true);
+ });
});
diff --git a/skyvern-frontend/src/routes/workflows/editor/helpContent.ts b/skyvern-frontend/src/routes/workflows/editor/helpContent.ts
index 076365645..af4e06c53 100644
--- a/skyvern-frontend/src/routes/workflows/editor/helpContent.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/helpContent.ts
@@ -218,6 +218,8 @@ export const helpTooltips = {
"The format of the file to parse. Auto-detected from the URL extension when possible. ZIP archives are always unzipped, and the block outputs the extracted files as file_name, file_path, and file_size. Data Schema is ignored for ZIPs. To parse contained files, loop over the output and pass each file_path to another File Parser block.",
worksheet:
"Excel only: the exact name of the worksheet to read. Leave empty to read the first worksheet. The run fails if the named worksheet is not in the workbook.",
+ forceOcr:
+ "PDF only: ignore the PDF's embedded text layer and read the pages with vision OCR instead. Skyvern already does this when the text layer is empty or its characters are scrambled; turn this on when a PDF's text layer is unreadable in some other way. OCR is slower, costs more than reading the text layer, and reads at most the first 50 pages, so a longer PDF loses the rest.",
},
terminate: {
...baseHelpTooltipContent,
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsx
index 9269bf8fb..2fd7d8b59 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserEditor.tsx
@@ -16,6 +16,7 @@ import {
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
+import { Switch } from "@/components/ui/switch";
import { WorkflowBlockInput } from "@/components/WorkflowBlockInput";
import { WorkflowDataSchemaInputGroup } from "@/components/DataSchemaInputGroup/WorkflowDataSchemaInputGroup";
@@ -235,6 +236,26 @@ function FileParserEditorBody({
update({ nextLoopOnFailure })
}
/>
+ <div className="flex items-center justify-between gap-2 pt-2">
+ <div className="flex min-w-0 items-center gap-2">
+ <Label className="truncate text-xs font-normal text-tertiary-foreground">
+ Force OCR
+ </Label>
+ <HelpTooltip content={helpTooltips["fileParser"]["forceOcr"]} />
+ </div>
+ <div className="w-52 shrink-0">
+ <Switch
+ checked={data.forceOcr ?? false}
+ data-testid="force-ocr-switch"
+ onCheckedChange={(checked) => {
+ if (!editable) {
+ return;
+ }
+ update({ forceOcr: checked });
+ }}
+ />
+ </div>
+ </div>
<IgnoreWorkflowSystemPrompt
ignoreWorkflowSystemPrompt={
data.ignoreWorkflowSystemPrompt ?? false
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/types.ts
index ef6663f07..68ea5d0df 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/types.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/types.ts
@@ -20,6 +20,7 @@ export type FileParserNodeData = NodeBaseData & {
fileType: FileParserFileType;
jsonSchema: string;
worksheet: string;
+ forceOcr: boolean;
model: WorkflowModel | null;
};
@@ -35,6 +36,7 @@ export const fileParserNodeDefaultData: FileParserNodeData = {
nextLoopOnFailure: false,
jsonSchema: "null",
worksheet: "",
+ forceOcr: false,
model: null,
ignoreWorkflowSystemPrompt: false,
} as const;
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.tsx
index 395520eb0..47d283be3 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/FileParserBlockForm.tsx
@@ -34,6 +34,7 @@ function FileParserBlockFormBody({
fileType,
jsonSchema,
worksheet,
+ forceOcr,
model,
continueOnFailure,
nextLoopOnFailure,
@@ -45,6 +46,7 @@ function FileParserBlockFormBody({
fileType,
jsonSchema,
worksheet,
+ forceOcr,
model,
continueOnFailure,
nextLoopOnFailure,
@@ -54,6 +56,7 @@ function FileParserBlockFormBody({
fileType,
jsonSchema,
worksheet,
+ forceOcr,
model,
continueOnFailure,
nextLoopOnFailure,
diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
index 3339cef5e..5502f7a08 100644
--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
@@ -1259,6 +1259,7 @@ function convertToNode(
fileType: block.file_type ?? "auto_detect",
jsonSchema: JSON.stringify(block.json_schema, null, 2),
worksheet: block.worksheet ?? "",
+ forceOcr: block.force_ocr ?? false,
model: block.model,
},
};
@@ -3741,6 +3742,7 @@ function getWorkflowBlock(
file_type: node.data.fileType,
json_schema: JSONParseSafe(node.data.jsonSchema),
worksheet: node.data.worksheet || null,
+ force_ocr: node.data.forceOcr ?? false,
};
}
case "textPrompt": {
@@ -5332,6 +5334,7 @@ function convertBlocksToBlockYAML(
file_type: block.file_type,
json_schema: block.json_schema,
worksheet: block.worksheet,
+ force_ocr: block.force_ocr,
};
return blockYaml;
}
diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
index 4ca4b65b3..8bb14caab 100644
--- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
@@ -498,6 +498,7 @@ export type FileURLParserBlock = WorkflowBlockBase & {
file_type: "auto_detect" | "csv" | "excel" | "pdf" | "image" | "docx" | "zip";
json_schema: Record<string, unknown> | null;
worksheet: string | null;
+ force_ocr: boolean;
};
export type ValidationBlock = WorkflowBlockBase & {
diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
index 9c5ae7aaa..8f6f293d4 100644
--- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
@@ -474,6 +474,7 @@ export type FileUrlParserBlockYAML = BlockYAMLBase & {
file_type: "auto_detect" | "csv" | "excel" | "pdf" | "image" | "docx" | "zip";
json_schema?: Record<string, unknown> | null;
worksheet?: string | null;
+ force_ocr?: boolean;
};
export type ForLoopBlockYAML = BlockYAMLBase & {
diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.py
index a6dd3f838..59859254c 100644
--- a/skyvern/core/script_generations/generate_script.py
+++ b/skyvern/core/script_generations/generate_script.py
@@ -2506,6 +2506,18 @@ def _build_file_url_parser_statement(block: dict[str, Any]) -> cst.SimpleStateme
)
)
+ if block.get("force_ocr"):
+ args.append(
+ cst.Arg(
+ keyword=cst.Name("force_ocr"),
+ value=_value(True),
+ whitespace_after_arg=cst.ParenthesizedWhitespace(
+ indent=True,
+ last_line=cst.SimpleWhitespace(INDENT),
+ ),
+ )
+ )
+
if block.get("label") is not None:
args.append(
cst.Arg(
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index be2e2372f..dc28efa58 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -148,6 +148,7 @@ from skyvern.forge.sdk.core.skyvern_context import (
MultiFieldTotpAttempt,
MultiFieldTotpRejection,
SkyvernContext,
+ UnconsumedOTPCode,
action_for_multi_field_totp_persistence,
canonical_url,
normalize_multi_field_totp_code,
@@ -224,6 +225,7 @@ from skyvern.services.credential_verdict_service import (
)
from skyvern.services.error_detection_service import detect_user_defined_errors_for_task
from skyvern.services.otp_service import (
+ OTPValue,
describe_webhook_contract_failure,
extract_totp_from_navigation_inputs,
is_usable_credential_totp_placeholder,
@@ -362,6 +364,11 @@ STEP_STALL_LOG_REPEAT_EVERY_SECONDS = 30 * 60
# a scrape plus one LLM call, and fewer than 1 in 100,000 of those LLM calls alone runs past 15 minutes.
PARALLEL_VERIFICATION_TIMEOUT_SECONDS = 15 * 60
+# How long a polled verification code may be carried to a later plan. Emailed codes outlive this by
+# minutes, but a code the page never accepted is better re-polled than submitted indefinitely, and
+# two steps' worth is enough to cover a code field that renders one step after the gate fires.
+UNCONSUMED_OTP_CODE_TTL_SECONDS = 120.0
+
EXTRACT_ACTION_TEMPLATE = "extract-action"
DECISIVE_CRITERION_VALIDATE_TEMPLATE = "decisive-criterion-validate"
@@ -866,6 +873,30 @@ def _replace_multi_field_totp_code(
return value
+def _plan_without_unresolved_code_entry(
+ task: Task, json_response: dict[str, Any], otp_value: OTPValue | None
+) -> dict[str, Any]:
+ """Drop a plan that would type into the code field when no code of the expected type resolved.
+
+ The planner writes a placeholder ("OTP code from email") into its own input_text action and
+ relies on the resolver to replace it, so returning that plan verbatim submits the placeholder.
+ A plan that only clicks or waits is still the right move and is returned untouched.
+ """
+ planned_actions = json_response.get("actions")
+ if not isinstance(planned_actions, list) or not any(
+ isinstance(action, dict) and str(action.get("action_type", "")).upper() == ActionType.INPUT_TEXT.value.upper()
+ for action in planned_actions
+ ):
+ return json_response
+ LOG.warning(
+ "No verification code resolved; dropping the plan that would have typed into the code field",
+ task_id=task.task_id,
+ workflow_run_id=task.workflow_run_id,
+ resolved_otp_type=otp_value.get_otp_type().value if otp_value else None,
+ )
+ return {**json_response, "actions": []}
+
+
def _replace_multi_field_totp_prompt_code(
task_id: str, prompt_kwargs: dict[str, Any], *, rejection: MultiFieldTotpRejection | None = None
) -> dict[str, Any]:
@@ -1025,6 +1056,43 @@ def recovery_credential_release_guard(
return guard
+def _unconsumed_otp_code(context: SkyvernContext, task_id: str) -> UnconsumedOTPCode | None:
+ """The polled code an earlier plan was given but never typed, while it is still worth submitting."""
+ held = context.unconsumed_otp_codes.get(task_id)
+ if held is None or context.totp_codes.get(task_id) != held.value:
+ return None
+ if time.time() - held.obtained_at > UNCONSUMED_OTP_CODE_TTL_SECONDS:
+ return None
+ return held
+
+
+def _release_planned_otp_code(context: SkyvernContext, task_id: str, actions: Sequence[Action]) -> None:
+ """Discard the verification code the plan was given, unless the plan never typed it.
+
+ The OTP gate can fire on a page whose code field has not rendered yet, so the plan that follows
+ is a click rather than an input. Discarding the code there means the next step polls again, and
+ a second poll of the same email can resolve to the sign-in link beside the code instead.
+ """
+ held = _unconsumed_otp_code(context, task_id)
+ if held is None:
+ context.pop_totp_code(task_id)
+ return
+ canonical = skyvern_context.strip_multi_field_totp_code_separators(held.value)
+ for action in actions:
+ if not isinstance(action, InputTextAction):
+ continue
+ if held.value in action.text or canonical in skyvern_context.strip_multi_field_totp_code_separators(
+ action.text
+ ):
+ context.pop_totp_code(task_id)
+ return
+ LOG.info(
+ "Verification code was not typed by the planned actions; carrying it to the next step",
+ task_id=task_id,
+ code_age_seconds=round(time.time() - held.obtained_at, 1),
+ )
+
+
def _first_plan_carries_consumable_totp(
actions: list[Any],
multi_field_source_ready: bool,
@@ -5868,7 +5936,7 @@ class ForgeAgent:
actions = [terminate_action]
if context:
- context.pop_totp_code(task.task_id)
+ _release_planned_otp_code(context, task.task_id, actions)
except NoTOTPVerificationCodeFound as e:
# Surface only the source poll_otp_value actually queried so
# the failure_reason tells the customer which delivery endpoint
@@ -8737,7 +8805,9 @@ class ForgeAgent:
final_navigation_payload.append(verification_code_dict)
elif final_navigation_payload is None:
final_navigation_payload = {SPECIAL_FIELD_VERIFICATION_CODE: verification_code}
- if expire_verification_code:
+ # A carried code is owned by the planning round that polled it, not by this prompt: expiring
+ # it here would send the step that finally has the code field back to the delivery source.
+ if expire_verification_code and _unconsumed_otp_code(current_context, task.task_id) is None:
current_context.pop_totp_code(task.task_id)
mask_values = (
current_context.multi_field_totp_mask_values.get(task.task_id, set())
@@ -11460,6 +11530,7 @@ class ForgeAgent:
current_context = skyvern_context.current()
attempt = current_context.multi_field_totp.get(task.task_id) if current_context else None
rejection = current_context.multi_field_totp_rejections.get(task.task_id) if current_context else None
+ held_code: UnconsumedOTPCode | None = None
if attempt is not None or rejection is not None:
expected_digits = (
attempt.expected_digits
@@ -11478,11 +11549,21 @@ class ForgeAgent:
multi_field_expected_digits=expected_digits,
)
else:
- otp_value = await resolve_otp_value(
- task,
- expected_otp_type=OTPType.TOTP,
- allowed_credential_parameter_keys=allowed_credential_parameter_keys,
- )
+ if current_context is not None:
+ held_code = _unconsumed_otp_code(current_context, task.task_id)
+ if held_code is not None:
+ LOG.info(
+ "Reusing the verification code fetched before the code field rendered",
+ task_id=task.task_id,
+ code_age_seconds=round(time.time() - held_code.obtained_at, 1),
+ )
+ otp_value = OTPValue(value=held_code.value, type=OTPType.TOTP)
+ else:
+ otp_value = await resolve_otp_value(
+ task,
+ expected_otp_type=OTPType.TOTP,
+ allowed_credential_parameter_keys=allowed_credential_parameter_keys,
+ )
if not otp_value or otp_value.get_otp_type() != OTPType.TOTP:
if (
@@ -11494,7 +11575,7 @@ class ForgeAgent:
)
):
return {**_replace_multi_field_totp_prompt_code(task.task_id, json_response), "actions": []}
- return json_response
+ return _plan_without_unresolved_code_entry(task, json_response, otp_value)
if current_context is not None and not otp_value.from_credential_seed:
code_digits = (
@@ -11521,7 +11602,13 @@ class ForgeAgent:
elif has_box_inputs and not skyvern_context.is_supported_multi_field_totp_code(otp_value.value):
normalize_multi_field_totp_code(otp_value.value, task_id=task.task_id)
return {**_replace_multi_field_totp_prompt_code(task.task_id, json_response), "actions": []}
- external_code_obtained_at = time.time() if not otp_value.from_credential_seed else None
+ # A carried-over code keeps its original fetch time, so passing it to another plan cannot
+ # stretch either its reuse window or the rejection timing derived from it.
+ external_code_obtained_at = (
+ None
+ if otp_value.from_credential_seed
+ else (held_code.obtained_at if held_code is not None else time.time())
+ )
if current_context is not None:
if otp_value.from_credential_seed:
current_context.seed_generated_totp_values.setdefault(task.task_id, set()).add(otp_value.value)
@@ -11542,8 +11629,16 @@ class ForgeAgent:
current_context.totp_codes[f"{task.task_id}_totp_cache"] = otp_value.value
current_context.register_secret_value(otp_value.value)
current_context.totp_codes[task.task_id] = attempt.hint_code
+ current_context.unconsumed_otp_codes.pop(task.task_id, None)
else:
current_context.totp_codes[task.task_id] = otp_value.value
+ if external_code_obtained_at is None:
+ # Seed-generated codes are regenerated on demand, so there is nothing to carry.
+ current_context.unconsumed_otp_codes.pop(task.task_id, None)
+ else:
+ current_context.unconsumed_otp_codes[task.task_id] = UnconsumedOTPCode(
+ value=otp_value.value, obtained_at=external_code_obtained_at
+ )
build_result = await self._build_extract_action_prompt(
task,
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index 9735601b6..fdc3f5370 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -61,6 +61,18 @@ class MultiFieldTotpAttempt:
observed_max_filled: int = 0
+@dataclass(frozen=True)
+class UnconsumedOTPCode:
+ """A polled verification code handed to a plan that did not type it.
+
+ ``obtained_at`` is the wall-clock time the code was fetched, not the time it was handed over, so
+ carrying it to a later plan cannot extend how long it is considered worth submitting.
+ """
+
+ value: str = field(repr=False)
+ obtained_at: float
+
+
@dataclass
class MultiFieldTotpRejection:
rejected_code_hash: str = field(repr=False)
@@ -702,6 +714,9 @@ class SkyvernContext:
# Keep source names with exact CDP targets; finalization must not infer provenance from affixes.
download_suffix_applied_files: dict[str, tuple[str, str]] = field(default_factory=dict, repr=False)
totp_codes: dict[str, str | None] = field(default_factory=dict)
+ # Polled codes placed in totp_codes that no plan has typed yet. Only the single-field polling
+ # path records one, so every other producer keeps the historical discard-after-planning behavior.
+ unconsumed_otp_codes: dict[str, UnconsumedOTPCode] = field(default_factory=dict, repr=False)
seed_generated_totp_values: dict[str, set[str]] = field(default_factory=dict, repr=False)
multi_field_totp: dict[str, MultiFieldTotpAttempt] = field(default_factory=dict)
multi_field_totp_rejections: dict[str, MultiFieldTotpRejection] = field(default_factory=dict)
@@ -1075,6 +1090,7 @@ class SkyvernContext:
return self.__repr__()
def pop_totp_code(self, task_id: str) -> None:
+ self.unconsumed_otp_codes.pop(task_id, None)
if task_id in self.totp_codes:
self.totp_codes.pop(task_id)
@@ -1104,6 +1120,7 @@ class SkyvernContext:
def clear_multi_field_totp_state(self, task_id: str, *, restore_unverified_external: bool = False) -> None:
attempt = self.multi_field_totp.pop(task_id, None)
self.seed_generated_totp_values.pop(task_id, None)
+ self.unconsumed_otp_codes.pop(task_id, None)
self.totp_codes.pop(task_id, None)
self.totp_codes.pop(f"{task_id}_secret", None)
cached_code = self.totp_codes.pop(f"{task_id}_totp_cache", None)
diff --git a/skyvern/forge/sdk/utils/pdf_parser.py b/skyvern/forge/sdk/utils/pdf_parser.py
index 7363e6f10..745bd4ece 100644
--- a/skyvern/forge/sdk/utils/pdf_parser.py
+++ b/skyvern/forge/sdk/utils/pdf_parser.py
@@ -4,7 +4,9 @@ Utility functions for PDF parsing with fallback support.
This module provides robust PDF parsing that tries pypdf first and falls back
to pdfplumber if pypdf fails, ensuring maximum compatibility with various PDF formats.
For image-based/scanned PDFs where text extraction yields no content, pages can be
-rendered as images via render_pdf_pages_as_images for vision-LLM OCR.
+rendered as images via render_pdf_pages_as_images for vision-LLM OCR. A text layer that
+is non-empty but glyph-scrambled needs the same OCR fallback; scrambled_text_layer_reason
+identifies it.
"""
import io
@@ -20,6 +22,79 @@ from skyvern.utils.token_counter import count_tokens
LOG = structlog.get_logger(__name__)
+# A font whose ToUnicode map is wrong substitutes its glyphs out of an unrelated Unicode
+# block. Latin Extended-A/B and the private-use area are where those substitutions land;
+# a Latin-script document's own characters only appear there as in-word diacritics.
+_REMAPPED_GLYPH_RANGES: tuple[tuple[int, int], ...] = ((0x0100, 0x024F), (0xE000, 0xF8FF), (0xFFFD, 0xFFFD))
+# Below these floors the shares computed from the text layer are too noisy to act on.
+_MIN_TEXT_LAYER_CHARS = 40
+_MIN_TEXT_LAYER_WORDS = 10
+_MIN_REMAPPED_GLYPHS = 8
+_MIN_ISOLATED_WORDS = 3
+# A remapped number is a run of glyphs; a checkbox or bullet drawn from an icon font is a
+# single glyph standing alone. Only runs count, so an ordinary form keeps its text layer.
+_MIN_ISOLATED_WORD_LENGTH = 2
+# Diacritics sit inside words next to ASCII letters, so a legitimately accented language
+# keeps this share low (measured ~0.3 for Vietnamese, 0.0 for Polish/Czech/Turkish) while a
+# scrambled digit run is a whole word of nothing else (~0.9).
+_MIN_ISOLATED_GLYPH_SHARE = 0.6
+# Non-Latin scripts are read as-is: their characters fall outside the ranges above, so the
+# shares would be measuring the wrong thing.
+_MIN_ASCII_LETTER_SHARE = 0.5
+
+
+def _is_remapped_glyph(char: str) -> bool:
+ code_point = ord(char)
+ return any(start <= code_point <= end for start, end in _REMAPPED_GLYPH_RANGES)
+
+
+def _is_isolated_word(word: str) -> bool:
+ return (
+ len(word) >= _MIN_ISOLATED_WORD_LENGTH
+ and any(_is_remapped_glyph(char) for char in word)
+ and not any(char.isascii() and char.isalnum() for char in word)
+ )
+
+
+def scrambled_text_layer_reason(text: str) -> str | None:
+ """Return why a PDF text layer looks glyph-scrambled, or None if it can be trusted.
+
+ A subset font with a broken ToUnicode map keeps the document's layout and its ASCII
+ words but replaces whole runs — typically every digit — with characters from an
+ unrelated Unicode block, so the extracted text is plausible-looking and wrong.
+ """
+ words = text.split()
+ if len(words) < _MIN_TEXT_LAYER_WORDS:
+ return None
+ if sum(1 for char in text if not char.isspace()) < _MIN_TEXT_LAYER_CHARS:
+ return None
+
+ # Remapped glyphs are Latin letters to str.isalpha, so counting them here would let the
+ # corruption vote on which script the document is written in — and a page of scrambled
+ # numbers would read as non-Latin and escape every check below.
+ letters = [char for char in text if char.isalpha() and not _is_remapped_glyph(char)]
+ if letters and sum(1 for char in letters if char.isascii()) / len(letters) < _MIN_ASCII_LETTER_SHARE:
+ return None
+
+ remapped_glyphs = sum(1 for char in text if _is_remapped_glyph(char))
+ if remapped_glyphs < _MIN_REMAPPED_GLYPHS:
+ return None
+
+ isolated_words = [word for word in words if _is_isolated_word(word)]
+ if len(isolated_words) < _MIN_ISOLATED_WORDS:
+ return None
+
+ isolated_glyphs = sum(1 for word in isolated_words for char in word if _is_remapped_glyph(char))
+ isolated_share = isolated_glyphs / remapped_glyphs
+ if isolated_share < _MIN_ISOLATED_GLYPH_SHARE:
+ return None
+
+ return (
+ f"{len(isolated_words)} of {len(words)} words in the PDF text layer are built only from characters "
+ f"outside the document's script ({isolated_share:.0%} of the {remapped_glyphs} such characters found), "
+ "which means the embedded font maps its glyphs to the wrong code points"
+ )
+
def extract_pdf_file(
file_path: str,
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 843930902..c22c09599 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -201,7 +201,12 @@ from skyvern.forge.sdk.services.bitwarden import BitwardenConstants
from skyvern.forge.sdk.services.credentials import AzureVaultConstants, OnePasswordConstants, generate_totp_code
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.trace import traced
-from skyvern.forge.sdk.utils.pdf_parser import extract_pdf_file, render_pdf_pages_as_images, validate_pdf_file
+from skyvern.forge.sdk.utils.pdf_parser import (
+ extract_pdf_file,
+ render_pdf_pages_as_images,
+ scrambled_text_layer_reason,
+ validate_pdf_file,
+)
from skyvern.forge.sdk.utils.sanitization import sanitize_postgres_text
from skyvern.forge.sdk.workflow import web_search
from skyvern.forge.sdk.workflow.code_block_authorized_files import (
@@ -313,6 +318,7 @@ from skyvern.schemas.workflows import (
ERROR_CODE_MAPPING_MAX_UTF8_BYTES,
ERROR_CODE_MAX_LENGTH,
ERROR_CODE_REASONING_MAX_LENGTH,
+ FORCE_OCR_DESCRIPTION,
AIFallbackMode,
BlockResult,
BlockStatus,
@@ -13758,6 +13764,7 @@ class FileParserBlock(Block):
file_type: FileType = FileType.AUTO_DETECT
json_schema: dict[str, Any] | None = None
worksheet: str | None = None
+ force_ocr: bool = Field(default=False, description=FORCE_OCR_DESCRIPTION)
schema_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTS
ocr_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTS
_telemetry: FileParserTelemetry = PrivateAttr(default_factory=FileParserTelemetry)
@@ -14138,32 +14145,52 @@ class FileParserBlock(Block):
) -> str:
"""Parse PDF file and return extracted text.
- Uses the shared PDF parsing utility that tries pypdf first,
- then falls back to pdfplumber if pypdf fails. If text extraction
- yields empty/minimal content (e.g. scanned or image-based PDFs),
- renders pages as images and sends them to a vision LLM for OCR.
+ Uses the shared PDF parsing utility that tries pypdf first, then falls back to
+ pdfplumber if pypdf fails. When the text layer cannot be trusted — it is empty
+ (scanned or image-based PDF), its glyphs are scrambled by a broken embedded font,
+ or the block asks for OCR outright — renders pages as images and sends them to a
+ vision LLM for OCR instead.
"""
- try:
- extracted_text = await _run_blocking_parse_step(
- "PDF text extraction",
- self.file_url,
- extract_pdf_file,
- file_path,
- file_identifier=self.file_url,
- )
- except PDFParsingError as e:
- raise InvalidFileType(file_url=self.file_url, file_type=self.file_type, error=str(e))
+ extracted_text = ""
+ ocr_reason: str | None = None
+ if self.force_ocr:
+ ocr_reason = "the block is configured to always OCR PDFs"
+ else:
+ try:
+ extracted_text = await _run_blocking_parse_step(
+ "PDF text extraction",
+ self.file_url,
+ extract_pdf_file,
+ file_path,
+ file_identifier=self.file_url,
+ )
+ except PDFParsingError as e:
+ raise InvalidFileType(file_url=self.file_url, file_type=self.file_type, error=str(e))
- # If text extraction returned meaningful content, use it directly
- if extracted_text.strip():
- return extracted_text
+ if not extracted_text.strip():
+ ocr_reason = "PDF text extraction returned empty content"
+ else:
+ # A scrambled text layer is worse than an empty one: it reaches the extraction
+ # LLM as plausible text, which then invents values for the characters it cannot
+ # read. Only OCR recovers the real ones. The scan is per-character over a text
+ # layer that can reach a million characters, so it stays off the event loop.
+ ocr_reason = await _run_blocking_parse_step(
+ "PDF text layer check",
+ self.file_url,
+ scrambled_text_layer_reason,
+ extracted_text,
+ )
+
+ if ocr_reason is None:
+ return extracted_text
- # Scanned / image-based PDF — render pages as images and OCR each page in
- # its own vision-LLM call. A single call covering every page collapses a
- # multi-page document down to the first page or two.
+ # Render pages as images and OCR each page in its own vision-LLM call. A single
+ # call covering every page collapses a multi-page document down to the first page
+ # or two.
LOG.info(
- "PDF text extraction returned empty content, falling back to vision LLM OCR",
+ "Falling back to vision LLM OCR for the PDF text layer",
file_url=self.file_url,
+ ocr_reason=ocr_reason,
)
try:
page_images = await _run_blocking_parse_step(
diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
index a717766d7..96ddcc7d0 100644
--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py
+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
@@ -715,6 +715,7 @@ def block_yaml_to_block(
file_type=block_yaml.file_type,
json_schema=block_yaml.json_schema,
worksheet=block_yaml.worksheet,
+ force_ocr=block_yaml.force_ocr,
)
elif block_yaml.block_type == BlockType.PDF_PARSER:
return PDFParserBlock(
diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.py
index e0cc2b6db..f4cfd4990 100644
--- a/skyvern/schemas/workflows.py
+++ b/skyvern/schemas/workflows.py
@@ -24,7 +24,7 @@ from pydantic import (
)
from skyvern.config import settings
-from skyvern.constants import ERROR_CODE_REASONING_MAX_LENGTH
+from skyvern.constants import ERROR_CODE_REASONING_MAX_LENGTH, MAX_PDF_OCR_PAGES
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_key
from skyvern.forge.sdk.settings_manager import SettingsManager
@@ -99,6 +99,13 @@ ENGINE_PINNED_DESCRIPTION = (
"Set to true only when skyvern-1.0 was explicitly chosen for this block; leave it unset otherwise."
)
+FORCE_OCR_DESCRIPTION = (
+ "PDF only: ignore the PDF's embedded text layer and read the pages with vision OCR instead. "
+ "Use it when a PDF's text layer is unreadable in a way Skyvern does not detect on its own. "
+ f"OCR reads at most the first {MAX_PDF_OCR_PAGES} pages, so a longer PDF loses the rest; "
+ "leave this off unless the text layer is unusable."
+)
+
class _LLMSelectionBlock(Protocol):
label: str
@@ -1270,6 +1277,7 @@ class FileParserBlockYAML(BlockYAML):
file_type: FileType = FileType.AUTO_DETECT
json_schema: dict[str, Any] | None = None
worksheet: str | None = None
+ force_ocr: bool = Field(default=False, description=FORCE_OCR_DESCRIPTION)
class PDFParserBlockYAML(BlockYAML):
diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.py
index 8b1d10d41..0208085e9 100644
--- a/skyvern/services/otp_service.py
+++ b/skyvern/services/otp_service.py
@@ -1085,6 +1085,7 @@ async def poll_otp_value(
workflow_run_id=workflow_run_id,
workflow_permanent_id=workflow_permanent_id,
context=webhook_otp_context,
+ expected_otp_type=expected_otp_type,
)
otp_value = _exclude_rejected_otp(
otp_value, rejected_code_hash, multi_field_expected_digits, task_id=task_id
@@ -1152,6 +1153,7 @@ async def _get_otp_value_from_url(
workflow_run_id: str | None = None,
workflow_permanent_id: str | None = None,
context: WebhookOTPVerificationContext | None = None,
+ expected_otp_type: OTPType | None = None,
) -> OTPValue | None:
request_data = {}
if task_id:
@@ -1241,7 +1243,7 @@ async def _get_otp_value_from_url(
otp_value: OTPValue | None = OTPValue(value=content, type=OTPType.TOTP)
if isinstance(content, str) and len(content) > 10:
try:
- otp_value = await parse_otp_login(content, organization_id)
+ otp_value = await parse_otp_login(content, organization_id, enforced_otp_type=expected_otp_type)
except InsufficientCreditsForOTPParse:
return None
except Exception as e:
@@ -1251,11 +1253,26 @@ async def _get_otp_value_from_url(
exception_type=type(e).__name__,
content_length=len(content),
)
+ if otp_value is not None and expected_otp_type is not None and otp_value.get_otp_type() != expected_otp_type:
+ # The enforcement is only a prompt instruction, so a parse can still answer with the
+ # other kind of OTP. The expected one may still arrive within the polling budget.
+ LOG.warning(
+ "TOTP webhook returned an OTP of another type; continuing to poll",
+ expected_otp_type=expected_otp_type.value,
+ otp_type=otp_value.get_otp_type().value,
+ task_id=task_id,
+ workflow_run_id=workflow_run_id,
+ )
+ return None
if not otp_value:
LOG.warning(
"Failed to parse otp login from the totp url",
content_length=len(content) if isinstance(content, str) else None,
+ expected_otp_type=expected_otp_type.value if expected_otp_type else None,
+ # An enforced parse reports "not found" rather than the type it did see, so a link
+ # answering a code poll is only visible by its shape.
+ content_is_bare_link=looks_like_magic_link(content) if isinstance(content, str) else None,
)
return None
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index b91a81928..010166bab 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -4041,6 +4041,7 @@ async def parse_file(
parameters: list[str] | None = None,
model: dict[str, Any] | None = None,
worksheet: str | None = None,
+ force_ocr: bool = False,
) -> None:
block_validation_output = await _validate_and_get_output_parameter(label, parameters)
file_url = _render_template_with_label(file_url, label)
@@ -4051,6 +4052,7 @@ async def parse_file(
file_type=file_type,
json_schema=schema,
worksheet=worksheet,
+ force_ocr=force_ocr,
label=block_validation_output.label,
output_parameter=block_validation_output.output_parameter,
parameters=block_validation_output.input_parameters,
diff --git a/tests/unit/test_agent_otp_routing.py b/tests/unit/test_agent_otp_routing.py
index e4ee5447d..e3a0094bb 100644
--- a/tests/unit/test_agent_otp_routing.py
+++ b/tests/unit/test_agent_otp_routing.py
@@ -15,6 +15,7 @@ from skyvern.forge.agent import (
ForgeAgent,
PromptBuildResult,
_model_is_abandoning_verification,
+ _release_planned_otp_code,
)
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import MultiFieldTotpAttempt, SkyvernContext
@@ -25,6 +26,7 @@ from skyvern.forge.sdk.workflow.models.parameter import CredentialParameter
from skyvern.schemas.run_enums import RunEngine
from skyvern.services import otp_service
from skyvern.services.otp_service import OTPValue
+from skyvern.webeye.actions.actions import ClickAction, InputTextAction
_VALID_TOTP_SEED = "JBSWY3DPEHPK3PXP"
_MULTI_FIELD_HINT = "907182"
@@ -912,6 +914,105 @@ def test_model_is_abandoning_verification_pure_batch_only(actions: list, expecte
assert _model_is_abandoning_verification({"actions": actions}) is expected
+async def test_code_fetched_before_the_field_renders_is_still_typed(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The OTP gate fires on a page whose code field has not rendered yet, so the plan that follows is
+ a click and the polled code goes unused. It must reach the step that does have the field: dropping
+ it there made the next step poll again, and a second poll of the same email can answer with the
+ sign-in link beside the code."""
+ from skyvern.forge import agent as agent_module
+ from tests.unit.test_multi_field_totp import _box_page
+
+ task = _make_task()
+ monkeypatch.setattr(agent_module.app, "WORKFLOW_CONTEXT_MANAGER", SimpleNamespace(workflow_run_contexts={}))
+ step = MagicMock()
+ browser_state = MagicMock()
+ resolver = AsyncMock(return_value=OTPValue(value="123456", type=OTPType.TOTP))
+ monkeypatch.setattr("skyvern.forge.agent.resolve_otp_value", resolver)
+ monkeypatch.setattr("skyvern.forge.agent.service_utils.is_cua_task", AsyncMock(return_value=False))
+ context = SkyvernContext(task_id=task.task_id)
+ replan_codes: list[str | None] = []
+
+ async def build_prompt(*_args: object, **_kwargs: object) -> PromptBuildResult:
+ replan_codes.append(context.totp_codes.get(task.task_id))
+ return PromptBuildResult(prompt="prompt", use_caching=False, prompt_name="test", without_page_information=True)
+
+ monkeypatch.setattr(ForgeAgent, "_build_extract_action_prompt", build_prompt)
+ click_plan = {"actions": [{"action_type": "CLICK", "id": "AAAA", "reasoning": "open the code form"}]}
+ typed_plan = {"actions": [{"action_type": "INPUT_TEXT", "id": "AABL", "text": "123456"}]}
+ llm_handler = AsyncMock(side_effect=[click_plan, typed_plan])
+ monkeypatch.setattr(
+ "skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler",
+ lambda *_args, **_kwargs: llm_handler,
+ )
+
+ agent = ForgeAgent.__new__(ForgeAgent)
+ with skyvern_context.scoped(context):
+ no_field_yet = await agent.handle_potential_verification_code(
+ task, step, _box_page(0), browser_state, _otp_json_response([])
+ )
+ _release_planned_otp_code(context, task.task_id, [ClickAction(element_id="AAAA")])
+ assert context.totp_codes[task.task_id] == "123456"
+
+ # The next step's first-pass prompt offers the code and would normally expire it there.
+ first_pass_payload = agent._build_navigation_payload(
+ task, expire_verification_code=True, step=step, scraped_page=_box_page(1)
+ )
+ field_rendered = await agent.handle_potential_verification_code(
+ task, step, _box_page(1), browser_state, _otp_json_response([])
+ )
+ _release_planned_otp_code(context, task.task_id, [InputTextAction(element_id="AABL", text="123456")])
+
+ assert (no_field_yet, field_rendered) == (click_plan, typed_plan)
+ assert first_pass_payload == {"verification_code": "123456"}
+ assert replan_codes == ["123456", "123456"]
+ assert resolver.await_count == 1
+ assert task.task_id not in context.totp_codes
+
+
+ ("planned_actions", "survives"),
+ [
+ pytest.param(
+ [
+ {"action_type": "INPUT_TEXT", "id": "AABL", "text": "OTP code from email"},
+ {"action_type": "CLICK", "id": "BBBB", "reasoning": "verify"},
+ ],
+ False,
+ id="placeholder_input_dropped",
+ ),
+ pytest.param(
+ [{"action_type": "CLICK", "id": "AAAA", "reasoning": "enter a verification code instead"}],
+ True,
+ id="click_only_plan_kept",
+ ),
+ ],
+)
+async def test_plan_typing_into_the_code_field_is_dropped_when_no_code_resolves(
+ monkeypatch: pytest.MonkeyPatch, planned_actions: list, survives: bool
+) -> None:
+ """The planner writes its own placeholder into the input_text action and expects the resolver to
+ replace it, so a plan returned verbatim after an unresolved poll submits "OTP code from email".
+ A plan that only clicks is still the right move and is returned untouched."""
+ from tests.unit.test_multi_field_totp import _box_page
+
+ task = _make_task()
+ monkeypatch.setattr(
+ "skyvern.forge.agent.resolve_otp_value",
+ AsyncMock(return_value=OTPValue(value="https://example.com/signin?token=abc", type=OTPType.MAGIC_LINK)),
+ )
+ json_response = _otp_json_response(planned_actions)
+
+ agent = ForgeAgent.__new__(ForgeAgent)
+ with skyvern_context.scoped(SkyvernContext(task_id=task.task_id)):
+ result = await agent.handle_potential_verification_code(
+ task, MagicMock(), _box_page(1), MagicMock(), json_response
+ )
+
+ assert result["actions"] == (planned_actions if survives else [])
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"give_up_actions",
diff --git a/tests/unit/test_otp_service.py b/tests/unit/test_otp_service.py
index 0d0f621aa..ca4004cb3 100644
--- a/tests/unit/test_otp_service.py
+++ b/tests/unit/test_otp_service.py
@@ -605,6 +605,84 @@ class TestGetOtpValueFromUrl:
assert result == OTPValue(value="654321", type="totp")
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ ("expected_otp_type", "resolved"),
+ [
+ pytest.param(OTPType.TOTP, OTPValue(value="654321", type="totp"), id="code_poll_gets_the_code"),
+ pytest.param(
+ OTPType.MAGIC_LINK,
+ OTPValue(value="https://example.com/signin?token=abc", type="magic_link"),
+ id="link_poll_gets_the_link",
+ ),
+ ],
+ )
+ async def test_webhook_parse_enforces_the_expected_otp_type(
+ self, monkeypatch: pytest.MonkeyPatch, expected_otp_type: OTPType, resolved: OTPValue
+ ) -> None:
+ """A relayed email carrying both a code and a sign-in link resolved to whichever the
+ unenforced parse happened to pick, so a code-entry login could be handed the link. The
+ expected type must reach the parser, as it already does on the stored-code path."""
+ from skyvern.services import otp_service
+
+ raw_email = "Your passcode is 654321. Or sign in directly: https://example.com/signin?token=abc"
+ _patch_totp_url_response(
+ monkeypatch,
+ status_code=200,
+ headers={"Content-Type": "application/json"},
+ response_body={"verification_code": raw_email},
+ is_json_response=True,
+ )
+
+ async def fake_parse_otp_login(
+ content: str, organization_id: str, enforced_otp_type: OTPType | None = None
+ ) -> OTPValue | None:
+ assert enforced_otp_type is expected_otp_type
+ if enforced_otp_type is OTPType.MAGIC_LINK:
+ return OTPValue(value="https://example.com/signin?token=abc", type="magic_link")
+ return OTPValue(value="654321", type="totp")
+
+ monkeypatch.setattr(otp_service, "parse_otp_login", fake_parse_otp_login)
+
+ result = await _get_otp_value_from_url(
+ organization_id="o_test",
+ url="https://example.com/totp",
+ api_key="api-key",
+ task_id="tsk_test",
+ expected_otp_type=expected_otp_type,
+ )
+
+ assert result == resolved
+
+ @pytest.mark.asyncio
+ async def test_webhook_otp_of_another_type_keeps_polling(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """A parse that answers with the other OTP type despite the enforcement must not be handed
+ back as the caller's answer: the code may still arrive within the polling budget."""
+ from skyvern.services import otp_service
+
+ _patch_totp_url_response(
+ monkeypatch,
+ status_code=200,
+ headers={"Content-Type": "application/json"},
+ response_body={"verification_code": "Sign in here: https://example.com/signin?token=abc"},
+ is_json_response=True,
+ )
+ monkeypatch.setattr(
+ otp_service,
+ "parse_otp_login",
+ AsyncMock(return_value=OTPValue(value="https://example.com/signin?token=abc", type="magic_link")),
+ )
+
+ result = await _get_otp_value_from_url(
+ organization_id="o_test",
+ url="https://example.com/totp",
+ api_key="api-key",
+ task_id="tsk_test",
+ expected_otp_type=OTPType.TOTP,
+ )
+
+ assert result is None
+
@pytest.mark.asyncio
async def test_parse_failure_warning_omits_raw_content_keeps_length(self, monkeypatch: pytest.MonkeyPatch) -> None:
from skyvern.services import otp_service
diff --git a/tests/unit/workflow/test_file_parser_block.py b/tests/unit/workflow/test_file_parser_block.py
index e69f2317c..b4dfb5767 100644
--- a/tests/unit/workflow/test_file_parser_block.py
+++ b/tests/unit/workflow/test_file_parser_block.py
@@ -32,6 +32,7 @@ from skyvern.core.script_generations.generate_script import _build_file_url_pars
from skyvern.forge import app
from skyvern.forge.forge_app_initializer import start_forge_app
from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormat
+from skyvern.forge.sdk.utils.pdf_parser import extract_pdf_file, scrambled_text_layer_reason
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.exceptions import FileParseTimeout, InvalidFileType, WorksheetNotFound
from skyvern.forge.sdk.workflow.models.block import BlockType, FileParserBlock, PDFParserBlock
@@ -1051,6 +1052,240 @@ class TestOcrPdfPages:
assert "page A" in result and "page B" in result
+# digit d -> U+0111 + d, ':' -> U+0146, '.' -> U+0144
+_DIGIT_REMAP = {ord(str(digit)): 0x0111 + digit for digit in range(10)} | {ord(":"): 0x0146, ord("."): 0x0144}
+
+_INVOICE_LINES = (
+ "Mobile Manager Invoice",
+ "Account Number: 7249163085",
+ "Date: 17 Aug 26",
+ "Invoice Total: 1234.56 GBP",
+ "Billing Period: 01 Jul 26 to 31 Jul 26",
+ "Please remit payment using the reference above.",
+)
+
+# An itemised page is mostly digits, so the scrambled glyphs outnumber the English labels.
+_ITEMISED_INVOICE_LINES = _INVOICE_LINES + tuple(
+ f"07700 9000{index % 10:02d} 17 Aug 26 09:1{index % 10} 00:0{index % 6}:12 0.0{index % 9} GBP"
+ for index in range(40)
+)
+
+
+def _build_pdf(lines: tuple[str, ...], remap: dict[int, int] | None = None) -> bytes:
+ """Build a one-page PDF, optionally giving its font a ToUnicode map that lies.
+
+ The page always draws ordinary ASCII character codes, so with ``remap`` set only the
+ ToUnicode map is wrong — which is what makes a subset font's text layer extract as
+ plausible-looking garbage instead of failing outright.
+ """
+ content = ("BT /F1 12 Tf 72 720 Td 16 TL\n" + "".join(f"({line}) Tj T*\n" for line in lines) + "ET").encode()
+ font = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica"
+ objects = [
+ b"<< /Type /Catalog /Pages 2 0 R >>",
+ b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+ b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
+ b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",
+ font + (b" /ToUnicode 6 0 R >>" if remap else b" >>"),
+ b"<< /Length %d >>\nstream\n" % len(content) + content + b"\nendstream",
+ ]
+ if remap:
+ entries = "\n".join(f"<{code:02X}> <{unicode_point:04X}>" for code, unicode_point in sorted(remap.items()))
+ cmap = (
+ "/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n/CMapName /Broken def\n"
+ "/CMapType 2 def\n1 begincodespacerange\n<00> <FF>\nendcodespacerange\n"
+ f"{len(remap)} beginbfchar\n{entries}\nendbfchar\nendcmap\n"
+ "CMapName currentdict /CMap defineresource pop\nend\nend"
+ ).encode()
+ objects.append(b"<< /Length %d >>\nstream\n" % len(cmap) + cmap + b"\nendstream")
+
+ out = bytearray(b"%PDF-1.4\n")
+ offsets = []
+ for number, payload in enumerate(objects, start=1):
+ offsets.append(len(out))
+ out += b"%d 0 obj\n" % number + payload + b"\nendobj\n"
+ xref_at = len(out)
+ out += b"xref\n0 %d\n" % (len(objects) + 1) + b"0000000000 65535 f \n"
+ out += b"".join(b"%010d 00000 n \n" % offset for offset in offsets)
+ out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (len(objects) + 1, xref_at)
+ return bytes(out)
+
+
+class TestScrambledTextLayerFallback:
+ """A text layer the embedded font has scrambled must reach OCR, not the extraction LLM.
+
+ Without this, every digit in the document arrives as an unrelated glyph and the
+ extraction LLM invents plausible values for dates and account numbers.
+ """
+
+ @staticmethod
+ def _patch_ocr(mp: pytest.MonkeyPatch, handler: AsyncMock) -> None:
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.LLMAPIHandlerFactory.get_override_llm_api_handler",
+ lambda *a, **kw: handler,
+ )
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.prompt_engine.load_prompt",
+ MagicMock(return_value="prompt"),
+ )
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.render_pdf_pages_as_images",
+ lambda *a, **kw: [b"page-1"],
+ )
+
+ async def test_remapped_digits_route_to_ocr(self, tmp_path: Path) -> None:
+ pdf_path = tmp_path / "invoice.pdf"
+ pdf_path.write_bytes(_build_pdf(_ITEMISED_INVOICE_LINES, remap=_DIGIT_REMAP))
+ # The real extractors run against the real broken font: no digit survives them.
+ text_layer = extract_pdf_file(str(pdf_path))
+ assert "17 Aug 26" not in text_layer and "7249163085" not in text_layer
+
+ block = _make_file_parser_block("https://example.com/invoice.pdf", FileType.PDF)
+ handler = AsyncMock(return_value={"extracted_text": "Date: 17 Aug 26"})
+
+ with pytest.MonkeyPatch.context() as mp:
+ self._patch_ocr(mp, handler)
+ result = await block._parse_pdf_file(str(pdf_path))
+
+ assert handler.await_count == 1
+ assert "17 Aug 26" in result
+
+ async def test_readable_text_layer_is_returned_without_ocr(self, tmp_path: Path) -> None:
+ pdf_path = tmp_path / "invoice.pdf"
+ pdf_path.write_bytes(_build_pdf(_INVOICE_LINES))
+ block = _make_file_parser_block("https://example.com/invoice.pdf", FileType.PDF)
+ handler = AsyncMock(return_value={"extracted_text": "transcribed"})
+
+ with pytest.MonkeyPatch.context() as mp:
+ self._patch_ocr(mp, handler)
+ result = await block._parse_pdf_file(str(pdf_path))
+
+ handler.assert_not_awaited()
+ assert "17 Aug 26" in result
+
+ async def test_force_ocr_skips_the_text_layer(self, tmp_path: Path) -> None:
+ pdf_path = tmp_path / "invoice.pdf"
+ pdf_path.write_bytes(_build_pdf(_INVOICE_LINES))
+ block = _make_file_parser_block("https://example.com/invoice.pdf", FileType.PDF)
+ block.force_ocr = True
+ handler = AsyncMock(return_value={"extracted_text": "Date: 17 Aug 26"})
+ extract = MagicMock(return_value="text layer that must not be read")
+
+ with pytest.MonkeyPatch.context() as mp:
+ self._patch_ocr(mp, handler)
+ mp.setattr("skyvern.forge.sdk.workflow.models.block.extract_pdf_file", extract)
+ result = await block._parse_pdf_file(str(pdf_path))
+
+ extract.assert_not_called()
+ assert result == "--- Page 1 ---\nDate: 17 Aug 26"
+
+
+class TestScrambledTextLayerDetection:
+ """Text layers that must stay trusted, so a real document is never re-read by OCR."""
+
+ @pytest.mark.parametrize(
+ "text",
+ [
+ pytest.param(
+ "Dear Sir or Madam,\nThank you for your continued business. Please find enclosed the "
+ "statement for your account.\nShould you have any questions, contact our support team.",
+ id="english_prose_without_digits",
+ ),
+ pytest.param(
+ "Faktura VAT nr 12/2026\nNabywca: Zakład Usług Łącznościowych\nKwota brutto: 1234,56 zł\n"
+ "Termin płatności: 17 sierpnia 2026\nDziękujemy za współpracę. Życzymy miłego dnia.",
+ id="polish_invoice",
+ ),
+ pytest.param(
+ "Zakład Usług Łącznościowych\nDziękujemy za współpracę i życzymy miłego dnia.\n"
+ "Prosimy o zapłatę należności w terminie wskazanym w umowie.\nPozdrawiamy serdecznie.",
+ id="polish_prose_without_digits",
+ ),
+ pytest.param(
+ "Děkujeme za vaši důvěru. Přejeme příjemný den.\nVáš účet byl úspěšně aktivován.\n"
+ "V případě dotazů se obraťte na naši zákaznickou linku.",
+ id="czech_prose_without_digits",
+ ),
+ pytest.param(
+ "Sayın müşterimiz, faturanız hazırlanmıştır. İlginiz için teşekkür ederiz.\n"
+ "Sorularınız için müşteri hizmetlerimize başvurabilirsiniz.",
+ id="turkish_prose_without_digits",
+ ),
+ pytest.param(
+ "Kính gửi quý khách hàng, hóa đơn của quý khách đã được lập và đã gửi tới địa chỉ đăng ký.\n"
+ "Để thanh toán, quý khách vui lòng chuyển khoản tới số tài khoản được ghi ở dưới.\n"
+ "Nếu quý khách đã thanh toán rồi thì xin vui lòng bỏ qua thông báo này.",
+ id="vietnamese_prose",
+ ),
+ pytest.param(
+ "Application for Service\n I confirm the details above are correct\n"
+ " I agree to the terms and conditions of supply\n"
+ " Please send my statements by email\n I am over 18 years of age\n"
+ " I would like marketing updates\n I have read the privacy notice\n"
+ " Direct debit\n Card payment\nSignature: ______ Date: 17 Aug 26",
+ id="icon_font_checkboxes",
+ ),
+ pytest.param("尊敬的客户,您的账单已经生成。请在到期日前付款。感谢您的支持。", id="chinese_prose"),
+ pytest.param("Уважаемый клиент, ваш счет сформирован. Оплатите до указанной даты.", id="russian_prose"),
+ pytest.param("Date: 17 Aug 26", id="too_short_to_judge"),
+ ],
+ )
+ def test_trustworthy_text_layers_are_not_flagged(self, text: str) -> None:
+ assert scrambled_text_layer_reason(text) is None
+
+ @pytest.mark.parametrize(
+ "lines",
+ [
+ pytest.param(_INVOICE_LINES, id="label_heavy_header"),
+ # The real document is an itemised invoice. Its scrambled digits are Latin letters
+ # to str.isalpha, so counting them as the document's own script sank the ASCII share
+ # below the Latin-script floor and the page escaped every check.
+ pytest.param(_ITEMISED_INVOICE_LINES, id="page_of_call_rows"),
+ ],
+ )
+ def test_remapped_digits_are_flagged(self, lines: tuple[str, ...]) -> None:
+ scrambled = "\n".join(line.translate(_DIGIT_REMAP) for line in lines)
+ assert scrambled_text_layer_reason(scrambled) is not None
+
+
+class TestForceOcrOnTheCachedScriptPath:
+ """A cached run must honor force OCR, or the fix only holds while the block runs via the agent."""
+
+ @pytest.mark.asyncio
+ async def test_cached_script_path_carries_force_ocr(self) -> None:
+ built: list[FileParserBlock] = []
+
+ async def capture(self: FileParserBlock, **kwargs: str | None) -> None:
+ built.append(self)
+
+ with (
+ patch.object(
+ script_service, "_validate_and_get_output_parameter", AsyncMock(return_value=_script_output())
+ ),
+ patch.object(script_service, "_render_template_with_label", lambda template, label=None: template),
+ patch.object(FileParserBlock, "execute_safe", capture),
+ ):
+ await script_service.parse_file(
+ file_url="https://example.com/invoice.pdf",
+ file_type=FileType.PDF,
+ label="parse_invoice",
+ force_ocr=True,
+ )
+
+ (block,) = built
+ assert block.force_ocr is True
+
+ def test_generated_script_carries_force_ocr(self) -> None:
+ base = {"file_url": "https://example.com/invoice.pdf", "file_type": "pdf", "label": "parse_invoice"}
+ module = cst.Module([])
+
+ with_force_ocr = module.code_for_node(_build_file_url_parser_statement({**base, "force_ocr": True}))
+ without_force_ocr = module.code_for_node(_build_file_url_parser_statement(base))
+
+ assert "force_ocr = True" in with_force_ocr
+ assert "force_ocr" not in without_force_ocr
+
+
BLOCKING_PARSE_SECONDS = 0.3
TICK_INTERVAL_SECONDS = 0.01
@@ -1143,6 +1378,20 @@ class TestParsingDoesNotBlockEventLoop:
assert text == "extracted text"
assert ticks > 0
+ async def test_text_layer_scramble_check_yields_to_event_loop(self) -> None:
+ """The check walks every character of a text layer that can reach a million of them."""
+ block = _make_file_parser_block("https://example.com/large.pdf", FileType.PDF)
+ text_layer = "Account Number: 7249163085 Date: 17 Aug 26 Please remit payment.\n" * 20_000
+
+ with pytest.MonkeyPatch.context() as mp:
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.extract_pdf_file",
+ lambda *a, **kw: text_layer,
+ )
+ _, ticks = await _count_event_loop_ticks_during(block._parse_pdf_file("/tmp/large.pdf"))
+
+ assert ticks > 0
+
async def test_docx_parse_yields_to_event_loop(self) -> None:
block = _make_file_parser_block("https://example.com/large.docx", FileType.DOCX)
document = MagicMock(paragraphs=[], tables=[])