Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11101: Improve PDF Fill field mapping with positional field labels (#6663)
Suchintan9659998eafd8
diff --git a/skyvern/forge/prompts/skyvern/pdf-fill-field-mapping.j2 b/skyvern/forge/prompts/skyvern/pdf-fill-field-mapping.j2index 4ab54b236..610ab7a74 100644--- a/skyvern/forge/prompts/skyvern/pdf-fill-field-mapping.j2+++ b/skyvern/forge/prompts/skyvern/pdf-fill-field-mapping.j2@@ -3,9 +3,21 @@ You are mapping user instructions and structured payload data to fields in a filReturn only strict JSON with this exact top-level shape:{"fields": {"<field_name>": <value>}, "thought": "..."}+Each field has a "context_label": the printed text nearest that field on the form. This is your primary+signal for what the field is for — e.g. a field labeled "Social security number" takes the SSN, and a+checkbox labeled "Limited liability company" is the one to check for an LLC. context_label may be+slightly noisy; rely on its meaningful words. When a value spans several adjacent boxes that share a+label (e.g. the three Social security number boxes), split the value across those fields in order.++The field "name" is an internal id that is usually opaque (e.g. "f1_03[0]") but is occasionally+human-meaningful (e.g. "first_name"). When a field's context_label is empty or uninformative, fall back+to its field name if the name carries clear meaning. If neither the context_label nor the name identifies+the field, leave it unfilled rather than guessing.+Rules:- Only include fields you can confidently fill from the user instructions and payload.- Field names must exactly match one of the PDF field names.+- Map each value primarily by context_label, falling back to a meaningful field name when context_label is blank.- Checkbox values must be booleans.- Radio and choice values must exactly match one of the field's allowed values.- Do not include explanatory text outside the JSON object.diff --git a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py b/skyvern/forge/sdk/workflow/models/pdf_fill_block.pyindex 94d544597..cb3cd8c2f 100644--- a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py+++ b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py@@ -13,6 +13,7 @@ from pathlib import Pathfrom typing import Any, Literal, castimport aiofiles+import pdfplumberimport structlogfrom pypdf import PdfReader, PdfWriterfrom pypdf.generic import DecodedStreamObject, DictionaryObject, NameObject@@ -67,6 +68,9 @@ class PdfFieldInventory:field_type: strcurrent_value: Anyallowed_values: list[str]+ # Printed label nearest the field's widget on the page; AcroForm field names like "f1_03[0]"+ # are opaque, so this is what lets the LLM map a value to the right field.+ context_label: str = ""@dataclass(frozen=True)@@ -231,8 +235,11 @@ class PdfFillBlock(Block):return "checkbox"return "text"- def _extract_field_inventory(self, reader: PdfReader) -> dict[str, PdfFieldInventory]:+ def _extract_field_inventory(self, reader: PdfReader, pdf_path: str | None = None) -> dict[str, PdfFieldInventory]:fields = reader.get_fields() or {}+ if not fields:+ return {}+ labels = self._extract_field_labels(pdf_path) if pdf_path else {}inventory: dict[str, PdfFieldInventory] = {}for name, field in fields.items():inventory[name] = PdfFieldInventory(@@ -240,9 +247,92 @@ class PdfFillBlock(Block):field_type=self._field_type(field),current_value=field.get("/V"),allowed_values=self._field_allowed_values(field),+ context_label=labels.get(name.split(".")[-1], ""),)return inventory+ @staticmethod+ def _extract_field_labels(pdf_path: str) -> dict[str, str]:+ """Best-effort map of leaf field name (e.g. 'f1_03[0]') -> nearest printed label on the page.++ AcroForm field names carry no meaning, so the LLM mapping mislocates values without this+ spatial context. Failures here are non-fatal: the fill falls back to name-only mapping.+ """+ try:+ reader = PdfReader(pdf_path)+ leaf_rects: dict[str, tuple[int, float, float, float, float, bool]] = {}+ for page_index, page in enumerate(reader.pages):+ page_height = float(page.mediabox.height)+ for annotation in page.get("/Annots") or []:+ widget = annotation.get_object()+ if widget.get("/Subtype") != "/Widget" or not widget.get("/Rect"):+ continue+ leaf = widget.get("/T")+ field_type = widget.get("/FT")+ if widget.get("/Parent"):+ parent = widget["/Parent"].get_object()+ leaf = leaf or parent.get("/T")+ field_type = field_type or parent.get("/FT")+ if not leaf or str(leaf) in leaf_rects:+ continue+ x0, y0, x1, y1 = (float(v) for v in widget["/Rect"])+ is_button = str(field_type) == "/Btn"+ leaf_rects[str(leaf)] = (page_index, x0, page_height - y1, x1, page_height - y0, is_button)++ if not leaf_rects:+ return {}++ with pdfplumber.open(pdf_path) as pdf:+ words_by_page = {+ pi: [w for w in pg.extract_words(extra_attrs=["upright"]) if w.get("upright", True)]+ for pi, pg in enumerate(pdf.pages)+ }++ labels: dict[str, str] = {}+ for leaf, (page_index, fx0, ftop, fx1, fbot, is_button) in leaf_rects.items():+ labels[leaf] = PdfFillBlock._nearest_label(+ words_by_page.get(page_index, []), fx0, ftop, fx1, fbot, is_button+ )+ return labels+ except Exception:+ LOG.warning("PdfFillBlock failed to derive field labels; falling back to name-only mapping", exc_info=True)+ return {}++ @staticmethod+ def _nearest_label(+ words: list[dict[str, Any]], fx0: float, ftop: float, fx1: float, fbot: float, is_button: bool = False+ ) -> str:+ field_height = fbot - ftop+ per_side: dict[str, list[tuple[float, dict[str, Any]]]] = {"above": [], "left": [], "right": []}+ for w in words:+ wx0, wtop, wx1, wbot = w["x0"], w["top"], w["x1"], w["bottom"]+ vertical_overlap = min(wbot, fbot) - max(wtop, ftop) > 0+ horizontal_overlap = min(wx1, fx1) - max(wx0, fx0) > -120+ if vertical_overlap and wx1 <= fx0 + 2:+ per_side["left"].append((fx0 - wx1, w))+ if vertical_overlap and wx0 >= fx1 - 2:+ per_side["right"].append((wx0 - fx1, w))+ if wbot <= ftop + 2 and horizontal_overlap:+ per_side["above"].append((ftop - wbot, w))++ # Checkbox/radio labels sit to the right of the box (the option text); text-field labels+ # sit above or to the left of the entry box.+ side_order = ("right", "above", "left") if is_button else ("above", "left", "right")+ for side in side_order:+ candidates = sorted(per_side[side], key=lambda c: c[0])+ # distances in PDF user-space points (~1/72"), calibrated on IRS AcroForms+ threshold = 220.0 if side in ("left", "right") else 26.0+ near = [c for c in candidates if c[0] <= threshold]+ if not near:+ continue+ anchor = near[0][1]+ anchor_cy = (anchor["top"] + anchor["bottom"]) / 2+ band = [c[1] for c in near if abs((c[1]["top"] + c[1]["bottom"]) / 2 - anchor_cy) < field_height * 1.5 + 4]+ text = " ".join(w["text"] for w in sorted(band, key=lambda w: w["x0"])).strip()+ if text:+ return text[:70]+ return ""+async def _map_fields_with_llm(self,inventory: dict[str, PdfFieldInventory],@@ -771,7 +861,7 @@ class PdfFillBlock(Block):try:validate_pdf_file(source_pdf_path, file_identifier=self.file_url)reader = PdfReader(source_pdf_path)- inventory = self._extract_field_inventory(reader)+ inventory = self._extract_field_inventory(reader, source_pdf_path)except PDFParsingError as e:return await self._record_failure(workflow_run_context,diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.pyindex a3a8f4168..5e5266851 100644--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py@@ -7,6 +7,7 @@ from pydantic import ValidationErrorfrom skyvern.config import settingsfrom skyvern.constants import DEFAULT_LOGIN_PROMPT+from skyvern.forge.sdk.copilot.code_block_steps import derive_code_block_stepsfrom skyvern.forge.sdk.db.enums import TaskTypefrom skyvern.forge.sdk.db.id import (generate_aws_secret_parameter_id,@@ -394,6 +395,64 @@ def _build_block_kwargs(}+def _code_block_step_span_issue(step: CodeBlockStep, code_line_count: int) -> str | None:+ """Return why a step's line span is malformed, or None when it is well-formed: a well-formed span+ is empty, a lone 1-based line_start, or a 1-based line_start <= line_end with both bounds within+ the code's line count (a lone line_end has no anchor for the editor to render from)."""+ line_start = step.line_start+ line_end = step.line_end+ if line_end is not None and line_start is None:+ return "line_end is set without line_start"+ for bound_name, bound in (("line_start", line_start), ("line_end", line_end)):+ if bound is None:+ continue+ if bound < 1:+ return f"{bound_name} must be >= 1, got {bound}"+ if bound > code_line_count:+ return f"{bound_name} ({bound}) exceeds the code length ({code_line_count} line(s))"+ if line_start is not None and line_end is not None and line_start > line_end:+ return f"line_start ({line_start}) must be <= line_end ({line_end})"+ return None+++def _reconcile_code_block_step_spans(+ *,+ block_label: str,+ steps: list[CodeBlockStep],+ code: str,+) -> list[CodeBlockStep]:+ """Repair malformed step spans against the code-derived spans instead of rejecting the save.++ Step spans are display-only line references the editor and run timeline render. A non-copilot+ save path (the copilot re-derives) can persist a stale or model-mangled span, e.g. a code edit+ that shortens the block below a step's line_end. Snap an invalid span to the synthesized span at+ the same position, or drop it to null when the code yields no step there; well-formed spans and+ all other fields are kept untouched."""+ code_line_count = len(code.splitlines())+ issues = [_code_block_step_span_issue(step, code_line_count) for step in steps]+ if not any(issues):+ return steps+ derived = derive_code_block_steps(code)+ reconciled: list[CodeBlockStep] = []+ for step_index, (step, issue) in enumerate(zip(steps, issues)):+ if issue is None:+ reconciled.append(step)+ continue+ derived_step = derived[step_index] if step_index < len(derived) else None+ repaired_start = derived_step["line_start"] if derived_step else None+ repaired_end = derived_step["line_end"] if derived_step else None+ LOG.info(+ "Reconciled malformed code-block step span",+ block_label=block_label,+ step_index=step_index,+ issue=issue,+ line_start=repaired_start,+ line_end=repaired_end,+ )+ reconciled.append(step.model_copy(update={"line_start": repaired_start, "line_end": repaired_end}))+ return reconciled++def block_yaml_to_block(block_yaml: BLOCK_YAML_TYPES,parameters: dict[str, PARAMETER_TYPE],@@ -519,7 +578,7 @@ def block_yaml_to_block(code_block_steps = []for step_index, step_yaml in enumerate(block_yaml.steps):try:- code_block_steps.append(CodeBlockStep(**step_yaml.model_dump()))+ step = CodeBlockStep(**step_yaml.model_dump())except ValidationError as exc:first_error = exc.errors()[0]field = ".".join(str(part) for part in first_error["loc"])@@ -528,6 +587,12 @@ def block_yaml_to_block(step_index=step_index,detail=f"{field}: {first_error['msg']}",) from exc+ code_block_steps.append(step)+ code_block_steps = _reconcile_code_block_step_spans(+ block_label=block_yaml.label,+ steps=code_block_steps,+ code=block_yaml.code,+ )return CodeBlock(**base_kwargs,code=block_yaml.code,diff --git a/tests/smoke_tests/test_pdf_fill_benchmark_smoke.py b/tests/smoke_tests/test_pdf_fill_benchmark_smoke.pynew file mode 100644index 000000000..784309c23--- /dev/null+++ b/tests/smoke_tests/test_pdf_fill_benchmark_smoke.py@@ -0,0 +1,65 @@+"""Real-LLM smoke gate for the pdf_fill benchmark.++Skipped by default (hermetic CI). Set RUN_PDF_FILL_BENCHMARK=1 with a local backend env that+can serve the LLM to run a bigger subset against the real model:++ RUN_PDF_FILL_BENCHMARK=1 PDF_FILL_BENCHMARK_ORG=o_... uv run pytest tests/smoke_tests/test_pdf_fill_benchmark_smoke.py -s++The curated cases are treated as a must-pass gate; the round-trip accuracy is an informational+threshold (the exploratory corpus is for finding opportunities — see the benchmark README).+"""++from __future__ import annotations++import os+import re+import subprocess+from pathlib import Path++import pytest++ENABLED = os.environ.get("RUN_PDF_FILL_BENCHMARK") == "1"+ORG = os.environ.get("PDF_FILL_BENCHMARK_ORG")+BENCH = Path(__file__).resolve().parents[1] / "benchmark" / "pdf_fill"+ROOT = Path(__file__).resolve().parents[2]+ROUNDTRIP_FLOOR = 0.80 # below this is a flagged opportunity, not a hard failure (see README)++pytestmark = pytest.mark.skipif(+ not (ENABLED and ORG),+ reason="set RUN_PDF_FILL_BENCHMARK=1 and PDF_FILL_BENCHMARK_ORG=o_... to run the real-LLM benchmark",+)+++def _ensure_corpus() -> None:+ subprocess.run(["uv", "run", "python", str(BENCH / "build_corpus.py")], cwd=ROOT, check=True, timeout=600)+++def test_curated_cases_pass() -> None:+ assert ORG is not None # guaranteed by pytestmark skip+ _ensure_corpus()+ proc = subprocess.run(+ ["uv", "run", "python", str(BENCH / "run_benchmark.py"), "--org-id", ORG],+ cwd=ROOT,+ capture_output=True,+ text=True,+ timeout=1800,+ )+ print(proc.stdout[-2000:])+ assert proc.returncode == 0, "a curated case regressed (must-pass gate)"+++def test_roundtrip_accuracy_above_floor() -> None:+ assert ORG is not None # guaranteed by pytestmark skip+ _ensure_corpus()+ proc = subprocess.run(+ ["uv", "run", "python", str(BENCH / "run_roundtrip.py"), "--org-id", ORG, "--glob", "irs_w9.pdf"],+ cwd=ROOT,+ capture_output=True,+ text=True,+ timeout=1800,+ )+ print(proc.stdout[-2000:])+ m = re.search(r"^TOTAL\s+\d+\s+(\d+)%", proc.stdout, re.MULTILINE)+ assert m, f"could not parse round-trip accuracy:\n{proc.stdout[-1000:]}"+ accuracy = int(m.group(1)) / 100+ assert accuracy >= ROUNDTRIP_FLOOR, f"round-trip accuracy {accuracy:.0%} < floor {ROUNDTRIP_FLOOR:.0%}"diff --git a/tests/unit/test_pdf_fill_block.py b/tests/unit/test_pdf_fill_block.pyindex ef014f48e..fee77a435 100644--- a/tests/unit/test_pdf_fill_block.py+++ b/tests/unit/test_pdf_fill_block.py@@ -258,6 +258,43 @@ async def test_downloaded_source_pdf_is_cleaned_up(monkeypatch: pytest.MonkeyPatassert not downloaded.exists(), "the downloaded source temp should be removed after execute"+def test_nearest_label_picks_aligned_side() -> None:+ # field box at x 100-200, top 100-114+ field = (100.0, 100.0, 200.0, 114.0)+ # label above the field+ above = [+ {"text": "Social", "x0": 100, "x1": 140, "top": 84, "bottom": 96, "upright": True},+ {"text": "security", "x0": 142, "x1": 190, "top": 84, "bottom": 96, "upright": True},+ ]+ assert PdfFillBlock._nearest_label(above, *field) == "Social security"+ # checkbox: label to the right+ right = [{"text": "LLC", "x0": 210, "x1": 240, "top": 100, "bottom": 114, "upright": True}]+ assert PdfFillBlock._nearest_label(right, *field) == "LLC"+ # nothing nearby+ assert PdfFillBlock._nearest_label([], *field) == ""+++def test_nearest_label_button_prefers_right_option_text() -> None:+ # A classification checkbox with a section header ABOVE and its option label to the RIGHT:+ # text fields should take the header, button fields should take the option text.+ field = (100.0, 100.0, 112.0, 112.0)+ words = [+ {"text": "Check", "x0": 60, "x1": 95, "top": 80, "bottom": 92, "upright": True},+ {"text": "one", "x0": 97, "x1": 120, "top": 80, "bottom": 92, "upright": True},+ {"text": "Individual", "x0": 120, "x1": 200, "top": 100, "bottom": 112, "upright": True},+ ]+ assert PdfFillBlock._nearest_label(words, *field, is_button=False) == "Check one"+ assert PdfFillBlock._nearest_label(words, *field, is_button=True) == "Individual"+++def test_extract_field_labels_on_fillable_fixture(tmp_path: Path) -> None:+ source_pdf = tmp_path / "labelled.pdf"+ _write_fillable_pdf(source_pdf)+ labels = PdfFillBlock._extract_field_labels(str(source_pdf))+ # the fixture has no printed text, so labels are empty strings but extraction must not crash+ assert isinstance(labels, dict)++def test_checkbox_without_known_state_is_skipped_not_silently_unchecked() -> None:block = _make_block()inventory = {diff --git a/tests/unit/workflow/test_code_block_schema.py b/tests/unit/workflow/test_code_block_schema.pyindex e8dcd544a..04b814722 100644--- a/tests/unit/workflow/test_code_block_schema.py+++ b/tests/unit/workflow/test_code_block_schema.py@@ -6,7 +6,7 @@ from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContextfrom skyvern.forge.sdk.workflow.exceptions import WorkflowDefinitionValidationExceptionfrom skyvern.forge.sdk.workflow.models.block import CodeBlock, CodeBlockStepfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter-from skyvern.forge.sdk.workflow.workflow_definition_converter import block_yaml_to_block+from skyvern.forge.sdk.workflow.workflow_definition_converter import _code_block_step_span_issue, block_yaml_to_blockfrom skyvern.schemas.workflows import CodeBlockYAMLfrom skyvern.webeye.actions.action_types import ActionType@@ -74,6 +74,115 @@ def test_converter_rejects_invalid_step_action_type_with_block_context() -> Noneassert "action_type" in message+def _convert_with_steps(code: str, steps: list[dict]) -> CodeBlock:+ block_yaml = CodeBlockYAML(label="code_1", code=code, steps=steps)+ block = block_yaml_to_block(block_yaml, {"code_1_output": _output_parameter()})+ assert isinstance(block, CodeBlock)+ return block+++def test_converter_accepts_valid_line_spans() -> None:+ block = _convert_with_steps(+ "a = 1\nb = 2\nc = 3",+ [+ {"description": "first", "action_type": "goto_url", "line_start": 1, "line_end": 2},+ {"description": "second", "action_type": "click", "line_start": 3, "line_end": 3},+ ],+ )+ assert block.steps is not None+ assert (block.steps[0].line_start, block.steps[0].line_end) == (1, 2)+ assert (block.steps[1].line_start, block.steps[1].line_end) == (3, 3)+++def test_converter_accepts_steps_without_line_spans() -> None:+ block = _convert_with_steps(+ "x = 1",+ [+ {"description": "no span", "action_type": "goto_url"},+ {"description": "still no span", "action_type": "extract"},+ ],+ )+ assert block.steps is not None+ assert block.steps[0].line_start is None and block.steps[0].line_end is None+++def test_converter_accepts_line_start_only_step() -> None:+ block = _convert_with_steps(+ "a = 1\nb = 2",+ [{"description": "lone start", "action_type": "click", "line_start": 2}],+ )+ assert block.steps is not None+ assert block.steps[0].line_start == 2 and block.steps[0].line_end is None+++ "line_start, line_end, code_line_count, expect_issue",+ [+ (1, 2, 3, False),+ (2, None, 3, False),+ (None, None, 3, False),+ (None, 2, 3, True),+ (3, 2, 3, True),+ (0, 1, 3, True),+ (1, -2, 3, True),+ (1, 4, 3, True),+ (5, None, 3, True),+ ],+)+def test_code_block_step_span_issue(+ line_start: int | None, line_end: int | None, code_line_count: int, expect_issue: bool+) -> None:+ step = CodeBlockStep(line_start=line_start, line_end=line_end)+ assert (_code_block_step_span_issue(step, code_line_count) is not None) == expect_issue+++def test_converter_snaps_out_of_range_span_to_synthesized_span() -> None:+ # Line 1 (x = 1) is not an action; the click is the only synthesized step, on line 2.+ block = _convert_with_steps(+ "x = 1\nawait page.click('#go')",+ [{"description": "click go", "action_type": "click", "line_start": 1, "line_end": 99}],+ )+ assert block.steps is not None+ # Snapped to the synthesized span (2, 2) rather than clamped to (1, 2).+ assert (block.steps[0].line_start, block.steps[0].line_end) == (2, 2)+ assert block.steps[0].description == "click go"+++ "step",+ [+ {"description": "lone end", "action_type": "click", "line_end": 2},+ {"description": "inverted", "action_type": "click", "line_start": 3, "line_end": 2},+ {"description": "zero", "action_type": "click", "line_start": 0, "line_end": 1},+ {"description": "negative", "action_type": "click", "line_start": 1, "line_end": -2},+ {"description": "beyond end", "action_type": "click", "line_start": 1, "line_end": 9},+ {"description": "lone start beyond end", "action_type": "click", "line_start": 5},+ ],+)+def test_converter_drops_unrepairable_span_to_null(step: dict) -> None:+ # Action-less code has no synthesized step to snap to, so an invalid span is dropped to+ # null (the step survives as display-only metadata) instead of 422-ing the whole save.+ block = _convert_with_steps("a = 1\nb = 2\nc = 3", [step])+ assert block.steps is not None+ assert block.steps[0].line_start is None and block.steps[0].line_end is None+ assert block.steps[0].description == step["description"]+++def test_converter_preserves_valid_span_and_repairs_invalid_neighbor() -> None:+ block = _convert_with_steps(+ "await page.goto('https://example.com')\nawait page.click('#go')",+ [+ {"description": "open", "action_type": "goto_url", "line_start": 1, "line_end": 1},+ {"description": "click", "action_type": "click", "line_start": 2, "line_end": 50},+ ],+ )+ assert block.steps is not None+ # A valid span is left untouched; only the invalid neighbor is reconciled.+ assert (block.steps[0].line_start, block.steps[0].line_end) == (1, 1)+ assert (block.steps[1].line_start, block.steps[1].line_end) == (2, 2)+ assert block.steps[1].action_type is ActionType.CLICK++def test_code_block_model_roundtrip() -> None:block = CodeBlock(label="code_1",