Skip to content

Automate browser based workflows with AI

SKY-11101: Improve PDF Fill field mapping with positional field labels (#6663)

Suchintan9659998eafd8
diff --git a/skyvern/forge/prompts/skyvern/pdf-fill-field-mapping.j2 b/skyvern/forge/prompts/skyvern/pdf-fill-field-mapping.j2
index 4ab54b236..610ab7a74 100644
--- a/skyvern/forge/prompts/skyvern/pdf-fill-field-mapping.j2
+++ b/skyvern/forge/prompts/skyvern/pdf-fill-field-mapping.j2
@@ -3,9 +3,21 @@ You are mapping user instructions and structured payload data to fields in a fil
Return only strict JSON with this exact top-level shape:
{"fields": {"<field_name>": <value>}, "thought": "..."}
+Each field has a "context_label": the printed text nearest that field on the form. This is your primary
+signal for what the field is for — e.g. a field labeled "Social security number" takes the SSN, and a
+checkbox labeled "Limited liability company" is the one to check for an LLC. context_label may be
+slightly noisy; rely on its meaningful words. When a value spans several adjacent boxes that share a
+label (e.g. the three Social security number boxes), split the value across those fields in order.
+
+The field "name" is an internal id that is usually opaque (e.g. "f1_03[0]") but is occasionally
+human-meaningful (e.g. "first_name"). When a field's context_label is empty or uninformative, fall back
+to its field name if the name carries clear meaning. If neither the context_label nor the name identifies
+the field, leave it unfilled rather than guessing.
+
Rules:
- Only include fields you can confidently fill from the user instructions and payload.
- Field names must exactly match one of the PDF field names.
+- Map each value primarily by context_label, falling back to a meaningful field name when context_label is blank.
- Checkbox values must be booleans.
- Radio and choice values must exactly match one of the field's allowed values.
- Do not include explanatory text outside the JSON object.
diff --git a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
index 94d544597..cb3cd8c2f 100644
--- a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
+++ b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
@@ -13,6 +13,7 @@ from pathlib import Path
from typing import Any, Literal, cast
import aiofiles
+import pdfplumber
import structlog
from pypdf import PdfReader, PdfWriter
from pypdf.generic import DecodedStreamObject, DictionaryObject, NameObject
@@ -67,6 +68,9 @@ class PdfFieldInventory:
field_type: str
current_value: Any
allowed_values: list[str]
+ # Printed label nearest the field's widget on the page; AcroForm field names like "f1_03[0]"
+ # are opaque, so this is what lets the LLM map a value to the right field.
+ context_label: str = ""
@dataclass(frozen=True)
@@ -231,8 +235,11 @@ class PdfFillBlock(Block):
return "checkbox"
return "text"
- def _extract_field_inventory(self, reader: PdfReader) -> dict[str, PdfFieldInventory]:
+ def _extract_field_inventory(self, reader: PdfReader, pdf_path: str | None = None) -> dict[str, PdfFieldInventory]:
fields = reader.get_fields() or {}
+ if not fields:
+ return {}
+ labels = self._extract_field_labels(pdf_path) if pdf_path else {}
inventory: dict[str, PdfFieldInventory] = {}
for name, field in fields.items():
inventory[name] = PdfFieldInventory(
@@ -240,9 +247,92 @@ class PdfFillBlock(Block):
field_type=self._field_type(field),
current_value=field.get("/V"),
allowed_values=self._field_allowed_values(field),
+ context_label=labels.get(name.split(".")[-1], ""),
)
return inventory
+ @staticmethod
+ def _extract_field_labels(pdf_path: str) -> dict[str, str]:
+ """Best-effort map of leaf field name (e.g. 'f1_03[0]') -> nearest printed label on the page.
+
+ AcroForm field names carry no meaning, so the LLM mapping mislocates values without this
+ spatial context. Failures here are non-fatal: the fill falls back to name-only mapping.
+ """
+ try:
+ reader = PdfReader(pdf_path)
+ leaf_rects: dict[str, tuple[int, float, float, float, float, bool]] = {}
+ for page_index, page in enumerate(reader.pages):
+ page_height = float(page.mediabox.height)
+ for annotation in page.get("/Annots") or []:
+ widget = annotation.get_object()
+ if widget.get("/Subtype") != "/Widget" or not widget.get("/Rect"):
+ continue
+ leaf = widget.get("/T")
+ field_type = widget.get("/FT")
+ if widget.get("/Parent"):
+ parent = widget["/Parent"].get_object()
+ leaf = leaf or parent.get("/T")
+ field_type = field_type or parent.get("/FT")
+ if not leaf or str(leaf) in leaf_rects:
+ continue
+ x0, y0, x1, y1 = (float(v) for v in widget["/Rect"])
+ is_button = str(field_type) == "/Btn"
+ leaf_rects[str(leaf)] = (page_index, x0, page_height - y1, x1, page_height - y0, is_button)
+
+ if not leaf_rects:
+ return {}
+
+ with pdfplumber.open(pdf_path) as pdf:
+ words_by_page = {
+ pi: [w for w in pg.extract_words(extra_attrs=["upright"]) if w.get("upright", True)]
+ for pi, pg in enumerate(pdf.pages)
+ }
+
+ labels: dict[str, str] = {}
+ for leaf, (page_index, fx0, ftop, fx1, fbot, is_button) in leaf_rects.items():
+ labels[leaf] = PdfFillBlock._nearest_label(
+ words_by_page.get(page_index, []), fx0, ftop, fx1, fbot, is_button
+ )
+ return labels
+ except Exception:
+ LOG.warning("PdfFillBlock failed to derive field labels; falling back to name-only mapping", exc_info=True)
+ return {}
+
+ @staticmethod
+ def _nearest_label(
+ words: list[dict[str, Any]], fx0: float, ftop: float, fx1: float, fbot: float, is_button: bool = False
+ ) -> str:
+ field_height = fbot - ftop
+ per_side: dict[str, list[tuple[float, dict[str, Any]]]] = {"above": [], "left": [], "right": []}
+ for w in words:
+ wx0, wtop, wx1, wbot = w["x0"], w["top"], w["x1"], w["bottom"]
+ vertical_overlap = min(wbot, fbot) - max(wtop, ftop) > 0
+ horizontal_overlap = min(wx1, fx1) - max(wx0, fx0) > -120
+ if vertical_overlap and wx1 <= fx0 + 2:
+ per_side["left"].append((fx0 - wx1, w))
+ if vertical_overlap and wx0 >= fx1 - 2:
+ per_side["right"].append((wx0 - fx1, w))
+ if wbot <= ftop + 2 and horizontal_overlap:
+ per_side["above"].append((ftop - wbot, w))
+
+ # Checkbox/radio labels sit to the right of the box (the option text); text-field labels
+ # sit above or to the left of the entry box.
+ side_order = ("right", "above", "left") if is_button else ("above", "left", "right")
+ for side in side_order:
+ candidates = sorted(per_side[side], key=lambda c: c[0])
+ # distances in PDF user-space points (~1/72"), calibrated on IRS AcroForms
+ threshold = 220.0 if side in ("left", "right") else 26.0
+ near = [c for c in candidates if c[0] <= threshold]
+ if not near:
+ continue
+ anchor = near[0][1]
+ anchor_cy = (anchor["top"] + anchor["bottom"]) / 2
+ band = [c[1] for c in near if abs((c[1]["top"] + c[1]["bottom"]) / 2 - anchor_cy) < field_height * 1.5 + 4]
+ text = " ".join(w["text"] for w in sorted(band, key=lambda w: w["x0"])).strip()
+ if text:
+ return text[:70]
+ return ""
+
async def _map_fields_with_llm(
self,
inventory: dict[str, PdfFieldInventory],
@@ -771,7 +861,7 @@ class PdfFillBlock(Block):
try:
validate_pdf_file(source_pdf_path, file_identifier=self.file_url)
reader = PdfReader(source_pdf_path)
- inventory = self._extract_field_inventory(reader)
+ inventory = self._extract_field_inventory(reader, source_pdf_path)
except PDFParsingError as e:
return await self._record_failure(
workflow_run_context,
diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
index a3a8f4168..5e5266851 100644
--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py
+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
@@ -7,6 +7,7 @@ from pydantic import ValidationError
from skyvern.config import settings
from skyvern.constants import DEFAULT_LOGIN_PROMPT
+from skyvern.forge.sdk.copilot.code_block_steps import derive_code_block_steps
from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.db.id import (
generate_aws_secret_parameter_id,
@@ -394,6 +395,64 @@ def _build_block_kwargs(
}
+def _code_block_step_span_issue(step: CodeBlockStep, code_line_count: int) -> str | None:
+ """Return why a step's line span is malformed, or None when it is well-formed: a well-formed span
+ is empty, a lone 1-based line_start, or a 1-based line_start <= line_end with both bounds within
+ the code's line count (a lone line_end has no anchor for the editor to render from)."""
+ line_start = step.line_start
+ line_end = step.line_end
+ if line_end is not None and line_start is None:
+ return "line_end is set without line_start"
+ for bound_name, bound in (("line_start", line_start), ("line_end", line_end)):
+ if bound is None:
+ continue
+ if bound < 1:
+ return f"{bound_name} must be >= 1, got {bound}"
+ if bound > code_line_count:
+ return f"{bound_name} ({bound}) exceeds the code length ({code_line_count} line(s))"
+ if line_start is not None and line_end is not None and line_start > line_end:
+ return f"line_start ({line_start}) must be <= line_end ({line_end})"
+ return None
+
+
+def _reconcile_code_block_step_spans(
+ *,
+ block_label: str,
+ steps: list[CodeBlockStep],
+ code: str,
+) -> list[CodeBlockStep]:
+ """Repair malformed step spans against the code-derived spans instead of rejecting the save.
+
+ Step spans are display-only line references the editor and run timeline render. A non-copilot
+ save path (the copilot re-derives) can persist a stale or model-mangled span, e.g. a code edit
+ that shortens the block below a step's line_end. Snap an invalid span to the synthesized span at
+ the same position, or drop it to null when the code yields no step there; well-formed spans and
+ all other fields are kept untouched."""
+ code_line_count = len(code.splitlines())
+ issues = [_code_block_step_span_issue(step, code_line_count) for step in steps]
+ if not any(issues):
+ return steps
+ derived = derive_code_block_steps(code)
+ reconciled: list[CodeBlockStep] = []
+ for step_index, (step, issue) in enumerate(zip(steps, issues)):
+ if issue is None:
+ reconciled.append(step)
+ continue
+ derived_step = derived[step_index] if step_index < len(derived) else None
+ repaired_start = derived_step["line_start"] if derived_step else None
+ repaired_end = derived_step["line_end"] if derived_step else None
+ LOG.info(
+ "Reconciled malformed code-block step span",
+ block_label=block_label,
+ step_index=step_index,
+ issue=issue,
+ line_start=repaired_start,
+ line_end=repaired_end,
+ )
+ reconciled.append(step.model_copy(update={"line_start": repaired_start, "line_end": repaired_end}))
+ return reconciled
+
+
def block_yaml_to_block(
block_yaml: BLOCK_YAML_TYPES,
parameters: dict[str, PARAMETER_TYPE],
@@ -519,7 +578,7 @@ def block_yaml_to_block(
code_block_steps = []
for step_index, step_yaml in enumerate(block_yaml.steps):
try:
- code_block_steps.append(CodeBlockStep(**step_yaml.model_dump()))
+ step = CodeBlockStep(**step_yaml.model_dump())
except ValidationError as exc:
first_error = exc.errors()[0]
field = ".".join(str(part) for part in first_error["loc"])
@@ -528,6 +587,12 @@ def block_yaml_to_block(
step_index=step_index,
detail=f"{field}: {first_error['msg']}",
) from exc
+ code_block_steps.append(step)
+ code_block_steps = _reconcile_code_block_step_spans(
+ block_label=block_yaml.label,
+ steps=code_block_steps,
+ code=block_yaml.code,
+ )
return CodeBlock(
**base_kwargs,
code=block_yaml.code,
diff --git a/tests/smoke_tests/test_pdf_fill_benchmark_smoke.py b/tests/smoke_tests/test_pdf_fill_benchmark_smoke.py
new file mode 100644
index 000000000..784309c23
--- /dev/null
+++ b/tests/smoke_tests/test_pdf_fill_benchmark_smoke.py
@@ -0,0 +1,65 @@
+"""Real-LLM smoke gate for the pdf_fill benchmark.
+
+Skipped by default (hermetic CI). Set RUN_PDF_FILL_BENCHMARK=1 with a local backend env that
+can serve the LLM to run a bigger subset against the real model:
+
+ RUN_PDF_FILL_BENCHMARK=1 PDF_FILL_BENCHMARK_ORG=o_... uv run pytest tests/smoke_tests/test_pdf_fill_benchmark_smoke.py -s
+
+The curated cases are treated as a must-pass gate; the round-trip accuracy is an informational
+threshold (the exploratory corpus is for finding opportunities — see the benchmark README).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+from pathlib import Path
+
+import pytest
+
+ENABLED = os.environ.get("RUN_PDF_FILL_BENCHMARK") == "1"
+ORG = os.environ.get("PDF_FILL_BENCHMARK_ORG")
+BENCH = Path(__file__).resolve().parents[1] / "benchmark" / "pdf_fill"
+ROOT = Path(__file__).resolve().parents[2]
+ROUNDTRIP_FLOOR = 0.80 # below this is a flagged opportunity, not a hard failure (see README)
+
+pytestmark = pytest.mark.skipif(
+ not (ENABLED and ORG),
+ reason="set RUN_PDF_FILL_BENCHMARK=1 and PDF_FILL_BENCHMARK_ORG=o_... to run the real-LLM benchmark",
+)
+
+
+def _ensure_corpus() -> None:
+ subprocess.run(["uv", "run", "python", str(BENCH / "build_corpus.py")], cwd=ROOT, check=True, timeout=600)
+
+
+def test_curated_cases_pass() -> None:
+ assert ORG is not None # guaranteed by pytestmark skip
+ _ensure_corpus()
+ proc = subprocess.run(
+ ["uv", "run", "python", str(BENCH / "run_benchmark.py"), "--org-id", ORG],
+ cwd=ROOT,
+ capture_output=True,
+ text=True,
+ timeout=1800,
+ )
+ print(proc.stdout[-2000:])
+ assert proc.returncode == 0, "a curated case regressed (must-pass gate)"
+
+
+def test_roundtrip_accuracy_above_floor() -> None:
+ assert ORG is not None # guaranteed by pytestmark skip
+ _ensure_corpus()
+ proc = subprocess.run(
+ ["uv", "run", "python", str(BENCH / "run_roundtrip.py"), "--org-id", ORG, "--glob", "irs_w9.pdf"],
+ cwd=ROOT,
+ capture_output=True,
+ text=True,
+ timeout=1800,
+ )
+ print(proc.stdout[-2000:])
+ m = re.search(r"^TOTAL\s+\d+\s+(\d+)%", proc.stdout, re.MULTILINE)
+ assert m, f"could not parse round-trip accuracy:\n{proc.stdout[-1000:]}"
+ accuracy = int(m.group(1)) / 100
+ assert accuracy >= ROUNDTRIP_FLOOR, f"round-trip accuracy {accuracy:.0%} < floor {ROUNDTRIP_FLOOR:.0%}"
diff --git a/tests/unit/test_pdf_fill_block.py b/tests/unit/test_pdf_fill_block.py
index ef014f48e..fee77a435 100644
--- a/tests/unit/test_pdf_fill_block.py
+++ b/tests/unit/test_pdf_fill_block.py
@@ -258,6 +258,43 @@ async def test_downloaded_source_pdf_is_cleaned_up(monkeypatch: pytest.MonkeyPat
assert not downloaded.exists(), "the downloaded source temp should be removed after execute"
+def test_nearest_label_picks_aligned_side() -> None:
+ # field box at x 100-200, top 100-114
+ field = (100.0, 100.0, 200.0, 114.0)
+ # label above the field
+ above = [
+ {"text": "Social", "x0": 100, "x1": 140, "top": 84, "bottom": 96, "upright": True},
+ {"text": "security", "x0": 142, "x1": 190, "top": 84, "bottom": 96, "upright": True},
+ ]
+ assert PdfFillBlock._nearest_label(above, *field) == "Social security"
+ # checkbox: label to the right
+ right = [{"text": "LLC", "x0": 210, "x1": 240, "top": 100, "bottom": 114, "upright": True}]
+ assert PdfFillBlock._nearest_label(right, *field) == "LLC"
+ # nothing nearby
+ assert PdfFillBlock._nearest_label([], *field) == ""
+
+
+def test_nearest_label_button_prefers_right_option_text() -> None:
+ # A classification checkbox with a section header ABOVE and its option label to the RIGHT:
+ # text fields should take the header, button fields should take the option text.
+ field = (100.0, 100.0, 112.0, 112.0)
+ words = [
+ {"text": "Check", "x0": 60, "x1": 95, "top": 80, "bottom": 92, "upright": True},
+ {"text": "one", "x0": 97, "x1": 120, "top": 80, "bottom": 92, "upright": True},
+ {"text": "Individual", "x0": 120, "x1": 200, "top": 100, "bottom": 112, "upright": True},
+ ]
+ assert PdfFillBlock._nearest_label(words, *field, is_button=False) == "Check one"
+ assert PdfFillBlock._nearest_label(words, *field, is_button=True) == "Individual"
+
+
+def test_extract_field_labels_on_fillable_fixture(tmp_path: Path) -> None:
+ source_pdf = tmp_path / "labelled.pdf"
+ _write_fillable_pdf(source_pdf)
+ labels = PdfFillBlock._extract_field_labels(str(source_pdf))
+ # the fixture has no printed text, so labels are empty strings but extraction must not crash
+ assert isinstance(labels, dict)
+
+
def test_checkbox_without_known_state_is_skipped_not_silently_unchecked() -> None:
block = _make_block()
inventory = {
diff --git a/tests/unit/workflow/test_code_block_schema.py b/tests/unit/workflow/test_code_block_schema.py
index e8dcd544a..04b814722 100644
--- a/tests/unit/workflow/test_code_block_schema.py
+++ b/tests/unit/workflow/test_code_block_schema.py
@@ -6,7 +6,7 @@ from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.exceptions import WorkflowDefinitionValidationException
from skyvern.forge.sdk.workflow.models.block import CodeBlock, CodeBlockStep
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
-from skyvern.forge.sdk.workflow.workflow_definition_converter import block_yaml_to_block
+from skyvern.forge.sdk.workflow.workflow_definition_converter import _code_block_step_span_issue, block_yaml_to_block
from skyvern.schemas.workflows import CodeBlockYAML
from skyvern.webeye.actions.action_types import ActionType
@@ -74,6 +74,115 @@ def test_converter_rejects_invalid_step_action_type_with_block_context() -> None
assert "action_type" in message
+def _convert_with_steps(code: str, steps: list[dict]) -> CodeBlock:
+ block_yaml = CodeBlockYAML(label="code_1", code=code, steps=steps)
+ block = block_yaml_to_block(block_yaml, {"code_1_output": _output_parameter()})
+ assert isinstance(block, CodeBlock)
+ return block
+
+
+def test_converter_accepts_valid_line_spans() -> None:
+ block = _convert_with_steps(
+ "a = 1\nb = 2\nc = 3",
+ [
+ {"description": "first", "action_type": "goto_url", "line_start": 1, "line_end": 2},
+ {"description": "second", "action_type": "click", "line_start": 3, "line_end": 3},
+ ],
+ )
+ assert block.steps is not None
+ assert (block.steps[0].line_start, block.steps[0].line_end) == (1, 2)
+ assert (block.steps[1].line_start, block.steps[1].line_end) == (3, 3)
+
+
+def test_converter_accepts_steps_without_line_spans() -> None:
+ block = _convert_with_steps(
+ "x = 1",
+ [
+ {"description": "no span", "action_type": "goto_url"},
+ {"description": "still no span", "action_type": "extract"},
+ ],
+ )
+ assert block.steps is not None
+ assert block.steps[0].line_start is None and block.steps[0].line_end is None
+
+
+def test_converter_accepts_line_start_only_step() -> None:
+ block = _convert_with_steps(
+ "a = 1\nb = 2",
+ [{"description": "lone start", "action_type": "click", "line_start": 2}],
+ )
+ assert block.steps is not None
+ assert block.steps[0].line_start == 2 and block.steps[0].line_end is None
+
+
+ "line_start, line_end, code_line_count, expect_issue",
+ [
+ (1, 2, 3, False),
+ (2, None, 3, False),
+ (None, None, 3, False),
+ (None, 2, 3, True),
+ (3, 2, 3, True),
+ (0, 1, 3, True),
+ (1, -2, 3, True),
+ (1, 4, 3, True),
+ (5, None, 3, True),
+ ],
+)
+def test_code_block_step_span_issue(
+ line_start: int | None, line_end: int | None, code_line_count: int, expect_issue: bool
+) -> None:
+ step = CodeBlockStep(line_start=line_start, line_end=line_end)
+ assert (_code_block_step_span_issue(step, code_line_count) is not None) == expect_issue
+
+
+def test_converter_snaps_out_of_range_span_to_synthesized_span() -> None:
+ # Line 1 (x = 1) is not an action; the click is the only synthesized step, on line 2.
+ block = _convert_with_steps(
+ "x = 1\nawait page.click('#go')",
+ [{"description": "click go", "action_type": "click", "line_start": 1, "line_end": 99}],
+ )
+ assert block.steps is not None
+ # Snapped to the synthesized span (2, 2) rather than clamped to (1, 2).
+ assert (block.steps[0].line_start, block.steps[0].line_end) == (2, 2)
+ assert block.steps[0].description == "click go"
+
+
+ "step",
+ [
+ {"description": "lone end", "action_type": "click", "line_end": 2},
+ {"description": "inverted", "action_type": "click", "line_start": 3, "line_end": 2},
+ {"description": "zero", "action_type": "click", "line_start": 0, "line_end": 1},
+ {"description": "negative", "action_type": "click", "line_start": 1, "line_end": -2},
+ {"description": "beyond end", "action_type": "click", "line_start": 1, "line_end": 9},
+ {"description": "lone start beyond end", "action_type": "click", "line_start": 5},
+ ],
+)
+def test_converter_drops_unrepairable_span_to_null(step: dict) -> None:
+ # Action-less code has no synthesized step to snap to, so an invalid span is dropped to
+ # null (the step survives as display-only metadata) instead of 422-ing the whole save.
+ block = _convert_with_steps("a = 1\nb = 2\nc = 3", [step])
+ assert block.steps is not None
+ assert block.steps[0].line_start is None and block.steps[0].line_end is None
+ assert block.steps[0].description == step["description"]
+
+
+def test_converter_preserves_valid_span_and_repairs_invalid_neighbor() -> None:
+ block = _convert_with_steps(
+ "await page.goto('https://example.com')\nawait page.click('#go')",
+ [
+ {"description": "open", "action_type": "goto_url", "line_start": 1, "line_end": 1},
+ {"description": "click", "action_type": "click", "line_start": 2, "line_end": 50},
+ ],
+ )
+ assert block.steps is not None
+ # A valid span is left untouched; only the invalid neighbor is reconciled.
+ assert (block.steps[0].line_start, block.steps[0].line_end) == (1, 1)
+ assert (block.steps[1].line_start, block.steps[1].line_end) == (2, 2)
+ assert block.steps[1].action_type is ActionType.CLICK
+
+
def test_code_block_model_roundtrip() -> None:
block = CodeBlock(
label="code_1",