Skip to content

Automate browser based workflows with AI

SKY-11025: structured keyed output for copilot extraction code blocks (#6580)

Andrew Neilsonedb298e0d5ed
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index 59ca7a3f1..0bdcc7ac5 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -529,8 +529,9 @@ def render_synthesized_offer_text(
parts = [
"SYNTHESIZED CODE BLOCK (offered once). The page interactions you scouted were compiled into a "
"deterministic Playwright snippet. Persist it VERBATIM as a `code` block labeled "
- f"`{_SYNTHESIZED_BLOCK_LABEL}` via update_workflow / update_and_run_blocks; only hand-author the "
- "extract/report steps it does not cover.",
+ f"`{_SYNTHESIZED_BLOCK_LABEL}` via update_workflow / update_and_run_blocks; hand-author the "
+ "data-capture step it does not cover so the block `return`s a keyed structure (a dict, or an array "
+ "of objects for repeated records) — never a flat `page.inner_text(...)` / `text_content(...)` blob.",
"```python",
synthesized.code.rstrip("\n"),
"```",
@@ -560,4 +561,12 @@ def render_synthesized_offer_text(
parts.append("```json")
parts.append(_render_artifact_metadata_block(metadata))
parts.append("```")
+ parts.append(
+ "Shape the data-capture `return` so each `goal_value_paths` entry resolves to a named scalar field. "
+ "For `records[].field` paths, return an array of objects; for a single record, return a keyed dict. "
+ "Bind each `<fill: ...>` to the page text you captured. For example:"
+ )
+ parts.append("```python")
+ parts.append('return {"records": [{"field_a": "...", "field_b": "..."}]}')
+ parts.append("```")
return "\n".join(parts)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index e7e728135..d5543af67 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import ast
import re
import textwrap
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from dataclasses import field as dataclass_field
from typing import Annotated, Any, Literal, NamedTuple
@@ -394,6 +394,9 @@ def _normalize_code_artifact_metadata_detailed(
reject_unfilled_goal_value_paths=impose_defaults,
)
)
+ return_shape_error = _extraction_return_shape_error(label, dumped, str(code_blocks[label].get("code") or ""))
+ if return_shape_error is not None:
+ item_violations.append(return_shape_error)
if item_violations:
violations.extend(item_violations)
offending_labels.append(label)
@@ -1101,6 +1104,172 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
)
+_FLAT_STRING_TEXT_METHODS = frozenset({"inner_text", "text_content", "inner_html", "content"})
+
+
+def _expr_is_flat_string(node: ast.expr, string_locals: set[str]) -> bool:
+ """True only for expressions that are unambiguously a single text blob: a string
+ literal/f-string, a `.inner_text()/.text_content()/.inner_html()` read, a
+ `str(...)` cast, a `str.join(...)`, string concatenation, or a local bound to one
+ of those. Anything ambiguous returns False so the validator never false-positives."""
+ if isinstance(node, ast.Await):
+ return _expr_is_flat_string(node.value, string_locals)
+ if isinstance(node, ast.Constant):
+ return isinstance(node.value, str)
+ if isinstance(node, ast.JoinedStr):
+ return True
+ if isinstance(node, ast.Name):
+ return node.id in string_locals
+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
+ return _expr_is_flat_string(node.left, string_locals) or _expr_is_flat_string(node.right, string_locals)
+ if isinstance(node, ast.Call):
+ func = node.func
+ if isinstance(func, ast.Attribute):
+ if func.attr in _FLAT_STRING_TEXT_METHODS:
+ return True
+ if func.attr == "join":
+ return True
+ if func.attr in {"strip", "lstrip", "rstrip", "lower", "upper"}:
+ return _expr_is_flat_string(func.value, string_locals)
+ if isinstance(func, ast.Name) and func.id == "str":
+ return True
+ return False
+
+
+def _expr_is_structured(node: ast.expr) -> bool:
+ if isinstance(node, ast.Await):
+ return _expr_is_structured(node.value)
+ return isinstance(node, (ast.Dict, ast.List, ast.DictComp, ast.ListComp, ast.SetComp, ast.Set, ast.Tuple))
+
+
+_NESTED_SCOPE_NODES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef)
+
+
+def _iter_top_level_scope(statements: list[ast.stmt]) -> Iterator[ast.stmt]:
+ """Yield statements in the snippet's own scope, descending into control flow
+ (if/for/while/with/try) but never into nested function/class bodies."""
+ for statement in statements:
+ yield statement
+ if isinstance(statement, _NESTED_SCOPE_NODES):
+ continue
+ for child in ast.iter_child_nodes(statement):
+ if isinstance(child, ast.stmt):
+ yield from _iter_top_level_scope([child])
+
+
+def _code_block_returns_flat_string(code: str) -> bool:
+ """True when every top-level `return` in the snippet yields a flat text blob and
+ none yields a structured value. Returns inside nested functions, and indeterminate
+ or structured returns, are not flagged."""
+ try:
+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")
+ except SyntaxError:
+ return False
+
+ scope_statements = list(_iter_top_level_scope(tree.body))
+ string_locals: set[str] = set()
+ for node in scope_statements:
+ if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+ if _expr_is_flat_string(node.value, string_locals):
+ string_locals.add(node.targets[0].id)
+
+ returns = [node for node in scope_statements if isinstance(node, ast.Return) and node.value is not None]
+ if not returns:
+ return False
+ if any(_expr_is_structured(node.value) for node in returns if node.value is not None):
+ return False
+ return all(_expr_is_flat_string(node.value, string_locals) for node in returns if node.value is not None)
+
+
+def _function_body_has_structured_return(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool:
+ structured_locals: set[str] = set()
+ for inner in ast.walk(node):
+ if isinstance(inner, ast.Assign) and len(inner.targets) == 1 and isinstance(inner.targets[0], ast.Name):
+ if _expr_is_structured(inner.value):
+ structured_locals.add(inner.targets[0].id)
+ for inner in ast.walk(node):
+ if not isinstance(inner, ast.Return) or inner.value is None:
+ continue
+ if _expr_is_structured(inner.value):
+ return True
+ if isinstance(inner.value, ast.Name) and inner.value.id in structured_locals:
+ return True
+ return False
+
+
+def _name_loaded_in(statements: list[ast.stmt], name: str, *, skip: ast.AST) -> bool:
+ skip_nodes = set(ast.walk(skip))
+ for statement in statements:
+ for inner in ast.walk(statement):
+ if inner in skip_nodes:
+ continue
+ if isinstance(inner, ast.Name) and isinstance(inner.ctx, ast.Load) and inner.id == name:
+ return True
+ return False
+
+
+def _code_block_returns_uninvoked_structured_function(code: str) -> bool:
+ """True when the snippet's only structured data lives in a nested function that the
+ top-level scope never invokes, returns, or binds — the shared wrapper then captures
+ the function object instead of its data. Top-level structured returns or structured
+ local bindings (legit implicit capture) are not flagged, and anything indeterminate
+ returns False."""
+ try:
+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")
+ except SyntaxError:
+ return False
+
+ scope_statements = list(_iter_top_level_scope(tree.body))
+ for statement in scope_statements:
+ if isinstance(statement, ast.Return) and statement.value is not None and _expr_is_structured(statement.value):
+ return False
+ if isinstance(statement, ast.Assign) and _expr_is_structured(statement.value):
+ return False
+
+ structured_functions = [
+ statement
+ for statement in scope_statements
+ if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef))
+ and _function_body_has_structured_return(statement)
+ ]
+ if not structured_functions:
+ return False
+ return all(not _name_loaded_in(scope_statements, function.name, skip=function) for function in structured_functions)
+
+
+def _artifact_declares_goal_values(artifact: Mapping[str, Any]) -> bool:
+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
+ for row in _artifact_rows(artifact.get(field_name)):
+ if _artifact_goal_value_paths(row.get("goal_value_paths")):
+ return True
+ return False
+
+
+def _extraction_return_shape_error(label: str, artifact: Mapping[str, Any], code: str) -> str | None:
+ """Reject an extraction-intent code block whose declared goal values do not reach
+ the block output as a keyed structure: a flat text blob, or structured data trapped
+ in an uninvoked nested function. Extraction-intent is the existing `goal_value_paths`
+ signal; non-extraction blocks are never subject to this."""
+ if not _artifact_declares_goal_values(artifact) or not code.strip():
+ return None
+ if _code_block_returns_flat_string(code):
+ return (
+ f"Code block `{label}` declares `goal_value_paths` but `return`s a flat text blob "
+ "(e.g. `page.inner_text(...)`/`text_content(...)`). Return a keyed structure instead: a dict, or an "
+ "array of objects for repeated records, whose declared goal values resolve to named scalar fields "
+ '(for example `return {"records": [{"number": "...", "expiration_date": "..."}]}`). A single value '
+ 'is fine as a keyed scalar (`{"<field>": value}`); do not array-wrap it.'
+ )
+ if _code_block_returns_uninvoked_structured_function(code):
+ return (
+ f"Code block `{label}` declares `goal_value_paths` but its structured `return` sits inside a nested "
+ "function the top level never calls, so the block captures the function object instead of the data. "
+ "Call that function and return its result (e.g. `return await run(page)`), or build the keyed "
+ "structure at the top level so the declared goal values reach the block output."
+ )
+ return None
+
+
def _code_artifact_metadata_shape_errors(
label: str,
artifact: Mapping[str, Any],
diff --git a/tests/unit/test_copilot_code_artifact_metadata_violations.py b/tests/unit/test_copilot_code_artifact_metadata_violations.py
index d43d15ed9..c7a3e7f11 100644
--- a/tests/unit/test_copilot_code_artifact_metadata_violations.py
+++ b/tests/unit/test_copilot_code_artifact_metadata_violations.py
@@ -15,7 +15,11 @@ from skyvern.forge.sdk.copilot.outcome_verification_trace import (
)
from skyvern.forge.sdk.copilot.output_utils import _sanitize_failure_text
from skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata
-from skyvern.forge.sdk.copilot.tools.workflow_update import _normalize_code_artifact_metadata_detailed
+from skyvern.forge.sdk.copilot.tools.workflow_update import (
+ _code_block_returns_flat_string,
+ _code_block_returns_uninvoked_structured_function,
+ _normalize_code_artifact_metadata_detailed,
+)
def _code_block_yaml(label: str) -> str:
@@ -495,6 +499,285 @@ class TestStaleLabelRekey:
assert normalized["my_block"]["declared_goal"] == "g"
+def _extraction_code_block_yaml(label: str, code: str) -> str:
+ indented = textwrap.indent(textwrap.dedent(code).strip(), " " * 16)
+ return textwrap.dedent(
+ f"""
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+{indented}
+ """
+ ).strip()
+
+
+def _extraction_metadata(label: str, goal_value_paths: list[str]) -> dict:
+ metadata = _valid_metadata(label)
+ metadata["claimed_outcomes"][0]["goal_value_paths"] = list(goal_value_paths)
+ metadata["terminal_verifier_expectations"][0]["goal_value_paths"] = list(goal_value_paths)
+ return metadata
+
+
+def _non_extraction_metadata(label: str) -> dict:
+ return {
+ "block_label": label,
+ "artifact_id": f"code_artifact:{label}",
+ "declared_goal": "click submit",
+ "claimed_outcomes": [
+ {
+ "id": "claim:x",
+ "scope": "outcome",
+ "text": "submitted",
+ "status": "observed_not_verified",
+ "depends_on": ["dependency:p"],
+ "covered_criteria": ["criterion:c"],
+ "observation_refs": ["obs1"],
+ }
+ ],
+ "page_dependencies": [
+ {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}
+ ],
+ "completion_criteria": [{"id": "criterion:c", "text": "submitted", "level": "outcome", "terminal": False}],
+ "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],
+ "observation_refs": [
+ {
+ "observation_ref": "obs1",
+ "dependency_id": "dependency:p",
+ "status": "observed_not_verified",
+ "source_tool": "scout_interaction",
+ }
+ ],
+ }
+
+
+class TestExtractionReturnShape:
+ def test_flat_inner_text_return_is_rejected(self) -> None:
+ code = """
+ await page.goto("https://example.com/")
+ return page.inner_text("#results")
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert normalized == {}
+ assert error is not None
+ assert "flat text blob" in error
+ assert "array of objects" in error
+
+ def test_flat_string_local_return_is_rejected(self) -> None:
+ code = """
+ await page.goto("https://example.com/")
+ text = await page.locator("#results").inner_text()
+ return text
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert normalized == {}
+ assert error is not None
+ assert "flat text blob" in error
+
+ def test_keyed_dict_return_passes(self) -> None:
+ code = """
+ await page.goto("https://example.com/")
+ return {"records": [{"number": "1-25-80030"}]}
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
+ def test_array_of_objects_comprehension_return_passes(self) -> None:
+ code = """
+ await page.goto("https://example.com/")
+ rows = await page.locator(".row").all()
+ return [{"number": await row.inner_text()} for row in rows]
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
+ def test_single_scalar_passes_as_keyed_field_without_array_wrapping(self) -> None:
+ code = """
+ await page.goto("https://example.com/")
+ return {"total": 5}
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["total"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
+ def test_non_extraction_block_with_flat_return_is_not_rejected(self) -> None:
+ code = """
+ await page.goto("https://example.com/")
+ return page.inner_text("#status")
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_non_extraction_metadata("my_block")],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
+
+class TestExtractionUninvokedNestedReturn:
+ def test_uninvoked_nested_structured_function_is_rejected(self) -> None:
+ code = """
+ async def run(page):
+ result = {"records": [{"number": "1-25-80030"}]}
+ return result
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert normalized == {}
+ assert error is not None
+ assert "nested function" in error
+ assert "captures the function object" in error
+
+ def test_invoked_and_returned_nested_function_passes(self) -> None:
+ code = """
+ async def run(page):
+ return {"records": [{"number": "1-25-80030"}]}
+ return await run(page)
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
+ def test_invoked_and_bound_nested_function_passes(self) -> None:
+ code = """
+ async def run(page):
+ return {"records": [{"number": "1-25-80030"}]}
+ data = await run(page)
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
+ def test_top_level_structured_local_passes(self) -> None:
+ code = """
+ await page.goto("https://example.com/")
+ records = [{"number": "1-25-80030"}]
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
+ def test_indeterminate_nested_function_is_not_flagged(self) -> None:
+ code = """
+ def helper():
+ return "text"
+ await page.goto("https://example.com/")
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
+
+class TestUninvokedStructuredFunctionClassifier:
+ def test_uninvoked_structured_function_with_literal_return_is_flagged(self) -> None:
+ assert _code_block_returns_uninvoked_structured_function("def run():\n return {'a': 1}") is True
+
+ def test_uninvoked_structured_function_with_local_return_is_flagged(self) -> None:
+ code = """
+ async def run(page):
+ result = {"records": []}
+ return result
+ """
+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is True
+
+ def test_invoked_function_is_not_flagged(self) -> None:
+ code = """
+ def run():
+ return {"a": 1}
+ data = run()
+ return data
+ """
+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False
+
+ def test_top_level_structured_return_is_not_flagged(self) -> None:
+ code = """
+ async def run(page):
+ return {"x": 1}
+ return {"records": []}
+ """
+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False
+
+ def test_top_level_structured_assignment_is_not_flagged(self) -> None:
+ assert _code_block_returns_uninvoked_structured_function("records = [{'number': '1'}]") is False
+
+ def test_function_returning_string_is_not_flagged(self) -> None:
+ code = """
+ def run():
+ return "text"
+ """
+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False
+
+ def test_function_referenced_but_not_called_is_not_flagged(self) -> None:
+ code = """
+ def build():
+ return {"records": []}
+ callbacks = [build]
+ """
+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False
+
+
+class TestFlatStringClassifier:
+ def test_string_literal_return_is_flat(self) -> None:
+ assert _code_block_returns_flat_string('return "hello"') is True
+
+ def test_fstring_return_is_flat(self) -> None:
+ assert _code_block_returns_flat_string('return f"{a} {b}"') is True
+
+ def test_join_return_is_flat(self) -> None:
+ assert _code_block_returns_flat_string('return " ".join(parts)') is True
+
+ def test_dict_return_is_not_flat(self) -> None:
+ assert _code_block_returns_flat_string('return {"a": 1}') is False
+
+ def test_list_return_is_not_flat(self) -> None:
+ assert _code_block_returns_flat_string("return [1, 2, 3]") is False
+
+ def test_unknown_name_return_is_indeterminate_not_flat(self) -> None:
+ assert _code_block_returns_flat_string("return some_unknown") is False
+
+ def test_no_return_is_not_flat(self) -> None:
+ assert _code_block_returns_flat_string('await page.goto("https://example.com/")') is False
+
+ def test_mixed_structured_and_flat_returns_are_not_flagged(self) -> None:
+ code = """
+ if condition:
+ return {"records": []}
+ return page.inner_text("#x")
+ """
+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is False
+
+
def _two_code_block_yaml(first: str, second: str) -> str:
return textwrap.dedent(
f"""
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index d3c98990f..6e4da46f1 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -521,6 +521,27 @@ class TestCompiledAuthoringImposition:
"prior_source": "workflow_yaml",
}
+ @pytest.mark.asyncio
+ async def test_imposition_appends_no_fill_skeleton_and_emits_no_skeleton_substitution(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+ metadata = [_terminal_metadata("search_registry", "search the registry")]
+
+ result = await _update_workflow(
+ {"workflow_yaml": _SUBMITTED_LITERAL_YAML, "code_artifact_metadata": metadata}, ctx
+ )
+
+ assert result["ok"] is True
+ assert "structured_return_skeleton" not in result["data"]["imposed_substitutions"]
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert "<fill" not in block["code"]
+ assert "<fill: captured value>" not in block["code"]
+ assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
+
@pytest.mark.asyncio
async def test_unchanged_prior_code_does_not_impose(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index 2f956f2ce..970ae7cb3 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -904,3 +904,26 @@ def test_code_block_preflight_restores_recursion_limit() -> None:
preflight_code_block("await page.locator('button[type=submit]').first.click(timeout=5000)\n")
assert sys.getrecursionlimit() == before
+
+
+class TestOfferDemonstratesStructuredReturn:
+ def test_offer_directs_keyed_return_not_inner_text_blob(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector="#search-submit",
+ source_url="https://example.com/search",
+ )
+ ]
+ )
+ assert result is not None
+ offer = render_synthesized_offer_text(
+ result,
+ trajectory=[
+ {"tool_name": "click", "selector": "#search-submit", "source_url": "https://example.com/search"}
+ ],
+ )
+ assert "keyed structure" in offer
+ assert "inner_text" in offer
+ assert 'return {"records":' in offer