Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11025: structured keyed output for copilot extraction code blocks (#6580)
Andrew Neilsonedb298e0d5ed
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.pyindex 59ca7a3f1..0bdcc7ac5 100644--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py@@ -529,8 +529,9 @@ def render_synthesized_offer_text(parts = ["SYNTHESIZED CODE BLOCK (offered once). The page interactions you scouted were compiled into a ""deterministic Playwright snippet. Persist it VERBATIM as a `code` block labeled "- f"`{_SYNTHESIZED_BLOCK_LABEL}` via update_workflow / update_and_run_blocks; only hand-author the "- "extract/report steps it does not cover.",+ f"`{_SYNTHESIZED_BLOCK_LABEL}` via update_workflow / update_and_run_blocks; hand-author the "+ "data-capture step it does not cover so the block `return`s a keyed structure (a dict, or an array "+ "of objects for repeated records) — never a flat `page.inner_text(...)` / `text_content(...)` blob.","```python",synthesized.code.rstrip("\n"),"```",@@ -560,4 +561,12 @@ def render_synthesized_offer_text(parts.append("```json")parts.append(_render_artifact_metadata_block(metadata))parts.append("```")+ parts.append(+ "Shape the data-capture `return` so each `goal_value_paths` entry resolves to a named scalar field. "+ "For `records[].field` paths, return an array of objects; for a single record, return a keyed dict. "+ "Bind each `<fill: ...>` to the page text you captured. For example:"+ )+ parts.append("```python")+ parts.append('return {"records": [{"field_a": "...", "field_b": "..."}]}')+ parts.append("```")return "\n".join(parts)diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex e7e728135..d5543af67 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport astimport reimport textwrap-from collections.abc import Mapping+from collections.abc import Iterator, Mappingfrom dataclasses import dataclassfrom dataclasses import field as dataclass_fieldfrom typing import Annotated, Any, Literal, NamedTuple@@ -394,6 +394,9 @@ def _normalize_code_artifact_metadata_detailed(reject_unfilled_goal_value_paths=impose_defaults,))+ return_shape_error = _extraction_return_shape_error(label, dumped, str(code_blocks[label].get("code") or ""))+ if return_shape_error is not None:+ item_violations.append(return_shape_error)if item_violations:violations.extend(item_violations)offending_labels.append(label)@@ -1101,6 +1104,172 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext))+_FLAT_STRING_TEXT_METHODS = frozenset({"inner_text", "text_content", "inner_html", "content"})+++def _expr_is_flat_string(node: ast.expr, string_locals: set[str]) -> bool:+ """True only for expressions that are unambiguously a single text blob: a string+ literal/f-string, a `.inner_text()/.text_content()/.inner_html()` read, a+ `str(...)` cast, a `str.join(...)`, string concatenation, or a local bound to one+ of those. Anything ambiguous returns False so the validator never false-positives."""+ if isinstance(node, ast.Await):+ return _expr_is_flat_string(node.value, string_locals)+ if isinstance(node, ast.Constant):+ return isinstance(node.value, str)+ if isinstance(node, ast.JoinedStr):+ return True+ if isinstance(node, ast.Name):+ return node.id in string_locals+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):+ return _expr_is_flat_string(node.left, string_locals) or _expr_is_flat_string(node.right, string_locals)+ if isinstance(node, ast.Call):+ func = node.func+ if isinstance(func, ast.Attribute):+ if func.attr in _FLAT_STRING_TEXT_METHODS:+ return True+ if func.attr == "join":+ return True+ if func.attr in {"strip", "lstrip", "rstrip", "lower", "upper"}:+ return _expr_is_flat_string(func.value, string_locals)+ if isinstance(func, ast.Name) and func.id == "str":+ return True+ return False+++def _expr_is_structured(node: ast.expr) -> bool:+ if isinstance(node, ast.Await):+ return _expr_is_structured(node.value)+ return isinstance(node, (ast.Dict, ast.List, ast.DictComp, ast.ListComp, ast.SetComp, ast.Set, ast.Tuple))+++_NESTED_SCOPE_NODES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef)+++def _iter_top_level_scope(statements: list[ast.stmt]) -> Iterator[ast.stmt]:+ """Yield statements in the snippet's own scope, descending into control flow+ (if/for/while/with/try) but never into nested function/class bodies."""+ for statement in statements:+ yield statement+ if isinstance(statement, _NESTED_SCOPE_NODES):+ continue+ for child in ast.iter_child_nodes(statement):+ if isinstance(child, ast.stmt):+ yield from _iter_top_level_scope([child])+++def _code_block_returns_flat_string(code: str) -> bool:+ """True when every top-level `return` in the snippet yields a flat text blob and+ none yields a structured value. Returns inside nested functions, and indeterminate+ or structured returns, are not flagged."""+ try:+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")+ except SyntaxError:+ return False++ scope_statements = list(_iter_top_level_scope(tree.body))+ string_locals: set[str] = set()+ for node in scope_statements:+ if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):+ if _expr_is_flat_string(node.value, string_locals):+ string_locals.add(node.targets[0].id)++ returns = [node for node in scope_statements if isinstance(node, ast.Return) and node.value is not None]+ if not returns:+ return False+ if any(_expr_is_structured(node.value) for node in returns if node.value is not None):+ return False+ return all(_expr_is_flat_string(node.value, string_locals) for node in returns if node.value is not None)+++def _function_body_has_structured_return(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool:+ structured_locals: set[str] = set()+ for inner in ast.walk(node):+ if isinstance(inner, ast.Assign) and len(inner.targets) == 1 and isinstance(inner.targets[0], ast.Name):+ if _expr_is_structured(inner.value):+ structured_locals.add(inner.targets[0].id)+ for inner in ast.walk(node):+ if not isinstance(inner, ast.Return) or inner.value is None:+ continue+ if _expr_is_structured(inner.value):+ return True+ if isinstance(inner.value, ast.Name) and inner.value.id in structured_locals:+ return True+ return False+++def _name_loaded_in(statements: list[ast.stmt], name: str, *, skip: ast.AST) -> bool:+ skip_nodes = set(ast.walk(skip))+ for statement in statements:+ for inner in ast.walk(statement):+ if inner in skip_nodes:+ continue+ if isinstance(inner, ast.Name) and isinstance(inner.ctx, ast.Load) and inner.id == name:+ return True+ return False+++def _code_block_returns_uninvoked_structured_function(code: str) -> bool:+ """True when the snippet's only structured data lives in a nested function that the+ top-level scope never invokes, returns, or binds — the shared wrapper then captures+ the function object instead of its data. Top-level structured returns or structured+ local bindings (legit implicit capture) are not flagged, and anything indeterminate+ returns False."""+ try:+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")+ except SyntaxError:+ return False++ scope_statements = list(_iter_top_level_scope(tree.body))+ for statement in scope_statements:+ if isinstance(statement, ast.Return) and statement.value is not None and _expr_is_structured(statement.value):+ return False+ if isinstance(statement, ast.Assign) and _expr_is_structured(statement.value):+ return False++ structured_functions = [+ statement+ for statement in scope_statements+ if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef))+ and _function_body_has_structured_return(statement)+ ]+ if not structured_functions:+ return False+ return all(not _name_loaded_in(scope_statements, function.name, skip=function) for function in structured_functions)+++def _artifact_declares_goal_values(artifact: Mapping[str, Any]) -> bool:+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):+ for row in _artifact_rows(artifact.get(field_name)):+ if _artifact_goal_value_paths(row.get("goal_value_paths")):+ return True+ return False+++def _extraction_return_shape_error(label: str, artifact: Mapping[str, Any], code: str) -> str | None:+ """Reject an extraction-intent code block whose declared goal values do not reach+ the block output as a keyed structure: a flat text blob, or structured data trapped+ in an uninvoked nested function. Extraction-intent is the existing `goal_value_paths`+ signal; non-extraction blocks are never subject to this."""+ if not _artifact_declares_goal_values(artifact) or not code.strip():+ return None+ if _code_block_returns_flat_string(code):+ return (+ f"Code block `{label}` declares `goal_value_paths` but `return`s a flat text blob "+ "(e.g. `page.inner_text(...)`/`text_content(...)`). Return a keyed structure instead: a dict, or an "+ "array of objects for repeated records, whose declared goal values resolve to named scalar fields "+ '(for example `return {"records": [{"number": "...", "expiration_date": "..."}]}`). A single value '+ 'is fine as a keyed scalar (`{"<field>": value}`); do not array-wrap it.'+ )+ if _code_block_returns_uninvoked_structured_function(code):+ return (+ f"Code block `{label}` declares `goal_value_paths` but its structured `return` sits inside a nested "+ "function the top level never calls, so the block captures the function object instead of the data. "+ "Call that function and return its result (e.g. `return await run(page)`), or build the keyed "+ "structure at the top level so the declared goal values reach the block output."+ )+ return None++def _code_artifact_metadata_shape_errors(label: str,artifact: Mapping[str, Any],diff --git a/tests/unit/test_copilot_code_artifact_metadata_violations.py b/tests/unit/test_copilot_code_artifact_metadata_violations.pyindex d43d15ed9..c7a3e7f11 100644--- a/tests/unit/test_copilot_code_artifact_metadata_violations.py+++ b/tests/unit/test_copilot_code_artifact_metadata_violations.py@@ -15,7 +15,11 @@ from skyvern.forge.sdk.copilot.outcome_verification_trace import ()from skyvern.forge.sdk.copilot.output_utils import _sanitize_failure_textfrom skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata-from skyvern.forge.sdk.copilot.tools.workflow_update import _normalize_code_artifact_metadata_detailed+from skyvern.forge.sdk.copilot.tools.workflow_update import (+ _code_block_returns_flat_string,+ _code_block_returns_uninvoked_structured_function,+ _normalize_code_artifact_metadata_detailed,+)def _code_block_yaml(label: str) -> str:@@ -495,6 +499,285 @@ class TestStaleLabelRekey:assert normalized["my_block"]["declared_goal"] == "g"+def _extraction_code_block_yaml(label: str, code: str) -> str:+ indented = textwrap.indent(textwrap.dedent(code).strip(), " " * 16)+ return textwrap.dedent(+ f"""+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+{indented}+ """+ ).strip()+++def _extraction_metadata(label: str, goal_value_paths: list[str]) -> dict:+ metadata = _valid_metadata(label)+ metadata["claimed_outcomes"][0]["goal_value_paths"] = list(goal_value_paths)+ metadata["terminal_verifier_expectations"][0]["goal_value_paths"] = list(goal_value_paths)+ return metadata+++def _non_extraction_metadata(label: str) -> dict:+ return {+ "block_label": label,+ "artifact_id": f"code_artifact:{label}",+ "declared_goal": "click submit",+ "claimed_outcomes": [+ {+ "id": "claim:x",+ "scope": "outcome",+ "text": "submitted",+ "status": "observed_not_verified",+ "depends_on": ["dependency:p"],+ "covered_criteria": ["criterion:c"],+ "observation_refs": ["obs1"],+ }+ ],+ "page_dependencies": [+ {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}+ ],+ "completion_criteria": [{"id": "criterion:c", "text": "submitted", "level": "outcome", "terminal": False}],+ "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],+ "observation_refs": [+ {+ "observation_ref": "obs1",+ "dependency_id": "dependency:p",+ "status": "observed_not_verified",+ "source_tool": "scout_interaction",+ }+ ],+ }+++class TestExtractionReturnShape:+ def test_flat_inner_text_return_is_rejected(self) -> None:+ code = """+ await page.goto("https://example.com/")+ return page.inner_text("#results")+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert normalized == {}+ assert error is not None+ assert "flat text blob" in error+ assert "array of objects" in error++ def test_flat_string_local_return_is_rejected(self) -> None:+ code = """+ await page.goto("https://example.com/")+ text = await page.locator("#results").inner_text()+ return text+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert normalized == {}+ assert error is not None+ assert "flat text blob" in error++ def test_keyed_dict_return_passes(self) -> None:+ code = """+ await page.goto("https://example.com/")+ return {"records": [{"number": "1-25-80030"}]}+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]++ def test_array_of_objects_comprehension_return_passes(self) -> None:+ code = """+ await page.goto("https://example.com/")+ rows = await page.locator(".row").all()+ return [{"number": await row.inner_text()} for row in rows]+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]++ def test_single_scalar_passes_as_keyed_field_without_array_wrapping(self) -> None:+ code = """+ await page.goto("https://example.com/")+ return {"total": 5}+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["total"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]++ def test_non_extraction_block_with_flat_return_is_not_rejected(self) -> None:+ code = """+ await page.goto("https://example.com/")+ return page.inner_text("#status")+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_non_extraction_metadata("my_block")],+ _extraction_code_block_yaml("my_block", code),+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]+++class TestExtractionUninvokedNestedReturn:+ def test_uninvoked_nested_structured_function_is_rejected(self) -> None:+ code = """+ async def run(page):+ result = {"records": [{"number": "1-25-80030"}]}+ return result+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert normalized == {}+ assert error is not None+ assert "nested function" in error+ assert "captures the function object" in error++ def test_invoked_and_returned_nested_function_passes(self) -> None:+ code = """+ async def run(page):+ return {"records": [{"number": "1-25-80030"}]}+ return await run(page)+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]++ def test_invoked_and_bound_nested_function_passes(self) -> None:+ code = """+ async def run(page):+ return {"records": [{"number": "1-25-80030"}]}+ data = await run(page)+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]++ def test_top_level_structured_local_passes(self) -> None:+ code = """+ await page.goto("https://example.com/")+ records = [{"number": "1-25-80030"}]+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]++ def test_indeterminate_nested_function_is_not_flagged(self) -> None:+ code = """+ def helper():+ return "text"+ await page.goto("https://example.com/")+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]+++class TestUninvokedStructuredFunctionClassifier:+ def test_uninvoked_structured_function_with_literal_return_is_flagged(self) -> None:+ assert _code_block_returns_uninvoked_structured_function("def run():\n return {'a': 1}") is True++ def test_uninvoked_structured_function_with_local_return_is_flagged(self) -> None:+ code = """+ async def run(page):+ result = {"records": []}+ return result+ """+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is True++ def test_invoked_function_is_not_flagged(self) -> None:+ code = """+ def run():+ return {"a": 1}+ data = run()+ return data+ """+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False++ def test_top_level_structured_return_is_not_flagged(self) -> None:+ code = """+ async def run(page):+ return {"x": 1}+ return {"records": []}+ """+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False++ def test_top_level_structured_assignment_is_not_flagged(self) -> None:+ assert _code_block_returns_uninvoked_structured_function("records = [{'number': '1'}]") is False++ def test_function_returning_string_is_not_flagged(self) -> None:+ code = """+ def run():+ return "text"+ """+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False++ def test_function_referenced_but_not_called_is_not_flagged(self) -> None:+ code = """+ def build():+ return {"records": []}+ callbacks = [build]+ """+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False+++class TestFlatStringClassifier:+ def test_string_literal_return_is_flat(self) -> None:+ assert _code_block_returns_flat_string('return "hello"') is True++ def test_fstring_return_is_flat(self) -> None:+ assert _code_block_returns_flat_string('return f"{a} {b}"') is True++ def test_join_return_is_flat(self) -> None:+ assert _code_block_returns_flat_string('return " ".join(parts)') is True++ def test_dict_return_is_not_flat(self) -> None:+ assert _code_block_returns_flat_string('return {"a": 1}') is False++ def test_list_return_is_not_flat(self) -> None:+ assert _code_block_returns_flat_string("return [1, 2, 3]") is False++ def test_unknown_name_return_is_indeterminate_not_flat(self) -> None:+ assert _code_block_returns_flat_string("return some_unknown") is False++ def test_no_return_is_not_flat(self) -> None:+ assert _code_block_returns_flat_string('await page.goto("https://example.com/")') is False++ def test_mixed_structured_and_flat_returns_are_not_flagged(self) -> None:+ code = """+ if condition:+ return {"records": []}+ return page.inner_text("#x")+ """+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is False++def _two_code_block_yaml(first: str, second: str) -> str:return textwrap.dedent(f"""diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex d3c98990f..6e4da46f1 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -521,6 +521,27 @@ class TestCompiledAuthoringImposition:"prior_source": "workflow_yaml",}+ @pytest.mark.asyncio+ async def test_imposition_appends_no_fill_skeleton_and_emits_no_skeleton_substitution(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = self._provider_search_ctx()+ metadata = [_terminal_metadata("search_registry", "search the registry")]++ result = await _update_workflow(+ {"workflow_yaml": _SUBMITTED_LITERAL_YAML, "code_artifact_metadata": metadata}, ctx+ )++ assert result["ok"] is True+ assert "structured_return_skeleton" not in result["data"]["imposed_substitutions"]+ parsed = parse_workflow_yaml(ctx.workflow_yaml)+ assert isinstance(parsed, dict)+ block = _single_code_block(parsed)+ assert "<fill" not in block["code"]+ assert "<fill: captured value>" not in block["code"]+ assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]+@pytest.mark.asyncioasync def test_unchanged_prior_code_does_not_impose(self, monkeypatch: pytest.MonkeyPatch) -> None:_stub_successful_update(monkeypatch)diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.pyindex 2f956f2ce..970ae7cb3 100644--- a/tests/unit/test_copilot_code_block_synthesis.py+++ b/tests/unit/test_copilot_code_block_synthesis.py@@ -904,3 +904,26 @@ def test_code_block_preflight_restores_recursion_limit() -> None:preflight_code_block("await page.locator('button[type=submit]').first.click(timeout=5000)\n")assert sys.getrecursionlimit() == before+++class TestOfferDemonstratesStructuredReturn:+ def test_offer_directs_keyed_return_not_inner_text_blob(self) -> None:+ result = synthesize_code_block(+ [+ _interaction(+ "click",+ selector="#search-submit",+ source_url="https://example.com/search",+ )+ ]+ )+ assert result is not None+ offer = render_synthesized_offer_text(+ result,+ trajectory=[+ {"tool_name": "click", "selector": "#search-submit", "source_url": "https://example.com/search"}+ ],+ )+ assert "keyed structure" in offer+ assert "inner_text" in offer+ assert 'return {"records":' in offer