Skip to content

Automate browser based workflows with AI

fix(SKY-11463): surface extraction/AI steps in code-block plain view (#6823)

Aaron Perezbd3aefeb1b21
diff --git a/skyvern/forge/sdk/copilot/code_block_steps.py b/skyvern/forge/sdk/copilot/code_block_steps.py
index 1123a26fb..d59f1c083 100644
--- a/skyvern/forge/sdk/copilot/code_block_steps.py
+++ b/skyvern/forge/sdk/copilot/code_block_steps.py
@@ -10,17 +10,19 @@ import yaml
LOG = structlog.get_logger()
-# Playwright/SkyvernPage method name -> step action_type (values are ActionType members;
+# Method name -> step action_type for the editor's static step preview (values are ActionType members,
# kept as string literals so this module does not import skyvern.webeye, matching code_block_synthesis.py).
-# Mirror the runtime recorder's maps (code_block_recorder._PAGE_ACTION_MAP / _LOCATOR_ACTION_MAP) so the
-# static editor preview surfaces the same calls the timeline records — otherwise a recorded call (e.g.
-# page.evaluate) renders in the timeline but is silently dropped from the editor step list.
+# Must stay consistent with the two surfaces that record actions at runtime, so preview matches timeline:
+# the runtime recorder's raw-Playwright maps (code_block_recorder._PAGE_ACTION_MAP / _LOCATOR_ACTION_MAP)
+# and the @action_wrap(ActionType.X) decorators on the SkyvernPage high-level API (skyvern_page.py, e.g.
+# page.extract / page.complete). null_action is excluded as a no-op probe.
_METHOD_ACTION_TYPES: dict[str, str] = {
+ # raw Playwright (mirrors code_block_recorder)
"goto": "goto_url",
"click": "click",
"dblclick": "click",
- "check": "click",
- "uncheck": "click",
+ "check": "checkbox",
+ "uncheck": "checkbox",
"tap": "click",
"fill": "input_text",
"type": "input_text",
@@ -34,7 +36,26 @@ _METHOD_ACTION_TYPES: dict[str, str] = {
"reload": "reload_page",
"evaluate": "execute_js",
"wait_for_timeout": "wait",
+ # SkyvernPage @action_wrap high-level API (mirrors skyvern_page.py)
+ "extract": "extract",
+ "fill_autocomplete": "input_text",
+ "upload_file": "upload_file",
+ "complete": "complete",
+ "terminate": "terminate",
+ "verification_code": "verification_code",
+ "solve_captcha": "solve_captcha",
+ "download_file": "download_file",
+ "wait": "wait",
+ "reload_page": "reload_page",
+ "scroll": "scroll",
+ "keypress": "keypress",
+ "move": "move",
+ "drag": "drag",
+ "left_mouse": "left_mouse",
}
+# Methods whose natural-language `prompt` is the first positional argument (it is keyword-only on the
+# interaction methods, which the keyword scan below already covers).
+_PROMPT_POSITIONAL_METHODS: frozenset[str] = frozenset({"extract", "complete", "solve_captcha", "verification_code"})
# Awaited calls that are sync/no-op helpers — never surfaced as their own step.
_IGNORED_METHODS: frozenset[str] = frozenset(
{"wait_for_load_state", "wait_for_selector", "wait_for_url", "wait_for_function"}
@@ -52,6 +73,8 @@ class CodeActionSpan:
method: str
receiver: str # source of the call receiver, e.g. "page" or "page.get_by_role('link', name='Login')"
first_arg: str | None # source of the first call arg, if any
+ prompt: str | None # natural-language `prompt` argument value, if a string literal
+ loop_var: str | None # name of the enclosing for-loop target, if the call is inside one
def analyze_code_actions(code: str) -> list[CodeActionSpan]:
@@ -63,6 +86,11 @@ def analyze_code_actions(code: str) -> list[CodeActionSpan]:
except SyntaxError:
return []
+ parents: dict[ast.AST, ast.AST] = {}
+ for parent in ast.walk(tree):
+ for child in ast.iter_child_nodes(parent):
+ parents[child] = parent
+
spans: list[CodeActionSpan] = []
for node in ast.walk(tree):
if not isinstance(node, ast.Await) or not isinstance(node.value, ast.Call):
@@ -84,12 +112,45 @@ def analyze_code_actions(code: str) -> list[CodeActionSpan]:
method=method,
receiver=_safe_unparse(call.func.value),
first_arg=_safe_unparse(call.args[0]) if call.args else None,
+ prompt=_prompt_literal(call, method),
+ loop_var=_enclosing_loop_var(node, parents),
)
)
spans.sort(key=lambda s: (s.line_start, s.line_end))
return spans
+def _prompt_literal(call: ast.Call, method: str) -> str | None:
+ """The natural-language `prompt` argument as a string literal, else None.
+
+ Decode the constant straight off the AST node — re-parsing ast.unparse output would
+ re-escape real newlines/tabs in a block-scalar prompt into literal "\\n" the copy can't collapse.
+ """
+ for keyword in call.keywords:
+ if keyword.arg == "prompt":
+ return _constant_str(keyword.value)
+ if method in _PROMPT_POSITIONAL_METHODS and call.args:
+ return _constant_str(call.args[0])
+ return None
+
+
+def _constant_str(node: ast.AST) -> str | None:
+ """The decoded value of a string-literal AST node, else None (e.g. a variable or f-string)."""
+ if isinstance(node, ast.Constant) and isinstance(node.value, str):
+ return node.value
+ return None
+
+
+def _enclosing_loop_var(node: ast.AST, parents: dict[ast.AST, ast.AST]) -> str | None:
+ """Name of the nearest enclosing for-loop's target, when it is a plain variable."""
+ current = parents.get(node)
+ while current is not None:
+ if isinstance(current, (ast.For, ast.AsyncFor)) and isinstance(current.target, ast.Name):
+ return current.target.id
+ current = parents.get(current)
+ return None
+
+
def _safe_unparse(node: ast.AST) -> str:
try:
return ast.unparse(node)
@@ -121,12 +182,37 @@ def _target_label(receiver: str, first_arg: str | None) -> str:
return "the element"
+def _normalize_whitespace(text: str) -> str:
+ return re.sub(r"\s+", " ", text).strip()
+
+
+def _humanize_identifier(name: str) -> str:
+ return name.replace("_", " ").strip()
+
+
def _describe(span: CodeActionSpan) -> str:
+ # A natural-language `prompt` is the author's own reader-facing intent (e.g.
+ # "Extract the URLs of the top 20 posts"); prefer it over a structural label, but
+ # ignore a whitespace-only prompt so a step never renders with blank copy.
+ if span.prompt:
+ normalized = _normalize_whitespace(span.prompt)
+ if normalized:
+ return normalized
value = _string_value(span.first_arg)
if span.action_type == "goto_url":
- return f"Open {value}" if value else "Open the page"
+ if value:
+ return f"Open {value}"
+ # A non-literal URL is a link discovered at runtime; say what/why instead of
+ # a repeated generic "Open the page". Inside a loop it opens each iterated item.
+ if span.loop_var:
+ return f"Open each {_humanize_identifier(span.loop_var)}"
+ return "Open the linked page"
+ if span.action_type == "extract":
+ return "Extract information from the page"
if span.action_type == "click":
return f"Click {_target_label(span.receiver, span.first_arg)}"
+ if span.action_type == "checkbox":
+ return f"Toggle {_target_label(span.receiver, span.first_arg)}"
if span.action_type == "hover":
return f"Hover over {_target_label(span.receiver, span.first_arg)}"
if span.action_type == "input_text":
@@ -148,6 +234,20 @@ def _describe(span: CodeActionSpan) -> str:
return "Reload the page"
if span.action_type == "execute_js":
return "Run a script"
+ if span.action_type == "complete":
+ return "Confirm the page is complete"
+ if span.action_type == "terminate":
+ return "Stop the workflow"
+ if span.action_type == "solve_captcha":
+ return "Solve the captcha"
+ if span.action_type == "verification_code":
+ return "Enter the verification code"
+ if span.action_type == "download_file":
+ return "Download a file"
+ if span.action_type == "scroll":
+ return "Scroll the page"
+ if span.action_type in ("move", "drag", "left_mouse"):
+ return "Move the cursor"
return "Run a step"
diff --git a/tests/unit/test_code_block_steps.py b/tests/unit/test_code_block_steps.py
index 30c925710..595bf7366 100644
--- a/tests/unit/test_code_block_steps.py
+++ b/tests/unit/test_code_block_steps.py
@@ -288,3 +288,103 @@ def test_get_by_role_without_name_falls_back_to_the_element():
code = "async def run(page):\n await page.get_by_role('button').click()\n"
steps = derive_code_block_steps(code)
assert steps[0]["description"] == "Click the element"
+
+
+def test_extract_call_surfaces_as_a_distinct_step_with_prompt_copy():
+ # page.extract() is the code-first extraction surface and must surface as its own step.
+ code = (
+ "async def run(page):\n"
+ " await page.goto('https://example.com/')\n"
+ " data = await page.extract(prompt='Extract the URLs of the top 20 posts', schema={'type': 'object'})\n"
+ )
+ steps = derive_code_block_steps(code)
+ assert [s["action_type"] for s in steps] == ["goto_url", "extract"]
+ assert steps[1]["description"] == "Extract the URLs of the top 20 posts"
+
+
+def test_multiline_prompt_literal_renders_without_backslash_escapes():
+ # A YAML block-scalar / multiline prompt must collapse to readable copy, not leak
+ # the source-level "\n" escape that ast.unparse would re-introduce.
+ code = "async def run(page):\n await page.extract(prompt='Extract the URLs\\nof the top 20 posts')\n"
+ steps = derive_code_block_steps(code)
+ assert steps[0]["action_type"] == "extract"
+ assert steps[0]["description"] == "Extract the URLs of the top 20 posts"
+
+
+def test_extract_without_prompt_literal_falls_back_to_a_generic_extract_label():
+ code = "async def run(page):\n data = await page.extract(prompt=goal)\n"
+ steps = derive_code_block_steps(code)
+ assert steps[0]["action_type"] == "extract"
+ assert steps[0]["description"] == "Extract information from the page"
+
+
+def test_whitespace_only_prompt_falls_back_instead_of_rendering_blank_copy():
+ code = "async def run(page):\n data = await page.extract(prompt=' \\n ')\n"
+ steps = derive_code_block_steps(code)
+ assert steps[0]["action_type"] == "extract"
+ assert steps[0]["description"] == "Extract information from the page"
+
+
+def test_check_and_uncheck_map_to_checkbox_matching_the_recorder():
+ code = (
+ "async def run(page):\n"
+ " await page.get_by_label('Remember me').check()\n"
+ " await page.get_by_label('Subscribe').uncheck()\n"
+ )
+ steps = derive_code_block_steps(code)
+ assert [s["action_type"] for s in steps] == ["checkbox", "checkbox"]
+ assert steps[0]["description"] == 'Toggle "Remember me"'
+
+
+def test_extraction_then_looped_navigation_are_distinguishable():
+ # The canonical failing case: open a site, extract a list of links, then for
+ # each link open it and extract its contents. Every step must read as the
+ # action it performs, and the two navigations must not be the same generic copy.
+ code = (
+ "async def run(page, limit):\n"
+ " await page.goto('https://example.com/')\n"
+ " posts = await page.extract(prompt='Extract the URLs of the top posts')\n"
+ " for post in posts['posts'][:limit]:\n"
+ " await page.goto(post['url'])\n"
+ " await page.extract(prompt='Extract the top comment on the post')\n"
+ )
+ steps = derive_code_block_steps(code)
+ assert [s["action_type"] for s in steps] == ["goto_url", "extract", "goto_url", "extract"]
+ descriptions = [s["description"] for s in steps]
+ assert descriptions == [
+ "Open https://example.com/",
+ "Extract the URLs of the top posts",
+ "Open each post",
+ "Extract the top comment on the post",
+ ]
+ # The follow-up navigation must be distinguishable from the first, not a repeated label.
+ assert descriptions[0] != descriptions[2]
+
+
+def test_goto_with_non_literal_url_outside_a_loop_describes_a_linked_page():
+ code = "async def run(page, target_url):\n await page.goto(target_url)\n"
+ steps = derive_code_block_steps(code)
+ assert steps[0]["action_type"] == "goto_url"
+ assert steps[0]["description"] == "Open the linked page"
+
+
+def test_prompt_kwarg_is_preferred_as_step_copy_for_interactions():
+ code = "async def run(page):\n await page.click('#login', prompt='Click the login button')\n"
+ steps = derive_code_block_steps(code)
+ assert steps[0]["action_type"] == "click"
+ assert steps[0]["description"] == "Click the login button"
+
+
+def test_skyvern_page_high_level_actions_surface_as_steps():
+ # The durable copilot surface is the @action_wrap-decorated SkyvernPage API,
+ # not only raw Playwright calls; these must each render as their own step.
+ code = (
+ "async def run(page, doc):\n"
+ " await page.select_option('#country', prompt='Choose the country')\n"
+ " await page.upload_file('#file', files=str(doc))\n"
+ " await page.complete(prompt='Confirm the form was submitted')\n"
+ )
+ steps = derive_code_block_steps(code)
+ assert [s["action_type"] for s in steps] == ["select_option", "upload_file", "complete"]
+ assert steps[0]["description"] == "Choose the country"
+ assert steps[2]["description"] == "Confirm the form was submitted"