Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11463): surface extraction/AI steps in code-block plain view (#6823)
Aaron Perezbd3aefeb1b21
diff --git a/skyvern/forge/sdk/copilot/code_block_steps.py b/skyvern/forge/sdk/copilot/code_block_steps.pyindex 1123a26fb..d59f1c083 100644--- a/skyvern/forge/sdk/copilot/code_block_steps.py+++ b/skyvern/forge/sdk/copilot/code_block_steps.py@@ -10,17 +10,19 @@ import yamlLOG = structlog.get_logger()-# Playwright/SkyvernPage method name -> step action_type (values are ActionType members;+# Method name -> step action_type for the editor's static step preview (values are ActionType members,# kept as string literals so this module does not import skyvern.webeye, matching code_block_synthesis.py).-# Mirror the runtime recorder's maps (code_block_recorder._PAGE_ACTION_MAP / _LOCATOR_ACTION_MAP) so the-# static editor preview surfaces the same calls the timeline records — otherwise a recorded call (e.g.-# page.evaluate) renders in the timeline but is silently dropped from the editor step list.+# Must stay consistent with the two surfaces that record actions at runtime, so preview matches timeline:+# the runtime recorder's raw-Playwright maps (code_block_recorder._PAGE_ACTION_MAP / _LOCATOR_ACTION_MAP)+# and the @action_wrap(ActionType.X) decorators on the SkyvernPage high-level API (skyvern_page.py, e.g.+# page.extract / page.complete). null_action is excluded as a no-op probe._METHOD_ACTION_TYPES: dict[str, str] = {+ # raw Playwright (mirrors code_block_recorder)"goto": "goto_url","click": "click","dblclick": "click",- "check": "click",- "uncheck": "click",+ "check": "checkbox",+ "uncheck": "checkbox","tap": "click","fill": "input_text","type": "input_text",@@ -34,7 +36,26 @@ _METHOD_ACTION_TYPES: dict[str, str] = {"reload": "reload_page","evaluate": "execute_js","wait_for_timeout": "wait",+ # SkyvernPage @action_wrap high-level API (mirrors skyvern_page.py)+ "extract": "extract",+ "fill_autocomplete": "input_text",+ "upload_file": "upload_file",+ "complete": "complete",+ "terminate": "terminate",+ "verification_code": "verification_code",+ "solve_captcha": "solve_captcha",+ "download_file": "download_file",+ "wait": "wait",+ "reload_page": "reload_page",+ "scroll": "scroll",+ "keypress": "keypress",+ "move": "move",+ "drag": "drag",+ "left_mouse": "left_mouse",}+# Methods whose natural-language `prompt` is the first positional argument (it is keyword-only on the+# interaction methods, which the keyword scan below already covers).+_PROMPT_POSITIONAL_METHODS: frozenset[str] = frozenset({"extract", "complete", "solve_captcha", "verification_code"})# Awaited calls that are sync/no-op helpers — never surfaced as their own step._IGNORED_METHODS: frozenset[str] = frozenset({"wait_for_load_state", "wait_for_selector", "wait_for_url", "wait_for_function"}@@ -52,6 +73,8 @@ class CodeActionSpan:method: strreceiver: str # source of the call receiver, e.g. "page" or "page.get_by_role('link', name='Login')"first_arg: str | None # source of the first call arg, if any+ prompt: str | None # natural-language `prompt` argument value, if a string literal+ loop_var: str | None # name of the enclosing for-loop target, if the call is inside onedef analyze_code_actions(code: str) -> list[CodeActionSpan]:@@ -63,6 +86,11 @@ def analyze_code_actions(code: str) -> list[CodeActionSpan]:except SyntaxError:return []+ parents: dict[ast.AST, ast.AST] = {}+ for parent in ast.walk(tree):+ for child in ast.iter_child_nodes(parent):+ parents[child] = parent+spans: list[CodeActionSpan] = []for node in ast.walk(tree):if not isinstance(node, ast.Await) or not isinstance(node.value, ast.Call):@@ -84,12 +112,45 @@ def analyze_code_actions(code: str) -> list[CodeActionSpan]:method=method,receiver=_safe_unparse(call.func.value),first_arg=_safe_unparse(call.args[0]) if call.args else None,+ prompt=_prompt_literal(call, method),+ loop_var=_enclosing_loop_var(node, parents),))spans.sort(key=lambda s: (s.line_start, s.line_end))return spans+def _prompt_literal(call: ast.Call, method: str) -> str | None:+ """The natural-language `prompt` argument as a string literal, else None.++ Decode the constant straight off the AST node — re-parsing ast.unparse output would+ re-escape real newlines/tabs in a block-scalar prompt into literal "\\n" the copy can't collapse.+ """+ for keyword in call.keywords:+ if keyword.arg == "prompt":+ return _constant_str(keyword.value)+ if method in _PROMPT_POSITIONAL_METHODS and call.args:+ return _constant_str(call.args[0])+ return None+++def _constant_str(node: ast.AST) -> str | None:+ """The decoded value of a string-literal AST node, else None (e.g. a variable or f-string)."""+ if isinstance(node, ast.Constant) and isinstance(node.value, str):+ return node.value+ return None+++def _enclosing_loop_var(node: ast.AST, parents: dict[ast.AST, ast.AST]) -> str | None:+ """Name of the nearest enclosing for-loop's target, when it is a plain variable."""+ current = parents.get(node)+ while current is not None:+ if isinstance(current, (ast.For, ast.AsyncFor)) and isinstance(current.target, ast.Name):+ return current.target.id+ current = parents.get(current)+ return None++def _safe_unparse(node: ast.AST) -> str:try:return ast.unparse(node)@@ -121,12 +182,37 @@ def _target_label(receiver: str, first_arg: str | None) -> str:return "the element"+def _normalize_whitespace(text: str) -> str:+ return re.sub(r"\s+", " ", text).strip()+++def _humanize_identifier(name: str) -> str:+ return name.replace("_", " ").strip()++def _describe(span: CodeActionSpan) -> str:+ # A natural-language `prompt` is the author's own reader-facing intent (e.g.+ # "Extract the URLs of the top 20 posts"); prefer it over a structural label, but+ # ignore a whitespace-only prompt so a step never renders with blank copy.+ if span.prompt:+ normalized = _normalize_whitespace(span.prompt)+ if normalized:+ return normalizedvalue = _string_value(span.first_arg)if span.action_type == "goto_url":- return f"Open {value}" if value else "Open the page"+ if value:+ return f"Open {value}"+ # A non-literal URL is a link discovered at runtime; say what/why instead of+ # a repeated generic "Open the page". Inside a loop it opens each iterated item.+ if span.loop_var:+ return f"Open each {_humanize_identifier(span.loop_var)}"+ return "Open the linked page"+ if span.action_type == "extract":+ return "Extract information from the page"if span.action_type == "click":return f"Click {_target_label(span.receiver, span.first_arg)}"+ if span.action_type == "checkbox":+ return f"Toggle {_target_label(span.receiver, span.first_arg)}"if span.action_type == "hover":return f"Hover over {_target_label(span.receiver, span.first_arg)}"if span.action_type == "input_text":@@ -148,6 +234,20 @@ def _describe(span: CodeActionSpan) -> str:return "Reload the page"if span.action_type == "execute_js":return "Run a script"+ if span.action_type == "complete":+ return "Confirm the page is complete"+ if span.action_type == "terminate":+ return "Stop the workflow"+ if span.action_type == "solve_captcha":+ return "Solve the captcha"+ if span.action_type == "verification_code":+ return "Enter the verification code"+ if span.action_type == "download_file":+ return "Download a file"+ if span.action_type == "scroll":+ return "Scroll the page"+ if span.action_type in ("move", "drag", "left_mouse"):+ return "Move the cursor"return "Run a step"diff --git a/tests/unit/test_code_block_steps.py b/tests/unit/test_code_block_steps.pyindex 30c925710..595bf7366 100644--- a/tests/unit/test_code_block_steps.py+++ b/tests/unit/test_code_block_steps.py@@ -288,3 +288,103 @@ def test_get_by_role_without_name_falls_back_to_the_element():code = "async def run(page):\n await page.get_by_role('button').click()\n"steps = derive_code_block_steps(code)assert steps[0]["description"] == "Click the element"+++def test_extract_call_surfaces_as_a_distinct_step_with_prompt_copy():+ # page.extract() is the code-first extraction surface and must surface as its own step.+ code = (+ "async def run(page):\n"+ " await page.goto('https://example.com/')\n"+ " data = await page.extract(prompt='Extract the URLs of the top 20 posts', schema={'type': 'object'})\n"+ )+ steps = derive_code_block_steps(code)+ assert [s["action_type"] for s in steps] == ["goto_url", "extract"]+ assert steps[1]["description"] == "Extract the URLs of the top 20 posts"+++def test_multiline_prompt_literal_renders_without_backslash_escapes():+ # A YAML block-scalar / multiline prompt must collapse to readable copy, not leak+ # the source-level "\n" escape that ast.unparse would re-introduce.+ code = "async def run(page):\n await page.extract(prompt='Extract the URLs\\nof the top 20 posts')\n"+ steps = derive_code_block_steps(code)+ assert steps[0]["action_type"] == "extract"+ assert steps[0]["description"] == "Extract the URLs of the top 20 posts"+++def test_extract_without_prompt_literal_falls_back_to_a_generic_extract_label():+ code = "async def run(page):\n data = await page.extract(prompt=goal)\n"+ steps = derive_code_block_steps(code)+ assert steps[0]["action_type"] == "extract"+ assert steps[0]["description"] == "Extract information from the page"+++def test_whitespace_only_prompt_falls_back_instead_of_rendering_blank_copy():+ code = "async def run(page):\n data = await page.extract(prompt=' \\n ')\n"+ steps = derive_code_block_steps(code)+ assert steps[0]["action_type"] == "extract"+ assert steps[0]["description"] == "Extract information from the page"+++def test_check_and_uncheck_map_to_checkbox_matching_the_recorder():+ code = (+ "async def run(page):\n"+ " await page.get_by_label('Remember me').check()\n"+ " await page.get_by_label('Subscribe').uncheck()\n"+ )+ steps = derive_code_block_steps(code)+ assert [s["action_type"] for s in steps] == ["checkbox", "checkbox"]+ assert steps[0]["description"] == 'Toggle "Remember me"'+++def test_extraction_then_looped_navigation_are_distinguishable():+ # The canonical failing case: open a site, extract a list of links, then for+ # each link open it and extract its contents. Every step must read as the+ # action it performs, and the two navigations must not be the same generic copy.+ code = (+ "async def run(page, limit):\n"+ " await page.goto('https://example.com/')\n"+ " posts = await page.extract(prompt='Extract the URLs of the top posts')\n"+ " for post in posts['posts'][:limit]:\n"+ " await page.goto(post['url'])\n"+ " await page.extract(prompt='Extract the top comment on the post')\n"+ )+ steps = derive_code_block_steps(code)+ assert [s["action_type"] for s in steps] == ["goto_url", "extract", "goto_url", "extract"]+ descriptions = [s["description"] for s in steps]+ assert descriptions == [+ "Open https://example.com/",+ "Extract the URLs of the top posts",+ "Open each post",+ "Extract the top comment on the post",+ ]+ # The follow-up navigation must be distinguishable from the first, not a repeated label.+ assert descriptions[0] != descriptions[2]+++def test_goto_with_non_literal_url_outside_a_loop_describes_a_linked_page():+ code = "async def run(page, target_url):\n await page.goto(target_url)\n"+ steps = derive_code_block_steps(code)+ assert steps[0]["action_type"] == "goto_url"+ assert steps[0]["description"] == "Open the linked page"+++def test_prompt_kwarg_is_preferred_as_step_copy_for_interactions():+ code = "async def run(page):\n await page.click('#login', prompt='Click the login button')\n"+ steps = derive_code_block_steps(code)+ assert steps[0]["action_type"] == "click"+ assert steps[0]["description"] == "Click the login button"+++def test_skyvern_page_high_level_actions_surface_as_steps():+ # The durable copilot surface is the @action_wrap-decorated SkyvernPage API,+ # not only raw Playwright calls; these must each render as their own step.+ code = (+ "async def run(page, doc):\n"+ " await page.select_option('#country', prompt='Choose the country')\n"+ " await page.upload_file('#file', files=str(doc))\n"+ " await page.complete(prompt='Confirm the form was submitted')\n"+ )+ steps = derive_code_block_steps(code)+ assert [s["action_type"] for s in steps] == ["select_option", "upload_file", "complete"]+ assert steps[0]["description"] == "Choose the country"+ assert steps[2]["description"] == "Confirm the form was submitted"