Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11462): surface raw DOM reads as an extraction step in code block outline (#6928)
Celal Zamanoğlu9770e8ecf4e8
diff --git a/skyvern/forge/sdk/copilot/code_block_steps.py b/skyvern/forge/sdk/copilot/code_block_steps.pyindex d59f1c083..5db19ed2b 100644--- a/skyvern/forge/sdk/copilot/code_block_steps.py+++ b/skyvern/forge/sdk/copilot/code_block_steps.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport astimport re-from dataclasses import dataclass+from dataclasses import dataclass, replacefrom typing import Any, Iteratorimport structlog@@ -53,6 +53,19 @@ _METHOD_ACTION_TYPES: dict[str, str] = {"drag": "drag","left_mouse": "left_mouse",}+# Non-state-changing DOM reads -> an extract step in the editor outline only; deliberately absent+# from the runtime recorder maps so a read never fires a run-timeline action. Predicates, count(),+# and all() are excluded so a control-flow check never fabricates an extraction step.+_READ_METHODS: dict[str, str] = {+ "text_content": "extract",+ "all_text_contents": "extract",+ "inner_text": "extract",+ "all_inner_texts": "extract",+ "inner_html": "extract",+ "get_attribute": "extract",+ "input_value": "extract",+ "content": "extract",+}# Methods whose natural-language `prompt` is the first positional argument (it is keyword-only on the# interaction methods, which the keyword scan below already covers)._PROMPT_POSITIONAL_METHODS: frozenset[str] = frozenset({"extract", "complete", "solve_captcha", "verification_code"})@@ -101,7 +114,7 @@ def analyze_code_actions(code: str) -> list[CodeActionSpan]:method = call.func.attrif method in _IGNORED_METHODS:continue- action_type = _METHOD_ACTION_TYPES.get(method)+ action_type = _METHOD_ACTION_TYPES.get(method) or _READ_METHODS.get(method)if action_type is None:continuespans.append(@@ -251,6 +264,18 @@ def _describe(span: CodeActionSpan) -> str:return "Run a step"+def _consolidate_read_spans(spans: list[CodeActionSpan]) -> list[CodeActionSpan]:+ """Merge adjacent raw DOM reads into one extract span; an explicit page.extract() is not a read and stays separate."""+ consolidated: list[CodeActionSpan] = []+ for span in spans:+ prev = consolidated[-1] if consolidated else None+ if span.method in _READ_METHODS and prev is not None and prev.method in _READ_METHODS:+ consolidated[-1] = replace(prev, line_end=max(prev.line_end, span.line_end))+ continue+ consolidated.append(span)+ return consolidated++def derive_code_block_steps(code: str, goal: str | None = None) -> list[dict[str, Any]]:"""Derive the ordered plain-language steps for a code block from its code (deterministic)."""return [@@ -260,7 +285,7 @@ def derive_code_block_steps(code: str, goal: str | None = None) -> list[dict[str"line_start": span.line_start,"line_end": span.line_end,}- for span in analyze_code_actions(code)+ for span in _consolidate_read_spans(analyze_code_actions(code))]diff --git a/tests/unit/test_code_block_steps.py b/tests/unit/test_code_block_steps.pyindex 595bf7366..47f28b3f0 100644--- a/tests/unit/test_code_block_steps.py+++ b/tests/unit/test_code_block_steps.py@@ -388,3 +388,77 @@ def test_skyvern_page_high_level_actions_surface_as_steps():assert [s["action_type"] for s in steps] == ["select_option", "upload_file", "complete"]assert steps[0]["description"] == "Choose the country"assert steps[2]["description"] == "Confirm the form was submitted"+++def test_raw_dom_reads_surface_as_an_extraction_step_not_just_navigation():+ # The reported case: a block navigates then scrapes the DOM with raw Playwright+ # reads (no page.extract()). It must read as "navigate, then extract", not as a+ # lone "Goto URL" that hides everything the code does.+ code = (+ "async def run(page):\n"+ " await page.goto('https://example.com/')\n"+ " rows = page.locator('tr.item')\n"+ " count = await rows.count()\n"+ " results = []\n"+ " for i in range(count):\n"+ " row = rows.nth(i)\n"+ " title = await row.locator('.title').text_content()\n"+ " href = await row.locator('a').get_attribute('href')\n"+ " results.append({'title': title, 'href': href})\n"+ )+ steps = derive_code_block_steps(code)+ assert [s["action_type"] for s in steps] == ["goto_url", "extract"]+ assert steps[1]["description"] == "Extract information from the page"+++def test_consecutive_dom_reads_collapse_into_one_extraction_step():+ # A scrape reads many fields; surfacing one step per read is noise. Collapse a+ # run of adjacent reads into a single step spanning their combined line range.+ code = (+ "async def run(page):\n"+ " a = await page.locator('#a').text_content()\n"+ " b = await page.locator('#b').inner_text()\n"+ " c = await page.locator('#c').get_attribute('value')\n"+ )+ steps = derive_code_block_steps(code)+ assert [s["action_type"] for s in steps] == ["extract"]+ assert steps[0]["line_start"] == 2+ assert steps[0]["line_end"] == 4+++def test_dom_reads_separated_by_an_action_are_distinct_steps():+ code = (+ "async def run(page):\n"+ " name = await page.locator('#name').text_content()\n"+ " await page.get_by_role('button', name='Next').click()\n"+ " price = await page.locator('#price').text_content()\n"+ )+ steps = derive_code_block_steps(code)+ assert [s["action_type"] for s in steps] == ["extract", "click", "extract"]+++def test_explicit_extract_prompt_is_preserved_alongside_raw_reads():+ # page.extract() carries the author's reader-facing prompt; consolidating raw+ # reads must never merge into it and clobber that copy.+ code = (+ "async def run(page):\n"+ " data = await page.extract(prompt='Extract the product names')\n"+ " extra = await page.locator('#x').text_content()\n"+ )+ steps = derive_code_block_steps(code)+ assert [s["action_type"] for s in steps] == ["extract", "extract"]+ assert steps[0]["description"] == "Extract the product names"+ assert steps[1]["description"] == "Extract information from the page"+++def test_control_flow_reads_do_not_fabricate_an_extraction_step():+ # Visibility checks and counts gate control flow; they are not data extraction+ # and must not invent an extract step.+ code = (+ "async def run(page):\n"+ " await page.goto('https://example.com/')\n"+ " if await page.locator('#banner').is_visible():\n"+ " await page.get_by_role('button', name='Close').click()\n"+ )+ steps = derive_code_block_steps(code)+ assert [s["action_type"] for s in steps] == ["goto_url", "click"]