Skip to content

Automate browser based workflows with AI

fix(SKY-11462): surface raw DOM reads as an extraction step in code block outline (#6928)

Celal Zamanoğlu9770e8ecf4e8
diff --git a/skyvern/forge/sdk/copilot/code_block_steps.py b/skyvern/forge/sdk/copilot/code_block_steps.py
index d59f1c083..5db19ed2b 100644
--- a/skyvern/forge/sdk/copilot/code_block_steps.py
+++ b/skyvern/forge/sdk/copilot/code_block_steps.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import ast
import re
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from typing import Any, Iterator
import structlog
@@ -53,6 +53,19 @@ _METHOD_ACTION_TYPES: dict[str, str] = {
"drag": "drag",
"left_mouse": "left_mouse",
}
+# Non-state-changing DOM reads -> an extract step in the editor outline only; deliberately absent
+# from the runtime recorder maps so a read never fires a run-timeline action. Predicates, count(),
+# and all() are excluded so a control-flow check never fabricates an extraction step.
+_READ_METHODS: dict[str, str] = {
+ "text_content": "extract",
+ "all_text_contents": "extract",
+ "inner_text": "extract",
+ "all_inner_texts": "extract",
+ "inner_html": "extract",
+ "get_attribute": "extract",
+ "input_value": "extract",
+ "content": "extract",
+}
# Methods whose natural-language `prompt` is the first positional argument (it is keyword-only on the
# interaction methods, which the keyword scan below already covers).
_PROMPT_POSITIONAL_METHODS: frozenset[str] = frozenset({"extract", "complete", "solve_captcha", "verification_code"})
@@ -101,7 +114,7 @@ def analyze_code_actions(code: str) -> list[CodeActionSpan]:
method = call.func.attr
if method in _IGNORED_METHODS:
continue
- action_type = _METHOD_ACTION_TYPES.get(method)
+ action_type = _METHOD_ACTION_TYPES.get(method) or _READ_METHODS.get(method)
if action_type is None:
continue
spans.append(
@@ -251,6 +264,18 @@ def _describe(span: CodeActionSpan) -> str:
return "Run a step"
+def _consolidate_read_spans(spans: list[CodeActionSpan]) -> list[CodeActionSpan]:
+ """Merge adjacent raw DOM reads into one extract span; an explicit page.extract() is not a read and stays separate."""
+ consolidated: list[CodeActionSpan] = []
+ for span in spans:
+ prev = consolidated[-1] if consolidated else None
+ if span.method in _READ_METHODS and prev is not None and prev.method in _READ_METHODS:
+ consolidated[-1] = replace(prev, line_end=max(prev.line_end, span.line_end))
+ continue
+ consolidated.append(span)
+ return consolidated
+
+
def derive_code_block_steps(code: str, goal: str | None = None) -> list[dict[str, Any]]:
"""Derive the ordered plain-language steps for a code block from its code (deterministic)."""
return [
@@ -260,7 +285,7 @@ def derive_code_block_steps(code: str, goal: str | None = None) -> list[dict[str
"line_start": span.line_start,
"line_end": span.line_end,
}
- for span in analyze_code_actions(code)
+ for span in _consolidate_read_spans(analyze_code_actions(code))
]
diff --git a/tests/unit/test_code_block_steps.py b/tests/unit/test_code_block_steps.py
index 595bf7366..47f28b3f0 100644
--- a/tests/unit/test_code_block_steps.py
+++ b/tests/unit/test_code_block_steps.py
@@ -388,3 +388,77 @@ def test_skyvern_page_high_level_actions_surface_as_steps():
assert [s["action_type"] for s in steps] == ["select_option", "upload_file", "complete"]
assert steps[0]["description"] == "Choose the country"
assert steps[2]["description"] == "Confirm the form was submitted"
+
+
+def test_raw_dom_reads_surface_as_an_extraction_step_not_just_navigation():
+ # The reported case: a block navigates then scrapes the DOM with raw Playwright
+ # reads (no page.extract()). It must read as "navigate, then extract", not as a
+ # lone "Goto URL" that hides everything the code does.
+ code = (
+ "async def run(page):\n"
+ " await page.goto('https://example.com/')\n"
+ " rows = page.locator('tr.item')\n"
+ " count = await rows.count()\n"
+ " results = []\n"
+ " for i in range(count):\n"
+ " row = rows.nth(i)\n"
+ " title = await row.locator('.title').text_content()\n"
+ " href = await row.locator('a').get_attribute('href')\n"
+ " results.append({'title': title, 'href': href})\n"
+ )
+ steps = derive_code_block_steps(code)
+ assert [s["action_type"] for s in steps] == ["goto_url", "extract"]
+ assert steps[1]["description"] == "Extract information from the page"
+
+
+def test_consecutive_dom_reads_collapse_into_one_extraction_step():
+ # A scrape reads many fields; surfacing one step per read is noise. Collapse a
+ # run of adjacent reads into a single step spanning their combined line range.
+ code = (
+ "async def run(page):\n"
+ " a = await page.locator('#a').text_content()\n"
+ " b = await page.locator('#b').inner_text()\n"
+ " c = await page.locator('#c').get_attribute('value')\n"
+ )
+ steps = derive_code_block_steps(code)
+ assert [s["action_type"] for s in steps] == ["extract"]
+ assert steps[0]["line_start"] == 2
+ assert steps[0]["line_end"] == 4
+
+
+def test_dom_reads_separated_by_an_action_are_distinct_steps():
+ code = (
+ "async def run(page):\n"
+ " name = await page.locator('#name').text_content()\n"
+ " await page.get_by_role('button', name='Next').click()\n"
+ " price = await page.locator('#price').text_content()\n"
+ )
+ steps = derive_code_block_steps(code)
+ assert [s["action_type"] for s in steps] == ["extract", "click", "extract"]
+
+
+def test_explicit_extract_prompt_is_preserved_alongside_raw_reads():
+ # page.extract() carries the author's reader-facing prompt; consolidating raw
+ # reads must never merge into it and clobber that copy.
+ code = (
+ "async def run(page):\n"
+ " data = await page.extract(prompt='Extract the product names')\n"
+ " extra = await page.locator('#x').text_content()\n"
+ )
+ steps = derive_code_block_steps(code)
+ assert [s["action_type"] for s in steps] == ["extract", "extract"]
+ assert steps[0]["description"] == "Extract the product names"
+ assert steps[1]["description"] == "Extract information from the page"
+
+
+def test_control_flow_reads_do_not_fabricate_an_extraction_step():
+ # Visibility checks and counts gate control flow; they are not data extraction
+ # and must not invent an extract step.
+ code = (
+ "async def run(page):\n"
+ " await page.goto('https://example.com/')\n"
+ " if await page.locator('#banner').is_visible():\n"
+ " await page.get_by_role('button', name='Close').click()\n"
+ )
+ steps = derive_code_block_steps(code)
+ assert [s["action_type"] for s in steps] == ["goto_url", "click"]