Skip to content

Automate browser based workflows with AI

feat: show all selector options to script reviewer (#5410)

pedrohsdb7c126b34fcf4
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.py
index 4f830b89e..143425725 100644
--- a/skyvern/core/script_generations/real_skyvern_page_ai.py
+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py
@@ -21,7 +21,7 @@ from skyvern.forge.sdk.schemas.totp_codes import OTPType
from skyvern.schemas.workflows import BlockStatus
from skyvern.services import script_service
from skyvern.services.otp_service import poll_otp_value
-from skyvern.utils.css_selector import compute_stable_selector
+from skyvern.utils.css_selector import compute_selector_options
from skyvern.utils.prompt_engine import load_prompt_with_elements
from skyvern.webeye.actions import handler_utils
from skyvern.webeye.actions.actions import (
@@ -735,11 +735,12 @@ class RealSkyvernPageAi(SkyvernPageAi):
"title",
)
}
- # el_data already has the shape compute_stable_selector expects
+ # el_data already has the shape compute_selector_options expects
# (tagName, attributes dict, text) — pass it directly.
- css_sel = compute_stable_selector(el_data)
- if css_sel:
- action_data["css_suggestion"] = css_sel
+ sel_options = compute_selector_options(el_data)
+ if sel_options:
+ action_data["css_suggestion"] = sel_options[0][0]
+ action_data["selector_options"] = sel_options
else:
# Fall back to xpath only if no CSS selector can be derived
xpath = action.get_xpath() if hasattr(action, "get_xpath") else None
diff --git a/skyvern/forge/prompts/skyvern/script-reviewer.j2 b/skyvern/forge/prompts/skyvern/script-reviewer.j2
index 03f9913a0..49067d1cb 100644
--- a/skyvern/forge/prompts/skyvern/script-reviewer.j2
+++ b/skyvern/forge/prompts/skyvern/script-reviewer.j2
@@ -116,7 +116,12 @@ The cached code failed and the AI agent took over. Here is what happened:
{%- if a.field %} [field: {{ a.field }}]{% endif %}
{%- if a.element_tag %} [tag: {{ a.element_tag }}]{% endif %}
{%- if a.element_text %} [text: "{{ a.element_text }}"]{% endif %}
-{%- if a.css_suggestion %} **[USE THIS SELECTOR: `{{ a.css_suggestion }}`]**{% endif %}
+{%- if a.selector_options and a.selector_options | length > 1 %}
+ Selector options (prefer the most robust — avoid IDs that look auto-generated):
+{%- for sel, desc in a.selector_options %}
+ {{ loop.index }}. `{{ sel }}` — {{ desc }}
+{%- endfor %}
+{%- elif a.css_suggestion %} **[USE THIS SELECTOR: `{{ a.css_suggestion }}`]**{% endif %}
{%- if a.all_attributes %} [attrs: {{ a.all_attributes | tojson }}]{% endif %}
{%- if not a.all_attributes and a.element_attributes %} [attrs: {{ a.element_attributes | tojson }}]{% endif %}
{%- if a.page_url %}{% set ns.prev_url = a.page_url %}{% endif %}
@@ -127,7 +132,12 @@ The cached code failed and the AI agent took over. Here is what happened:
- Action: {{ episode.agent_actions.action_type or "click" }}: {{ episode.agent_actions.intention or "N/A" }}
{%- if episode.agent_actions.element_tag %} [tag: {{ episode.agent_actions.element_tag }}]{% endif %}
{%- if episode.agent_actions.element_text %} [text: "{{ episode.agent_actions.element_text }}"]{% endif %}
-{%- if episode.agent_actions.css_suggestion %} **[USE THIS SELECTOR: `{{ episode.agent_actions.css_suggestion }}`]**{% endif %}
+{%- if episode.agent_actions.selector_options and episode.agent_actions.selector_options | length > 1 %}
+ Selector options (prefer the most robust — avoid IDs that look auto-generated):
+{%- for sel, desc in episode.agent_actions.selector_options %}
+ {{ loop.index }}. `{{ sel }}` — {{ desc }}
+{%- endfor %}
+{%- elif episode.agent_actions.css_suggestion %} **[USE THIS SELECTOR: `{{ episode.agent_actions.css_suggestion }}`]**{% endif %}
{%- if episode.agent_actions.all_attributes %} [attrs: {{ episode.agent_actions.all_attributes | tojson }}]{% endif %}
{%- if episode.agent_actions.reasoning %}
- Reasoning: {{ episode.agent_actions.reasoning }}{% endif %}
diff --git a/skyvern/utils/css_selector.py b/skyvern/utils/css_selector.py
index feaf7124d..1b00f7275 100644
--- a/skyvern/utils/css_selector.py
+++ b/skyvern/utils/css_selector.py
@@ -17,69 +17,89 @@ if TYPE_CHECKING:
def compute_stable_selector(element_data: dict | None) -> str | None:
- """Derive a stable CSS selector from a scraped element dictionary.
-
- Priority order (highest confidence first):
- 1. #id — unique by definition
- 2. [data-testid="..."] — stable test attribute
- 3. tag[aria-label="..."] — accessibility attribute
- 4. tag[name="..."] — form element name
- 5. tag[placeholder="..."] — visible hint text
- 6. tag:has-text("...") — visible text content
- 7. tag[role="..."] — ARIA role
-
- Returns None if no reliable selector can be built.
+ """Derive the BEST stable CSS selector from a scraped element dictionary.
+
+ Returns the highest-ranked selector that isn't obviously dynamic.
+ For the full ranked list (shown to the reviewer), use compute_selector_options.
"""
- if not element_data:
+ options = compute_selector_options(element_data)
+ if not options:
return None
+ # Return the top-ranked option
+ return options[0][0]
+
+
+def compute_selector_options(element_data: dict | None) -> list[tuple[str, str]]:
+ """Compute ALL viable CSS selectors ranked by stability.
+
+ Returns a list of (selector, description) tuples, most stable first.
+ The reviewer sees all options and picks the best one.
+
+ Ranking (most stable first):
+ 1. [data-testid="..."] — stable test attribute
+ 2. tag[aria-label="..."] — accessibility attribute
+ 3. tag[name="..."] — form element name
+ 4. tag[placeholder="..."] — visible hint text
+ 5. tag:has-text("...") — visible text content
+ 6. tag[role="..."] — ARIA role
+ 7. #id (stable) — unique, not auto-generated
+ 8. input[type="..."] — type attribute (weak)
+ 9. #id (possibly dynamic) — ID that looks auto-generated
+ """
+ if not element_data:
+ return []
tag = (element_data.get("tagName") or "").lower()
attrs = element_data.get("attributes") or {}
text = (element_data.get("text") or "").strip()
+ options: list[tuple[str, str]] = []
- # 1. ID attribute (strongest — unique per page by spec)
- elem_id = attrs.get("id", "").strip()
- if elem_id and not _looks_dynamic(elem_id):
- return f"#{_css_escape(elem_id)}"
-
- # 2. data-testid (stable testing attribute)
+ # data-testid (most stable — added by developers for testing)
testid = attrs.get("data-testid", "").strip()
if testid:
- return f'[data-testid="{_css_escape_attr(testid)}"]'
+ options.append((f'[data-testid="{_css_escape_attr(testid)}"]', "stable test attribute"))
- # 3. aria-label + tag
+ # aria-label + tag
aria = attrs.get("aria-label", "").strip()
if aria and tag:
- return f'{tag}[aria-label="{_css_escape_attr(aria)}"]'
+ options.append((f'{tag}[aria-label="{_css_escape_attr(aria)}"]', "accessibility label"))
- # 4. name + tag (for form elements)
+ # name + tag (for form elements)
name = attrs.get("name", "").strip()
if name and tag in ("input", "select", "textarea", "button"):
- return f'{tag}[name="{_css_escape_attr(name)}"]'
+ options.append((f'{tag}[name="{_css_escape_attr(name)}"]', "form element name"))
- # 5. placeholder + tag
+ # placeholder + tag
placeholder = attrs.get("placeholder", "").strip()
if placeholder and tag in ("input", "textarea"):
- return f'{tag}[placeholder="{_css_escape_attr(placeholder)}"]'
+ options.append((f'{tag}[placeholder="{_css_escape_attr(placeholder)}"]', "placeholder text"))
- # 6. Visible text content + tag (for buttons, links)
+ # Visible text content + tag (for buttons, links)
if text and tag in ("button", "a") and len(text) <= 50:
- # Use :has-text() which is a case-insensitive substring match
clean_text = text.replace("\n", " ").replace("\r", "").strip()
safe_text = _css_escape_attr(clean_text)
- return f'{tag}:has-text("{safe_text}")'
+ options.append((f'{tag}:has-text("{safe_text}")', "visible text"))
- # 7. role + tag
+ # role + tag
role = attrs.get("role", "").strip()
if role and tag:
- return f'{tag}[role="{_css_escape_attr(role)}"]'
+ options.append((f'{tag}[role="{_css_escape_attr(role)}"]', "ARIA role"))
- # 8. type + tag (for inputs — weak but better than nothing)
+ # ID attribute — ranked AFTER semantic selectors
+ elem_id = attrs.get("id", "").strip()
+ if elem_id:
+ if not _looks_dynamic(elem_id):
+ options.append((f"#{_css_escape(elem_id)}", "HTML id"))
+ else:
+ # Still include dynamic IDs but clearly marked — let the reviewer decide
+ options.append((f"#{_css_escape(elem_id)}", "HTML id (likely auto-generated — may change between runs)"))
+
+ # Input type attribute (weak selector — many inputs share the same type)
input_type = attrs.get("type", "").strip()
if input_type and tag == "input" and input_type not in ("text", "hidden"):
- return f'input[type="{_css_escape_attr(input_type)}"]'
+ options.append((f'input[type="{_css_escape_attr(input_type)}"]', "input type"))
- return None
+ return options
def _looks_dynamic(value: str) -> bool:
@@ -176,6 +196,7 @@ def build_action_summary(action: Action) -> dict:
"element_text": (elem.get("text") or "")[:100] or None,
"all_attributes": useful_attrs or None,
"css_suggestion": compute_stable_selector(elem),
+ "selector_options": compute_selector_options(elem) or None,
}
diff --git a/tests/unit/test_action_url.py b/tests/unit/test_action_url.py
index b3f8271d2..4acc57b7d 100644
--- a/tests/unit/test_action_url.py
+++ b/tests/unit/test_action_url.py
@@ -58,6 +58,31 @@ def test_build_action_summary_page_url_with_element_data():
assert summary["all_attributes"]["id"] == "email"
+def test_build_action_summary_includes_selector_options():
+ """selector_options shows ALL viable selectors ranked by stability."""
+ action = Action(
+ action_type=ActionType.INPUT_TEXT,
+ status="completed",
+ intention="Fill passcode",
+ skyvern_element_data={
+ "tagName": "input",
+ "attributes": {"id": "input61", "name": "credentials.passcode", "type": "password"},
+ },
+ )
+ summary = build_action_summary(action)
+ assert summary["selector_options"] is not None
+ selectors = [s for s, _ in summary["selector_options"]]
+ assert 'input[name="credentials.passcode"]' in selectors
+ assert "#input61" in selectors
+
+
+def test_build_action_summary_selector_options_none_when_no_element():
+ """selector_options is None when no element data."""
+ action = _make_action()
+ summary = build_action_summary(action)
+ assert summary["selector_options"] is None
+
+
def test_template_url_rendering():
"""The reviewer template renders [url: ...] and [url changed: ...] correctly."""
from skyvern.forge.prompts import prompt_engine
diff --git a/tests/unit/test_css_selector.py b/tests/unit/test_css_selector.py
index 83a51a0ec..3c3766b39 100644
--- a/tests/unit/test_css_selector.py
+++ b/tests/unit/test_css_selector.py
@@ -1,6 +1,6 @@
-"""Tests for compute_stable_selector and _looks_dynamic."""
+"""Tests for compute_stable_selector, compute_selector_options, and _looks_dynamic."""
-from skyvern.utils.css_selector import _looks_dynamic, compute_stable_selector
+from skyvern.utils.css_selector import _looks_dynamic, compute_selector_options, compute_stable_selector
class TestLooksDynamic:
@@ -38,76 +38,147 @@ class TestLooksDynamic:
class TestComputeStableSelector:
- """Tests for compute_stable_selector priority and output."""
+ """Tests for compute_stable_selector — returns top-ranked option."""
- def test_prefers_text_over_dynamic_id(self):
- """When ID is dynamic and text is available, use :has-text()."""
+ def test_prefers_name_over_dynamic_id(self):
+ """When ID is dynamic and name is available, use name."""
elem = {
- "tagName": "button",
- "text": "Apply Filters",
- "attributes": {"id": "ember12345678", "type": "submit"},
+ "tagName": "input",
+ "text": "",
+ "attributes": {"id": "input61", "name": "credentials.passcode", "type": "password"},
}
result = compute_stable_selector(elem)
- assert result == 'button:has-text("Apply Filters")'
+ assert result == 'input[name="credentials.passcode"]'
- def test_static_id_preferred_over_text(self):
- """When ID looks stable, use it (it's higher priority)."""
+ def test_static_id_still_returned_when_no_semantic(self):
+ """When only a stable ID is available, it's returned."""
elem = {
- "tagName": "button",
- "text": "Submit",
+ "tagName": "div",
+ "text": "",
"attributes": {"id": "signOnButton"},
}
result = compute_stable_selector(elem)
assert result == "#signOnButton"
- def test_aria_label_preferred_over_text(self):
+ def test_aria_label_ranked_above_id(self):
+ """aria-label is preferred over even stable IDs."""
elem = {
"tagName": "input",
"text": "",
- "attributes": {"aria-label": "Email Address", "id": "uid-12345"},
+ "attributes": {"aria-label": "Email Address", "id": "email-field"},
}
result = compute_stable_selector(elem)
assert result == 'input[aria-label="Email Address"]'
- def test_realistic_scraper_shape(self):
- """Scraper output shape (from domUtils.js buildElementObject) should work directly.
+ def test_no_data_returns_none(self):
+ assert compute_stable_selector(None) is None
+ assert compute_stable_selector({}) is None
+
+
+class TestComputeSelectorOptions:
+ """Tests for compute_selector_options — returns ALL viable selectors ranked."""
- el_data has top-level 'id' (Skyvern uniqueId), 'tagName', 'text',
- and nested 'attributes' with real HTML attrs. The scraper stores
- unique_id as a separate attribute, not as 'id'.
- """
- el_data = {
- "id": "AAGD", # Skyvern unique_id — NOT the HTML id
+ def test_returns_all_options_ranked(self):
+ """Element with multiple attributes produces multiple options."""
+ elem = {
+ "tagName": "input",
+ "text": "",
+ "attributes": {
+ "name": "credentials.passcode",
+ "id": "input61",
+ "placeholder": "Enter passcode",
+ "type": "password",
+ },
+ }
+ options = compute_selector_options(elem)
+ selectors = [s for s, _ in options]
+
+ # name should be ranked above the ID (input61 is not caught by _looks_dynamic
+ # but name is ranked higher than IDs regardless)
+ assert 'input[name="credentials.passcode"]' in selectors
+ assert "#input61" in selectors
+ assert 'input[placeholder="Enter passcode"]' in selectors
+ assert 'input[type="password"]' in selectors
+
+ # name should come before ID in the ranking
+ name_idx = selectors.index('input[name="credentials.passcode"]')
+ id_idx = selectors.index("#input61")
+ assert name_idx < id_idx
+
+ def test_dynamic_id_marked_as_possibly_generated(self):
+ """IDs that look auto-generated are included but flagged."""
+ elem = {
"tagName": "button",
- "attributes": {"unique_id": "AAGD", "type": "submit"},
- "text": "Apply Filters",
- "xpath": "/html/body/div/button",
+ "text": "Submit",
+ "attributes": {"id": "ember12345678", "type": "submit"},
}
- result = compute_stable_selector(el_data)
- assert result == 'button:has-text("Apply Filters")'
+ options = compute_selector_options(elem)
+ id_options = [(s, d) for s, d in options if s == "#ember12345678"]
+ assert len(id_options) == 1
+ assert "auto-generated" in id_options[0][1]
- def test_realistic_scraper_shape_with_aria_label(self):
- """Scraper shape with aria-label should produce aria-label selector."""
- el_data = {
- "id": "AABC",
+ def test_stable_id_not_marked_dynamic(self):
+ """Stable IDs get a clean description."""
+ elem = {
+ "tagName": "button",
+ "text": "Sign On",
+ "attributes": {"id": "signOnButton"},
+ }
+ options = compute_selector_options(elem)
+ id_options = [(s, d) for s, d in options if s == "#signOnButton"]
+ assert len(id_options) == 1
+ assert "auto-generated" not in id_options[0][1]
+ assert id_options[0][1] == "HTML id"
+
+ def test_data_testid_ranked_first(self):
+ """data-testid is the most stable selector."""
+ elem = {
"tagName": "input",
- "attributes": {"unique_id": "AABC", "aria-label": "Email Address", "type": "email"},
"text": "",
+ "attributes": {"data-testid": "email-input", "name": "email", "id": "field-1"},
}
- result = compute_stable_selector(el_data)
- assert result == 'input[aria-label="Email Address"]'
+ options = compute_selector_options(elem)
+ assert options[0] == ('[data-testid="email-input"]', "stable test attribute")
+
+ def test_empty_element_returns_empty(self):
+ assert compute_selector_options(None) == []
+ assert compute_selector_options({}) == []
+
+ def test_single_option_element(self):
+ """Element with only one viable attribute produces one option."""
+ elem = {
+ "tagName": "div",
+ "text": "",
+ "attributes": {"role": "dialog"},
+ }
+ options = compute_selector_options(elem)
+ assert len(options) == 1
+ assert options[0] == ('div[role="dialog"]', "ARIA role")
- def test_realistic_scraper_shape_with_stable_html_id(self):
- """When the HTML id is stable (same as Skyvern id only by coincidence), use it."""
- el_data = {
- "id": "AAGD",
+ def test_button_text_included(self):
+ """Buttons with short text get a :has-text() option."""
+ elem = {
"tagName": "button",
- "attributes": {"id": "signOnButton", "type": "submit"},
- "text": "Sign On",
+ "text": "Apply Filters",
+ "attributes": {},
}
- result = compute_stable_selector(el_data)
- assert result == "#signOnButton"
+ options = compute_selector_options(elem)
+ assert ('button:has-text("Apply Filters")', "visible text") in options
- def test_no_data_returns_none(self):
- assert compute_stable_selector(None) is None
- assert compute_stable_selector({}) is None
+ def test_input61_case(self):
+ """The exact case that motivated this change — input61 with a name attribute."""
+ elem = {
+ "tagName": "input",
+ "text": "",
+ "attributes": {
+ "id": "input61",
+ "name": "credentials.passcode",
+ "type": "password",
+ },
+ }
+ options = compute_selector_options(elem)
+ # Should have at least name, id, and type options
+ assert len(options) >= 3
+ # Name should be #1 (ranked above ID)
+ assert options[0][0] == 'input[name="credentials.passcode"]'
+ assert options[0][1] == "form element name"