Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: show all selector options to script reviewer (#5410)
pedrohsdb7c126b34fcf4
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.pyindex 4f830b89e..143425725 100644--- a/skyvern/core/script_generations/real_skyvern_page_ai.py+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py@@ -21,7 +21,7 @@ from skyvern.forge.sdk.schemas.totp_codes import OTPTypefrom skyvern.schemas.workflows import BlockStatusfrom skyvern.services import script_servicefrom skyvern.services.otp_service import poll_otp_value-from skyvern.utils.css_selector import compute_stable_selector+from skyvern.utils.css_selector import compute_selector_optionsfrom skyvern.utils.prompt_engine import load_prompt_with_elementsfrom skyvern.webeye.actions import handler_utilsfrom skyvern.webeye.actions.actions import (@@ -735,11 +735,12 @@ class RealSkyvernPageAi(SkyvernPageAi):"title",)}- # el_data already has the shape compute_stable_selector expects+ # el_data already has the shape compute_selector_options expects# (tagName, attributes dict, text) — pass it directly.- css_sel = compute_stable_selector(el_data)- if css_sel:- action_data["css_suggestion"] = css_sel+ sel_options = compute_selector_options(el_data)+ if sel_options:+ action_data["css_suggestion"] = sel_options[0][0]+ action_data["selector_options"] = sel_optionselse:# Fall back to xpath only if no CSS selector can be derivedxpath = action.get_xpath() if hasattr(action, "get_xpath") else Nonediff --git a/skyvern/forge/prompts/skyvern/script-reviewer.j2 b/skyvern/forge/prompts/skyvern/script-reviewer.j2index 03f9913a0..49067d1cb 100644--- a/skyvern/forge/prompts/skyvern/script-reviewer.j2+++ b/skyvern/forge/prompts/skyvern/script-reviewer.j2@@ -116,7 +116,12 @@ The cached code failed and the AI agent took over. Here is what happened:{%- if a.field %} [field: {{ a.field }}]{% endif %}{%- if a.element_tag %} [tag: {{ a.element_tag }}]{% endif %}{%- if a.element_text %} [text: "{{ a.element_text }}"]{% endif %}-{%- if a.css_suggestion %} **[USE THIS SELECTOR: `{{ a.css_suggestion }}`]**{% endif %}+{%- if a.selector_options and a.selector_options | length > 1 %}+ Selector options (prefer the most robust — avoid IDs that look auto-generated):+{%- for sel, desc in a.selector_options %}+ {{ loop.index }}. `{{ sel }}` — {{ desc }}+{%- endfor %}+{%- elif a.css_suggestion %} **[USE THIS SELECTOR: `{{ a.css_suggestion }}`]**{% endif %}{%- if a.all_attributes %} [attrs: {{ a.all_attributes | tojson }}]{% endif %}{%- if not a.all_attributes and a.element_attributes %} [attrs: {{ a.element_attributes | tojson }}]{% endif %}{%- if a.page_url %}{% set ns.prev_url = a.page_url %}{% endif %}@@ -127,7 +132,12 @@ The cached code failed and the AI agent took over. Here is what happened:- Action: {{ episode.agent_actions.action_type or "click" }}: {{ episode.agent_actions.intention or "N/A" }}{%- if episode.agent_actions.element_tag %} [tag: {{ episode.agent_actions.element_tag }}]{% endif %}{%- if episode.agent_actions.element_text %} [text: "{{ episode.agent_actions.element_text }}"]{% endif %}-{%- if episode.agent_actions.css_suggestion %} **[USE THIS SELECTOR: `{{ episode.agent_actions.css_suggestion }}`]**{% endif %}+{%- if episode.agent_actions.selector_options and episode.agent_actions.selector_options | length > 1 %}+ Selector options (prefer the most robust — avoid IDs that look auto-generated):+{%- for sel, desc in episode.agent_actions.selector_options %}+ {{ loop.index }}. `{{ sel }}` — {{ desc }}+{%- endfor %}+{%- elif episode.agent_actions.css_suggestion %} **[USE THIS SELECTOR: `{{ episode.agent_actions.css_suggestion }}`]**{% endif %}{%- if episode.agent_actions.all_attributes %} [attrs: {{ episode.agent_actions.all_attributes | tojson }}]{% endif %}{%- if episode.agent_actions.reasoning %}- Reasoning: {{ episode.agent_actions.reasoning }}{% endif %}diff --git a/skyvern/utils/css_selector.py b/skyvern/utils/css_selector.pyindex feaf7124d..1b00f7275 100644--- a/skyvern/utils/css_selector.py+++ b/skyvern/utils/css_selector.py@@ -17,69 +17,89 @@ if TYPE_CHECKING:def compute_stable_selector(element_data: dict | None) -> str | None:- """Derive a stable CSS selector from a scraped element dictionary.-- Priority order (highest confidence first):- 1. #id — unique by definition- 2. [data-testid="..."] — stable test attribute- 3. tag[aria-label="..."] — accessibility attribute- 4. tag[name="..."] — form element name- 5. tag[placeholder="..."] — visible hint text- 6. tag:has-text("...") — visible text content- 7. tag[role="..."] — ARIA role-- Returns None if no reliable selector can be built.+ """Derive the BEST stable CSS selector from a scraped element dictionary.++ Returns the highest-ranked selector that isn't obviously dynamic.+ For the full ranked list (shown to the reviewer), use compute_selector_options."""- if not element_data:+ options = compute_selector_options(element_data)+ if not options:return None+ # Return the top-ranked option+ return options[0][0]+++def compute_selector_options(element_data: dict | None) -> list[tuple[str, str]]:+ """Compute ALL viable CSS selectors ranked by stability.++ Returns a list of (selector, description) tuples, most stable first.+ The reviewer sees all options and picks the best one.++ Ranking (most stable first):+ 1. [data-testid="..."] — stable test attribute+ 2. tag[aria-label="..."] — accessibility attribute+ 3. tag[name="..."] — form element name+ 4. tag[placeholder="..."] — visible hint text+ 5. tag:has-text("...") — visible text content+ 6. tag[role="..."] — ARIA role+ 7. #id (stable) — unique, not auto-generated+ 8. input[type="..."] — type attribute (weak)+ 9. #id (possibly dynamic) — ID that looks auto-generated+ """+ if not element_data:+ return []tag = (element_data.get("tagName") or "").lower()attrs = element_data.get("attributes") or {}text = (element_data.get("text") or "").strip()+ options: list[tuple[str, str]] = []- # 1. ID attribute (strongest — unique per page by spec)- elem_id = attrs.get("id", "").strip()- if elem_id and not _looks_dynamic(elem_id):- return f"#{_css_escape(elem_id)}"-- # 2. data-testid (stable testing attribute)+ # data-testid (most stable — added by developers for testing)testid = attrs.get("data-testid", "").strip()if testid:- return f'[data-testid="{_css_escape_attr(testid)}"]'+ options.append((f'[data-testid="{_css_escape_attr(testid)}"]', "stable test attribute"))- # 3. aria-label + tag+ # aria-label + tagaria = attrs.get("aria-label", "").strip()if aria and tag:- return f'{tag}[aria-label="{_css_escape_attr(aria)}"]'+ options.append((f'{tag}[aria-label="{_css_escape_attr(aria)}"]', "accessibility label"))- # 4. name + tag (for form elements)+ # name + tag (for form elements)name = attrs.get("name", "").strip()if name and tag in ("input", "select", "textarea", "button"):- return f'{tag}[name="{_css_escape_attr(name)}"]'+ options.append((f'{tag}[name="{_css_escape_attr(name)}"]', "form element name"))- # 5. placeholder + tag+ # placeholder + tagplaceholder = attrs.get("placeholder", "").strip()if placeholder and tag in ("input", "textarea"):- return f'{tag}[placeholder="{_css_escape_attr(placeholder)}"]'+ options.append((f'{tag}[placeholder="{_css_escape_attr(placeholder)}"]', "placeholder text"))- # 6. Visible text content + tag (for buttons, links)+ # Visible text content + tag (for buttons, links)if text and tag in ("button", "a") and len(text) <= 50:- # Use :has-text() which is a case-insensitive substring matchclean_text = text.replace("\n", " ").replace("\r", "").strip()safe_text = _css_escape_attr(clean_text)- return f'{tag}:has-text("{safe_text}")'+ options.append((f'{tag}:has-text("{safe_text}")', "visible text"))- # 7. role + tag+ # role + tagrole = attrs.get("role", "").strip()if role and tag:- return f'{tag}[role="{_css_escape_attr(role)}"]'+ options.append((f'{tag}[role="{_css_escape_attr(role)}"]', "ARIA role"))- # 8. type + tag (for inputs — weak but better than nothing)+ # ID attribute — ranked AFTER semantic selectors+ elem_id = attrs.get("id", "").strip()+ if elem_id:+ if not _looks_dynamic(elem_id):+ options.append((f"#{_css_escape(elem_id)}", "HTML id"))+ else:+ # Still include dynamic IDs but clearly marked — let the reviewer decide+ options.append((f"#{_css_escape(elem_id)}", "HTML id (likely auto-generated — may change between runs)"))++ # Input type attribute (weak selector — many inputs share the same type)input_type = attrs.get("type", "").strip()if input_type and tag == "input" and input_type not in ("text", "hidden"):- return f'input[type="{_css_escape_attr(input_type)}"]'+ options.append((f'input[type="{_css_escape_attr(input_type)}"]', "input type"))- return None+ return optionsdef _looks_dynamic(value: str) -> bool:@@ -176,6 +196,7 @@ def build_action_summary(action: Action) -> dict:"element_text": (elem.get("text") or "")[:100] or None,"all_attributes": useful_attrs or None,"css_suggestion": compute_stable_selector(elem),+ "selector_options": compute_selector_options(elem) or None,}diff --git a/tests/unit/test_action_url.py b/tests/unit/test_action_url.pyindex b3f8271d2..4acc57b7d 100644--- a/tests/unit/test_action_url.py+++ b/tests/unit/test_action_url.py@@ -58,6 +58,31 @@ def test_build_action_summary_page_url_with_element_data():assert summary["all_attributes"]["id"] == "email"+def test_build_action_summary_includes_selector_options():+ """selector_options shows ALL viable selectors ranked by stability."""+ action = Action(+ action_type=ActionType.INPUT_TEXT,+ status="completed",+ intention="Fill passcode",+ skyvern_element_data={+ "tagName": "input",+ "attributes": {"id": "input61", "name": "credentials.passcode", "type": "password"},+ },+ )+ summary = build_action_summary(action)+ assert summary["selector_options"] is not None+ selectors = [s for s, _ in summary["selector_options"]]+ assert 'input[name="credentials.passcode"]' in selectors+ assert "#input61" in selectors+++def test_build_action_summary_selector_options_none_when_no_element():+ """selector_options is None when no element data."""+ action = _make_action()+ summary = build_action_summary(action)+ assert summary["selector_options"] is None++def test_template_url_rendering():"""The reviewer template renders [url: ...] and [url changed: ...] correctly."""from skyvern.forge.prompts import prompt_enginediff --git a/tests/unit/test_css_selector.py b/tests/unit/test_css_selector.pyindex 83a51a0ec..3c3766b39 100644--- a/tests/unit/test_css_selector.py+++ b/tests/unit/test_css_selector.py@@ -1,6 +1,6 @@-"""Tests for compute_stable_selector and _looks_dynamic."""+"""Tests for compute_stable_selector, compute_selector_options, and _looks_dynamic."""-from skyvern.utils.css_selector import _looks_dynamic, compute_stable_selector+from skyvern.utils.css_selector import _looks_dynamic, compute_selector_options, compute_stable_selectorclass TestLooksDynamic:@@ -38,76 +38,147 @@ class TestLooksDynamic:class TestComputeStableSelector:- """Tests for compute_stable_selector priority and output."""+ """Tests for compute_stable_selector — returns top-ranked option."""- def test_prefers_text_over_dynamic_id(self):- """When ID is dynamic and text is available, use :has-text()."""+ def test_prefers_name_over_dynamic_id(self):+ """When ID is dynamic and name is available, use name."""elem = {- "tagName": "button",- "text": "Apply Filters",- "attributes": {"id": "ember12345678", "type": "submit"},+ "tagName": "input",+ "text": "",+ "attributes": {"id": "input61", "name": "credentials.passcode", "type": "password"},}result = compute_stable_selector(elem)- assert result == 'button:has-text("Apply Filters")'+ assert result == 'input[name="credentials.passcode"]'- def test_static_id_preferred_over_text(self):- """When ID looks stable, use it (it's higher priority)."""+ def test_static_id_still_returned_when_no_semantic(self):+ """When only a stable ID is available, it's returned."""elem = {- "tagName": "button",- "text": "Submit",+ "tagName": "div",+ "text": "","attributes": {"id": "signOnButton"},}result = compute_stable_selector(elem)assert result == "#signOnButton"- def test_aria_label_preferred_over_text(self):+ def test_aria_label_ranked_above_id(self):+ """aria-label is preferred over even stable IDs."""elem = {"tagName": "input","text": "",- "attributes": {"aria-label": "Email Address", "id": "uid-12345"},+ "attributes": {"aria-label": "Email Address", "id": "email-field"},}result = compute_stable_selector(elem)assert result == 'input[aria-label="Email Address"]'- def test_realistic_scraper_shape(self):- """Scraper output shape (from domUtils.js buildElementObject) should work directly.+ def test_no_data_returns_none(self):+ assert compute_stable_selector(None) is None+ assert compute_stable_selector({}) is None+++class TestComputeSelectorOptions:+ """Tests for compute_selector_options — returns ALL viable selectors ranked."""- el_data has top-level 'id' (Skyvern uniqueId), 'tagName', 'text',- and nested 'attributes' with real HTML attrs. The scraper stores- unique_id as a separate attribute, not as 'id'.- """- el_data = {- "id": "AAGD", # Skyvern unique_id — NOT the HTML id+ def test_returns_all_options_ranked(self):+ """Element with multiple attributes produces multiple options."""+ elem = {+ "tagName": "input",+ "text": "",+ "attributes": {+ "name": "credentials.passcode",+ "id": "input61",+ "placeholder": "Enter passcode",+ "type": "password",+ },+ }+ options = compute_selector_options(elem)+ selectors = [s for s, _ in options]++ # name should be ranked above the ID (input61 is not caught by _looks_dynamic+ # but name is ranked higher than IDs regardless)+ assert 'input[name="credentials.passcode"]' in selectors+ assert "#input61" in selectors+ assert 'input[placeholder="Enter passcode"]' in selectors+ assert 'input[type="password"]' in selectors++ # name should come before ID in the ranking+ name_idx = selectors.index('input[name="credentials.passcode"]')+ id_idx = selectors.index("#input61")+ assert name_idx < id_idx++ def test_dynamic_id_marked_as_possibly_generated(self):+ """IDs that look auto-generated are included but flagged."""+ elem = {"tagName": "button",- "attributes": {"unique_id": "AAGD", "type": "submit"},- "text": "Apply Filters",- "xpath": "/html/body/div/button",+ "text": "Submit",+ "attributes": {"id": "ember12345678", "type": "submit"},}- result = compute_stable_selector(el_data)- assert result == 'button:has-text("Apply Filters")'+ options = compute_selector_options(elem)+ id_options = [(s, d) for s, d in options if s == "#ember12345678"]+ assert len(id_options) == 1+ assert "auto-generated" in id_options[0][1]- def test_realistic_scraper_shape_with_aria_label(self):- """Scraper shape with aria-label should produce aria-label selector."""- el_data = {- "id": "AABC",+ def test_stable_id_not_marked_dynamic(self):+ """Stable IDs get a clean description."""+ elem = {+ "tagName": "button",+ "text": "Sign On",+ "attributes": {"id": "signOnButton"},+ }+ options = compute_selector_options(elem)+ id_options = [(s, d) for s, d in options if s == "#signOnButton"]+ assert len(id_options) == 1+ assert "auto-generated" not in id_options[0][1]+ assert id_options[0][1] == "HTML id"++ def test_data_testid_ranked_first(self):+ """data-testid is the most stable selector."""+ elem = {"tagName": "input",- "attributes": {"unique_id": "AABC", "aria-label": "Email Address", "type": "email"},"text": "",+ "attributes": {"data-testid": "email-input", "name": "email", "id": "field-1"},}- result = compute_stable_selector(el_data)- assert result == 'input[aria-label="Email Address"]'+ options = compute_selector_options(elem)+ assert options[0] == ('[data-testid="email-input"]', "stable test attribute")++ def test_empty_element_returns_empty(self):+ assert compute_selector_options(None) == []+ assert compute_selector_options({}) == []++ def test_single_option_element(self):+ """Element with only one viable attribute produces one option."""+ elem = {+ "tagName": "div",+ "text": "",+ "attributes": {"role": "dialog"},+ }+ options = compute_selector_options(elem)+ assert len(options) == 1+ assert options[0] == ('div[role="dialog"]', "ARIA role")- def test_realistic_scraper_shape_with_stable_html_id(self):- """When the HTML id is stable (same as Skyvern id only by coincidence), use it."""- el_data = {- "id": "AAGD",+ def test_button_text_included(self):+ """Buttons with short text get a :has-text() option."""+ elem = {"tagName": "button",- "attributes": {"id": "signOnButton", "type": "submit"},- "text": "Sign On",+ "text": "Apply Filters",+ "attributes": {},}- result = compute_stable_selector(el_data)- assert result == "#signOnButton"+ options = compute_selector_options(elem)+ assert ('button:has-text("Apply Filters")', "visible text") in options- def test_no_data_returns_none(self):- assert compute_stable_selector(None) is None- assert compute_stable_selector({}) is None+ def test_input61_case(self):+ """The exact case that motivated this change — input61 with a name attribute."""+ elem = {+ "tagName": "input",+ "text": "",+ "attributes": {+ "id": "input61",+ "name": "credentials.passcode",+ "type": "password",+ },+ }+ options = compute_selector_options(elem)+ # Should have at least name, id, and type options+ assert len(options) >= 3+ # Name should be #1 (ranked above ID)+ assert options[0][0] == 'input[name="credentials.passcode"]'+ assert options[0][1] == "form element name"