Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(mcp): disambiguate ref selectors for unnamed duplicate widgets (#SKY-9701) (#5941)
Aaron Perezc2dc316c3a3f
diff --git a/skyvern/cli/core/browser_ops.py b/skyvern/cli/core/browser_ops.pyindex ff68c0096..1585f8fd1 100644--- a/skyvern/cli/core/browser_ops.py+++ b/skyvern/cli/core/browser_ops.py@@ -577,6 +577,8 @@ class ObservedElement:tag: strvalue: str | None = Noneoptions: list[str] | None = None+ match_index: int = 0+ needs_disambiguation: bool = False@dataclass@@ -635,9 +637,19 @@ async def do_observe(all_elements = [e for e in all_elements if e.get("role") in INTERACTIVE_ROLES]total = len(all_elements)++ # Compute group sizes against the FULL filtered list (pre-cap) so that+ # kept elements colliding with off-cap siblings still get disambiguated.+ full_group_size: dict[tuple[str, str], int] = {}+ for elem in all_elements:+ full_group_size[(elem.get("role", ""), elem.get("name", ""))] = (+ full_group_size.get((elem.get("role", ""), elem.get("name", "")), 0) + 1+ )+capped = all_elements[:max_elements]observed: list[ObservedElement] = []+ group_counts: dict[tuple[str, str], int] = {}for i, elem in enumerate(capped):role = elem.get("role", "")name = elem.get("name", "")@@ -647,6 +659,10 @@ async def do_observe(if value and _is_password_field(role, name):value = "***"+ key = (role, name)+ match_index = group_counts.get(key, 0)+ group_counts[key] = match_index + 1+observed.append(ObservedElement(ref=f"e{i}",@@ -655,6 +671,8 @@ async def do_observe(tag=_ROLE_TO_TAG.get(role, ""),value=value,options=_extract_options(elem),+ match_index=match_index,+ needs_disambiguation=full_group_size[key] > 1,))@@ -669,21 +687,20 @@ async def do_observe(def serialize_elements(elements: list[ObservedElement]) -> list[dict[str, Any]]:"""Serialize observed elements to dicts, filtering empty display fields."""- return [- {- k: v- for k, v in {- "ref": e.ref,- "role": e.role,- "name": e.name,- "tag": e.tag,- "value": e.value,- "options": e.options,- }.items()- if k in _ELEMENT_KEEP_ALWAYS or (v is not None and v != "")+ result: list[dict[str, Any]] = []+ for e in elements:+ fields: dict[str, Any] = {+ "ref": e.ref,+ "role": e.role,+ "name": e.name,+ "tag": e.tag,+ "value": e.value,+ "options": e.options,}- for e in elements- ]+ if e.needs_disambiguation:+ fields["match_index"] = e.match_index+ result.append({k: v for k, v in fields.items() if k in _ELEMENT_KEEP_ALWAYS or (v is not None and v != "")})+ return resultdef ref_to_selector(elem: dict[str, Any]) -> str:@@ -692,8 +709,12 @@ def ref_to_selector(elem: dict[str, Any]) -> str:name = elem.get("name", "")if name:escaped = name.replace('"', '\\"')- return f'role={role}[name="{escaped}"]'- return f"role={role}"+ base = f'role={role}[name="{escaped}"]'+ else:+ base = f"role={role}"+ if "match_index" in elem:+ return f"{base} >> nth={elem['match_index']}"+ return base# ---------------------------------------------------------------------------diff --git a/tests/unit/test_mcp_observe_execute.py b/tests/unit/test_mcp_observe_execute.pyindex a63989a67..c1d045936 100644--- a/tests/unit/test_mcp_observe_execute.py+++ b/tests/unit/test_mcp_observe_execute.py@@ -16,6 +16,7 @@ from skyvern.cli.core.browser_ops import (do_execute,do_observe,ref_to_selector,+ serialize_elements,)from skyvern.cli.core.result import BrowserContextfrom skyvern.cli.mcp_tools import browser as mcp_browser@@ -138,6 +139,22 @@ class TestRefToSelector:result = ref_to_selector({"role": "button", "name": 'Click "here"'})assert result == 'role=button[name="Click \\"here\\""]'+ def test_match_index_zero_emits_nth(self) -> None:+ """Presence of match_index in the dict signals a duplicate group; emit nth even for 0."""+ elem = {"role": "combobox", "name": "", "match_index": 0}+ assert ref_to_selector(elem) == "role=combobox >> nth=0"++ def test_unnamed_duplicate_appends_nth(self) -> None:+ elem = {"role": "combobox", "name": "", "match_index": 2}+ assert ref_to_selector(elem) == "role=combobox >> nth=2"++ def test_named_duplicate_appends_nth(self) -> None:+ elem = {"role": "button", "name": "Edit", "match_index": 1}+ assert ref_to_selector(elem) == 'role=button[name="Edit"] >> nth=1'++ def test_missing_match_index_is_backward_compatible(self) -> None:+ assert ref_to_selector({"role": "textbox", "name": "Email"}) == 'role=textbox[name="Email"]'+# ---------------------------------------------------------------------------# Unit tests: do_observe@@ -242,6 +259,108 @@ class TestDoObserve:assert tags["Sign In"] == "button"assert tags["Forgot password?"] == "a"+ @pytest.mark.asyncio+ async def test_unique_elements_have_match_index_zero(self) -> None:+ page = _make_page()+ result = await do_observe(page)+ assert all(e.match_index == 0 for e in result.elements)++ @pytest.mark.asyncio+ async def test_unnamed_duplicates_get_distinct_match_indices(self) -> None:+ """SKY-9701: multiple unnamed widgets of the same role must each get a unique index."""+ tree = _make_a11y_tree(+ children=[+ {"role": "combobox", "name": ""},+ {"role": "combobox", "name": ""},+ {"role": "combobox", "name": ""},+ {"role": "combobox", "name": ""},+ ]+ )+ page = _make_page(tree)+ result = await do_observe(page)+ assert [e.match_index for e in result.elements] == [0, 1, 2, 3]++ @pytest.mark.asyncio+ async def test_named_duplicates_get_distinct_match_indices(self) -> None:+ """Elements sharing both role and name also collide; disambiguate by index."""+ tree = _make_a11y_tree(+ children=[+ {"role": "button", "name": "Edit"},+ {"role": "button", "name": "Delete"},+ {"role": "button", "name": "Edit"},+ ]+ )+ page = _make_page(tree)+ result = await do_observe(page)+ grouped = [(e.role, e.name, e.match_index) for e in result.elements]+ assert grouped == [+ ("button", "Edit", 0),+ ("button", "Delete", 0),+ ("button", "Edit", 1),+ ]++ @pytest.mark.asyncio+ async def test_serialized_output_includes_match_index_for_every_member_of_duplicate_group(self) -> None:+ """Every member of a multi-element (role, name) group carries match_index, including the first."""+ tree = _make_a11y_tree(+ children=[+ {"role": "combobox", "name": ""},+ {"role": "combobox", "name": ""},+ ]+ )+ page = _make_page(tree)+ result = await do_observe(page)+ serialized = serialize_elements(result.elements)+ assert serialized[0]["match_index"] == 0+ assert serialized[1]["match_index"] == 1++ @pytest.mark.asyncio+ async def test_serialized_output_omits_match_index_for_unique_elements(self) -> None:+ page = _make_page()+ result = await do_observe(page)+ serialized = serialize_elements(result.elements)+ assert all("match_index" not in d for d in serialized)++ @pytest.mark.asyncio+ async def test_disambiguation_counts_collisions_beyond_max_elements_cap(self) -> None:+ """SKY-9701 follow-up: duplicates beyond the cap must still trigger disambiguation+ for kept refs, otherwise selector-only clicks on capped duplicates fall back to the+ ambiguous base selector."""+ children = [{"role": "combobox", "name": ""} for _ in range(6)]+ page = _make_page(_make_a11y_tree(children=children))+ result = await do_observe(page, max_elements=2)+ assert result.element_count == 2+ assert result.total_on_page == 6+ serialized = serialize_elements(result.elements)+ # Both kept refs must carry match_index so the selector emits nth=N,+ # even though only 2 of 6 are returned.+ assert serialized[0]["match_index"] == 0+ assert serialized[1]["match_index"] == 1+ selectors = [ref_to_selector(elem) for elem in serialized]+ assert selectors == ["role=combobox >> nth=0", "role=combobox >> nth=1"]++ @pytest.mark.asyncio+ async def test_unnamed_duplicate_resolves_to_distinct_selectors(self) -> None:+ """End-to-end: observe -> serialize -> ref_to_selector produces unique selectors for every ref, including e0."""+ tree = _make_a11y_tree(+ children=[+ {"role": "combobox", "name": ""},+ {"role": "combobox", "name": ""},+ {"role": "combobox", "name": ""},+ {"role": "combobox", "name": ""},+ ]+ )+ page = _make_page(tree)+ result = await do_observe(page)+ serialized = serialize_elements(result.elements)+ selectors = [ref_to_selector(elem) for elem in serialized]+ assert selectors == [+ "role=combobox >> nth=0",+ "role=combobox >> nth=1",+ "role=combobox >> nth=2",+ "role=combobox >> nth=3",+ ]+# ---------------------------------------------------------------------------# Unit tests: do_execute@@ -505,3 +624,59 @@ class TestSkyvernExecuteMCP:result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e99"}}])assert result["ok"] is Falseassert "unknown ref" in result["data"]["results"][0]["error"].lower()++ @pytest.mark.asyncio+ async def test_execute_ref_to_unnamed_duplicate_uses_nth(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """SKY-9701: clicking ref=e2 on a page with 4 unnamed comboboxes resolves to the 3rd, not the 1st."""+ tree = _make_a11y_tree(+ children=[+ {"role": "combobox", "name": ""},+ {"role": "combobox", "name": ""},+ {"role": "combobox", "name": ""},+ {"role": "combobox", "name": ""},+ ]+ )+ page = _make_page(tree)+ ctx = BrowserContext(mode="local")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ click_result = {"ok": True, "data": None}+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))++ result = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "click", "params": {"ref": "e2"}},+ ]+ )+ assert result["ok"] is True+ click_kwargs = mcp_browser.skyvern_click.call_args.kwargs+ assert click_kwargs["selector"] == "role=combobox >> nth=2"++ @pytest.mark.asyncio+ async def test_execute_ref_e0_on_unnamed_duplicate_emits_nth_zero(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """SKY-9701 follow-up: the first duplicate (e0) must also emit `nth=0` so Playwright+ strict mode does not raise and the click does not silently resolve to whichever+ element happens to be first."""+ tree = _make_a11y_tree(+ children=[+ {"role": "combobox", "name": ""},+ {"role": "combobox", "name": ""},+ ]+ )+ page = _make_page(tree)+ ctx = BrowserContext(mode="local")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ click_result = {"ok": True, "data": None}+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))++ result = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "click", "params": {"ref": "e0"}},+ ]+ )+ assert result["ok"] is True+ click_kwargs = mcp_browser.skyvern_click.call_args.kwargs+ assert click_kwargs["selector"] == "role=combobox >> nth=0"