Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9402): preserve class and readable pseudo-text for icon-only navigation elements (#6092)
LawyZheng0aa6cadf8236
diff --git a/pyproject.toml b/pyproject.tomlindex c6cc99cdf..d8250c188 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -254,6 +254,7 @@ override-dependencies = [constraint-dependencies = ["authlib>=1.7.1","flask>=3.1.3",+ "idna>=3.15","joserfc>=1.6.3","mako>=1.3.12","pyasn1>=0.6.3",diff --git a/skyvern/webeye/scraper/scraped_page.py b/skyvern/webeye/scraper/scraped_page.pyindex c83f41955..c62386a84 100644--- a/skyvern/webeye/scraper/scraped_page.py+++ b/skyvern/webeye/scraper/scraped_page.py@@ -1,5 +1,6 @@import copyimport json+import reimport typingfrom abc import ABC, abstractmethodfrom enum import StrEnum@@ -25,6 +26,14 @@ ELEMENT_NODE_ATTRIBUTES = {"id",}+_PUA_PATTERN = re.compile(r"[\uE000-\uF8FF\U000F0000-\U000FFFFD\U00100000-\U0010FFFD]+")+++def _replace_pua_with_marker(text: str | None) -> str:+ if not text:+ return ""+ return _PUA_PATTERN.sub("[icon]", text)+def build_attribute(key: str, value: Any) -> str:if isinstance(value, bool) or isinstance(value, int):@@ -89,8 +98,8 @@ def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:if element.get("purgeable", False):return children_html + option_html- before_pseudo_text = element.get("beforePseudoText") or ""- after_pseudo_text = element.get("afterPseudoText") or ""+ before_pseudo_text = _replace_pua_with_marker(element.get("beforePseudoText"))+ after_pseudo_text = _replace_pua_with_marker(element.get("afterPseudoText"))# Check if the element is self-closingif (diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 0049975ca..09200f996 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -810,7 +810,11 @@ def trim_element(element: dict) -> dict:del queue_ele["attributes"]if "attributes" in queue_ele and not queue_ele.get("keepAllAttr", False):- new_attributes = _trimmed_attributes(queue_ele["attributes"])+ has_pseudo = bool(queue_ele.get("beforePseudoText") or queue_ele.get("afterPseudoText"))+ is_icon_only = (+ queue_ele.get("interactable", False) and not str(queue_ele.get("text", "")).strip() and has_pseudo+ )+ new_attributes = _trimmed_attributes(queue_ele["attributes"], keep_class=is_icon_only)if new_attributes:queue_ele["attributes"] = new_attributeselse:@@ -861,7 +865,7 @@ def _trimmed_base64_data(attributes: dict) -> dict:return new_attributes-def _trimmed_attributes(attributes: dict) -> dict:+def _trimmed_attributes(attributes: dict, *, keep_class: bool = False) -> dict:new_attributes: dict = {}for key in attributes:@@ -870,6 +874,13 @@ def _trimmed_attributes(attributes: dict) -> dict:if key in RESERVED_ATTRIBUTES:new_attributes[key] = attributes[key]+ if keep_class and "class" in attributes:+ cls = str(attributes["class"])+ if len(cls) > 100:+ last_space = cls.rfind(" ", 0, 100)+ cls = cls[: last_space if last_space > 0 else 100]+ new_attributes["class"] = cls+return new_attributesdiff --git a/tests/unit/test_icon_only_element_serialization.py b/tests/unit/test_icon_only_element_serialization.pynew file mode 100644index 000000000..37901b558--- /dev/null+++ b/tests/unit/test_icon_only_element_serialization.py@@ -0,0 +1,177 @@+"""+Tests for icon-only interactable element serialization.++Icon-only navigation controls (e.g. CSS-rendered arrow buttons with no visible text)+need their class attribute and pseudo-text preserved through the trim/serialization+pipeline so LLMs can identify them.+"""++import copy++import pytest++from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.webeye.scraper.scraped_page import _replace_pua_with_marker, json_to_html+from skyvern.webeye.scraper.scraper import _trimmed_attributes, trim_element++[email protected](autouse=True)+def _skyvern_ctx():+ with skyvern_context.scoped(SkyvernContext(organization_id="o_test")):+ yield+++ICON_ONLY_NAV_ELEMENT = {+ "id": "AACO",+ "frame": "main.frame",+ "frame_index": 0,+ "interactable": True,+ "hoverOnly": False,+ "tagName": "div",+ "attributes": {+ "class": "icon-nav forward",+ "id": "widget_10",+ "lang": "en",+ "unique_id": "AACO",+ },+ "beforePseudoText": None,+ "text": "",+ "afterPseudoText": "\uf061",+ "children": [],+ "purgeable": False,+ "keepAllAttr": False,+ "isSelectable": False,+}++NORMAL_TEXT_BUTTON = {+ "id": "NEXT1",+ "frame": "main.frame",+ "frame_index": 0,+ "interactable": True,+ "tagName": "button",+ "attributes": {+ "class": "btn-primary large-button extra-styles",+ "type": "submit",+ "unique_id": "NEXT1",+ },+ "text": "Submit",+ "beforePseudoText": None,+ "afterPseudoText": None,+ "children": [],+ "purgeable": False,+ "keepAllAttr": False,+ "isSelectable": False,+}++NON_INTERACTABLE_DIV = {+ "id": "DIV1",+ "frame": "main.frame",+ "frame_index": 0,+ "interactable": False,+ "tagName": "div",+ "attributes": {+ "class": "layout-wrapper",+ "unique_id": "DIV1",+ },+ "text": "",+ "beforePseudoText": None,+ "afterPseudoText": "\uf061",+ "children": [],+ "purgeable": False,+ "keepAllAttr": False,+ "isSelectable": False,+}+++class TestTrimmedAttributesClassPreservation:+ def test_icon_only_interactable_preserves_class(self):+ attrs = {"class": "icon-nav forward", "data-dojo-type": "dojosite/Noop", "lang": "en"}+ result = _trimmed_attributes(attrs, keep_class=True)+ assert result["class"] == "icon-nav forward"++ def test_normal_element_strips_class(self):+ attrs = {"class": "btn-primary", "type": "submit"}+ result = _trimmed_attributes(attrs, keep_class=False)+ assert "class" not in result+ assert result["type"] == "submit"++ def test_long_class_truncated(self):+ long_class = "tw-" + "x" * 200+ attrs = {"class": long_class}+ result = _trimmed_attributes(attrs, keep_class=True)+ assert len(result["class"]) == 100++ def test_reserved_attrs_always_kept(self):+ attrs = {"aria-label": "Next", "class": "appGo", "lang": "en"}+ result = _trimmed_attributes(attrs, keep_class=False)+ assert result["aria-label"] == "Next"+ assert "class" not in result+ assert "lang" not in result+++class TestTrimElementIconOnly:+ def test_icon_nav_element_retains_class_after_trim(self):+ el = copy.deepcopy(ICON_ONLY_NAV_ELEMENT)+ trimmed = trim_element(el)+ assert "attributes" in trimmed+ assert trimmed["attributes"].get("class") == "icon-nav forward"++ def test_text_button_does_not_retain_class(self):+ el = copy.deepcopy(NORMAL_TEXT_BUTTON)+ trimmed = trim_element(el)+ attrs = trimmed.get("attributes", {})+ assert "class" not in attrs+ assert attrs.get("type") == "submit"++ def test_non_interactable_icon_does_not_retain_class(self):+ el = copy.deepcopy(NON_INTERACTABLE_DIV)+ trimmed = trim_element(el)+ attrs = trimmed.get("attributes", {})+ assert "class" not in attrs+++class TestPUAReplacement:+ def test_fontawesome_arrow_right(self):+ assert _replace_pua_with_marker("\uf061") == "[icon]"++ def test_multiple_pua_chars(self):+ assert _replace_pua_with_marker("\uf061\uf062") == "[icon]"++ def test_mixed_pua_and_text(self):+ assert _replace_pua_with_marker("Next \uf061") == "Next [icon]"++ def test_no_pua(self):+ assert _replace_pua_with_marker("Next") == "Next"++ def test_empty_string(self):+ assert _replace_pua_with_marker("") == ""++ def test_none_returns_empty(self):+ assert _replace_pua_with_marker(None) == ""+++class TestJsonToHtmlWithIconElements:+ def test_icon_nav_after_trim_produces_readable_output(self):+ el = copy.deepcopy(ICON_ONLY_NAV_ELEMENT)+ trimmed = trim_element(el)+ html = json_to_html(trimmed)+ assert 'class="icon-nav forward"' in html+ assert "[icon]" in html+ assert "AACO" in html++ def test_text_button_no_icon_marker(self):+ el = copy.deepcopy(NORMAL_TEXT_BUTTON)+ trimmed = trim_element(el)+ html = json_to_html(trimmed)+ assert "Submit" in html+ assert "[icon]" not in html+ assert "class=" not in html++ def test_before_pseudo_pua_also_replaced(self):+ el = copy.deepcopy(ICON_ONLY_NAV_ELEMENT)+ el["beforePseudoText"] = "\uf060"+ el["afterPseudoText"] = None+ trimmed = trim_element(el)+ html = json_to_html(trimmed)+ assert "[icon]" in htmldiff --git a/uv.lock b/uv.lockindex 85c68c5e8..59bb2dbfc 100644--- a/uv.lock+++ b/uv.lock@@ -17,6 +17,7 @@ resolution-markers = [constraints = [{ name = "authlib", specifier = ">=1.7.1" },{ name = "flask", specifier = ">=3.1.3" },+ { name = "idna", specifier = ">=3.15" },{ name = "joserfc", specifier = ">=1.6.3" },{ name = "mako", specifier = ">=1.3.12" },{ name = "pyasn1", specifier = ">=0.6.3" },@@ -1976,11 +1977,11 @@ wheels = [[[package]]name = "idna"-version = "3.13"+version = "3.15"source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/ce/cc/762dfb036166873f0059f3b7de4565e1b5bc3d6f28a414c13da27e442f99/idna-3.13.tar.gz", hash = "sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242", size = 194210, upload-time = "2026-04-22T16:42:42.314Z" }+sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/5d/13/ad7d7ca3808a898b4612b6fe93cde56b53f3034dcde235acb1f0e1df24c6/idna-3.13-py3-none-any.whl", hash = "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3", size = 68629, upload-time = "2026-04-22T16:42:40.909Z" },+ { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" },][[package]]