Skip to content

Automate browser based workflows with AI

fix(SKY-9402): preserve class and readable pseudo-text for icon-only navigation elements (#6092)

LawyZheng0aa6cadf8236
diff --git a/pyproject.toml b/pyproject.toml
index c6cc99cdf..d8250c188 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -254,6 +254,7 @@ override-dependencies = [
constraint-dependencies = [
"authlib>=1.7.1",
"flask>=3.1.3",
+ "idna>=3.15",
"joserfc>=1.6.3",
"mako>=1.3.12",
"pyasn1>=0.6.3",
diff --git a/skyvern/webeye/scraper/scraped_page.py b/skyvern/webeye/scraper/scraped_page.py
index c83f41955..c62386a84 100644
--- a/skyvern/webeye/scraper/scraped_page.py
+++ b/skyvern/webeye/scraper/scraped_page.py
@@ -1,5 +1,6 @@
import copy
import json
+import re
import typing
from abc import ABC, abstractmethod
from enum import StrEnum
@@ -25,6 +26,14 @@ ELEMENT_NODE_ATTRIBUTES = {
"id",
}
+_PUA_PATTERN = re.compile(r"[\uE000-\uF8FF\U000F0000-\U000FFFFD\U00100000-\U0010FFFD]+")
+
+
+def _replace_pua_with_marker(text: str | None) -> str:
+ if not text:
+ return ""
+ return _PUA_PATTERN.sub("[icon]", text)
+
def build_attribute(key: str, value: Any) -> str:
if isinstance(value, bool) or isinstance(value, int):
@@ -89,8 +98,8 @@ def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:
if element.get("purgeable", False):
return children_html + option_html
- before_pseudo_text = element.get("beforePseudoText") or ""
- after_pseudo_text = element.get("afterPseudoText") or ""
+ before_pseudo_text = _replace_pua_with_marker(element.get("beforePseudoText"))
+ after_pseudo_text = _replace_pua_with_marker(element.get("afterPseudoText"))
# Check if the element is self-closing
if (
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 0049975ca..09200f996 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -810,7 +810,11 @@ def trim_element(element: dict) -> dict:
del queue_ele["attributes"]
if "attributes" in queue_ele and not queue_ele.get("keepAllAttr", False):
- new_attributes = _trimmed_attributes(queue_ele["attributes"])
+ has_pseudo = bool(queue_ele.get("beforePseudoText") or queue_ele.get("afterPseudoText"))
+ is_icon_only = (
+ queue_ele.get("interactable", False) and not str(queue_ele.get("text", "")).strip() and has_pseudo
+ )
+ new_attributes = _trimmed_attributes(queue_ele["attributes"], keep_class=is_icon_only)
if new_attributes:
queue_ele["attributes"] = new_attributes
else:
@@ -861,7 +865,7 @@ def _trimmed_base64_data(attributes: dict) -> dict:
return new_attributes
-def _trimmed_attributes(attributes: dict) -> dict:
+def _trimmed_attributes(attributes: dict, *, keep_class: bool = False) -> dict:
new_attributes: dict = {}
for key in attributes:
@@ -870,6 +874,13 @@ def _trimmed_attributes(attributes: dict) -> dict:
if key in RESERVED_ATTRIBUTES:
new_attributes[key] = attributes[key]
+ if keep_class and "class" in attributes:
+ cls = str(attributes["class"])
+ if len(cls) > 100:
+ last_space = cls.rfind(" ", 0, 100)
+ cls = cls[: last_space if last_space > 0 else 100]
+ new_attributes["class"] = cls
+
return new_attributes
diff --git a/tests/unit/test_icon_only_element_serialization.py b/tests/unit/test_icon_only_element_serialization.py
new file mode 100644
index 000000000..37901b558
--- /dev/null
+++ b/tests/unit/test_icon_only_element_serialization.py
@@ -0,0 +1,177 @@
+"""
+Tests for icon-only interactable element serialization.
+
+Icon-only navigation controls (e.g. CSS-rendered arrow buttons with no visible text)
+need their class attribute and pseudo-text preserved through the trim/serialization
+pipeline so LLMs can identify them.
+"""
+
+import copy
+
+import pytest
+
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.webeye.scraper.scraped_page import _replace_pua_with_marker, json_to_html
+from skyvern.webeye.scraper.scraper import _trimmed_attributes, trim_element
+
+
[email protected](autouse=True)
+def _skyvern_ctx():
+ with skyvern_context.scoped(SkyvernContext(organization_id="o_test")):
+ yield
+
+
+ICON_ONLY_NAV_ELEMENT = {
+ "id": "AACO",
+ "frame": "main.frame",
+ "frame_index": 0,
+ "interactable": True,
+ "hoverOnly": False,
+ "tagName": "div",
+ "attributes": {
+ "class": "icon-nav forward",
+ "id": "widget_10",
+ "lang": "en",
+ "unique_id": "AACO",
+ },
+ "beforePseudoText": None,
+ "text": "",
+ "afterPseudoText": "\uf061",
+ "children": [],
+ "purgeable": False,
+ "keepAllAttr": False,
+ "isSelectable": False,
+}
+
+NORMAL_TEXT_BUTTON = {
+ "id": "NEXT1",
+ "frame": "main.frame",
+ "frame_index": 0,
+ "interactable": True,
+ "tagName": "button",
+ "attributes": {
+ "class": "btn-primary large-button extra-styles",
+ "type": "submit",
+ "unique_id": "NEXT1",
+ },
+ "text": "Submit",
+ "beforePseudoText": None,
+ "afterPseudoText": None,
+ "children": [],
+ "purgeable": False,
+ "keepAllAttr": False,
+ "isSelectable": False,
+}
+
+NON_INTERACTABLE_DIV = {
+ "id": "DIV1",
+ "frame": "main.frame",
+ "frame_index": 0,
+ "interactable": False,
+ "tagName": "div",
+ "attributes": {
+ "class": "layout-wrapper",
+ "unique_id": "DIV1",
+ },
+ "text": "",
+ "beforePseudoText": None,
+ "afterPseudoText": "\uf061",
+ "children": [],
+ "purgeable": False,
+ "keepAllAttr": False,
+ "isSelectable": False,
+}
+
+
+class TestTrimmedAttributesClassPreservation:
+ def test_icon_only_interactable_preserves_class(self):
+ attrs = {"class": "icon-nav forward", "data-dojo-type": "dojosite/Noop", "lang": "en"}
+ result = _trimmed_attributes(attrs, keep_class=True)
+ assert result["class"] == "icon-nav forward"
+
+ def test_normal_element_strips_class(self):
+ attrs = {"class": "btn-primary", "type": "submit"}
+ result = _trimmed_attributes(attrs, keep_class=False)
+ assert "class" not in result
+ assert result["type"] == "submit"
+
+ def test_long_class_truncated(self):
+ long_class = "tw-" + "x" * 200
+ attrs = {"class": long_class}
+ result = _trimmed_attributes(attrs, keep_class=True)
+ assert len(result["class"]) == 100
+
+ def test_reserved_attrs_always_kept(self):
+ attrs = {"aria-label": "Next", "class": "appGo", "lang": "en"}
+ result = _trimmed_attributes(attrs, keep_class=False)
+ assert result["aria-label"] == "Next"
+ assert "class" not in result
+ assert "lang" not in result
+
+
+class TestTrimElementIconOnly:
+ def test_icon_nav_element_retains_class_after_trim(self):
+ el = copy.deepcopy(ICON_ONLY_NAV_ELEMENT)
+ trimmed = trim_element(el)
+ assert "attributes" in trimmed
+ assert trimmed["attributes"].get("class") == "icon-nav forward"
+
+ def test_text_button_does_not_retain_class(self):
+ el = copy.deepcopy(NORMAL_TEXT_BUTTON)
+ trimmed = trim_element(el)
+ attrs = trimmed.get("attributes", {})
+ assert "class" not in attrs
+ assert attrs.get("type") == "submit"
+
+ def test_non_interactable_icon_does_not_retain_class(self):
+ el = copy.deepcopy(NON_INTERACTABLE_DIV)
+ trimmed = trim_element(el)
+ attrs = trimmed.get("attributes", {})
+ assert "class" not in attrs
+
+
+class TestPUAReplacement:
+ def test_fontawesome_arrow_right(self):
+ assert _replace_pua_with_marker("\uf061") == "[icon]"
+
+ def test_multiple_pua_chars(self):
+ assert _replace_pua_with_marker("\uf061\uf062") == "[icon]"
+
+ def test_mixed_pua_and_text(self):
+ assert _replace_pua_with_marker("Next \uf061") == "Next [icon]"
+
+ def test_no_pua(self):
+ assert _replace_pua_with_marker("Next") == "Next"
+
+ def test_empty_string(self):
+ assert _replace_pua_with_marker("") == ""
+
+ def test_none_returns_empty(self):
+ assert _replace_pua_with_marker(None) == ""
+
+
+class TestJsonToHtmlWithIconElements:
+ def test_icon_nav_after_trim_produces_readable_output(self):
+ el = copy.deepcopy(ICON_ONLY_NAV_ELEMENT)
+ trimmed = trim_element(el)
+ html = json_to_html(trimmed)
+ assert 'class="icon-nav forward"' in html
+ assert "[icon]" in html
+ assert "AACO" in html
+
+ def test_text_button_no_icon_marker(self):
+ el = copy.deepcopy(NORMAL_TEXT_BUTTON)
+ trimmed = trim_element(el)
+ html = json_to_html(trimmed)
+ assert "Submit" in html
+ assert "[icon]" not in html
+ assert "class=" not in html
+
+ def test_before_pseudo_pua_also_replaced(self):
+ el = copy.deepcopy(ICON_ONLY_NAV_ELEMENT)
+ el["beforePseudoText"] = "\uf060"
+ el["afterPseudoText"] = None
+ trimmed = trim_element(el)
+ html = json_to_html(trimmed)
+ assert "[icon]" in html
diff --git a/uv.lock b/uv.lock
index 85c68c5e8..59bb2dbfc 100644
--- a/uv.lock
+++ b/uv.lock
@@ -17,6 +17,7 @@ resolution-markers = [
constraints = [
{ name = "authlib", specifier = ">=1.7.1" },
{ name = "flask", specifier = ">=3.1.3" },
+ { name = "idna", specifier = ">=3.15" },
{ name = "joserfc", specifier = ">=1.6.3" },
{ name = "mako", specifier = ">=1.3.12" },
{ name = "pyasn1", specifier = ">=0.6.3" },
@@ -1976,11 +1977,11 @@ wheels = [
[[package]]
name = "idna"
-version = "3.13"
+version = "3.15"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/ce/cc/762dfb036166873f0059f3b7de4565e1b5bc3d6f28a414c13da27e442f99/idna-3.13.tar.gz", hash = "sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242", size = 194210, upload-time = "2026-04-22T16:42:42.314Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/5d/13/ad7d7ca3808a898b4612b6fe93cde56b53f3034dcde235acb1f0e1df24c6/idna-3.13-py3-none-any.whl", hash = "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3", size = 68629, upload-time = "2026-04-22T16:42:40.909Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" },
]
[[package]]