Skip to content

Automate browser based workflows with AI

remove the blocked svg element from the prompt (#1624)

Shuchang Zhengd336fbf28609
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 020b4bd8d..90a24c3ed 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -20,6 +20,8 @@ from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
from skyvern.forge.sdk.workflow.models.block import BlockTypeVar
from skyvern.webeye.browser_factory import BrowserState
from skyvern.webeye.scraper.scraper import ELEMENT_NODE_ATTRIBUTES, CleanupElementTreeFunc, json_to_html
+from skyvern.webeye.utils.dom import SkyvernElement
+from skyvern.webeye.utils.page import SkyvernFrame
LOG = structlog.get_logger()
@@ -103,6 +105,7 @@ def _remove_skyvern_attributes(element: Dict) -> Dict:
async def _convert_svg_to_string(
+ skyvern_frame: SkyvernFrame,
element: Dict,
task: Task | None = None,
step: Step | None = None,
@@ -116,6 +119,24 @@ async def _convert_svg_to_string(
task_id = task.task_id if task else None
step_id = step.step_id if step else None
element_id = element.get("id", "")
+
+ try:
+ locater = skyvern_frame.get_frame().locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')
+ skyvern_element = SkyvernElement(locator=locater, frame=skyvern_frame.get_frame(), static_element=element)
+
+ _, blocked = await skyvern_frame.get_blocking_element_id(await skyvern_element.get_element_handler())
+ if blocked:
+ del element["children"]
+ element["isDropped"] = True
+ return
+ except Exception:
+ LOG.warning(
+ "Failed to get the blocking element for the svg, going to continue parsing the svg",
+ exc_info=True,
+ task_id=task_id,
+ step_id=step_id,
+ )
+
svg_element = _remove_skyvern_attributes(element)
svg_html = json_to_html(svg_element)
hash_object = hashlib.sha256()
@@ -422,13 +443,14 @@ class AgentFunction:
:param elements: List of elements to remove xpaths from.
:return: List of elements without xpaths.
"""
+ skyvern_frame = await SkyvernFrame.create_instance(frame=frame)
queue = []
for element in element_tree:
queue.append(element)
while queue:
queue_ele = queue.pop(0)
_remove_rect(queue_ele)
- await _convert_svg_to_string(queue_ele, task, step)
+ await _convert_svg_to_string(skyvern_frame, queue_ele, task, step)
if _should_css_shape_convert(element=queue_ele):
await _convert_css_shape_to_string(