Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix invisible iframe scraping (#1723)
Shuchang Zheng6f3dcd13b8a5
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 3c029f4c6..dc528ce30 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -248,7 +248,7 @@ async def _convert_svg_to_string(async def _convert_css_shape_to_string(- frame: Page | Frame,+ skyvern_frame: SkyvernFrame,element: Dict,task: Task | None = None,step: Step | None = None,@@ -279,28 +279,18 @@ async def _convert_css_shape_to_string(if css_shape:LOG.debug("CSS shape loaded from cache", element_id=element_id, key=shape_key, shape=css_shape)else:- # FIXME: support element in iframe- locater = frame.locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')- if await locater.count() == 0:- LOG.info(- "No locater found to convert css shape",- task_id=task_id,- step_id=step_id,- element_id=element_id,- )- return None-- if await locater.count() > 1:- LOG.info(- "multiple locaters found to convert css shape",- task_id=task_id,- step_id=step_id,- element_id=element_id,- )- return None-try:- LOG.debug("call LLM to convert css shape to string shape", element_id=element_id)+ locater = skyvern_frame.get_frame().locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')+ if await locater.count() == 0:+ LOG.info(+ "No locater found to convert css shape",+ task_id=task_id,+ step_id=step_id,+ element_id=element_id,+ key=shape_key,+ )+ return None+if not await locater.is_visible(timeout=settings.BROWSER_ACTION_TIMEOUT_MS):LOG.info("element is not visible on the page, going to abort conversion",@@ -309,9 +299,22 @@ async def _convert_css_shape_to_string(element_id=element_id,key=shape_key,)++ skyvern_element = SkyvernElement(locator=locater, frame=skyvern_frame.get_frame(), static_element=element)++ _, blocked = await skyvern_frame.get_blocking_element_id(await skyvern_element.get_element_handler())+ if blocked:+ LOG.info(+ "element is blocked by another element, going to abort conversion",+ task_id=task_id,+ step_id=step_id,+ element_id=element_id,+ key=shape_key,+ )return None- screenshot = await locater.screenshot(timeout=settings.BROWSER_SCREENSHOT_TIMEOUT_MS)+ LOG.debug("call LLM to convert css shape to string shape", element_id=element_id)+ screenshot = await locater.screenshot(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)prompt = prompt_engine.load_prompt("css-shape-convert")# TODO: we don't retry the css shape conversion today@@ -466,7 +469,7 @@ class AgentFunction:if _should_css_shape_convert(element=queue_ele):await _convert_css_shape_to_string(- frame=frame,+ skyvern_frame=skyvern_frame,element=queue_ele,task=task,step=step,diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 84976cc37..922c67a89 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -470,19 +470,17 @@ async def get_interactable_element_tree_in_frame(try:frame_element = await frame.frame_element()+ # it will get stuck when we `frame.evaluate()` on an invisible iframe+ if not await frame_element.is_visible():+ continue+ unique_id = await frame_element.get_attribute("unique_id")except Exception:LOG.warning(- "Unable to get frame_element",+ "Unable to get unique_id from frame_element",exc_info=True,)continue- # it will get stuck when we `frame.evaluate()` on an invisible iframe- if not await frame_element.is_visible():- continue-- unique_id = await frame_element.get_attribute("unique_id")-frame_js_script = f"() => buildTreeFromBody('{unique_id}')"await SkyvernFrame.evaluate(frame=frame, expression=JS_FUNCTION_DEFS)