Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
do not parse the dropped element twice (#2801)
Shuchang Zheng9dd62072864d
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex d633f4be7..bc71c924d 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -106,7 +106,19 @@ def _remove_skyvern_attributes(element: Dict) -> Dict:return element_copied-def _mark_element_as_dropped(element: dict) -> None:+def _add_to_dropped_css_svg_element_map(hashed_key: str | None) -> None:+ context = skyvern_context.ensure_context()+ if hashed_key:+ context.dropped_css_svg_element_map[hashed_key] = True+++def _is_element_already_dropped(hashed_key: str) -> bool:+ context = skyvern_context.ensure_context()+ return hashed_key in context.dropped_css_svg_element_map+++def _mark_element_as_dropped(element: dict, *, hashed_key: str | None) -> None:+ _add_to_dropped_css_svg_element_map(hashed_key)if "children" in element:del element["children"]element["isDropped"] = True@@ -127,7 +139,7 @@ async def _check_svg_eligibility(return Falseif always_drop:- _mark_element_as_dropped(element)+ _mark_element_as_dropped(element, hashed_key=None)return Falsetask_id = task.task_id if task else None@@ -137,11 +149,11 @@ async def _check_svg_eligibility(try:locater = skyvern_frame.get_frame().locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')if await locater.count() == 0:- _mark_element_as_dropped(element)+ _mark_element_as_dropped(element, hashed_key=None)return Falseif not await locater.is_visible(timeout=settings.BROWSER_ACTION_TIMEOUT_MS):- _mark_element_as_dropped(element)+ _mark_element_as_dropped(element, hashed_key=None)return Falseskyvern_element = SkyvernElement(locator=locater, frame=skyvern_frame.get_frame(), static_element=element)@@ -150,7 +162,7 @@ async def _check_svg_eligibility(await skyvern_element.get_element_handler(timeout=1000))if not skyvern_element.is_interactable() and blocked:- _mark_element_as_dropped(element)+ _mark_element_as_dropped(element, hashed_key=None)return Falseexcept Exception:LOG.warning(@@ -195,6 +207,11 @@ async def _convert_svg_to_string(if svg_shape:LOG.debug("SVG loaded from cache", element_id=element_id, key=svg_key, shape=svg_shape)else:+ if _is_element_already_dropped(svg_key):+ LOG.debug("SVG is already dropped, going to abort conversion", element_id=element_id, key=svg_key)+ _mark_element_as_dropped(element, hashed_key=svg_key)+ return+if len(svg_html) > settings.SVG_MAX_LENGTH:# TODO: implement a fallback solution for "too large" case, maybe convert by screenshotLOG.warning(@@ -205,7 +222,7 @@ async def _convert_svg_to_string(length=len(svg_html),key=svg_key,)- _mark_element_as_dropped(element)+ _mark_element_as_dropped(element, hashed_key=svg_key)returnLOG.debug("call LLM to convert SVG to string shape", element_id=element_id)@@ -244,7 +261,7 @@ async def _convert_svg_to_string(element_id=element_id,key=svg_key,)- _mark_element_as_dropped(element)+ _mark_element_as_dropped(element, hashed_key=svg_key)returnexcept Exception:LOG.info(@@ -268,7 +285,7 @@ async def _convert_svg_to_string(key=svg_key,length=len(svg_html),)- _mark_element_as_dropped(element)+ _mark_element_as_dropped(element, hashed_key=svg_key)returnelement["attributes"] = dict()@@ -313,6 +330,9 @@ async def _convert_css_shape_to_string(if css_shape:LOG.debug("CSS shape loaded from cache", element_id=element_id, key=shape_key, shape=css_shape)else:+ if _is_element_already_dropped(shape_key):+ LOG.debug("CSS shape is already dropped, going to abort conversion", element_id=element_id, key=shape_key)+ return Nonetry:locater = skyvern_frame.get_frame().locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')if await locater.count() == 0:@@ -399,6 +419,7 @@ async def _convert_css_shape_to_string(element_id=element_id,key=shape_key,)+ _add_to_dropped_css_svg_element_map(shape_key)return Noneexcept Exception:LOG.info(@@ -422,6 +443,7 @@ async def _convert_css_shape_to_string(element_id=element_id,key=shape_key,)+ _add_to_dropped_css_svg_element_map(shape_key)return Noneexcept Exception:LOG.warning(@@ -432,6 +454,7 @@ async def _convert_css_shape_to_string(element_id=element_id,exc_info=True,)+ _add_to_dropped_css_svg_element_map(shape_key)return Noneif "attributes" not in element:diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex 63e4c1200..195a7d0b1 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -25,6 +25,7 @@ class SkyvernContext:refresh_working_page: bool = Falseframe_index_map: dict[Frame, int] = field(default_factory=dict)max_screenshot_scrolling_times: int | None = None+ dropped_css_svg_element_map: dict[str, bool] = field(default_factory=dict)def __repr__(self) -> str:return f"SkyvernContext(request_id={self.request_id}, organization_id={self.organization_id}, task_id={self.task_id}, workflow_id={self.workflow_id}, workflow_run_id={self.workflow_run_id}, task_v2_id={self.task_v2_id}, max_steps_override={self.max_steps_override}, run_id={self.run_id})"