Skip to content

Automate browser based workflows with AI

decrease parse input prompt token (#3210)

LawyZheng9a359ebfdec7
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index f842dc426..d7bfea434 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -741,7 +741,8 @@ async def handle_sequential_click_for_dropdown(
reasoning=action.reasoning, intention=action.intention, element_id=action.element_id
),
step=step,
- scraped_page=scraped_page,
+ element_tree_builder=scraped_page,
+ skyvern_element=anchor_element,
)
if dropdown_select_context.is_date_related:
@@ -934,7 +935,8 @@ async def handle_input_text_action(
input_or_select_context = await _get_input_or_select_context(
action=action,
- scraped_page=scraped_page,
+ element_tree_builder=scraped_page,
+ skyvern_element=skyvern_element,
step=step,
)
@@ -1538,7 +1540,7 @@ async def handle_select_option_action(
)
input_or_select_context = await _get_input_or_select_context(
- action=action, scraped_page=scraped_page, step=step
+ action=action, element_tree_builder=scraped_page, step=step, skyvern_element=skyvern_element
)
if len(incremental_element) == 0:
@@ -3332,26 +3334,9 @@ async def normal_select(
action_result: List[ActionResult] = []
is_success = False
locator = skyvern_element.get_locator()
-
- prompt = load_prompt_with_elements(
- element_tree_builder=builder,
- prompt_engine=prompt_engine,
- template_name="parse-input-or-select-context",
- action_reasoning=action.reasoning,
- element_id=action.element_id,
- )
- json_response = await app.SECONDARY_LLM_API_HANDLER(
- prompt=prompt, step=step, prompt_name="parse-input-or-select-context"
- )
- json_response["intention"] = action.intention
- input_or_select_context = InputOrSelectContext.model_validate(json_response)
- LOG.info(
- "Parsed input/select context",
- context=input_or_select_context,
- task_id=task.task_id,
- step_id=step.step_id,
+ input_or_select_context = await _get_input_or_select_context(
+ action=action, element_tree_builder=builder, step=step, skyvern_element=skyvern_element
)
-
await skyvern_element.refresh_select_options()
options_html = skyvern_element.build_HTML()
field_information = (
@@ -3694,10 +3679,46 @@ class AbstractActionForContextParse(BaseModel):
async def _get_input_or_select_context(
- action: InputTextAction | SelectOptionAction | AbstractActionForContextParse, scraped_page: ScrapedPage, step: Step
+ action: InputTextAction | SelectOptionAction | AbstractActionForContextParse,
+ skyvern_element: SkyvernElement,
+ element_tree_builder: ElementTreeBuilder,
+ step: Step,
+ ancestor_depth: int = 5,
) -> InputOrSelectContext:
+ skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())
+ try:
+ depth = await skyvern_frame.get_element_dom_depth(await skyvern_element.get_element_handler())
+ except Exception:
+ LOG.warning("Failed to get element depth, using the original element tree", exc_info=True)
+ depth = 0
+
+ if depth > ancestor_depth:
+ # use ancestor to build the context
+ path = "/".join([".."] * ancestor_depth)
+ locator = skyvern_element.get_locator().locator(path)
+ try:
+ element_handle = await locator.element_handle(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
+ if element_handle is not None:
+ elements, element_tree = await skyvern_frame.build_tree_from_element(
+ starter=element_handle,
+ frame=skyvern_element.get_frame_id(),
+ )
+ clean_up_func = app.AGENT_FUNCTION.cleanup_element_tree_factory()
+ element_tree = await clean_up_func(skyvern_element.get_frame(), "", copy.deepcopy(element_tree))
+ element_tree_trimmed = trim_element_tree(copy.deepcopy(element_tree))
+ element_tree_builder = ScrapedPage(
+ elements=elements,
+ element_tree=element_tree,
+ element_tree_trimmed=element_tree_trimmed,
+ _browser_state=None,
+ _clean_up_func=None,
+ _scrape_exclude=None,
+ )
+ except Exception:
+ LOG.warning("Failed to get sub element tree, using the original element tree", exc_info=True, path=path)
+
prompt = load_prompt_with_elements(
- element_tree_builder=scraped_page,
+ element_tree_builder=element_tree_builder,
prompt_engine=prompt_engine,
template_name="parse-input-or-select-context",
action_reasoning=action.reasoning,
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 797511392..cb364ad8e 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -257,16 +257,16 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):
elements: list[dict]
id_to_element_dict: dict[str, dict] = {}
id_to_frame_dict: dict[str, str] = {}
- id_to_css_dict: dict[str, str]
- id_to_element_hash: dict[str, str]
- hash_to_element_ids: dict[str, list[str]]
+ id_to_css_dict: dict[str, str] = {}
+ id_to_element_hash: dict[str, str] = {}
+ hash_to_element_ids: dict[str, list[str]] = {}
element_tree: list[dict]
element_tree_trimmed: list[dict]
economy_element_tree: list[dict] | None = None
last_used_element_tree: list[dict] | None = None
- screenshots: list[bytes]
- url: str
- html: str
+ screenshots: list[bytes] = []
+ url: str = ""
+ html: str = ""
extracted_text: str | None = None
window_dimension: dict[str, int] | None = None
_browser_state: BrowserState = PrivateAttr()
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 04ac68d57..122bb1086 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -477,6 +477,10 @@ class SkyvernFrame:
js_script = "([element]) => getSelectOptions(element)"
return await self.evaluate(frame=self.frame, expression=js_script, arg=[element])
+ async def get_element_dom_depth(self, element: ElementHandle) -> int:
+ js_script = "([element]) => getElementDomDepth(element)"
+ return await self.evaluate(frame=self.frame, expression=js_script, arg=[element])
+
@TraceManager.traced_async()
async def build_tree_from_body(
self,
@@ -500,6 +504,19 @@ class SkyvernFrame:
frame=self.frame, expression=js_script, timeout_ms=timeout_ms, arg=[wait_until_finished]
)
+ @TraceManager.traced_async()
+ async def build_tree_from_element(
+ self,
+ starter: ElementHandle,
+ frame: str,
+ full_tree: bool = False,
+ timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
+ ) -> tuple[list[dict], list[dict]]:
+ js_script = "async ([starter, frame, full_tree]) => await buildElementTree(starter, frame, full_tree)"
+ return await self.evaluate(
+ frame=self.frame, expression=js_script, timeout_ms=timeout_ms, arg=[starter, frame, full_tree]
+ )
+
async def safe_wait_for_animation_end(self, timeout_ms: float = 3000) -> None:
try:
async with asyncio.timeout(timeout_ms / 1000):