Skip to content

Automate browser based workflows with AI

speed up extraction (#1617)

Shuchang Zhenged4d0c59e7eb
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 2f88cd53c..9d8968d5c 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -2684,7 +2684,7 @@ async def extract_information_for_navigation_goal(
# TODO: we only use HTML element for now, introduce a way to switch in the future
element_tree_format = ElementTreeFormat.HTML
- element_tree_in_prompt: str = scraped_page.build_element_tree(element_tree_format)
+ element_tree_in_prompt: str = scraped_page.build_element_tree(element_tree_format, html_need_skyvern_attrs=False)
scraped_page_refreshed = await scraped_page.refresh()
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index b27f18d66..5e1dc0513 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -703,9 +703,19 @@ function isInteractable(element, hoverStylesMap) {
// Check if element has hover styles that change cursor to pointer
// This is to handle the case where an element's cursor is "auto", but resolves to "pointer" on hover
if (elementCursor === "auto") {
+ // TODO: we need a better algorithm to match the selector with better performance
for (const [selector, styles] of hoverStylesMap) {
- if (element.matches(selector) && styles.cursor === "pointer") {
- return true;
+ let shouldMatch = false;
+ for (const className of element.classList) {
+ if (selector.includes(className)) {
+ shouldMatch = true;
+ break;
+ }
+ }
+ if (shouldMatch || selector.includes(tagName)) {
+ if (element.matches(selector) && styles.cursor === "pointer") {
+ return true;
+ }
}
}
}
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 43b7c0878..6355d7a41 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -247,12 +247,17 @@ class ScrapedPage(BaseModel):
self._clean_up_func = clean_up_func
self._scrape_exclude = scrape_exclude
- def build_element_tree(self, fmt: ElementTreeFormat = ElementTreeFormat.HTML) -> str:
+ def build_element_tree(
+ self, fmt: ElementTreeFormat = ElementTreeFormat.HTML, html_need_skyvern_attrs: bool = True
+ ) -> str:
if fmt == ElementTreeFormat.JSON:
return json.dumps(self.element_tree_trimmed)
if fmt == ElementTreeFormat.HTML:
- return "".join(json_to_html(element) for element in self.element_tree_trimmed)
+ return "".join(
+ json_to_html(element, need_skyvern_attrs=html_need_skyvern_attrs)
+ for element in self.element_tree_trimmed
+ )
raise UnknownElementTreeFormat(fmt=fmt)