Skip to content

Automate browser based workflows with AI

abort scraping when too many elements (#3678)

LawyZhenge08778993eef
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index 2bc7758f9..f6631086c 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -1533,9 +1533,13 @@ async function buildTreeFromBody(
) {
window.GlobalSkyvernFrameIndex = frame_index;
}
+ const maxElementNumber = 15000;
const elementsAndResultArray = await buildElementTree(
document.documentElement,
frame,
+ false,
+ undefined,
+ maxElementNumber,
);
DomUtils.elementListCache = elementsAndResultArray[0];
return elementsAndResultArray;
@@ -1546,6 +1550,7 @@ async function buildElementTree(
frame,
full_tree = false,
hoverStylesMap = undefined,
+ maxElementNumber = 0,
) {
// Generate hover styles map at the start
if (hoverStylesMap === undefined) {
@@ -1570,6 +1575,13 @@ async function buildElementTree(
return;
}
+ if (maxElementNumber > 0 && elements.length >= maxElementNumber) {
+ _jsConsoleWarn(
+ "Max element number reached, aborting the element tree building",
+ );
+ return;
+ }
+
const tagName = element.tagName?.toLowerCase();
if (!tagName) {
_jsConsoleLog("get a null tagName");